lael-labs/linux

Documentation/cgroup-v1/cgroups.txt+1−1

@@ -8,7 +8,7 @@ Original copyright statements from cpusets.txt: · 7 unmodified lines
8	Portions Copyright (C) 2004 BULL SA.	8	Portions Copyright (C) 2004 BULL SA.
9	Portions Copyright (c) 2004-2006 Silicon Graphics, Inc.	9	Portions Copyright (c) 2004-2006 Silicon Graphics, Inc.
10	Modified by Paul Jackson <pj@sgi.com>	10	Modified by Paul Jackson <pj@sgi.com>
11	Modified by Christoph Lameter <clameter@sgi.com>	11	Modified by Christoph Lameter <cl@linux.com>
12		12
13	CONTENTS:	13	CONTENTS:
14	=========	14	=========

Documentation/cgroup-v1/cpusets.txt+1−1

@@ -6,7 +6,7 @@ Written by Simon.Derr@bull.net · 5 unmodified lines
6		6
7	Portions Copyright (c) 2004-2006 Silicon Graphics, Inc.	7	Portions Copyright (c) 2004-2006 Silicon Graphics, Inc.
8	Modified by Paul Jackson <pj@sgi.com>	8	Modified by Paul Jackson <pj@sgi.com>
9	Modified by Christoph Lameter <clameter@sgi.com>	9	Modified by Christoph Lameter <cl@linux.com>
10	Modified by Paul Menage <menage@google.com>	10	Modified by Paul Menage <menage@google.com>
11	Modified by Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>	11	Modified by Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
12		12

Documentation/cgroup-v2.txt+25−0

@@ -843,6 +843,15 @@ PAGE_SIZE multiple when read back. · 842 unmodified lines
843	Amount of memory used to cache filesystem data,	843	Amount of memory used to cache filesystem data,
844	including tmpfs and shared memory.	844	including tmpfs and shared memory.
845		845
		846	kernel_stack
		847
		848	Amount of memory allocated to kernel stacks.
		849
		850	slab
		851
		852	Amount of memory used for storing in-kernel data
		853	structures.
		854
846	sock	855	sock
847		856
848	Amount of memory used in network transmission buffers	857	Amount of memory used in network transmission buffers
@@ -871,6 +880,16 @@ PAGE_SIZE multiple when read back. · 22 unmodified lines
871	on the internal memory management lists used by the	880	on the internal memory management lists used by the
872	page reclaim algorithm	881	page reclaim algorithm
873		882
		883	slab_reclaimable
		884
		885	Part of "slab" that might be reclaimed, such as
		886	dentries and inodes.
		887
		888	slab_unreclaimable
		889
		890	Part of "slab" that cannot be reclaimed on memory
		891	pressure.
		892
874	pgfault	893	pgfault
875		894
876	Total number of page faults incurred	895	Total number of page faults incurred
@@ -1368,6 +1387,12 @@ system than killing the group. Otherwise, memory.max is there to · 491 unmodified lines
1368	limit this type of spillover and ultimately contain buggy or even	1387	limit this type of spillover and ultimately contain buggy or even
1369	malicious applications.	1388	malicious applications.
1370		1389
		1390	Setting the original memory.limit_in_bytes below the current usage was
		1391	subject to a race condition, where concurrent charges could cause the
		1392	limit setting to fail. memory.max on the other hand will first set the
		1393	limit to prevent new charges, and then reclaim and OOM kill until the
		1394	new limit is met - or the task writing to memory.max is killed.
		1395
1371	The combined memory+swap accounting and limiting is replaced by real	1396	The combined memory+swap accounting and limiting is replaced by real
1372	control over swap space.	1397	control over swap space.
1373		1398

Documentation/filesystems/proc.txt+18−0

@@ -43,6 +43,7 @@ Table of Contents · 42 unmodified lines
43	3.7 /proc/<pid>/task/<tid>/children - Information about task children	43	3.7 /proc/<pid>/task/<tid>/children - Information about task children
44	3.8 /proc/<pid>/fdinfo/<fd> - Information about opened file	44	3.8 /proc/<pid>/fdinfo/<fd> - Information about opened file
45	3.9 /proc/<pid>/map_files - Information about memory mapped files	45	3.9 /proc/<pid>/map_files - Information about memory mapped files
		46	3.10 /proc/<pid>/timerslack_ns - Task timerslack value
46		47
47	4 Configuring procfs	48	4 Configuring procfs
48	4.1 Mount options	49	4.1 Mount options
@@ -1862,6 +1863,23 @@ time one can open(2) mappings from the listings of two processes and · 1813 unmodified lines
1862	comparing their inode numbers to figure out which anonymous memory areas	1863	comparing their inode numbers to figure out which anonymous memory areas
1863	are actually shared.	1864	are actually shared.
1864		1865
		1866	3.10 /proc/<pid>/timerslack_ns - Task timerslack value
		1867	---------------------------------------------------------
		1868	This file provides the value of the task's timerslack value in nanoseconds.
		1869	This value specifies a amount of time that normal timers may be deferred
		1870	in order to coalesce timers and avoid unnecessary wakeups.
		1871
		1872	This allows a task's interactivity vs power consumption trade off to be
		1873	adjusted.
		1874
		1875	Writing 0 to the file will set the tasks timerslack to the default value.
		1876
		1877	Valid values are from 0 - ULLONG_MAX
		1878
		1879	An application setting the value must have PTRACE_MODE_ATTACH_FSCREDS level
		1880	permissions on the task specified to change its timerslack_ns value.
		1881
		1882
1865	------------------------------------------------------------------------------	1883	------------------------------------------------------------------------------
1866	Configuring procfs	1884	Configuring procfs
1867	------------------------------------------------------------------------------	1885	------------------------------------------------------------------------------

Documentation/sysctl/vm.txt+18−0

@@ -803,6 +803,24 @@ performance impact. Reclaim code needs to take various locks to find freeable · 802 unmodified lines
803	directory and inode objects. With vfs_cache_pressure=1000, it will look for	803	directory and inode objects. With vfs_cache_pressure=1000, it will look for
804	ten times more freeable objects than there are.	804	ten times more freeable objects than there are.
805		805
		806	=============================================================
		807
		808	watermark_scale_factor:
		809
		810	This factor controls the aggressiveness of kswapd. It defines the
		811	amount of memory left in a node/system before kswapd is woken up and
		812	how much memory needs to be free before kswapd goes back to sleep.
		813
		814	The unit is in fractions of 10,000. The default value of 10 means the
		815	distances between watermarks are 0.1% of the available memory in the
		816	node/system. The maximum value is 1000, or 10% of memory.
		817
		818	A high rate of threads entering direct reclaim (allocstall) or kswapd
		819	going to sleep prematurely (kswapd_low_wmark_hit_quickly) can indicate
		820	that the number of free pages kswapd maintains for latency reasons is
		821	too small for the allocation bursts occurring in the system. This knob
		822	can then be used to tune kswapd aggressiveness accordingly.
		823
806	==============================================================	824	==============================================================
807		825
808	zone_reclaim_mode:	826	zone_reclaim_mode:

Documentation/vm/transhuge.txt+22−0

@@ -113,9 +113,26 @@ guaranteed, but it may be more likely in case the allocation is for a · 112 unmodified lines
113	MADV_HUGEPAGE region.	113	MADV_HUGEPAGE region.
114		114
115	echo always >/sys/kernel/mm/transparent_hugepage/defrag	115	echo always >/sys/kernel/mm/transparent_hugepage/defrag
		116	echo defer >/sys/kernel/mm/transparent_hugepage/defrag
116	echo madvise >/sys/kernel/mm/transparent_hugepage/defrag	117	echo madvise >/sys/kernel/mm/transparent_hugepage/defrag
117	echo never >/sys/kernel/mm/transparent_hugepage/defrag	118	echo never >/sys/kernel/mm/transparent_hugepage/defrag
118		119
		120	"always" means that an application requesting THP will stall on allocation
		121	failure and directly reclaim pages and compact memory in an effort to
		122	allocate a THP immediately. This may be desirable for virtual machines
		123	that benefit heavily from THP use and are willing to delay the VM start
		124	to utilise them.
		125
		126	"defer" means that an application will wake kswapd in the background
		127	to reclaim pages and wake kcompact to compact memory so that THP is
		128	available in the near future. It's the responsibility of khugepaged
		129	to then install the THP pages later.
		130
		131	"madvise" will enter direct reclaim like "always" but only for regions
		132	that are have used madvise(MADV_HUGEPAGE). This is the default behaviour.
		133
		134	"never" should be self-explanatory.
		135
119	By default kernel tries to use huge zero page on read page fault.	136	By default kernel tries to use huge zero page on read page fault.
120	It's possible to disable huge zero page by writing 0 or enable it	137	It's possible to disable huge zero page by writing 0 or enable it
121	back by writing 1:	138	back by writing 1:
@@ -229,6 +246,11 @@ thp_split_page is incremented every time a huge page is split into base · 107 unmodified lines
229	thp_split_page_failed is is incremented if kernel fails to split huge	246	thp_split_page_failed is is incremented if kernel fails to split huge
230	page. This can happen if the page was pinned by somebody.	247	page. This can happen if the page was pinned by somebody.
231		248
		249	thp_deferred_split_page is incremented when a huge page is put onto split
		250	queue. This happens when a huge page is partially unmapped and
		251	splitting it would free up some memory. Pages on split queue are
		252	going to be split under memory pressure.
		253
232	thp_split_pmd is incremented every time a PMD split into table of PTEs.	254	thp_split_pmd is incremented every time a PMD split into table of PTEs.
233	This can happen, for instance, when application calls mprotect() or	255	This can happen, for instance, when application calls mprotect() or
234	munmap() on part of huge page. It doesn't split huge page, only	256	munmap() on part of huge page. It doesn't split huge page, only

MAINTAINERS+1−2

 PER-CPU MEMORY ALLOCATOR
 M:	Tejun Heo <tj@kernel.org>
-M:	Christoph Lameter <cl@linux-foundation.org>
+M:	Christoph Lameter <cl@linux.com>
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git
 S:	Maintained
 F:	include/linux/percpu*.h
 F:	include/uapi/linux/cdrom.h
 UNISYS S-PAR DRIVERS
-M:	Benjamin Romer <benjamin.romer@unisys.com>
 M:	David Kershner <david.kershner@unisys.com>
 L:	sparmaintainer@unisys.com (Unisys internal)
 S:	Supported

arch/arc/include/asm/hugepage.h+0−3

 #define pmd_mkyoung(pmd)	pte_pmd(pte_mkyoung(pmd_pte(pmd)))
 #define pmd_mkhuge(pmd)		pte_pmd(pte_mkhuge(pmd_pte(pmd)))
 #define pmd_mknotpresent(pmd)	pte_pmd(pte_mknotpresent(pmd_pte(pmd)))
-#define pmd_mksplitting(pmd)	pte_pmd(pte_mkspecial(pmd_pte(pmd)))
 #define pmd_mkclean(pmd)	pte_pmd(pte_mkclean(pmd_pte(pmd)))
 #define pmd_write(pmd)		pte_write(pmd_pte(pmd))
 #define pmd_young(pmd)		pte_young(pmd_pte(pmd))
 #define pmd_pfn(pmd)		pte_pfn(pmd_pte(pmd))
 #define pmd_dirty(pmd)		pte_dirty(pmd_pte(pmd))
-#define pmd_special(pmd)	pte_special(pmd_pte(pmd))
 #define mk_pmd(page, prot)	pte_pmd(mk_pte(page, prot))
 #define pmd_trans_huge(pmd)	(pmd_val(pmd) & _PAGE_HW_SZ)
-#define pmd_trans_splitting(pmd)	(pmd_trans_huge(pmd) && pmd_special(pmd))
 #define pfn_pmd(pfn, prot)	(__pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)))

arch/arm/mm/fault.c+1−1

@@ -346,7 +346,7 @@ retry: · 345 unmodified lines
346	up_read(&mm->mmap_sem);	346	up_read(&mm->mmap_sem);
347		347
348	/*	348	/*
349	* Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR	349	* Handle the "normal" case first - VM_FAULT_MAJOR
350	*/	350	*/
351	if (likely(!(fault & (VM_FAULT_ERROR \| VM_FAULT_BADMAP \| VM_FAULT_BADACCESS))))	351	if (likely(!(fault & (VM_FAULT_ERROR \| VM_FAULT_BADMAP \| VM_FAULT_BADACCESS))))
352	return 0;	352	return 0;

arch/arm/mm/mmu.c+3−3

@@ -732,7 +732,7 @@ static void *__init late_alloc(unsigned long sz) · 731 unmodified lines
732	return ptr;	732	return ptr;
733	}	733	}
734		734
735	static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr,	735	static pte_t * __init arm_pte_alloc(pmd_t *pmd, unsigned long addr,
736	unsigned long prot,	736	unsigned long prot,
737	void (alloc)(unsigned long sz))	737	void (alloc)(unsigned long sz))
738	{	738	{
@@ -747,7 +747,7 @@ static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr, · 8 unmodified lines
747	static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr,	747	static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr,
748	unsigned long prot)	748	unsigned long prot)
749	{	749	{
750	return pte_alloc(pmd, addr, prot, early_alloc);	750	return arm_pte_alloc(pmd, addr, prot, early_alloc);
751	}	751	}
752		752
753	static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,	753	static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
@@ -756,7 +756,7 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, · 2 unmodified lines
756	void (alloc)(unsigned long sz),	756	void (alloc)(unsigned long sz),
757	bool ng)	757	bool ng)
758	{	758	{
759	pte_t *pte = pte_alloc(pmd, addr, type->prot_l1, alloc);	759	pte_t *pte = arm_pte_alloc(pmd, addr, type->prot_l1, alloc);
760	do {	760	do {
761	set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)),	761	set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)),
762	ng ? PTE_EXT_NG : 0);	762	ng ? PTE_EXT_NG : 0);

arch/arm/mm/pgd.c+1−1

@@ -80,7 +80,7 @@ pgd_t pgd_alloc(struct mm_struct mm) · 79 unmodified lines
80	if (!new_pmd)	80	if (!new_pmd)
81	goto no_pmd;	81	goto no_pmd;
82		82
83	new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);	83	new_pte = pte_alloc_map(mm, new_pmd, 0);
84	if (!new_pte)	84	if (!new_pte)
85	goto no_pte;	85	goto no_pte;
86		86

arch/arm64/mm/fault.c+1−1

@@ -304,7 +304,7 @@ retry: · 303 unmodified lines
304	up_read(&mm->mmap_sem);	304	up_read(&mm->mmap_sem);
305		305
306	/*	306	/*
307	* Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR	307	* Handle the "normal" case first - VM_FAULT_MAJOR
308	*/	308	*/
309	if (likely(!(fault & (VM_FAULT_ERROR \| VM_FAULT_BADMAP \|	309	if (likely(!(fault & (VM_FAULT_ERROR \| VM_FAULT_BADMAP \|
310	VM_FAULT_BADACCESS))))	310	VM_FAULT_BADACCESS))))

arch/arm64/mm/hugetlbpage.c+1−1

@@ -124,7 +124,7 @@ pte_t huge_pte_alloc(struct mm_struct mm, · 123 unmodified lines
124	* will be no pte_unmap() to correspond with this	124	* will be no pte_unmap() to correspond with this
125	* pte_alloc_map().	125	* pte_alloc_map().
126	*/	126	*/
127	pte = pte_alloc_map(mm, NULL, pmd, addr);	127	pte = pte_alloc_map(mm, pmd, addr);
128	} else if (sz == PMD_SIZE) {	128	} else if (sz == PMD_SIZE) {
129	if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&	129	if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
130	pud_none(*pud))	130	pud_none(*pud))

arch/c6x/Kconfig+1−0

@@ -36,6 +36,7 @@ config GENERIC_HWEIGHT · 35 unmodified lines
36		36
37	config GENERIC_BUG	37	config GENERIC_BUG
38	def_bool y	38	def_bool y
		39	depends on BUG
39		40
40	config C6X_BIG_KERNEL	41	config C6X_BIG_KERNEL
41	bool "Build a big kernel"	42	bool "Build a big kernel"

arch/ia64/include/asm/io.h+1−0

@@ -433,6 +433,7 @@ static inline void __iomem * ioremap_cache (unsigned long phys_addr, unsigned lo · 432 unmodified lines
433	return ioremap(phys_addr, size);	433	return ioremap(phys_addr, size);
434	}	434	}
435	#define ioremap_cache ioremap_cache	435	#define ioremap_cache ioremap_cache
		436	#define ioremap_uc ioremap_nocache
436		437
437		438
438	/*	439	/*

arch/ia64/include/asm/rwsem.h+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3	*	3	*
4	* Copyright (C) 2003 Ken Chen <kenneth.w.chen@intel.com>	4	* Copyright (C) 2003 Ken Chen <kenneth.w.chen@intel.com>
5	* Copyright (C) 2003 Asit Mallick <asit.k.mallick@intel.com>	5	* Copyright (C) 2003 Asit Mallick <asit.k.mallick@intel.com>
6	* Copyright (C) 2005 Christoph Lameter <clameter@sgi.com>	6	* Copyright (C) 2005 Christoph Lameter <cl@linux.com>
7	*	7	*
8	* Based on asm-i386/rwsem.h and other architecture implementation.	8	* Based on asm-i386/rwsem.h and other architecture implementation.
9	*	9	*

arch/ia64/mm/hugetlbpage.c+1−1

@@ -38,7 +38,7 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) · 37 unmodified lines
38	if (pud) {	38	if (pud) {
39	pmd = pmd_alloc(mm, pud, taddr);	39	pmd = pmd_alloc(mm, pud, taddr);
40	if (pmd)	40	if (pmd)
41	pte = pte_alloc_map(mm, NULL, pmd, taddr);	41	pte = pte_alloc_map(mm, pmd, taddr);
42	}	42	}
43	return pte;	43	return pte;
44	}	44	}

arch/metag/mm/hugetlbpage.c+1−1

@@ -67,7 +67,7 @@ pte_t huge_pte_alloc(struct mm_struct mm, · 66 unmodified lines
67	pgd = pgd_offset(mm, addr);	67	pgd = pgd_offset(mm, addr);
68	pud = pud_offset(pgd, addr);	68	pud = pud_offset(pgd, addr);
69	pmd = pmd_offset(pud, addr);	69	pmd = pmd_offset(pud, addr);
70	pte = pte_alloc_map(mm, NULL, pmd, addr);	70	pte = pte_alloc_map(mm, pmd, addr);
71	pgd->pgd &= ~_PAGE_SZ_MASK;	71	pgd->pgd &= ~_PAGE_SZ_MASK;
72	pgd->pgd \|= _PAGE_SZHUGE;	72	pgd->pgd \|= _PAGE_SZHUGE;
73		73

arch/mips/mm/gup.c+1−1

@@ -64,7 +64,7 @@ static inline void get_head_page_multiple(struct page *page, int nr) · 63 unmodified lines
64	{	64	{
65	VM_BUG_ON(page != compound_head(page));	65	VM_BUG_ON(page != compound_head(page));
66	VM_BUG_ON(page_count(page) == 0);	66	VM_BUG_ON(page_count(page) == 0);
67	atomic_add(nr, &page->_count);	67	page_ref_add(page, nr);
68	SetPageReferenced(page);	68	SetPageReferenced(page);
69	}	69	}
70		70

arch/mn10300/Kconfig+1−0

@@ -53,6 +53,7 @@ config GENERIC_HWEIGHT · 52 unmodified lines
53		53
54	config GENERIC_BUG	54	config GENERIC_BUG
55	def_bool y	55	def_bool y
		56	depends on BUG
56		57
57	config QUICKLIST	58	config QUICKLIST
58	def_bool y	59	def_bool y

arch/mn10300/kernel/fpu-nofpu.c+1−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	* 2 of the Licence, or (at your option) any later version.	9	* 2 of the Licence, or (at your option) any later version.
10	*/	10	*/
11	#include <asm/fpu.h>	11	#include <asm/fpu.h>
		12	#include <asm/elf.h>
12		13
13	/*	14	/*
14	* handle an FPU operational exception	15	* handle an FPU operational exception

arch/parisc/mm/hugetlbpage.c+1−1

@@ -63,7 +63,7 @@ pte_t huge_pte_alloc(struct mm_struct mm, · 62 unmodified lines
63	if (pud) {	63	if (pud) {
64	pmd = pmd_alloc(mm, pud, addr);	64	pmd = pmd_alloc(mm, pud, addr);
65	if (pmd)	65	if (pmd)
66	pte = pte_alloc_map(mm, NULL, pmd, addr);	66	pte = pte_alloc_map(mm, pmd, addr);
67	}	67	}
68	return pte;	68	return pte;
69	}	69	}

arch/powerpc/Kconfig+1−0

@@ -158,6 +158,7 @@ config PPC · 157 unmodified lines
158	select ARCH_HAS_DEVMEM_IS_ALLOWED	158	select ARCH_HAS_DEVMEM_IS_ALLOWED
159	select HAVE_ARCH_SECCOMP_FILTER	159	select HAVE_ARCH_SECCOMP_FILTER
160	select ARCH_HAS_UBSAN_SANITIZE_ALL	160	select ARCH_HAS_UBSAN_SANITIZE_ALL
		161	select ARCH_SUPPORTS_DEFERRED_STRUCT_PAGE_INIT
161		162
162	config GENERIC_CSUM	163	config GENERIC_CSUM
163	def_bool CPU_LITTLE_ENDIAN	164	def_bool CPU_LITTLE_ENDIAN

arch/powerpc/kernel/rtasd.c+2−7

 static unsigned int event_scan;
 static unsigned int rtas_event_scan_rate;
-static int full_rtas_msgs = 0;
+static bool full_rtas_msgs;
 /* Stop logging to nvram after first fatal error */
 static int logging_enabled; /* Until we initialize everything,
 static int __init rtasmsgs_setup(char *str)
 {
-	if (strcmp(str, "on") == 0)
+	return (kstrtobool(str, &full_rtas_msgs) == 0);
-		full_rtas_msgs = 1;
-	else if (strcmp(str, "off") == 0)
-		full_rtas_msgs = 0;
-	return 1;
 }
 __setup("rtasmsgs=", rtasmsgs_setup);

arch/powerpc/kernel/traps.c+2−3

 #ifdef CONFIG_SMP
 	printk("SMP NR_CPUS=%d ", NR_CPUS);
 #endif
-#ifdef CONFIG_DEBUG_PAGEALLOC
+	if (debug_pagealloc_enabled())
-	printk("DEBUG_PAGEALLOC ");
+		printk("DEBUG_PAGEALLOC ");
-#endif
 #ifdef CONFIG_NUMA
 	printk("NUMA ");
 #endif

arch/powerpc/mm/hash_utils_64.c+20−16

@@ -255,8 +255,10 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend, · 254 unmodified lines
255		255
256	if (ret < 0)	256	if (ret < 0)
257	break;	257	break;
		258
258	#ifdef CONFIG_DEBUG_PAGEALLOC	259	#ifdef CONFIG_DEBUG_PAGEALLOC
259	if ((paddr >> PAGE_SHIFT) < linear_map_hash_count)	260	if (debug_pagealloc_enabled() &&
		261	(paddr >> PAGE_SHIFT) < linear_map_hash_count)
260	linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret \| 0x80;	262	linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret \| 0x80;
261	#endif /* CONFIG_DEBUG_PAGEALLOC */	263	#endif /* CONFIG_DEBUG_PAGEALLOC */
262	}	264	}
@@ -512,17 +514,17 @@ static void __init htab_init_page_sizes(void) · 249 unmodified lines
512	if (mmu_has_feature(MMU_FTR_16M_PAGE))	514	if (mmu_has_feature(MMU_FTR_16M_PAGE))
513	memcpy(mmu_psize_defs, mmu_psize_defaults_gp,	515	memcpy(mmu_psize_defs, mmu_psize_defaults_gp,
514	sizeof(mmu_psize_defaults_gp));	516	sizeof(mmu_psize_defaults_gp));
515	found:	517	found:
516	#ifndef CONFIG_DEBUG_PAGEALLOC	518	if (!debug_pagealloc_enabled()) {
517	/*	519	/*
518	* Pick a size for the linear mapping. Currently, we only support	520	* Pick a size for the linear mapping. Currently, we only
519	* 16M, 1M and 4K which is the default	521	* support 16M, 1M and 4K which is the default
520	*/	522	*/
521	if (mmu_psize_defs[MMU_PAGE_16M].shift)	523	if (mmu_psize_defs[MMU_PAGE_16M].shift)
522	mmu_linear_psize = MMU_PAGE_16M;	524	mmu_linear_psize = MMU_PAGE_16M;
523	else if (mmu_psize_defs[MMU_PAGE_1M].shift)	525	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
524	mmu_linear_psize = MMU_PAGE_1M;	526	mmu_linear_psize = MMU_PAGE_1M;
525	#endif /* CONFIG_DEBUG_PAGEALLOC */	527	}
526		528
527	#ifdef CONFIG_PPC_64K_PAGES	529	#ifdef CONFIG_PPC_64K_PAGES
528	/*	530	/*
@@ -721,10 +723,12 @@ static void __init htab_initialize(void) · 192 unmodified lines
721	prot = pgprot_val(PAGE_KERNEL);	723	prot = pgprot_val(PAGE_KERNEL);
722		724
723	#ifdef CONFIG_DEBUG_PAGEALLOC	725	#ifdef CONFIG_DEBUG_PAGEALLOC
724	linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT;	726	if (debug_pagealloc_enabled()) {
725	linear_map_hash_slots = __va(memblock_alloc_base(linear_map_hash_count,	727	linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT;
726	1, ppc64_rma_size));	728	linear_map_hash_slots = __va(memblock_alloc_base(
727	memset(linear_map_hash_slots, 0, linear_map_hash_count);	729	linear_map_hash_count, 1, ppc64_rma_size));
		730	memset(linear_map_hash_slots, 0, linear_map_hash_count);
		731	}
728	#endif /* CONFIG_DEBUG_PAGEALLOC */	732	#endif /* CONFIG_DEBUG_PAGEALLOC */
729		733
730	/* On U3 based machines, we need to reserve the DART area and	734	/* On U3 based machines, we need to reserve the DART area and

arch/powerpc/mm/init_32.c+4−4

@@ -112,10 +112,10 @@ void __init MMU_setup(void) · 111 unmodified lines
112	if (strstr(boot_command_line, "noltlbs")) {	112	if (strstr(boot_command_line, "noltlbs")) {
113	__map_without_ltlbs = 1;	113	__map_without_ltlbs = 1;
114	}	114	}
115	#ifdef CONFIG_DEBUG_PAGEALLOC	115	if (debug_pagealloc_enabled()) {
116	__map_without_bats = 1;	116	__map_without_bats = 1;
117	__map_without_ltlbs = 1;	117	__map_without_ltlbs = 1;
118	#endif	118	}
119	}	119	}
120		120
121	/*	121	/*

arch/powerpc/mm/mmu_context_hash64.c+1−2

 	/* drop all the pending references */
 	count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
 	/* We allow PTE_FRAG_NR fragments from a PTE page */
-	count = atomic_sub_return(PTE_FRAG_NR - count, &page->_count);
+	if (page_ref_sub_and_test(page, PTE_FRAG_NR - count)) {
-	if (!count) {
 		pgtable_page_dtor(page);
 		free_hot_cold_page(page, 0);
 	}

arch/powerpc/mm/pgtable_64.c+1−1

@@ -403,7 +403,7 @@ static pte_t __alloc_for_cache(struct mm_struct mm, int kernel) · 402 unmodified lines
403	* count.	403	* count.
404	*/	404	*/
405	if (likely(!mm->context.pte_frag)) {	405	if (likely(!mm->context.pte_frag)) {
406	atomic_set(&page->_count, PTE_FRAG_NR);	406	set_page_count(page, PTE_FRAG_NR);
407	mm->context.pte_frag = ret + PTE_FRAG_SIZE;	407	mm->context.pte_frag = ret + PTE_FRAG_SIZE;
408	}	408	}
409	spin_unlock(&mm->page_table_lock);	409	spin_unlock(&mm->page_table_lock);

arch/powerpc/platforms/512x/mpc512x_shared.c+1−1

@@ -188,7 +188,7 @@ static struct fsl_diu_shared_fb __attribute__ ((__aligned__(8))) diu_shared_fb; · 187 unmodified lines
188	static inline void mpc512x_free_bootmem(struct page *page)	188	static inline void mpc512x_free_bootmem(struct page *page)
189	{	189	{
190	BUG_ON(PageTail(page));	190	BUG_ON(PageTail(page));
191	BUG_ON(atomic_read(&page->_count) > 1);	191	BUG_ON(page_ref_count(page) > 1);
192	free_reserved_page(page);	192	free_reserved_page(page);
193	}	193	}
194		194

arch/powerpc/platforms/pseries/hotplug-cpu.c+2−8

 static enum cpu_state_vals default_offline_state = CPU_STATE_OFFLINE;
-static int cede_offline_enabled __read_mostly = 1;
+static bool cede_offline_enabled __read_mostly = true;
 /*
  * Enable/disable cede_offline when available.
  */
 static int __init setup_cede_offline(char *str)
 {
-	if (!strcmp(str, "off"))
+	return (kstrtobool(str, &cede_offline_enabled) == 0);
-		cede_offline_enabled = 0;
-	else if (!strcmp(str, "on"))
-		cede_offline_enabled = 1;
-	else
-		return 0;
-	return 1;
 }
 __setup("cede_offline=", setup_cede_offline);

arch/s390/kernel/time.c+2−6

 /*
  * Server Time Protocol (STP) code.
  */
-static int stp_online;
+static bool stp_online;
 static struct stp_sstpi stp_info;
 static void *stp_page;
 static int __init early_parse_stp(char *p)
 {
-	if (strncmp(p, "off", 3) == 0)
+	return kstrtobool(p, &stp_online);
-		stp_online = 0;
-	else if (strncmp(p, "on", 2) == 0)
-		stp_online = 1;
-	return 0;
 }
 early_param("stp", early_parse_stp);

arch/s390/kernel/topology.c+2−5

 static void topology_work_fn(struct work_struct *work);
 static struct sysinfo_15_1_x *tl_info;
-static int topology_enabled = 1;
+static bool topology_enabled = true;
 static DECLARE_WORK(topology_work, topology_work_fn);
 /*
 static int __init early_parse_topology(char *p)
 {
-	if (strncmp(p, "off", 3))
+	return kstrtobool(p, &topology_enabled);
-		return 0;
-	topology_enabled = 0;
-	return 0;
 }
 early_param("topology", early_parse_topology);

arch/sh/mm/hugetlbpage.c+1−1

@@ -35,7 +35,7 @@ pte_t huge_pte_alloc(struct mm_struct mm, · 34 unmodified lines
35	if (pud) {	35	if (pud) {
36	pmd = pmd_alloc(mm, pud, addr);	36	pmd = pmd_alloc(mm, pud, addr);
37	if (pmd)	37	if (pmd)
38	pte = pte_alloc_map(mm, NULL, pmd, addr);	38	pte = pte_alloc_map(mm, pmd, addr);
39	}	39	}
40	}	40	}
41		41

arch/sparc/mm/hugetlbpage.c+1−1

@@ -146,7 +146,7 @@ pte_t huge_pte_alloc(struct mm_struct mm, · 145 unmodified lines
146	if (pud) {	146	if (pud) {
147	pmd = pmd_alloc(mm, pud, addr);	147	pmd = pmd_alloc(mm, pud, addr);
148	if (pmd)	148	if (pmd)
149	pte = pte_alloc_map(mm, NULL, pmd, addr);	149	pte = pte_alloc_map(mm, pmd, addr);
150	}	150	}
151	return pte;	151	return pte;
152	}	152	}

arch/tile/mm/hugetlbpage.c+1−1

@@ -77,7 +77,7 @@ pte_t huge_pte_alloc(struct mm_struct mm, · 76 unmodified lines
77	else {	77	else {
78	if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])	78	if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
79	panic("Unexpected page size %#lx\n", sz);	79	panic("Unexpected page size %#lx\n", sz);
80	return pte_alloc_map(mm, NULL, pmd, addr);	80	return pte_alloc_map(mm, pmd, addr);
81	}	81	}
82	}	82	}
83	#else	83	#else

arch/tile/mm/init.c+7−4

 		panic("pgtable_cache_init(): Cannot create pgd cache");
 }
-#ifdef CONFIG_DEBUG_PAGEALLOC
-static long __write_once initfree;
-#else
 static long __write_once initfree = 1;
-#endif
+static bool __write_once set_initfree_done;
 /* Select whether to free (1) or mark unusable (0) the __init pages. */
 static int __init set_initfree(char *str)
 {
 	long val;
 	if (kstrtol(str, 0, &val) == 0) {
+		set_initfree_done = true;
 		initfree = val;
 		pr_info("initfree: %s free init pages\n",
 			initfree ? "will" : "won't");
 {
 	unsigned long addr = (unsigned long) begin;
+	/* Prefer user request first */
+	if (!set_initfree_done) {
+		if (debug_pagealloc_enabled())
+			initfree = 0;
+	}
 	if (kdata_huge && !initfree) {
 		pr_warn("Warning: ignoring initfree=0: incompatible with kdata=huge\n");
 		initfree = 1;

arch/um/kernel/skas/mmu.c+1−1

@@ -31,7 +31,7 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc, · 30 unmodified lines
31	if (!pmd)	31	if (!pmd)
32	goto out_pmd;	32	goto out_pmd;
33		33
34	pte = pte_alloc_map(mm, NULL, pmd, proc);	34	pte = pte_alloc_map(mm, pmd, proc);
35	if (!pte)	35	if (!pte)
36	goto out_pte;	36	goto out_pte;
37		37

arch/unicore32/mm/fault.c+1−1

@@ -276,7 +276,7 @@ retry: · 275 unmodified lines
276	up_read(&mm->mmap_sem);	276	up_read(&mm->mmap_sem);
277		277
278	/*	278	/*
279	* Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR	279	* Handle the "normal" case first - VM_FAULT_MAJOR
280	*/	280	*/
281	if (likely(!(fault &	281	if (likely(!(fault &
282	(VM_FAULT_ERROR \| VM_FAULT_BADMAP \| VM_FAULT_BADACCESS))))	282	(VM_FAULT_ERROR \| VM_FAULT_BADMAP \| VM_FAULT_BADACCESS))))

arch/unicore32/mm/pgd.c+1−1

@@ -54,7 +54,7 @@ pgd_t get_pgd_slow(struct mm_struct mm) · 53 unmodified lines
54	if (!new_pmd)	54	if (!new_pmd)
55	goto no_pmd;	55	goto no_pmd;
56		56
57	new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);	57	new_pte = pte_alloc_map(mm, new_pmd, 0);
58	if (!new_pte)	58	if (!new_pte)
59	goto no_pte;	59	goto no_pte;
60		60

arch/x86/kernel/aperture_64.c+2−10

 	return 0;
 }
-static int gart_fix_e820 __initdata = 1;
+static bool gart_fix_e820 __initdata = true;
 static int __init parse_gart_mem(char *p)
 {
-	if (!p)
+	return kstrtobool(p, &gart_fix_e820);
-		return -EINVAL;
-	if (!strncmp(p, "off", 3))
-		gart_fix_e820 = 0;
-	else if (!strncmp(p, "on", 2))
-		gart_fix_e820 = 1;
-	return 0;
 }
 early_param("gart_fix_e820", parse_gart_mem);

arch/x86/kernel/tboot.c+1−1

@@ -135,7 +135,7 @@ static int map_tboot_page(unsigned long vaddr, unsigned long pfn, · 134 unmodified lines
135	pmd = pmd_alloc(&tboot_mm, pud, vaddr);	135	pmd = pmd_alloc(&tboot_mm, pud, vaddr);
136	if (!pmd)	136	if (!pmd)
137	return -1;	137	return -1;
138	pte = pte_alloc_map(&tboot_mm, NULL, pmd, vaddr);	138	pte = pte_alloc_map(&tboot_mm, pmd, vaddr);
139	if (!pte)	139	if (!pte)
140	return -1;	140	return -1;
141	set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));	141	set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));

arch/x86/mm/gup.c+1−1

@@ -131,7 +131,7 @@ static inline void get_head_page_multiple(struct page *page, int nr) · 130 unmodified lines
131	{	131	{
132	VM_BUG_ON_PAGE(page != compound_head(page), page);	132	VM_BUG_ON_PAGE(page != compound_head(page), page);
133	VM_BUG_ON_PAGE(page_count(page) == 0, page);	133	VM_BUG_ON_PAGE(page_count(page) == 0, page);
134	atomic_add(nr, &page->_count);	134	page_ref_add(page, nr);
135	SetPageReferenced(page);	135	SetPageReferenced(page);
136	}	136	}
137		137

arch/xtensa/mm/fault.c+1−1

@@ -146,7 +146,7 @@ good_area: · 145 unmodified lines
146	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);	146	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
147	if (flags & VM_FAULT_MAJOR)	147	if (flags & VM_FAULT_MAJOR)
148	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);	148	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
149	else if (flags & VM_FAULT_MINOR)	149	else
150	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);	150	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
151		151
152	return;	152	return;

crypto/async_tx/async_pq.c+1−1

@@ -444,7 +444,7 @@ static int __init async_pq_init(void) · 443 unmodified lines
444		444
445	static void __exit async_pq_exit(void)	445	static void __exit async_pq_exit(void)
446	{	446	{
447	put_page(pq_scribble_page);	447	__free_page(pq_scribble_page);
448	}	448	}
449		449
450	module_init(async_pq_init);	450	module_init(async_pq_init);

drivers/ata/pata_hpt366.c+5−8

 			       const char * const list[])
 {
 	unsigned char model_num[ATA_ID_PROD_LEN + 1];
-	int i = 0;
+	int i;
 	ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
-	while (list[i] != NULL) {
+	i = match_string(list, -1, model_num);
-		if (!strcmp(list[i], model_num)) {
+	if (i >= 0) {
-			pr_warn("%s is not supported for %s\n",
+		pr_warn("%s is not supported for %s\n", modestr, list[i]);
-				modestr, list[i]);
+		return 1;
-			return 1;
-		}
-		i++;
 	}
 	return 0;
 }

drivers/base/property.c+4−8

 	const char *propname, const char *string)
 {
 	const char **values;
-	int nval, ret, i;
+	int nval, ret;
 	nval = fwnode_property_read_string_array(fwnode, propname, NULL, 0);
 	if (nval < 0)
 	if (ret < 0)
 		goto out;
-	ret = -ENODATA;
+	ret = match_string(values, nval, string);
-	for (i = 0; i < nval; i++) {
+	if (ret < 0)
-		if (!strcmp(values[i], string)) {
+		ret = -ENODATA;
-			ret = i;
-			break;
-		}
-	}
 out:
 	kfree(values);
 	return ret;

drivers/block/aoe/aoecmd.c+2−2

@@ -875,7 +875,7 @@ bio_pageinc(struct bio *bio) · 874 unmodified lines
875	* compound pages is no longer allowed by the kernel.	875	* compound pages is no longer allowed by the kernel.
876	*/	876	*/
877	page = compound_head(bv.bv_page);	877	page = compound_head(bv.bv_page);
878	atomic_inc(&page->_count);	878	page_ref_inc(page);
879	}	879	}
880	}	880	}
881		881
@@ -888,7 +888,7 @@ bio_pagedec(struct bio *bio) · 6 unmodified lines
888		888
889	bio_for_each_segment(bv, bio, iter) {	889	bio_for_each_segment(bv, bio, iter) {
890	page = compound_head(bv.bv_page);	890	page = compound_head(bv.bv_page);
891	atomic_dec(&page->_count);	891	page_ref_dec(page);
892	}	892	}
893	}	893	}
894		894

drivers/firmware/broadcom/bcm47xx_nvram.c+2−3

 found:
 	__ioread32_copy(nvram_buf, header, sizeof(*header) / 4);
-	header = (struct nvram_header *)nvram_buf;
+	nvram_len = ((struct nvram_header *)(nvram_buf))->len;
-	nvram_len = header->len;
 	if (nvram_len > size) {
 		pr_err("The nvram size according to the header seems to be bigger than the partition on flash\n");
 		nvram_len = size;
 	}
 	if (nvram_len >= NVRAM_SPACE) {
 		pr_err("nvram on flash (%i bytes) is bigger than the reserved space in memory, will just copy the first %i bytes\n",
-		       header->len, NVRAM_SPACE - 1);
+		       nvram_len, NVRAM_SPACE - 1);
 		nvram_len = NVRAM_SPACE - 1;
 	}
 	/* proceed reading data after header */

drivers/gpu/drm/drm_edid_load.c+6−11

 	int i, valid_extensions = 0;
 	bool print_bad_edid = !connector->bad_edid_counter || (drm_debug & DRM_UT_KMS);
-	builtin = 0;
+	builtin = match_string(generic_edid_name, GENERIC_EDIDS, name);
-	for (i = 0; i < GENERIC_EDIDS; i++) {
+	if (builtin >= 0) {
-		if (strcmp(name, generic_edid_name[i]) == 0) {
+		fwdata = generic_edid[builtin];
-			fwdata = generic_edid[i];
+		fwsize = sizeof(generic_edid[builtin]);
-			fwsize = sizeof(generic_edid[i]);
+	} else {
-			builtin = 1;
-			break;
-		}
-	}
-	if (!builtin) {
 		struct platform_device *pdev;
 		int err;
 	}
 	DRM_INFO("Got %s EDID base block and %d extension%s from "
-	    "\"%s\" for connector \"%s\"\n", builtin ? "built-in" :
+	    "\"%s\" for connector \"%s\"\n", (builtin >= 0) ? "built-in" :
 	    "external", valid_extensions, valid_extensions == 1 ? "" : "s",
 	    name, connector_name);

drivers/ide/hpt366.c+2−7

 	.timings	= &hpt37x_timings
 };
-static int check_in_drive_list(ide_drive_t *drive, const char **list)
+static bool check_in_drive_list(ide_drive_t *drive, const char **list)
 {
-	char *m = (char *)&drive->id[ATA_ID_PROD];
+	return match_string(list, -1, (char *)&drive->id[ATA_ID_PROD]) >= 0;
-	while (*list)
-		if (!strcmp(*list++, m))
-			return 1;
-	return 0;
 }
 static struct hpt_info *hpt3xx_get_info(struct device *dev)

drivers/net/ethernet/freescale/gianfar.c+1−1

@@ -2944,7 +2944,7 @@ static bool gfar_add_rx_frag(struct gfar_rx_buff *rxb, u32 lstatus, · 2943 unmodified lines
2944	/* change offset to the other half */	2944	/* change offset to the other half */
2945	rxb->page_offset ^= GFAR_RXB_TRUESIZE;	2945	rxb->page_offset ^= GFAR_RXB_TRUESIZE;
2946		2946
2947	atomic_inc(&page->_count);	2947	page_ref_inc(page);
2948		2948
2949	return true;	2949	return true;
2950	}	2950	}

drivers/net/ethernet/intel/fm10k/fm10k_main.c+1−1

@@ -243,7 +243,7 @@ static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer, · 242 unmodified lines
243	/* Even if we own the page, we are not allowed to use atomic_set()	243	/* Even if we own the page, we are not allowed to use atomic_set()
244	* This would break get_page_unless_zero() users.	244	* This would break get_page_unless_zero() users.
245	*/	245	*/
246	atomic_inc(&page->_count);	246	page_ref_inc(page);
247		247
248	return true;	248	return true;
249	}	249	}

drivers/net/ethernet/intel/igb/igb_main.c+1−1

@@ -6630,7 +6630,7 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer, · 6629 unmodified lines
6630	/* Even if we own the page, we are not allowed to use atomic_set()	6630	/* Even if we own the page, we are not allowed to use atomic_set()
6631	* This would break get_page_unless_zero() users.	6631	* This would break get_page_unless_zero() users.
6632	*/	6632	*/
6633	atomic_inc(&page->_count);	6633	page_ref_inc(page);
6634		6634
6635	return true;	6635	return true;
6636	}	6636	}

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c+1−1

@@ -1942,7 +1942,7 @@ static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, · 1941 unmodified lines
1942	/* Even if we own the page, we are not allowed to use atomic_set()	1942	/* Even if we own the page, we are not allowed to use atomic_set()
1943	* This would break get_page_unless_zero() users.	1943	* This would break get_page_unless_zero() users.
1944	*/	1944	*/
1945	atomic_inc(&page->_count);	1945	page_ref_inc(page);
1946		1946
1947	return true;	1947	return true;
1948	}	1948	}

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c+1−1

@@ -837,7 +837,7 @@ add_tail_frag: · 836 unmodified lines
837	/* Even if we own the page, we are not allowed to use atomic_set()	837	/* Even if we own the page, we are not allowed to use atomic_set()
838	* This would break get_page_unless_zero() users.	838	* This would break get_page_unless_zero() users.
839	*/	839	*/
840	atomic_inc(&page->_count);	840	page_ref_inc(page);
841		841
842	return true;	842	return true;
843	}	843	}

drivers/net/ethernet/mellanox/mlx4/en_rx.c+4−5

 	/* Not doing get_page() for each frag is a big win
 	 * on asymetric workloads. Note we can not use atomic_set().
 	 */
-	atomic_add(page_alloc->page_size / frag_info->frag_stride - 1,
+	page_ref_add(page, page_alloc->page_size / frag_info->frag_stride - 1);
-		   &page->_count);
 	return 0;
 }
 			dma_unmap_page(priv->ddev, page_alloc[i].dma,
 				page_alloc[i].page_size, PCI_DMA_FROMDEVICE);
 			page = page_alloc[i].page;
-			atomic_set(&page->_count, 1);
+			set_page_count(page, 1);
 			put_page(page);
 		}
 	}
 		en_dbg(DRV, priv, "  frag %d allocator: - size:%d frags:%d\n",
 		       i, ring->page_alloc[i].page_size,
-		       atomic_read(&ring->page_alloc[i].page->_count));
+		       page_ref_count(ring->page_alloc[i].page));
 	}
 	return 0;
 		dma_unmap_page(priv->ddev, page_alloc->dma,
 			       page_alloc->page_size, PCI_DMA_FROMDEVICE);
 		page = page_alloc->page;
-		atomic_set(&page->_count, 1);
+		set_page_count(page, 1);
 		put_page(page);
 		page_alloc->page = NULL;
 	}

drivers/net/ethernet/sun/niu.c+1−1

@@ -3341,7 +3341,7 @@ static int niu_rbr_add_page(struct niu np, struct rx_ring_info rp, · 3340 unmodified lines
3341		3341
3342	niu_hash_page(rp, page, addr);	3342	niu_hash_page(rp, page, addr);
3343	if (rp->rbr_blocks_per_page > 1)	3343	if (rp->rbr_blocks_per_page > 1)
3344	atomic_add(rp->rbr_blocks_per_page - 1, &page->_count);	3344	page_ref_add(page, rp->rbr_blocks_per_page - 1);
3345		3345
3346	for (i = 0; i < rp->rbr_blocks_per_page; i++) {	3346	for (i = 0; i < rp->rbr_blocks_per_page; i++) {
3347	__le32 *rbr = &rp->rbr[start_index + i];	3347	__le32 *rbr = &rp->rbr[start_index + i];

drivers/net/wireless/marvell/mwifiex/debugfs.c+4−6

@@ -880,14 +880,12 @@ mwifiex_reset_write(struct file *file, · 879 unmodified lines
880	{	880	{
881	struct mwifiex_private *priv = file->private_data;	881	struct mwifiex_private *priv = file->private_data;
882	struct mwifiex_adapter *adapter = priv->adapter;	882	struct mwifiex_adapter *adapter = priv->adapter;
883	char cmd;
884	bool result;	883	bool result;
		884	int rc;
885		885
886	if (copy_from_user(&cmd, ubuf, sizeof(cmd)))	886	rc = kstrtobool_from_user(ubuf, count, &result);
887	return -EFAULT;	887	if (rc)
888		888	return rc;
889	if (strtobool(&cmd, &result))
890	return -EINVAL;
891		889
892	if (!result)	890	if (!result)
893	return -EINVAL;	891	return -EINVAL;

drivers/pinctrl/pinmux.c+3−10

 	unsigned num_groups;
 	int ret;
 	const char *group;
-	int i;
 	if (!pmxops) {
 		dev_err(pctldev->dev, "does not support mux function\n");
 		return -EINVAL;
 	}
 	if (map->data.mux.group) {
-		bool found = false;
 		group = map->data.mux.group;
-		for (i = 0; i < num_groups; i++) {
+		ret = match_string(groups, num_groups, group);
-			if (!strcmp(group, groups[i])) {
+		if (ret < 0) {
-				found = true;
-				break;
-			}
-		}
-		if (!found) {
 			dev_err(pctldev->dev,
 				"invalid group \"%s\" for function \"%s\"\n",
 				group, map->data.mux.function);
-			return -EINVAL;
+			return ret;
 		}
 	} else {
 		group = groups[0];

drivers/power/ab8500_btemp.c+5−10

 static int ab8500_btemp_get_ext_psy_data(struct device *dev, void *data)
 {
 	struct power_supply *psy;
-	struct power_supply *ext;
+	struct power_supply *ext = dev_get_drvdata(dev);
+	const char **supplicants = (const char **)ext->supplied_to;
 	struct ab8500_btemp *di;
 	union power_supply_propval ret;
-	int i, j;
+	int j;
-	bool psy_found = false;
 	psy = (struct power_supply *)data;
-	ext = dev_get_drvdata(dev);
 	di = power_supply_get_drvdata(psy);
 	/*
 	 * For all psy where the name of your driver
 	 * appears in any supplied_to
 	 */
-	for (i = 0; i < ext->num_supplicants; i++) {
+	j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
-		if (!strcmp(ext->supplied_to[i], psy->desc->name))
+	if (j < 0)
-			psy_found = true;
-	}
-	if (!psy_found)
 		return 0;
 	/* Go through all properties for the psy */

drivers/power/ab8500_charger.c+5−11

 static int ab8500_charger_get_ext_psy_data(struct device *dev, void *data)
 {
 	struct power_supply *psy;
-	struct power_supply *ext;
+	struct power_supply *ext = dev_get_drvdata(dev);
+	const char **supplicants = (const char **)ext->supplied_to;
 	struct ab8500_charger *di;
 	union power_supply_propval ret;
-	int i, j;
+	int j;
-	bool psy_found = false;
 	struct ux500_charger *usb_chg;
 	usb_chg = (struct ux500_charger *)data;
 	di = to_ab8500_charger_usb_device_info(usb_chg);
-	ext = dev_get_drvdata(dev);
 	/* For all psy where the driver name appears in any supplied_to */
-	for (i = 0; i < ext->num_supplicants; i++) {
+	j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
-		if (!strcmp(ext->supplied_to[i], psy->desc->name))
+	if (j < 0)
-			psy_found = true;
-	}
-	if (!psy_found)
 		return 0;
 	/* Go through all properties for the psy */

drivers/power/ab8500_fg.c+5−10

 static int ab8500_fg_get_ext_psy_data(struct device *dev, void *data)
 {
 	struct power_supply *psy;
-	struct power_supply *ext;
+	struct power_supply *ext = dev_get_drvdata(dev);
+	const char **supplicants = (const char **)ext->supplied_to;
 	struct ab8500_fg *di;
 	union power_supply_propval ret;
-	int i, j;
+	int j;
-	bool psy_found = false;
 	psy = (struct power_supply *)data;
-	ext = dev_get_drvdata(dev);
 	di = power_supply_get_drvdata(psy);
 	/*
 	 * For all psy where the name of your driver
 	 * appears in any supplied_to
 	 */
-	for (i = 0; i < ext->num_supplicants; i++) {
+	j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
-		if (!strcmp(ext->supplied_to[i], psy->desc->name))
+	if (j < 0)
-			psy_found = true;
-	}
-	if (!psy_found)
 		return 0;
 	/* Go through all properties for the psy */

drivers/power/abx500_chargalg.c+5−9

 static int abx500_chargalg_get_ext_psy_data(struct device *dev, void *data)
 {
 	struct power_supply *psy;
-	struct power_supply *ext;
+	struct power_supply *ext = dev_get_drvdata(dev);
+	const char **supplicants = (const char **)ext->supplied_to;
 	struct abx500_chargalg *di;
 	union power_supply_propval ret;
-	int i, j;
+	int j;
-	bool psy_found = false;
 	bool capacity_updated = false;
 	psy = (struct power_supply *)data;
-	ext = dev_get_drvdata(dev);
 	di = power_supply_get_drvdata(psy);
 	/* For all psy where the driver name appears in any supplied_to */
-	for (i = 0; i < ext->num_supplicants; i++) {
+	j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
-		if (!strcmp(ext->supplied_to[i], psy->desc->name))
+	if (j < 0)
-			psy_found = true;
-	}
-	if (!psy_found)
 		return 0;
 	/*

drivers/power/charger-manager.c+4−23

@@ -2019,27 +2019,6 @@ static void __exit charger_manager_cleanup(void) · 2018 unmodified lines
2019	}	2019	}
2020	module_exit(charger_manager_cleanup);	2020	module_exit(charger_manager_cleanup);
2021		2021
2022	/**
2023	* find_power_supply - find the associated power_supply of charger
2024	* @cm: the Charger Manager representing the battery
2025	* @psy: pointer to instance of charger's power_supply
2026	*/
2027	static bool find_power_supply(struct charger_manager *cm,
2028	struct power_supply *psy)
2029	{
2030	int i;
2031	bool found = false;
2032
2033	for (i = 0; cm->desc->psy_charger_stat[i]; i++) {
2034	if (!strcmp(psy->desc->name, cm->desc->psy_charger_stat[i])) {
2035	found = true;
2036	break;
2037	}
2038	}
2039
2040	return found;
2041	}
2042
2043	/**	2022	/**
2044	* cm_notify_event - charger driver notify Charger Manager of charger event	2023	* cm_notify_event - charger driver notify Charger Manager of charger event
2045	* @psy: pointer to instance of charger's power_supply	2024	* @psy: pointer to instance of charger's power_supply
@@ -2057,9 +2036,11 @@ void cm_notify_event(struct power_supply *psy, enum cm_event_types type, · 11 unmodified lines
2057		2036
2058	mutex_lock(&cm_list_mtx);	2037	mutex_lock(&cm_list_mtx);
2059	list_for_each_entry(cm, &cm_list, entry) {	2038	list_for_each_entry(cm, &cm_list, entry) {
2060	found_power_supply = find_power_supply(cm, psy);	2039	if (match_string(cm->desc->psy_charger_stat, -1,
2061	if (found_power_supply)	2040	psy->desc->name) >= 0) {
		2041	found_power_supply = true;
2062	break;	2042	break;
		2043	}
2063	}	2044	}
2064	mutex_unlock(&cm_list_mtx);	2045	mutex_unlock(&cm_list_mtx);
2065		2046

drivers/usb/common/common.c+8−14

 enum usb_device_speed usb_get_maximum_speed(struct device *dev)
 {
 	const char *maximum_speed;
-	int err;
+	int ret;
-	int i;
-	err = device_property_read_string(dev, "maximum-speed", &maximum_speed);
+	ret = device_property_read_string(dev, "maximum-speed", &maximum_speed);
-	if (err < 0)
+	if (ret < 0)
 		return USB_SPEED_UNKNOWN;
-	for (i = 0; i < ARRAY_SIZE(speed_names); i++)
+	ret = match_string(speed_names, ARRAY_SIZE(speed_names), maximum_speed);
-		if (strcmp(maximum_speed, speed_names[i]) == 0)
-			return i;
-	return USB_SPEED_UNKNOWN;
+	return (ret < 0) ? USB_SPEED_UNKNOWN : ret;
 }
 EXPORT_SYMBOL_GPL(usb_get_maximum_speed);
 static enum usb_dr_mode usb_get_dr_mode_from_string(const char *str)
 {
-	int i;
+	int ret;
-	for (i = 0; i < ARRAY_SIZE(usb_dr_modes); i++)
-		if (!strcmp(usb_dr_modes[i], str))
-			return i;
-	return USB_DR_MODE_UNKNOWN;
+	ret = match_string(usb_dr_modes, ARRAY_SIZE(usb_dr_modes), str);
+	return (ret < 0) ? USB_DR_MODE_UNKNOWN : ret;
 }
 enum usb_dr_mode usb_get_dr_mode(struct device *dev)

drivers/virtio/virtio_balloon.c+6−0

 #include <linux/balloon_compaction.h>
 #include <linux/oom.h>
 #include <linux/wait.h>
+#include <linux/mm.h>
 /*
  * Balloon device works in 4K page units.  So each page is pointed to by
 	unsigned long events[NR_VM_EVENT_ITEMS];
 	struct sysinfo i;
 	int idx = 0;
+	long available;
 	all_vm_events(events);
 	si_meminfo(&i);
+	available = si_mem_available();
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
 				pages_to_bytes(events[PSWPIN]));
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
 				pages_to_bytes(i.freeram));
 	update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
 				pages_to_bytes(i.totalram));
+	update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
+				pages_to_bytes(available));
 }
 /*

fs/btrfs/tests/btrfs-tests.c+1−2

 	void **slot;
 	spin_lock(&fs_info->buffer_lock);
-restart:
 	radix_tree_for_each_slot(slot, &fs_info->buffer_radix, &iter, 0) {
 		struct extent_buffer *eb;
 		/* Shouldn't happen but that kind of thinking creates CVE's */
 		if (radix_tree_exception(eb)) {
 			if (radix_tree_deref_retry(eb))
-				goto restart;
+				slot = radix_tree_iter_retry(&iter);
 			continue;
 		}
 		spin_unlock(&fs_info->buffer_lock);

fs/cifs/cifs_debug.c+14−42

 static ssize_t cifs_stats_proc_write(struct file *file,
 		const char __user *buffer, size_t count, loff_t *ppos)
 {
-	char c;
 	bool bv;
 	int rc;
 	struct list_head *tmp1, *tmp2, *tmp3;
 	struct cifs_ses *ses;
 	struct cifs_tcon *tcon;
-	rc = get_user(c, buffer);
+	rc = kstrtobool_from_user(buffer, count, &bv);
-	if (rc)
+	if (rc == 0) {
-		return rc;
-	if (strtobool(&c, &bv) == 0) {
 #ifdef CONFIG_CIFS_STATS2
 		atomic_set(&totBufAllocCount, 0);
 		atomic_set(&totSmBufAllocCount, 0);
 			}
 		}
 		spin_unlock(&cifs_tcp_ses_lock);
+	} else {
+		return rc;
 	}
 	return count;
 static ssize_t cifsFYI_proc_write(struct file *file, const char __user *buffer,
 		size_t count, loff_t *ppos)
 {
-	char c;
+	char c[2] = { '\0' };
 	bool bv;
 	int rc;
-	rc = get_user(c, buffer);
+	rc = get_user(c[0], buffer);
 	if (rc)
 		return rc;
-	if (strtobool(&c, &bv) == 0)
+	if (strtobool(c, &bv) == 0)
 		cifsFYI = bv;
-	else if ((c > '1') && (c <= '9'))
+	else if ((c[0] > '1') && (c[0] <= '9'))
-		cifsFYI = (int) (c - '0'); /* see cifs_debug.h for meanings */
+		cifsFYI = (int) (c[0] - '0'); /* see cifs_debug.h for meanings */
 	return count;
 }
 static ssize_t cifs_linux_ext_proc_write(struct file *file,
 		const char __user *buffer, size_t count, loff_t *ppos)
 {
-	char c;
-	bool bv;
 	int rc;
-	rc = get_user(c, buffer);
+	rc = kstrtobool_from_user(buffer, count, &linuxExtEnabled);
 	if (rc)
 		return rc;
-	rc = strtobool(&c, &bv);
-	if (rc)
-		return rc;
-	linuxExtEnabled = bv;
 	return count;
 }
 static ssize_t cifs_lookup_cache_proc_write(struct file *file,
 		const char __user *buffer, size_t count, loff_t *ppos)
 {
-	char c;
-	bool bv;
 	int rc;
-	rc = get_user(c, buffer);
+	rc = kstrtobool_from_user(buffer, count, &lookupCacheEnabled);
 	if (rc)
 		return rc;
-	rc = strtobool(&c, &bv);
-	if (rc)
-		return rc;
-	lookupCacheEnabled = bv;
 	return count;
 }
 static ssize_t traceSMB_proc_write(struct file *file, const char __user *buffer,
 		size_t count, loff_t *ppos)
 {
-	char c;
-	bool bv;
 	int rc;
-	rc = get_user(c, buffer);
+	rc = kstrtobool_from_user(buffer, count, &traceSMB);
 	if (rc)
 		return rc;
-	rc = strtobool(&c, &bv);
-	if (rc)
-		return rc;
-	traceSMB = bv;
 	return count;
 }
 	int rc;
 	unsigned int flags;
 	char flags_string[12];
-	char c;
 	bool bv;
 	if ((count < 1) || (count > 11))
 	if (count < 3) {
 		/* single char or single char followed by null */
-		c = flags_string[0];
+		if (strtobool(flags_string, &bv) == 0) {
-		if (strtobool(&c, &bv) == 0) {
 			global_secflags = bv ? CIFSSEC_MAX : CIFSSEC_DEF;
 			return count;
-		} else if (!isdigit(c)) {
+		} else if (!isdigit(flags_string[0])) {
 			cifs_dbg(VFS, "Invalid SecurityFlags: %s\n",
 					flags_string);
 			return -EINVAL;

fs/cifs/cifs_debug.h+1−1

@@ -25,7 +25,7 @@ · 24 unmodified lines
25	void cifs_dump_mem(char label, void data, int length);	25	void cifs_dump_mem(char label, void data, int length);
26	void cifs_dump_detail(void *);	26	void cifs_dump_detail(void *);
27	void cifs_dump_mids(struct TCP_Server_Info *);	27	void cifs_dump_mids(struct TCP_Server_Info *);
28	extern int traceSMB; /* flag which enables the function below */	28	extern bool traceSMB; /* flag which enables the function below */
29	void dump_smb(void *, int);	29	void dump_smb(void *, int);
30	#define CIFS_INFO 0x01	30	#define CIFS_INFO 0x01
31	#define CIFS_RC 0x02	31	#define CIFS_RC 0x02

fs/cifs/cifsfs.c+3−3

@@ -54,10 +54,10 @@ · 53 unmodified lines
54	#endif	54	#endif
55		55
56	int cifsFYI = 0;	56	int cifsFYI = 0;
57	int traceSMB = 0;	57	bool traceSMB;
58	bool enable_oplocks = true;	58	bool enable_oplocks = true;
59	unsigned int linuxExtEnabled = 1;	59	bool linuxExtEnabled = true;
60	unsigned int lookupCacheEnabled = 1;	60	bool lookupCacheEnabled = true;
61	unsigned int global_secflags = CIFSSEC_DEF;	61	unsigned int global_secflags = CIFSSEC_DEF;
62	/* unsigned int ntlmv2_support = 0; */	62	/* unsigned int ntlmv2_support = 0; */
63	unsigned int sign_CIFS_PDUs = 1;	63	unsigned int sign_CIFS_PDUs = 1;

fs/cifs/cifsglob.h+2−2

@@ -1596,11 +1596,11 @@ GLOBAL_EXTERN atomic_t midCount; · 1595 unmodified lines
1596		1596
1597	/* Misc globals */	1597	/* Misc globals */
1598	GLOBAL_EXTERN bool enable_oplocks; /* enable or disable oplocks */	1598	GLOBAL_EXTERN bool enable_oplocks; /* enable or disable oplocks */
1599	GLOBAL_EXTERN unsigned int lookupCacheEnabled;	1599	GLOBAL_EXTERN bool lookupCacheEnabled;
1600	GLOBAL_EXTERN unsigned int global_secflags; /* if on, session setup sent	1600	GLOBAL_EXTERN unsigned int global_secflags; /* if on, session setup sent
1601	with more secure ntlmssp2 challenge/resp */	1601	with more secure ntlmssp2 challenge/resp */
1602	GLOBAL_EXTERN unsigned int sign_CIFS_PDUs; /* enable smb packet signing */	1602	GLOBAL_EXTERN unsigned int sign_CIFS_PDUs; /* enable smb packet signing */
1603	GLOBAL_EXTERN unsigned int linuxExtEnabled;/enable Linux/Unix CIFS extensions/	1603	GLOBAL_EXTERN bool linuxExtEnabled;/enable Linux/Unix CIFS extensions/
1604	GLOBAL_EXTERN unsigned int CIFSMaxBufSize; /* max size not including hdr */	1604	GLOBAL_EXTERN unsigned int CIFSMaxBufSize; /* max size not including hdr */
1605	GLOBAL_EXTERN unsigned int cifs_min_rcv; /* min size of big ntwrk buf pool */	1605	GLOBAL_EXTERN unsigned int cifs_min_rcv; /* min size of big ntwrk buf pool */
1606	GLOBAL_EXTERN unsigned int cifs_min_small; /* min size of small buf pool */	1606	GLOBAL_EXTERN unsigned int cifs_min_small; /* min size of small buf pool */

fs/eventpoll.c+1−1

@@ -1616,7 +1616,7 @@ static int ep_poll(struct eventpoll ep, struct epoll_event __user events, · 1615 unmodified lines
1616	{	1616	{
1617	int res = 0, eavail, timed_out = 0;	1617	int res = 0, eavail, timed_out = 0;
1618	unsigned long flags;	1618	unsigned long flags;
1619	long slack = 0;	1619	u64 slack = 0;
1620	wait_queue_t wait;	1620	wait_queue_t wait;
1621	ktime_t expires, *to = NULL;	1621	ktime_t expires, *to = NULL;
1622		1622

fs/nilfs2/page.c+1−1

@@ -180,7 +180,7 @@ void nilfs_page_bug(struct page *page) · 179 unmodified lines
180		180
181	printk(KERN_CRIT "NILFS_PAGE_BUG(%p): cnt=%d index#=%llu flags=0x%lx "	181	printk(KERN_CRIT "NILFS_PAGE_BUG(%p): cnt=%d index#=%llu flags=0x%lx "
182	"mapping=%p ino=%lu\n",	182	"mapping=%p ino=%lu\n",
183	page, atomic_read(&page->_count),	183	page, page_ref_count(page),
184	(unsigned long long)page->index, page->flags, m, ino);	184	(unsigned long long)page->index, page->flags, m, ino);
185		185
186	if (page_has_buffers(page)) {	186	if (page_has_buffers(page)) {

fs/proc/base.c+70−1

@@ -434,7 +434,7 @@ static int proc_pid_wchan(struct seq_file m, struct pid_namespace ns, · 433 unmodified lines
434	&& !lookup_symbol_name(wchan, symname))	434	&& !lookup_symbol_name(wchan, symname))
435	seq_printf(m, "%s", symname);	435	seq_printf(m, "%s", symname);
436	else	436	else
437	seq_putc(m, '0');	437	seq_puts(m, "0\n");
438		438
439	return 0;	439	return 0;
440	}	440	}
@@ -2158,6 +2158,7 @@ static const struct file_operations proc_map_files_operations = { · 1717 unmodified lines
2158	.llseek = default_llseek,	2158	.llseek = default_llseek,
2159	};	2159	};
2160		2160
		2161	#ifdef CONFIG_CHECKPOINT_RESTORE
2161	struct timers_private {	2162	struct timers_private {
2162	struct pid *pid;	2163	struct pid *pid;
2163	struct task_struct *task;	2164	struct task_struct *task;
@@ -2256,6 +2257,73 @@ static const struct file_operations proc_timers_operations = { · 92 unmodified lines
2256	.llseek = seq_lseek,	2257	.llseek = seq_lseek,
2257	.release = seq_release_private,	2258	.release = seq_release_private,
2258	};	2259	};
		2260	#endif
		2261
		2262	static ssize_t timerslack_ns_write(struct file file, const char __user buf,
		2263	size_t count, loff_t *offset)
		2264	{
		2265	struct inode *inode = file_inode(file);
		2266	struct task_struct *p;
		2267	u64 slack_ns;
		2268	int err;
		2269
		2270	err = kstrtoull_from_user(buf, count, 10, &slack_ns);
		2271	if (err < 0)
		2272	return err;
		2273
		2274	p = get_proc_task(inode);
		2275	if (!p)
		2276	return -ESRCH;
		2277
		2278	if (ptrace_may_access(p, PTRACE_MODE_ATTACH_FSCREDS)) {
		2279	task_lock(p);
		2280	if (slack_ns == 0)
		2281	p->timer_slack_ns = p->default_timer_slack_ns;
		2282	else
		2283	p->timer_slack_ns = slack_ns;
		2284	task_unlock(p);
		2285	} else
		2286	count = -EPERM;
		2287
		2288	put_task_struct(p);
		2289
		2290	return count;
		2291	}
		2292
		2293	static int timerslack_ns_show(struct seq_file m, void v)
		2294	{
		2295	struct inode *inode = m->private;
		2296	struct task_struct *p;
		2297	int err = 0;
		2298
		2299	p = get_proc_task(inode);
		2300	if (!p)
		2301	return -ESRCH;
		2302
		2303	if (ptrace_may_access(p, PTRACE_MODE_ATTACH_FSCREDS)) {
		2304	task_lock(p);
		2305	seq_printf(m, "%llu\n", p->timer_slack_ns);
		2306	task_unlock(p);
		2307	} else
		2308	err = -EPERM;
		2309
		2310	put_task_struct(p);
		2311
		2312	return err;
		2313	}
		2314
		2315	static int timerslack_ns_open(struct inode inode, struct file filp)
		2316	{
		2317	return single_open(filp, timerslack_ns_show, inode);
		2318	}
		2319
		2320	static const struct file_operations proc_pid_set_timerslack_ns_operations = {
		2321	.open = timerslack_ns_open,
		2322	.read = seq_read,
		2323	.write = timerslack_ns_write,
		2324	.llseek = seq_lseek,
		2325	.release = single_release,
		2326	};
2259		2327
2260	static int proc_pident_instantiate(struct inode *dir,	2328	static int proc_pident_instantiate(struct inode *dir,
2261	struct dentry dentry, struct task_struct task, const void *ptr)	2329	struct dentry dentry, struct task_struct task, const void *ptr)
@@ -2831,6 +2899,7 @@ static const struct pid_entry tgid_base_stuff[] = { · 569 unmodified lines
2831	#ifdef CONFIG_CHECKPOINT_RESTORE	2899	#ifdef CONFIG_CHECKPOINT_RESTORE
2832	REG("timers", S_IRUGO, proc_timers_operations),	2900	REG("timers", S_IRUGO, proc_timers_operations),
2833	#endif	2901	#endif
		2902	REG("timerslack_ns", S_IRUGO\|S_IWUGO, proc_pid_set_timerslack_ns_operations),
2834	};	2903	};
2835		2904
2836	static int proc_tgid_base_readdir(struct file file, struct dir_context ctx)	2905	static int proc_tgid_base_readdir(struct file file, struct dir_context ctx)

fs/proc/meminfo.c+1−30

@@ -29,10 +29,7 @@ static int meminfo_proc_show(struct seq_file m, void v) · 28 unmodified lines
29	unsigned long committed;	29	unsigned long committed;
30	long cached;	30	long cached;
31	long available;	31	long available;
32	unsigned long pagecache;
33	unsigned long wmark_low = 0;
34	unsigned long pages[NR_LRU_LISTS];	32	unsigned long pages[NR_LRU_LISTS];
35	struct zone *zone;
36	int lru;	33	int lru;
37		34
38	/*	35	/*
@@ -51,33 +48,7 @@ static int meminfo_proc_show(struct seq_file m, void v) · 12 unmodified lines
51	for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)	48	for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
52	pages[lru] = global_page_state(NR_LRU_BASE + lru);	49	pages[lru] = global_page_state(NR_LRU_BASE + lru);
53		50
54	for_each_zone(zone)	51	available = si_mem_available();
55	wmark_low += zone->watermark[WMARK_LOW];
56
57	/*
58	* Estimate the amount of memory available for userspace allocations,
59	* without causing swapping.
60	*/
61	available = i.freeram - totalreserve_pages;
62
63	/*
64	* Not all the page cache can be freed, otherwise the system will
65	* start swapping. Assume at least half of the page cache, or the
66	* low watermark worth of cache, needs to stay.
67	*/
68	pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
69	pagecache -= min(pagecache / 2, wmark_low);
70	available += pagecache;
71
72	/*
73	* Part of the reclaimable slab consists of items that are in use,
74	* and cannot be freed. Cap this estimate at the low watermark.
75	*/
76	available += global_page_state(NR_SLAB_RECLAIMABLE) -
77	min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
78
79	if (available < 0)
80	available = 0;
81		52
82	/*	53	/*
83	* Tagged format, for easy grepping and expansion.	54	* Tagged format, for easy grepping and expansion.

fs/proc/page.c+6−2

 	 * pseudo flags for the well known (anonymous) memory mapped pages
 	 *
 	 * Note that page->_mapcount is overloaded in SLOB/SLUB/SLQB, so the
-	 * simple test in page_mapcount() is not enough.
+	 * simple test in page_mapped() is not enough.
 	 */
-	if (!PageSlab(page) && page_mapcount(page))
+	if (!PageSlab(page) && page_mapped(page))
 		u |= 1 << KPF_MMAP;
 	if (PageAnon(page))
 		u |= 1 << KPF_ANON;
 	 */
 	if (PageBuddy(page))
 		u |= 1 << KPF_BUDDY;
+	else if (page_count(page) == 0 && is_free_buddy_page(page))
+		u |= 1 << KPF_BUDDY;
 	if (PageBalloon(page))
 		u |= 1 << KPF_BALLOON;
 	u |= kpf_copy_bit(k, KPF_LOCKED,	PG_locked);
 	u |= kpf_copy_bit(k, KPF_SLAB,		PG_slab);
+	if (PageTail(page) && PageSlab(compound_head(page)))
+		u |= 1 << KPF_SLAB;
 	u |= kpf_copy_bit(k, KPF_ERROR,		PG_error);
 	u |= kpf_copy_bit(k, KPF_DIRTY,		PG_dirty);

fs/proc/vmcore.c+5−2

 	list_for_each_entry(m, &vmcore_list, list) {
 		if (*fpos < m->offset + m->size) {
-			tsz = min_t(size_t, m->offset + m->size - *fpos, buflen);
+			tsz = (size_t)min_t(unsigned long long,
+					    m->offset + m->size - *fpos,
+					    buflen);
 			start = m->paddr + *fpos - m->offset;
 			tmp = read_from_oldmem(buffer, tsz, &start, userbuf);
 			if (tmp < 0)
 		if (start < m->offset + m->size) {
 			u64 paddr = 0;
-			tsz = min_t(size_t, m->offset + m->size - start, size);
+			tsz = (size_t)min_t(unsigned long long,
+					    m->offset + m->size - start, size);
 			paddr = m->paddr + start - m->offset;
 			if (vmcore_remap_oldmem_pfn(vma, vma->vm_start + len,
 						    paddr >> PAGE_SHIFT, tsz,

fs/select.c+4−4

@@ -70,9 +70,9 @@ static long __estimate_accuracy(struct timespec *tv) · 69 unmodified lines
70	return slack;	70	return slack;
71	}	71	}
72		72
73	long select_estimate_accuracy(struct timespec *tv)	73	u64 select_estimate_accuracy(struct timespec *tv)
74	{	74	{
75	unsigned long ret;	75	u64 ret;
76	struct timespec now;	76	struct timespec now;
77		77
78	/*	78	/*
@@ -402,7 +402,7 @@ int do_select(int n, fd_set_bits fds, struct timespec end_time) · 323 unmodified lines
402	struct poll_wqueues table;	402	struct poll_wqueues table;
403	poll_table *wait;	403	poll_table *wait;
404	int retval, i, timed_out = 0;	404	int retval, i, timed_out = 0;
405	unsigned long slack = 0;	405	u64 slack = 0;
406	unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;	406	unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
407	unsigned long busy_end = 0;	407	unsigned long busy_end = 0;
408		408
@@ -784,7 +784,7 @@ static int do_poll(struct poll_list list, struct poll_wqueues wait, · 375 unmodified lines
784	poll_table* pt = &wait->pt;	784	poll_table* pt = &wait->pt;
785	ktime_t expire, *to = NULL;	785	ktime_t expire, *to = NULL;
786	int timed_out = 0, count = 0;	786	int timed_out = 0, count = 0;
787	unsigned long slack = 0;	787	u64 slack = 0;
788	unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;	788	unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
789	unsigned long busy_end = 0;	789	unsigned long busy_end = 0;
790		790

include/asm-generic/atomic-long.h+3−3

@@ -98,14 +98,14 @@ ATOMIC_LONG_ADD_SUB_OP(sub, _release) · 97 unmodified lines
98	#define atomic_long_xchg(v, new) \	98	#define atomic_long_xchg(v, new) \
99	(ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new)))	99	(ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
100		100
101	static inline void atomic_long_inc(atomic_long_t *l)	101	static __always_inline void atomic_long_inc(atomic_long_t *l)
102	{	102	{
103	ATOMIC_LONG_PFX(_t) v = (ATOMIC_LONG_PFX(_t) )l;	103	ATOMIC_LONG_PFX(_t) v = (ATOMIC_LONG_PFX(_t) )l;
104		104
105	ATOMIC_LONG_PFX(_inc)(v);	105	ATOMIC_LONG_PFX(_inc)(v);
106	}	106	}
107		107
108	static inline void atomic_long_dec(atomic_long_t *l)	108	static __always_inline void atomic_long_dec(atomic_long_t *l)
109	{	109	{
110	ATOMIC_LONG_PFX(_t) v = (ATOMIC_LONG_PFX(_t) )l;	110	ATOMIC_LONG_PFX(_t) v = (ATOMIC_LONG_PFX(_t) )l;
111		111
@@ -113,7 +113,7 @@ static inline void atomic_long_dec(atomic_long_t *l) · 1 unmodified line
113	}	113	}
114		114
115	#define ATOMIC_LONG_OP(op) \	115	#define ATOMIC_LONG_OP(op) \
116	static inline void \	116	static __always_inline void \
117	atomic_long_##op(long i, atomic_long_t *l) \	117	atomic_long_##op(long i, atomic_long_t *l) \
118	{ \	118	{ \
119	ATOMIC_LONG_PFX(_t) v = (ATOMIC_LONG_PFX(_t) )l; \	119	ATOMIC_LONG_PFX(_t) v = (ATOMIC_LONG_PFX(_t) )l; \

include/asm-generic/bug.h+18−9

 	do { printk(arg); __WARN_TAINT(taint); } while (0)
 #endif
+/* used internally by panic.c */
+struct warn_args;
+void __warn(const char *file, int line, void *caller, unsigned taint,
+	    struct pt_regs *regs, struct warn_args *args);
 #ifndef WARN_ON
 #define WARN_ON(condition) ({						\
 	int __ret_warn_on = !!(condition);				\
 	static bool __section(.data.unlikely) __warned;		\
 	int __ret_warn_once = !!(condition);			\
 								\
-	if (unlikely(__ret_warn_once))				\
+	if (unlikely(__ret_warn_once && !__warned)) {		\
-		if (WARN_ON(!__warned)) 			\
+		__warned = true;				\
-			__warned = true;			\
+		WARN_ON(1);					\
+	}							\
 	unlikely(__ret_warn_once);				\
 })
 	static bool __section(.data.unlikely) __warned;		\
 	int __ret_warn_once = !!(condition);			\
 								\
-	if (unlikely(__ret_warn_once))				\
+	if (unlikely(__ret_warn_once && !__warned)) {		\
-		if (WARN(!__warned, format)) 			\
+		__warned = true;				\
-			__warned = true;			\
+		WARN(1, format);				\
+	}							\
 	unlikely(__ret_warn_once);				\
 })
 	static bool __section(.data.unlikely) __warned;		\
 	int __ret_warn_once = !!(condition);			\
 								\
-	if (unlikely(__ret_warn_once))				\
+	if (unlikely(__ret_warn_once && !__warned)) {		\
-		if (WARN_TAINT(!__warned, taint, format))	\
+		__warned = true;				\
-			__warned = true;			\
+		WARN_TAINT(1, taint, format);			\
+	}							\
 	unlikely(__ret_warn_once);				\
 })

include/asm-generic/pgtable.h+17−0

@@ -783,6 +783,23 @@ static inline int pmd_clear_huge(pmd_t *pmd) · 782 unmodified lines
783	}	783	}
784	#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */	784	#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
785		785
		786	#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
		787	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		788	/*
		789	* ARCHes with special requirements for evicting THP backing TLB entries can
		790	* implement this. Otherwise also, it can help optimize normal TLB flush in
		791	* THP regime. stock flush_tlb_range() typically has optimization to nuke the
		792	* entire TLB TLB if flush span is greater than a threshold, which will
		793	* likely be true for a single huge page. Thus a single thp flush will
		794	* invalidate the entire TLB which is not desitable.
		795	* e.g. see arch/arc: flush_pmd_tlb_range
		796	*/
		797	#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
		798	#else
		799	#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()
		800	#endif
		801	#endif
		802
786	#endif /* !__ASSEMBLY__ */	803	#endif /* !__ASSEMBLY__ */
787		804
788	#ifndef io_remap_pfn_range	805	#ifndef io_remap_pfn_range

include/linux/buffer_head.h+5−5

include/linux/compaction.h+16−0

@@ -52,6 +52,10 @@ extern void compaction_defer_reset(struct zone *zone, int order, · 51 unmodified lines
52	bool alloc_success);	52	bool alloc_success);
53	extern bool compaction_restarting(struct zone *zone, int order);	53	extern bool compaction_restarting(struct zone *zone, int order);
54		54
		55	extern int kcompactd_run(int nid);
		56	extern void kcompactd_stop(int nid);
		57	extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx);
		58
55	#else	59	#else
56	static inline unsigned long try_to_compact_pages(gfp_t gfp_mask,	60	static inline unsigned long try_to_compact_pages(gfp_t gfp_mask,
57	unsigned int order, int alloc_flags,	61	unsigned int order, int alloc_flags,
@@ -84,6 +88,18 @@ static inline bool compaction_deferred(struct zone *zone, int order) · 26 unmodified lines
84	return true;	88	return true;
85	}	89	}
86		90
		91	static inline int kcompactd_run(int nid)
		92	{
		93	return 0;
		94	}
		95	static inline void kcompactd_stop(int nid)
		96	{
		97	}
		98
		99	static inline void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
		100	{
		101	}
		102
87	#endif /* CONFIG_COMPACTION */	103	#endif /* CONFIG_COMPACTION */
88		104
89	#if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)	105	#if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)

include/linux/freezer.h+1−1

@@ -231,7 +231,7 @@ static inline long freezable_schedule_timeout_killable_unsafe(long timeout) · 230 unmodified lines
231	* call this with locks held.	231	* call this with locks held.
232	*/	232	*/
233	static inline int freezable_schedule_hrtimeout_range(ktime_t *expires,	233	static inline int freezable_schedule_hrtimeout_range(ktime_t *expires,
234	unsigned long delta, const enum hrtimer_mode mode)	234	u64 delta, const enum hrtimer_mode mode)
235	{	235	{
236	int __retval;	236	int __retval;
237	freezer_do_not_count();	237	freezer_do_not_count();

include/linux/gfp.h+21−16

  *
  * __GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves.
  *   This takes precedence over the __GFP_MEMALLOC flag if both are set.
- *
- * __GFP_NOACCOUNT ignores the accounting for kmemcg limit enforcement.
  */
 #define __GFP_ATOMIC	((__force gfp_t)___GFP_ATOMIC)
 #define __GFP_HIGH	((__force gfp_t)___GFP_HIGH)
 #define GFP_HIGHUSER_MOVABLE	(GFP_HIGHUSER | __GFP_MOVABLE)
 #define GFP_TRANSHUGE	((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
 			 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN) & \
-			 ~__GFP_KSWAPD_RECLAIM)
+			 ~__GFP_RECLAIM)
 /* Convert GFP flags to their corresponding migrate type */
 #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
  *       0xe    => BAD (MOVABLE+DMA32+HIGHMEM)
  *       0xf    => BAD (MOVABLE+DMA32+HIGHMEM+DMA)
  *
- * ZONES_SHIFT must be <= 2 on 32 bit platforms.
+ * GFP_ZONES_SHIFT must be <= 2 on 32 bit platforms.
  */
-#if 16 * ZONES_SHIFT > BITS_PER_LONG
+#if defined(CONFIG_ZONE_DEVICE) && (MAX_NR_ZONES-1) <= 4
-#error ZONES_SHIFT too large to create GFP_ZONE_TABLE integer
+/* ZONE_DEVICE is not a valid GFP zone specifier */
+#define GFP_ZONES_SHIFT 2
+#else
+#define GFP_ZONES_SHIFT ZONES_SHIFT
+#endif
+#if 16 * GFP_ZONES_SHIFT > BITS_PER_LONG
+#error GFP_ZONES_SHIFT too large to create GFP_ZONE_TABLE integer
 #endif
 #define GFP_ZONE_TABLE ( \
-	(ZONE_NORMAL << 0 * ZONES_SHIFT)				      \
+	(ZONE_NORMAL << 0 * GFP_ZONES_SHIFT)				       \
-	| (OPT_ZONE_DMA << ___GFP_DMA * ZONES_SHIFT)			      \
+	| (OPT_ZONE_DMA << ___GFP_DMA * GFP_ZONES_SHIFT)		       \
-	| (OPT_ZONE_HIGHMEM << ___GFP_HIGHMEM * ZONES_SHIFT)		      \
+	| (OPT_ZONE_HIGHMEM << ___GFP_HIGHMEM * GFP_ZONES_SHIFT)	       \
-	| (OPT_ZONE_DMA32 << ___GFP_DMA32 * ZONES_SHIFT)		      \
+	| (OPT_ZONE_DMA32 << ___GFP_DMA32 * GFP_ZONES_SHIFT)		       \
-	| (ZONE_NORMAL << ___GFP_MOVABLE * ZONES_SHIFT)			      \
+	| (ZONE_NORMAL << ___GFP_MOVABLE * GFP_ZONES_SHIFT)		       \
-	| (OPT_ZONE_DMA << (___GFP_MOVABLE | ___GFP_DMA) * ZONES_SHIFT)	      \
+	| (OPT_ZONE_DMA << (___GFP_MOVABLE | ___GFP_DMA) * GFP_ZONES_SHIFT)    \
-	| (ZONE_MOVABLE << (___GFP_MOVABLE | ___GFP_HIGHMEM) * ZONES_SHIFT)   \
+	| (ZONE_MOVABLE << (___GFP_MOVABLE | ___GFP_HIGHMEM) * GFP_ZONES_SHIFT)\
-	| (OPT_ZONE_DMA32 << (___GFP_MOVABLE | ___GFP_DMA32) * ZONES_SHIFT)   \
+	| (OPT_ZONE_DMA32 << (___GFP_MOVABLE | ___GFP_DMA32) * GFP_ZONES_SHIFT)\
 )
 /*
 	enum zone_type z;
 	int bit = (__force int) (flags & GFP_ZONEMASK);
-	z = (GFP_ZONE_TABLE >> (bit * ZONES_SHIFT)) &
+	z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) &
-					 ((1 << ZONES_SHIFT) - 1);
+					 ((1 << GFP_ZONES_SHIFT) - 1);
 	VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1);
 	return z;
 }

include/linux/hrtimer.h+7−5

 	timer->node.expires = ktime_add_safe(time, delta);
 }
-static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, unsigned long delta)
+static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
 {
 	timer->_softexpires = time;
 	timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
 /* Basic timer operations: */
 extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
-			unsigned long range_ns, const enum hrtimer_mode mode);
+				   u64 range_ns, const enum hrtimer_mode mode);
 /**
  * hrtimer_start - (re)start an hrtimer on the current CPU
 static inline void hrtimer_start_expires(struct hrtimer *timer,
 					 enum hrtimer_mode mode)
 {
-	unsigned long delta;
+	u64 delta;
 	ktime_t soft, hard;
 	soft = hrtimer_get_softexpires(timer);
 	hard = hrtimer_get_expires(timer);
 extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
 				 struct task_struct *tsk);
-extern int schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
+extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
 						const enum hrtimer_mode mode);
 extern int schedule_hrtimeout_range_clock(ktime_t *expires,
-		unsigned long delta, const enum hrtimer_mode mode, int clock);
+					  u64 delta,
+					  const enum hrtimer_mode mode,
+					  int clock);
 extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
 /* Soft interrupt function to run the hrtimer queues: */

include/linux/huge_mm.h+13−9

 enum transparent_hugepage_flag {
 	TRANSPARENT_HUGEPAGE_FLAG,
 	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
-	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
+	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
+	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
 	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
 	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
 	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
 	   ((__vma)->vm_flags & VM_HUGEPAGE))) &&			\
 	 !((__vma)->vm_flags & VM_NOHUGEPAGE) &&			\
 	 !is_vma_temporary_stack(__vma))
-#define transparent_hugepage_defrag(__vma)				\
-	((transparent_hugepage_flags &					\
-	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) ||			\
-	 (transparent_hugepage_flags &					\
-	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) &&		\
-	  (__vma)->vm_flags & VM_HUGEPAGE))
 #define transparent_hugepage_use_zero_page()				\
 	(transparent_hugepage_flags &					\
 	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
 void deferred_split_huge_page(struct page *page);
 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
-		unsigned long address);
+		unsigned long address, bool freeze);
 #define split_huge_pmd(__vma, __pmd, __address)				\
 	do {								\
 		pmd_t *____pmd = (__pmd);				\
 		if (pmd_trans_huge(*____pmd)				\
 					|| pmd_devmap(*____pmd))	\
-			__split_huge_pmd(__vma, __pmd, __address);	\
+			__split_huge_pmd(__vma, __pmd, __address,	\
+						false);			\
 	}  while (0)
+void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
+		bool freeze, struct page *page);
 #if HPAGE_PMD_ORDER >= MAX_ORDER
 #error "hugepages can't be allocated by the buddy allocator"
 #endif
 static inline void deferred_split_huge_page(struct page *page) {}
 #define split_huge_pmd(__vma, __pmd, __address)	\
 	do { } while (0)
+static inline void split_huge_pmd_address(struct vm_area_struct *vma,
+		unsigned long address, bool freeze, struct page *page) {}
 static inline int hugepage_madvise(struct vm_area_struct *vma,
 				   unsigned long *vm_flags, int advice)
 {

include/linux/kernel.h+2−0

 int __must_check kstrtos16(const char *s, unsigned int base, s16 *res);
 int __must_check kstrtou8(const char *s, unsigned int base, u8 *res);
 int __must_check kstrtos8(const char *s, unsigned int base, s8 *res);
+int __must_check kstrtobool(const char *s, bool *res);
 int __must_check kstrtoull_from_user(const char __user *s, size_t count, unsigned int base, unsigned long long *res);
 int __must_check kstrtoll_from_user(const char __user *s, size_t count, unsigned int base, long long *res);
 int __must_check kstrtos16_from_user(const char __user *s, size_t count, unsigned int base, s16 *res);
 int __must_check kstrtou8_from_user(const char __user *s, size_t count, unsigned int base, u8 *res);
 int __must_check kstrtos8_from_user(const char __user *s, size_t count, unsigned int base, s8 *res);
+int __must_check kstrtobool_from_user(const char __user *s, size_t count, bool *res);
 static inline int __must_check kstrtou64_from_user(const char __user *s, size_t count, unsigned int base, u64 *res)
 {

include/linux/list_bl.h+2−2

@@ -48,7 +48,7 @@ static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) · 47 unmodified lines
48		48
49	#define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member)	49	#define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member)
50		50
51	static inline int hlist_bl_unhashed(const struct hlist_bl_node *h)	51	static inline bool hlist_bl_unhashed(const struct hlist_bl_node *h)
52	{	52	{
53	return !h->pprev;	53	return !h->pprev;
54	}	54	}
@@ -68,7 +68,7 @@ static inline void hlist_bl_set_first(struct hlist_bl_head *h, · 13 unmodified lines
68	h->first = (struct hlist_bl_node *)((unsigned long)n \| LIST_BL_LOCKMASK);	68	h->first = (struct hlist_bl_node *)((unsigned long)n \| LIST_BL_LOCKMASK);
69	}	69	}
70		70
71	static inline int hlist_bl_empty(const struct hlist_bl_head *h)	71	static inline bool hlist_bl_empty(const struct hlist_bl_head *h)
72	{	72	{
73	return !((unsigned long)READ_ONCE(h->first) & ~LIST_BL_LOCKMASK);	73	return !((unsigned long)READ_ONCE(h->first) & ~LIST_BL_LOCKMASK);
74	}	74	}

include/linux/memcontrol.h+33−11

 	MEM_CGROUP_STAT_SWAP,		/* # of pages, swapped out */
 	MEM_CGROUP_STAT_NSTATS,
 	/* default hierarchy stats */
-	MEMCG_SOCK = MEM_CGROUP_STAT_NSTATS,
+	MEMCG_KERNEL_STACK = MEM_CGROUP_STAT_NSTATS,
+	MEMCG_SLAB_RECLAIMABLE,
+	MEMCG_SLAB_UNRECLAIMABLE,
+	MEMCG_SOCK,
 	MEMCG_NR_STAT,
 };
 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
 		int nr_pages);
+unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
+					   int nid, unsigned int lru_mask);
 static inline
 unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
 {
 {
 }
+static inline unsigned long
+mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
+			     int nid, unsigned int lru_mask)
+{
+	return 0;
+}
 static inline void
 mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
 {
 	return static_branch_unlikely(&memcg_kmem_enabled_key);
 }
-static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
-{
-	return memcg->kmem_state == KMEM_ONLINE;
-}
 /*
  * In general, we'll do everything in our power to not incur in any overhead
  * for non-memcg users for the kmem functions. Not even a function call, if we
 	if (memcg_kmem_enabled())
 		__memcg_kmem_put_cache(cachep);
 }
+/**
+ * memcg_kmem_update_page_stat - update kmem page state statistics
+ * @page: the page
+ * @idx: page state item to account
+ * @val: number of pages (positive or negative)
+ */
+static inline void memcg_kmem_update_page_stat(struct page *page,
+				enum mem_cgroup_stat_index idx, int val)
+{
+	if (memcg_kmem_enabled() && page->mem_cgroup)
+		this_cpu_add(page->mem_cgroup->stat->count[idx], val);
+}
 #else
 #define for_each_memcg_cache_index(_idx)	\
 	for (; NULL; )
 	return false;
 }
-static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
-{
-	return false;
-}
 static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
 {
 	return 0;
 static inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
 {
 }
+static inline void memcg_kmem_update_page_stat(struct page *page,
+				enum mem_cgroup_stat_index idx, int val)
+{
+}
 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */
 #endif /* _LINUX_MEMCONTROL_H */

include/linux/mm.h+38−30

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	#include <linux/resource.h>	22	#include <linux/resource.h>
23	#include <linux/page_ext.h>	23	#include <linux/page_ext.h>
24	#include <linux/err.h>	24	#include <linux/err.h>
		25	#include <linux/page_ref.h>
25		26
26	struct mempolicy;	27	struct mempolicy;
27	struct anon_vma;	28	struct anon_vma;
@@ -82,6 +83,27 @@ extern int mmap_rnd_compat_bits __read_mostly; · 54 unmodified lines
82	#define mm_forbids_zeropage(X) (0)	83	#define mm_forbids_zeropage(X) (0)
83	#endif	84	#endif
84		85
		86	/*
		87	* Default maximum number of active map areas, this limits the number of vmas
		88	* per mm struct. Users can overwrite this number by sysctl but there is a
		89	* problem.
		90	*
		91	* When a program's coredump is generated as ELF format, a section is created
		92	* per a vma. In ELF, the number of sections is represented in unsigned short.
		93	* This means the number of sections should be smaller than 65535 at coredump.
		94	* Because the kernel adds some informative sections to a image of program at
		95	* generating coredump, we need some margin. The number of extra sections is
		96	* 1-3 now and depends on arch. We use "5" as safe margin, here.
		97	*
		98	* ELF extended numbering allows more than 65535 sections, so 16-bit bound is
		99	* not a hard limit any more. Although some userspace tools can be surprised by
		100	* that.
		101	*/
		102	#define MAPCOUNT_ELF_CORE_MARGIN (5)
		103	#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
		104
		105	extern int sysctl_max_map_count;
		106
85	extern unsigned long sysctl_user_reserve_kbytes;	107	extern unsigned long sysctl_user_reserve_kbytes;
86	extern unsigned long sysctl_admin_reserve_kbytes;	108	extern unsigned long sysctl_admin_reserve_kbytes;
87		109
@@ -122,6 +144,7 @@ extern unsigned int kobjsize(const void *objp); · 34 unmodified lines
122		144
123	/*	145	/*
124	* vm_flags in vm_area_struct, see mm_types.h.	146	* vm_flags in vm_area_struct, see mm_types.h.
		147	* When changing, update also include/trace/events/mmflags.h
125	*/	148	*/
126	#define VM_NONE 0x00000000	149	#define VM_NONE 0x00000000
127		150
@@ -364,8 +387,8 @@ static inline int pmd_devmap(pmd_t pmd) · 236 unmodified lines
364	*/	387	*/
365	static inline int put_page_testzero(struct page *page)	388	static inline int put_page_testzero(struct page *page)
366	{	389	{
367	VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);	390	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
368	return atomic_dec_and_test(&page->_count);	391	return page_ref_dec_and_test(page);
369	}	392	}
370		393
371	/*	394	/*
@@ -376,7 +399,7 @@ static inline int put_page_testzero(struct page *page) · 4 unmodified lines
376	*/	399	*/
377	static inline int get_page_unless_zero(struct page *page)	400	static inline int get_page_unless_zero(struct page *page)
378	{	401	{
379	return atomic_inc_not_zero(&page->_count);	402	return page_ref_add_unless(page, 1, 0);
380	}	403	}
381		404
382	extern int page_is_ram(unsigned long pfn);	405	extern int page_is_ram(unsigned long pfn);
@@ -464,11 +487,6 @@ static inline int total_mapcount(struct page *page) · 81 unmodified lines
464	}	487	}
465	#endif	488	#endif
466		489
467	static inline int page_count(struct page *page)
468	{
469	return atomic_read(&compound_head(page)->_count);
470	}
471
472	static inline struct page virt_to_head_page(const void x)	490	static inline struct page virt_to_head_page(const void x)
473	{	491	{
474	struct page *page = virt_to_page(x);	492	struct page *page = virt_to_page(x);
@@ -476,15 +494,6 @@ static inline struct page virt_to_head_page(const void x) · 1 unmodified line
476	return compound_head(page);	494	return compound_head(page);
477	}	495	}
478		496
479	/*
480	* Setup the page count before being freed into the page allocator for
481	* the first time (boot or memory hotplug)
482	*/
483	static inline void init_page_count(struct page *page)
484	{
485	atomic_set(&page->_count, 1);
486	}
487
488	void __put_page(struct page *page);	497	void __put_page(struct page *page);
489		498
490	void put_pages_list(struct list_head *pages);	499	void put_pages_list(struct list_head *pages);
@@ -694,8 +703,8 @@ static inline void get_page(struct page *page) · 203 unmodified lines
694	* Getting a normal page or the head of a compound page	703	* Getting a normal page or the head of a compound page
695	* requires to already have an elevated page->_count.	704	* requires to already have an elevated page->_count.
696	*/	705	*/
697	VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);	706	VM_BUG_ON_PAGE(page_ref_count(page) <= 0, page);
698	atomic_inc(&page->_count);	707	page_ref_inc(page);
699		708
700	if (unlikely(is_zone_device_page(page)))	709	if (unlikely(is_zone_device_page(page)))
701	get_zone_device_page(page);	710	get_zone_device_page(page);
@@ -1043,8 +1052,6 @@ static inline void clear_page_pfmemalloc(struct page *page) · 341 unmodified lines
1043	* just gets major/minor fault counters bumped up.	1052	* just gets major/minor fault counters bumped up.
1044	*/	1053	*/
1045		1054
1046	#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
1047
1048	#define VM_FAULT_OOM 0x0001	1055	#define VM_FAULT_OOM 0x0001
1049	#define VM_FAULT_SIGBUS 0x0002	1056	#define VM_FAULT_SIGBUS 0x0002
1050	#define VM_FAULT_MAJOR 0x0004	1057	#define VM_FAULT_MAJOR 0x0004
@@ -1523,8 +1530,7 @@ static inline void mm_dec_nr_pmds(struct mm_struct *mm) · 472 unmodified lines
1523	}	1530	}
1524	#endif	1531	#endif
1525		1532
1526	int __pte_alloc(struct mm_struct mm, struct vm_area_struct vma,	1533	int __pte_alloc(struct mm_struct mm, pmd_t pmd, unsigned long address);
1527	pmd_t *pmd, unsigned long address);
1528	int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);	1534	int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1529		1535
1530	/*	1536	/*
@@ -1650,15 +1656,15 @@ static inline void pgtable_page_dtor(struct page *page) · 119 unmodified lines
1650	pte_unmap(pte); \	1656	pte_unmap(pte); \
1651	} while (0)	1657	} while (0)
1652		1658
1653	#define pte_alloc_map(mm, vma, pmd, address) \	1659	#define pte_alloc(mm, pmd, address) \
1654	((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \	1660	(unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd, address))
1655	pmd, address))? \	1661
1656	NULL: pte_offset_map(pmd, address))	1662	#define pte_alloc_map(mm, pmd, address) \
		1663	(pte_alloc(mm, pmd, address) ? NULL : pte_offset_map(pmd, address))
1657		1664
1658	#define pte_alloc_map_lock(mm, pmd, address, ptlp) \	1665	#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
1659	((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \	1666	(pte_alloc(mm, pmd, address) ? \
1660	pmd, address))? \	1667	NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
1661	NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
1662		1668
1663	#define pte_alloc_kernel(pmd, address) \	1669	#define pte_alloc_kernel(pmd, address) \
1664	((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \	1670	((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
@@ -1853,6 +1859,7 @@ extern int __meminit init_per_zone_wmark_min(void); · 188 unmodified lines
1853	extern void mem_init(void);	1859	extern void mem_init(void);
1854	extern void __init mmap_init(void);	1860	extern void __init mmap_init(void);
1855	extern void show_mem(unsigned int flags);	1861	extern void show_mem(unsigned int flags);
		1862	extern long si_mem_available(void);
1856	extern void si_meminfo(struct sysinfo * val);	1863	extern void si_meminfo(struct sysinfo * val);
1857	extern void si_meminfo_node(struct sysinfo *val, int nid);	1864	extern void si_meminfo_node(struct sysinfo *val, int nid);
1858		1865
@@ -1867,6 +1874,7 @@ extern void zone_pcp_reset(struct zone *zone); · 8 unmodified lines
1867		1874
1868	/* page_alloc.c */	1875	/* page_alloc.c */
1869	extern int min_free_kbytes;	1876	extern int min_free_kbytes;
		1877	extern int watermark_scale_factor;
1870		1878
1871	/* nommu.c */	1879	/* nommu.c */
1872	extern atomic_long_t mmap_pages_allocated;	1880	extern atomic_long_t mmap_pages_allocated;

include/linux/mmzone.h+8−0

 					   mem_hotplug_begin/end() */
 	int kswapd_max_order;
 	enum zone_type classzone_idx;
+#ifdef CONFIG_COMPACTION
+	int kcompactd_max_order;
+	enum zone_type kcompactd_classzone_idx;
+	wait_queue_head_t kcompactd_wait;
+	struct task_struct *kcompactd;
+#endif
 #ifdef CONFIG_NUMA_BALANCING
 	/* Lock serializing the migrate rate limiting window */
 	spinlock_t numabalancing_migrate_lock;
 struct ctl_table;
 int min_free_kbytes_sysctl_handler(struct ctl_table *, int,
 					void __user *, size_t *, loff_t *);
+int watermark_scale_factor_sysctl_handler(struct ctl_table *, int,
+					void __user *, size_t *, loff_t *);
 extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1];
 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int,
 					void __user *, size_t *, loff_t *);

include/linux/page-flags-layout.h+2−0

 #define ZONES_SHIFT 1
 #elif MAX_NR_ZONES <= 4
 #define ZONES_SHIFT 2
+#elif MAX_NR_ZONES <= 8
+#define ZONES_SHIFT 3
 #else
 #error ZONES_SHIFT -- too many zones configured adjust calculation
 #endif

include/linux/page-flags.h+17−15

 	return page;
 }
-static inline int PageTail(struct page *page)
+static __always_inline int PageTail(struct page *page)
 {
 	return READ_ONCE(page->compound_head) & 1;
 }
-static inline int PageCompound(struct page *page)
+static __always_inline int PageCompound(struct page *page)
 {
 	return test_bit(PG_head, &page->flags) || PageTail(page);
 }
  * Macros to create function definitions for page flags
  */
 #define TESTPAGEFLAG(uname, lname, policy)				\
-static inline int Page##uname(struct page *page)			\
+static __always_inline int Page##uname(struct page *page)		\
 	{ return test_bit(PG_##lname, &policy(page, 0)->flags); }
 #define SETPAGEFLAG(uname, lname, policy)				\
-static inline void SetPage##uname(struct page *page)			\
+static __always_inline void SetPage##uname(struct page *page)		\
 	{ set_bit(PG_##lname, &policy(page, 1)->flags); }
 #define CLEARPAGEFLAG(uname, lname, policy)				\
-static inline void ClearPage##uname(struct page *page)			\
+static __always_inline void ClearPage##uname(struct page *page)		\
 	{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
 #define __SETPAGEFLAG(uname, lname, policy)				\
-static inline void __SetPage##uname(struct page *page)			\
+static __always_inline void __SetPage##uname(struct page *page)		\
 	{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
 #define __CLEARPAGEFLAG(uname, lname, policy)				\
-static inline void __ClearPage##uname(struct page *page)		\
+static __always_inline void __ClearPage##uname(struct page *page)	\
 	{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
 #define TESTSETFLAG(uname, lname, policy)				\
-static inline int TestSetPage##uname(struct page *page)			\
+static __always_inline int TestSetPage##uname(struct page *page)	\
 	{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
 #define TESTCLEARFLAG(uname, lname, policy)				\
-static inline int TestClearPage##uname(struct page *page)		\
+static __always_inline int TestClearPage##uname(struct page *page)	\
 	{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
 #define PAGEFLAG(uname, lname, policy)					\
 #define PAGE_MAPPING_KSM	2
 #define PAGE_MAPPING_FLAGS	(PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
-static inline int PageAnon(struct page *page)
+static __always_inline int PageAnon(struct page *page)
 {
 	page = compound_head(page);
 	return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
  * is found in VM_MERGEABLE vmas.  It's a PageAnon page, pointing not to any
  * anon_vma, but to that page's node of the stable tree.
  */
-static inline int PageKsm(struct page *page)
+static __always_inline int PageKsm(struct page *page)
 {
 	page = compound_head(page);
 	return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
 	return ret;
 }
-static inline void __SetPageUptodate(struct page *page)
+static __always_inline void __SetPageUptodate(struct page *page)
 {
 	VM_BUG_ON_PAGE(PageTail(page), page);
 	smp_wmb();
 	__set_bit(PG_uptodate, &page->flags);
 }
-static inline void SetPageUptodate(struct page *page)
+static __always_inline void SetPageUptodate(struct page *page)
 {
 	VM_BUG_ON_PAGE(PageTail(page), page);
 	/*
 __PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
-static inline void set_compound_head(struct page *page, struct page *head)
+static __always_inline void set_compound_head(struct page *page, struct page *head)
 {
 	WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
 }
-static inline void clear_compound_head(struct page *page)
+static __always_inline void clear_compound_head(struct page *page)
 {
 	WRITE_ONCE(page->compound_head, 0);
 }
 	atomic_set(&page->_mapcount, -1);
 }
+extern bool is_free_buddy_page(struct page *page);
 #define PAGE_BALLOON_MAPCOUNT_VALUE (-256)
 static inline int PageBalloon(struct page *page)

include/linux/page_ref.h+173−0

@@ -0,0 +1,173 @@
	1	#ifndef _LINUX_PAGE_REF_H
	2	#define _LINUX_PAGE_REF_H
	3
	4	#include <linux/atomic.h>
	5	#include <linux/mm_types.h>
	6	#include <linux/page-flags.h>
	7	#include <linux/tracepoint-defs.h>
	8
	9	extern struct tracepoint __tracepoint_page_ref_set;
	10	extern struct tracepoint __tracepoint_page_ref_mod;
	11	extern struct tracepoint __tracepoint_page_ref_mod_and_test;
	12	extern struct tracepoint __tracepoint_page_ref_mod_and_return;
	13	extern struct tracepoint __tracepoint_page_ref_mod_unless;
	14	extern struct tracepoint __tracepoint_page_ref_freeze;
	15	extern struct tracepoint __tracepoint_page_ref_unfreeze;
	16
	17	#ifdef CONFIG_DEBUG_PAGE_REF
	18
	19	/*
	20	* Ideally we would want to use the trace_<tracepoint>_enabled() helper
	21	* functions. But due to include header file issues, that is not
	22	* feasible. Instead we have to open code the static key functions.
	23	*
	24	* See trace_##name##_enabled(void) in include/linux/tracepoint.h
	25	*/
	26	#define page_ref_tracepoint_active(t) static_key_false(&(t).key)
	27
	28	extern void __page_ref_set(struct page *page, int v);
	29	extern void __page_ref_mod(struct page *page, int v);
	30	extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
	31	extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
	32	extern void __page_ref_mod_unless(struct page *page, int v, int u);
	33	extern void __page_ref_freeze(struct page *page, int v, int ret);
	34	extern void __page_ref_unfreeze(struct page *page, int v);
	35
	36	#else
	37
	38	#define page_ref_tracepoint_active(t) false
	39
	40	static inline void __page_ref_set(struct page *page, int v)
	41	{
	42	}
	43	static inline void __page_ref_mod(struct page *page, int v)
	44	{
	45	}
	46	static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)
	47	{
	48	}
	49	static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)
	50	{
	51	}
	52	static inline void __page_ref_mod_unless(struct page *page, int v, int u)
	53	{
	54	}
	55	static inline void __page_ref_freeze(struct page *page, int v, int ret)
	56	{
	57	}
	58	static inline void __page_ref_unfreeze(struct page *page, int v)
	59	{
	60	}
	61
	62	#endif
	63
	64	static inline int page_ref_count(struct page *page)
	65	{
	66	return atomic_read(&page->_count);
	67	}
	68
	69	static inline int page_count(struct page *page)
	70	{
	71	return atomic_read(&compound_head(page)->_count);
	72	}
	73
	74	static inline void set_page_count(struct page *page, int v)
	75	{
	76	atomic_set(&page->_count, v);
	77	if (page_ref_tracepoint_active(__tracepoint_page_ref_set))
	78	__page_ref_set(page, v);
	79	}
	80
	81	/*
	82	* Setup the page count before being freed into the page allocator for
	83	* the first time (boot or memory hotplug)
	84	*/
	85	static inline void init_page_count(struct page *page)
	86	{
	87	set_page_count(page, 1);
	88	}
	89
	90	static inline void page_ref_add(struct page *page, int nr)
	91	{
	92	atomic_add(nr, &page->_count);
	93	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
	94	__page_ref_mod(page, nr);
	95	}
	96
	97	static inline void page_ref_sub(struct page *page, int nr)
	98	{
	99	atomic_sub(nr, &page->_count);
	100	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
	101	__page_ref_mod(page, -nr);
	102	}
	103
	104	static inline void page_ref_inc(struct page *page)
	105	{
	106	atomic_inc(&page->_count);
	107	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
	108	__page_ref_mod(page, 1);
	109	}
	110
	111	static inline void page_ref_dec(struct page *page)
	112	{
	113	atomic_dec(&page->_count);
	114	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
	115	__page_ref_mod(page, -1);
	116	}
	117
	118	static inline int page_ref_sub_and_test(struct page *page, int nr)
	119	{
	120	int ret = atomic_sub_and_test(nr, &page->_count);
	121
	122	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
	123	__page_ref_mod_and_test(page, -nr, ret);
	124	return ret;
	125	}
	126
	127	static inline int page_ref_dec_and_test(struct page *page)
	128	{
	129	int ret = atomic_dec_and_test(&page->_count);
	130
	131	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
	132	__page_ref_mod_and_test(page, -1, ret);
	133	return ret;
	134	}
	135
	136	static inline int page_ref_dec_return(struct page *page)
	137	{
	138	int ret = atomic_dec_return(&page->_count);
	139
	140	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return))
	141	__page_ref_mod_and_return(page, -1, ret);
	142	return ret;
	143	}
	144
	145	static inline int page_ref_add_unless(struct page *page, int nr, int u)
	146	{
	147	int ret = atomic_add_unless(&page->_count, nr, u);
	148
	149	if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_unless))
	150	__page_ref_mod_unless(page, nr, ret);
	151	return ret;
	152	}
	153
	154	static inline int page_ref_freeze(struct page *page, int count)
	155	{
	156	int ret = likely(atomic_cmpxchg(&page->_count, count, 0) == count);
	157
	158	if (page_ref_tracepoint_active(__tracepoint_page_ref_freeze))
	159	__page_ref_freeze(page, count, ret);
	160	return ret;
	161	}
	162
	163	static inline void page_ref_unfreeze(struct page *page, int count)
	164	{
	165	VM_BUG_ON_PAGE(page_count(page) != 0, page);
	166	VM_BUG_ON(count == 0);
	167
	168	atomic_set(&page->_count, count);
	169	if (page_ref_tracepoint_active(__tracepoint_page_ref_unfreeze))
	170	__page_ref_unfreeze(page, count);
	171	}
	172
	173	#endif

include/linux/pagemap.h+3−16

 	 * SMP requires.
 	 */
 	VM_BUG_ON_PAGE(page_count(page) == 0, page);
-	atomic_inc(&page->_count);
+	page_ref_inc(page);
 #else
 	if (unlikely(!get_page_unless_zero(page))) {
 	VM_BUG_ON(!in_atomic());
 # endif
 	VM_BUG_ON_PAGE(page_count(page) == 0, page);
-	atomic_add(count, &page->_count);
+	page_ref_add(page, count);
 #else
-	if (unlikely(!atomic_add_unless(&page->_count, count, 0)))
+	if (unlikely(!page_ref_add_unless(page, count, 0)))
 		return 0;
 #endif
 	VM_BUG_ON_PAGE(PageCompound(page) && page != compound_head(page), page);
 	return 1;
 }
-static inline int page_freeze_refs(struct page *page, int count)
-{
-	return likely(atomic_cmpxchg(&page->_count, count, 0) == count);
-}
-static inline void page_unfreeze_refs(struct page *page, int count)
-{
-	VM_BUG_ON_PAGE(page_count(page) != 0, page);
-	VM_BUG_ON(count == 0);
-	atomic_set(&page->_count, count);
-}
 #ifdef CONFIG_NUMA
 extern struct page *__page_cache_alloc(gfp_t gfp);
 #else

include/linux/poll.h+1−1

@@ -96,7 +96,7 @@ extern void poll_initwait(struct poll_wqueues *pwq); · 95 unmodified lines
96	extern void poll_freewait(struct poll_wqueues *pwq);	96	extern void poll_freewait(struct poll_wqueues *pwq);
97	extern int poll_schedule_timeout(struct poll_wqueues *pwq, int state,	97	extern int poll_schedule_timeout(struct poll_wqueues *pwq, int state,
98	ktime_t *expires, unsigned long slack);	98	ktime_t *expires, unsigned long slack);
99	extern long select_estimate_accuracy(struct timespec *tv);	99	extern u64 select_estimate_accuracy(struct timespec *tv);
100		100
101		101
102	static inline int poll_schedule(struct poll_wqueues *pwq, int state)	102	static inline int poll_schedule(struct poll_wqueues *pwq, int state)

include/linux/quicklist.h+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	* as needed after allocation when they are freed. Per cpu lists of pages	5	* as needed after allocation when they are freed. Per cpu lists of pages
6	* are kept that only contain node local pages.	6	* are kept that only contain node local pages.
7	*	7	*
8	* (C) 2007, SGI. Christoph Lameter <clameter@sgi.com>	8	* (C) 2007, SGI. Christoph Lameter <cl@linux.com>
9	*/	9	*/
10	#include <linux/kernel.h>	10	#include <linux/kernel.h>
11	#include <linux/gfp.h>	11	#include <linux/gfp.h>

include/linux/radix-tree.h+26−2

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	#ifndef _LINUX_RADIX_TREE_H	21	#ifndef _LINUX_RADIX_TREE_H
22	#define _LINUX_RADIX_TREE_H	22	#define _LINUX_RADIX_TREE_H
23		23
		24	#include <linux/bitops.h>
24	#include <linux/preempt.h>	25	#include <linux/preempt.h>
25	#include <linux/types.h>	26	#include <linux/types.h>
26	#include <linux/bug.h>	27	#include <linux/bug.h>
@@ -270,8 +271,15 @@ static inline void radix_tree_replace_slot(void *pslot, void item) · 243 unmodified lines
270	}	271	}
271		272
272	int __radix_tree_create(struct radix_tree_root *root, unsigned long index,	273	int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
273	struct radix_tree_node nodep, void *slotp);	274	unsigned order, struct radix_tree_node **nodep,
274	int radix_tree_insert(struct radix_tree_root , unsigned long, void );	275	void ***slotp);
		276	int __radix_tree_insert(struct radix_tree_root *, unsigned long index,
		277	unsigned order, void *);
		278	static inline int radix_tree_insert(struct radix_tree_root *root,
		279	unsigned long index, void *entry)
		280	{
		281	return __radix_tree_insert(root, index, 0, entry);
		282	}
275	void __radix_tree_lookup(struct radix_tree_root root, unsigned long index,	283	void __radix_tree_lookup(struct radix_tree_root root, unsigned long index,
276	struct radix_tree_node nodep, void *slotp);	284	struct radix_tree_node nodep, void *slotp);
277	void radix_tree_lookup(struct radix_tree_root , unsigned long);	285	void radix_tree_lookup(struct radix_tree_root , unsigned long);
@@ -394,6 +402,22 @@ void *radix_tree_iter_retry(struct radix_tree_iter iter) · 116 unmodified lines
394	return NULL;	402	return NULL;
395	}	403	}
396		404
		405	/**
		406	* radix_tree_iter_next - resume iterating when the chunk may be invalid
		407	* @iter: iterator state
		408	*
		409	* If the iterator needs to release then reacquire a lock, the chunk may
		410	* have been invalidated by an insertion or deletion. Call this function
		411	* to continue the iteration from the next index.
		412	*/
		413	static inline __must_check
		414	void *radix_tree_iter_next(struct radix_tree_iter iter)
		415	{
		416	iter->next_index = iter->index + 1;
		417	iter->tags = 0;
		418	return NULL;
		419	}
		420
397	/**	421	/**
398	* radix_tree_chunk_size - get current chunk size	422	* radix_tree_chunk_size - get current chunk size
399	*	423	*

include/linux/rmap.h+6−0

 	TTU_MIGRATION = 2,		/* migration mode */
 	TTU_MUNLOCK = 4,		/* munlock mode */
 	TTU_LZFREE = 8,			/* lazy free mode */
+	TTU_SPLIT_HUGE_PMD = 16,	/* split huge PMD if any */
 	TTU_IGNORE_MLOCK = (1 << 8),	/* ignore mlock */
 	TTU_IGNORE_ACCESS = (1 << 9),	/* don't age */
 	TTU_BATCH_FLUSH = (1 << 11),	/* Batch TLB flushes where possible
 					 * and caller guarantees they will
 					 * do a final flush if necessary */
+	TTU_RMAP_LOCKED = (1 << 12)	/* do not grab rmap lock:
+					 * caller holds it */
 };
 #ifdef CONFIG_MMU
  */
 int try_to_munlock(struct page *);
+void remove_migration_ptes(struct page *old, struct page *new, bool locked);
 /*
  * Called by memory-failure.c to kill processes.
  */
 };
 int rmap_walk(struct page *page, struct rmap_walk_control *rwc);
+int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc);
 #else	/* !CONFIG_MMU */

include/linux/sched.h+2−2

@@ -1791,8 +1791,8 @@ struct task_struct { · 1790 unmodified lines
1791	* time slack values; these are used to round up poll() and	1791	* time slack values; these are used to round up poll() and
1792	* select() etc timeout values. These are in nanoseconds.	1792	* select() etc timeout values. These are in nanoseconds.
1793	*/	1793	*/
1794	unsigned long timer_slack_ns;	1794	u64 timer_slack_ns;
1795	unsigned long default_timer_slack_ns;	1795	u64 default_timer_slack_ns;
1796		1796
1797	#ifdef CONFIG_KASAN	1797	#ifdef CONFIG_KASAN
1798	unsigned int kasan_depth;	1798	unsigned int kasan_depth;

include/linux/sched/sysctl.h+0−21

@@ -14,27 +14,6 @@ extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, · 13 unmodified lines
14	enum { sysctl_hung_task_timeout_secs = 0 };	14	enum { sysctl_hung_task_timeout_secs = 0 };
15	#endif	15	#endif
16		16
17	/*
18	* Default maximum number of active map areas, this limits the number of vmas
19	* per mm struct. Users can overwrite this number by sysctl but there is a
20	* problem.
21	*
22	* When a program's coredump is generated as ELF format, a section is created
23	* per a vma. In ELF, the number of sections is represented in unsigned short.
24	* This means the number of sections should be smaller than 65535 at coredump.
25	* Because the kernel adds some informative sections to a image of program at
26	* generating coredump, we need some margin. The number of extra sections is
27	* 1-3 now and depends on arch. We use "5" as safe margin, here.
28	*
29	* ELF extended numbering allows more than 65535 sections, so 16-bit bound is
30	* not a hard limit any more. Although some userspace tools can be surprised by
31	* that.
32	*/
33	#define MAPCOUNT_ELF_CORE_MARGIN (5)
34	#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
35
36	extern int sysctl_max_map_count;
37
38	extern unsigned int sysctl_sched_latency;	17	extern unsigned int sysctl_sched_latency;
39	extern unsigned int sysctl_sched_min_granularity;	18	extern unsigned int sysctl_sched_min_granularity;
40	extern unsigned int sysctl_sched_wakeup_granularity;	19	extern unsigned int sysctl_sched_wakeup_granularity;

include/linux/string.h+7−1

 extern void argv_free(char **argv);
 extern bool sysfs_streq(const char *s1, const char *s2);
-extern int strtobool(const char *s, bool *res);
+extern int kstrtobool(const char *s, bool *res);
+static inline int strtobool(const char *s, bool *res)
+{
+	return kstrtobool(s, res);
+}
+int match_string(const char * const *array, size_t n, const char *string);
 #ifdef CONFIG_BINARY_PRINTF
 int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args);

include/linux/tick.h+1−1

@@ -111,7 +111,7 @@ enum tick_dep_bits { · 110 unmodified lines
111	#define TICK_DEP_MASK_CLOCK_UNSTABLE (1 << TICK_DEP_BIT_CLOCK_UNSTABLE)	111	#define TICK_DEP_MASK_CLOCK_UNSTABLE (1 << TICK_DEP_BIT_CLOCK_UNSTABLE)
112		112
113	#ifdef CONFIG_NO_HZ_COMMON	113	#ifdef CONFIG_NO_HZ_COMMON
114	extern int tick_nohz_enabled;	114	extern bool tick_nohz_enabled;
115	extern int tick_nohz_tick_stopped(void);	115	extern int tick_nohz_tick_stopped(void);
116	extern void tick_nohz_idle_enter(void);	116	extern void tick_nohz_idle_enter(void);
117	extern void tick_nohz_idle_exit(void);	117	extern void tick_nohz_idle_exit(void);

include/linux/unaligned/access_ok.h+12−12

include/linux/vm_event_item.h+2−0

 		COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED,
 		COMPACTISOLATED,
 		COMPACTSTALL, COMPACTFAIL, COMPACTSUCCESS,
+		KCOMPACTD_WAKE,
 #endif
 #ifdef CONFIG_HUGETLB_PAGE
 		HTLB_BUDDY_PGALLOC, HTLB_BUDDY_PGALLOC_FAIL,
 		THP_COLLAPSE_ALLOC_FAILED,
 		THP_SPLIT_PAGE,
 		THP_SPLIT_PAGE_FAILED,
+		THP_DEFERRED_SPLIT_PAGE,
 		THP_SPLIT_PMD,
 		THP_ZERO_PAGE_ALLOC,
 		THP_ZERO_PAGE_ALLOC_FAILED,

include/trace/events/compaction.h+55−0

@@ -350,6 +350,61 @@ DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_defer_reset, · 349 unmodified lines
350	);	350	);
351	#endif	351	#endif
352		352
		353	TRACE_EVENT(mm_compaction_kcompactd_sleep,
		354
		355	TP_PROTO(int nid),
		356
		357	TP_ARGS(nid),
		358
		359	TP_STRUCT__entry(
		360	__field(int, nid)
		361	),
		362
		363	TP_fast_assign(
		364	__entry->nid = nid;
		365	),
		366
		367	TP_printk("nid=%d", __entry->nid)
		368	);
		369
		370	DECLARE_EVENT_CLASS(kcompactd_wake_template,
		371
		372	TP_PROTO(int nid, int order, enum zone_type classzone_idx),
		373
		374	TP_ARGS(nid, order, classzone_idx),
		375
		376	TP_STRUCT__entry(
		377	__field(int, nid)
		378	__field(int, order)
		379	__field(enum zone_type, classzone_idx)
		380	),
		381
		382	TP_fast_assign(
		383	__entry->nid = nid;
		384	__entry->order = order;
		385	__entry->classzone_idx = classzone_idx;
		386	),
		387
		388	TP_printk("nid=%d order=%d classzone_idx=%-8s",
		389	__entry->nid,
		390	__entry->order,
		391	__print_symbolic(__entry->classzone_idx, ZONE_TYPE))
		392	);
		393
		394	DEFINE_EVENT(kcompactd_wake_template, mm_compaction_wakeup_kcompactd,
		395
		396	TP_PROTO(int nid, int order, enum zone_type classzone_idx),
		397
		398	TP_ARGS(nid, order, classzone_idx)
		399	);
		400
		401	DEFINE_EVENT(kcompactd_wake_template, mm_compaction_kcompactd_wake,
		402
		403	TP_PROTO(int nid, int order, enum zone_type classzone_idx),
		404
		405	TP_ARGS(nid, order, classzone_idx)
		406	);
		407
353	#endif /* _TRACE_COMPACTION_H */	408	#endif /* _TRACE_COMPACTION_H */
354		409
355	/* This part must be outside protection */	410	/* This part must be outside protection */

include/trace/events/mmflags.h+16−7

 	) : "none"
 #if defined(CONFIG_X86)
-#define __VM_ARCH_SPECIFIC {VM_PAT,     "pat"           }
+#define __VM_ARCH_SPECIFIC_1 {VM_PAT,     "pat"           }
 #elif defined(CONFIG_PPC)
-#define __VM_ARCH_SPECIFIC {VM_SAO,     "sao"           }
+#define __VM_ARCH_SPECIFIC_1 {VM_SAO,     "sao"           }
 #elif defined(CONFIG_PARISC) || defined(CONFIG_METAG) || defined(CONFIG_IA64)
-#define __VM_ARCH_SPECIFIC {VM_GROWSUP,	"growsup"	}
+#define __VM_ARCH_SPECIFIC_1 {VM_GROWSUP,	"growsup"	}
 #elif !defined(CONFIG_MMU)
-#define __VM_ARCH_SPECIFIC {VM_MAPPED_COPY,"mappedcopy"	}
+#define __VM_ARCH_SPECIFIC_1 {VM_MAPPED_COPY,"mappedcopy"	}
 #else
-#define __VM_ARCH_SPECIFIC {VM_ARCH_1,	"arch_1"	}
+#define __VM_ARCH_SPECIFIC_1 {VM_ARCH_1,	"arch_1"	}
+#endif
+#if defined(CONFIG_X86)
+#define __VM_ARCH_SPECIFIC_2 {VM_MPX,		"mpx"		}
+#else
+#define __VM_ARCH_SPECIFIC_2 {VM_ARCH_2,	"arch_2"	}
 #endif
 #ifdef CONFIG_MEM_SOFT_DIRTY
 	{VM_MAYEXEC,			"mayexec"	},		\
 	{VM_MAYSHARE,			"mayshare"	},		\
 	{VM_GROWSDOWN,			"growsdown"	},		\
+	{VM_UFFD_MISSING,		"uffd_missing"	},		\
 	{VM_PFNMAP,			"pfnmap"	},		\
 	{VM_DENYWRITE,			"denywrite"	},		\
-	{VM_LOCKONFAULT,		"lockonfault"	},		\
+	{VM_UFFD_WP,			"uffd_wp"	},		\
 	{VM_LOCKED,			"locked"	},		\
 	{VM_IO,				"io"		},		\
 	{VM_SEQ_READ,			"seqread"	},		\
 	{VM_RAND_READ,			"randread"	},		\
 	{VM_DONTCOPY,			"dontcopy"	},		\
 	{VM_DONTEXPAND,			"dontexpand"	},		\
+	{VM_LOCKONFAULT,		"lockonfault"	},		\
 	{VM_ACCOUNT,			"account"	},		\
 	{VM_NORESERVE,			"noreserve"	},		\
 	{VM_HUGETLB,			"hugetlb"	},		\
-	__VM_ARCH_SPECIFIC				,		\
+	__VM_ARCH_SPECIFIC_1				,		\
+	__VM_ARCH_SPECIFIC_2				,		\
 	{VM_DONTDUMP,			"dontdump"	},		\
 IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY,	"softdirty"	)		\
 	{VM_MIXEDMAP,			"mixedmap"	},		\

include/trace/events/page_ref.h+134−0

@@ -0,0 +1,134 @@
	1	#undef TRACE_SYSTEM
	2	#define TRACE_SYSTEM page_ref
	3
	4	#if !defined(_TRACE_PAGE_REF_H) \|\| defined(TRACE_HEADER_MULTI_READ)
	5	#define _TRACE_PAGE_REF_H
	6
	7	#include <linux/types.h>
	8	#include <linux/page_ref.h>
	9	#include <linux/tracepoint.h>
	10	#include <trace/events/mmflags.h>
	11
	12	DECLARE_EVENT_CLASS(page_ref_mod_template,
	13
	14	TP_PROTO(struct page *page, int v),
	15
	16	TP_ARGS(page, v),
	17
	18	TP_STRUCT__entry(
	19	__field(unsigned long, pfn)
	20	__field(unsigned long, flags)
	21	__field(int, count)
	22	__field(int, mapcount)
	23	__field(void *, mapping)
	24	__field(int, mt)
	25	__field(int, val)
	26	),
	27
	28	TP_fast_assign(
	29	__entry->pfn = page_to_pfn(page);
	30	__entry->flags = page->flags;
	31	__entry->count = page_ref_count(page);
	32	__entry->mapcount = page_mapcount(page);
	33	__entry->mapping = page->mapping;
	34	__entry->mt = get_pageblock_migratetype(page);
	35	__entry->val = v;
	36	),
	37
	38	TP_printk("pfn=0x%lx flags=%s count=%d mapcount=%d mapping=%p mt=%d val=%d",
	39	__entry->pfn,
	40	show_page_flags(__entry->flags & ((1UL << NR_PAGEFLAGS) - 1)),
	41	__entry->count,
	42	__entry->mapcount, __entry->mapping, __entry->mt,
	43	__entry->val)
	44	);
	45
	46	DEFINE_EVENT(page_ref_mod_template, page_ref_set,
	47
	48	TP_PROTO(struct page *page, int v),
	49
	50	TP_ARGS(page, v)
	51	);
	52
	53	DEFINE_EVENT(page_ref_mod_template, page_ref_mod,
	54
	55	TP_PROTO(struct page *page, int v),
	56
	57	TP_ARGS(page, v)
	58	);
	59
	60	DECLARE_EVENT_CLASS(page_ref_mod_and_test_template,
	61
	62	TP_PROTO(struct page *page, int v, int ret),
	63
	64	TP_ARGS(page, v, ret),
	65
	66	TP_STRUCT__entry(
	67	__field(unsigned long, pfn)
	68	__field(unsigned long, flags)
	69	__field(int, count)
	70	__field(int, mapcount)
	71	__field(void *, mapping)
	72	__field(int, mt)
	73	__field(int, val)
	74	__field(int, ret)
	75	),
	76
	77	TP_fast_assign(
	78	__entry->pfn = page_to_pfn(page);
	79	__entry->flags = page->flags;
	80	__entry->count = page_ref_count(page);
	81	__entry->mapcount = page_mapcount(page);
	82	__entry->mapping = page->mapping;
	83	__entry->mt = get_pageblock_migratetype(page);
	84	__entry->val = v;
	85	__entry->ret = ret;
	86	),
	87
	88	TP_printk("pfn=0x%lx flags=%s count=%d mapcount=%d mapping=%p mt=%d val=%d ret=%d",
	89	__entry->pfn,
	90	show_page_flags(__entry->flags & ((1UL << NR_PAGEFLAGS) - 1)),
	91	__entry->count,
	92	__entry->mapcount, __entry->mapping, __entry->mt,
	93	__entry->val, __entry->ret)
	94	);
	95
	96	DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_mod_and_test,
	97
	98	TP_PROTO(struct page *page, int v, int ret),
	99
	100	TP_ARGS(page, v, ret)
	101	);
	102
	103	DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_mod_and_return,
	104
	105	TP_PROTO(struct page *page, int v, int ret),
	106
	107	TP_ARGS(page, v, ret)
	108	);
	109
	110	DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_mod_unless,
	111
	112	TP_PROTO(struct page *page, int v, int ret),
	113
	114	TP_ARGS(page, v, ret)
	115	);
	116
	117	DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_freeze,
	118
	119	TP_PROTO(struct page *page, int v, int ret),
	120
	121	TP_ARGS(page, v, ret)
	122	);
	123
	124	DEFINE_EVENT(page_ref_mod_template, page_ref_unfreeze,
	125
	126	TP_PROTO(struct page *page, int v),
	127
	128	TP_ARGS(page, v)
	129	);
	130
	131	#endif /* _TRACE_PAGE_COUNT_H */
	132
	133	/* This part must be outside protection */
	134	#include <trace/define_trace.h>

include/uapi/linux/byteorder/big_endian.h+12−12

include/uapi/linux/byteorder/little_endian.h+12−12

include/uapi/linux/elf-em.h+0−3

 #define EM_X86_64	62	/* AMD x86-64 */
 #define EM_S390		22	/* IBM S/390 */
 #define EM_CRIS		76	/* Axis Communications 32-bit embedded processor */
-#define EM_V850		87	/* NEC v850 */
 #define EM_M32R		88	/* Renesas M32R */
 #define EM_MN10300	89	/* Panasonic/MEI MN10300, AM33 */
 #define EM_OPENRISC     92     /* OpenRISC 32-bit embedded processor */
  */
 #define EM_ALPHA	0x9026
-/* Bogus old v850 magic number, used by old tools. */
-#define EM_CYGNUS_V850	0x9080
 /* Bogus old m32r magic number, used by old tools. */
 #define EM_CYGNUS_M32R	0x9041
 /* This is the old interim value for S/390 architecture */

include/uapi/linux/swab.h+5−5

@@ -151,7 +151,7 @@ static inline __attribute_const__ __u32 __fswahb32(__u32 val) · 150 unmodified lines
151	* __swab16p - return a byteswapped 16-bit value from a pointer	151	* __swab16p - return a byteswapped 16-bit value from a pointer
152	* @p: pointer to a naturally-aligned 16-bit value	152	* @p: pointer to a naturally-aligned 16-bit value
153	*/	153	*/
154	static inline __u16 __swab16p(const __u16 *p)	154	static __always_inline __u16 __swab16p(const __u16 *p)
155	{	155	{
156	#ifdef __arch_swab16p	156	#ifdef __arch_swab16p
157	return __arch_swab16p(p);	157	return __arch_swab16p(p);
@@ -164,7 +164,7 @@ static inline __u16 __swab16p(const __u16 *p) · 6 unmodified lines
164	* __swab32p - return a byteswapped 32-bit value from a pointer	164	* __swab32p - return a byteswapped 32-bit value from a pointer
165	* @p: pointer to a naturally-aligned 32-bit value	165	* @p: pointer to a naturally-aligned 32-bit value
166	*/	166	*/
167	static inline __u32 __swab32p(const __u32 *p)	167	static __always_inline __u32 __swab32p(const __u32 *p)
168	{	168	{
169	#ifdef __arch_swab32p	169	#ifdef __arch_swab32p
170	return __arch_swab32p(p);	170	return __arch_swab32p(p);
@@ -177,7 +177,7 @@ static inline __u32 __swab32p(const __u32 *p) · 6 unmodified lines
177	* __swab64p - return a byteswapped 64-bit value from a pointer	177	* __swab64p - return a byteswapped 64-bit value from a pointer
178	* @p: pointer to a naturally-aligned 64-bit value	178	* @p: pointer to a naturally-aligned 64-bit value
179	*/	179	*/
180	static inline __u64 __swab64p(const __u64 *p)	180	static __always_inline __u64 __swab64p(const __u64 *p)
181	{	181	{
182	#ifdef __arch_swab64p	182	#ifdef __arch_swab64p
183	return __arch_swab64p(p);	183	return __arch_swab64p(p);
@@ -232,7 +232,7 @@ static inline void __swab16s(__u16 *p) · 48 unmodified lines
232	* __swab32s - byteswap a 32-bit value in-place	232	* __swab32s - byteswap a 32-bit value in-place
233	* @p: pointer to a naturally-aligned 32-bit value	233	* @p: pointer to a naturally-aligned 32-bit value
234	*/	234	*/
235	static inline void __swab32s(__u32 *p)	235	static __always_inline void __swab32s(__u32 *p)
236	{	236	{
237	#ifdef __arch_swab32s	237	#ifdef __arch_swab32s
238	__arch_swab32s(p);	238	__arch_swab32s(p);
@@ -245,7 +245,7 @@ static inline void __swab32s(__u32 *p) · 6 unmodified lines
245	* __swab64s - byteswap a 64-bit value in-place	245	* __swab64s - byteswap a 64-bit value in-place
246	* @p: pointer to a naturally-aligned 64-bit value	246	* @p: pointer to a naturally-aligned 64-bit value
247	*/	247	*/
248	static inline void __swab64s(__u64 *p)	248	static __always_inline void __swab64s(__u64 *p)
249	{	249	{
250	#ifdef __arch_swab64s	250	#ifdef __arch_swab64s
251	__arch_swab64s(p);	251	__arch_swab64s(p);

include/uapi/linux/virtio_balloon.h+2−1

@@ -51,7 +51,8 @@ struct virtio_balloon_config { · 50 unmodified lines
51	#define VIRTIO_BALLOON_S_MINFLT 3 /* Number of minor faults */	51	#define VIRTIO_BALLOON_S_MINFLT 3 /* Number of minor faults */
52	#define VIRTIO_BALLOON_S_MEMFREE 4 /* Total amount of free memory */	52	#define VIRTIO_BALLOON_S_MEMFREE 4 /* Total amount of free memory */
53	#define VIRTIO_BALLOON_S_MEMTOT 5 /* Total amount of memory */	53	#define VIRTIO_BALLOON_S_MEMTOT 5 /* Total amount of memory */
54	#define VIRTIO_BALLOON_S_NR 6	54	#define VIRTIO_BALLOON_S_AVAIL 6 /* Available memory as in /proc */
		55	#define VIRTIO_BALLOON_S_NR 7
55		56
56	/*	57	/*
57	* Memory statistics structure.	58	* Memory statistics structure.

kernel/fork.c+9−1

 	struct page *page = alloc_kmem_pages_node(node, THREADINFO_GFP,
 						  THREAD_SIZE_ORDER);
+	if (page)
+		memcg_kmem_update_page_stat(page, MEMCG_KERNEL_STACK,
+					    1 << THREAD_SIZE_ORDER);
 	return page ? page_address(page) : NULL;
 }
 static inline void free_thread_info(struct thread_info *ti)
 {
-	free_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER);
+	struct page *page = virt_to_page(ti);
+	memcg_kmem_update_page_stat(page, MEMCG_KERNEL_STACK,
+				    -(1 << THREAD_SIZE_ORDER));
+	__free_kmem_pages(page, THREAD_SIZE_ORDER);
 }
 # else
 static struct kmem_cache *thread_info_cache;

kernel/panic.c+26−15

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	#include <linux/init.h>	24	#include <linux/init.h>
25	#include <linux/nmi.h>	25	#include <linux/nmi.h>
26	#include <linux/console.h>	26	#include <linux/console.h>
		27	#include <linux/bug.h>
27		28
28	#define PANIC_TIMER_STEP 100	29	#define PANIC_TIMER_STEP 100
29	#define PANIC_BLINK_SPD 18	30	#define PANIC_BLINK_SPD 18
@@ -449,20 +450,25 @@ void oops_exit(void) · 419 unmodified lines
449	kmsg_dump(KMSG_DUMP_OOPS);	450	kmsg_dump(KMSG_DUMP_OOPS);
450	}	451	}
451		452
452	#ifdef WANT_WARN_ON_SLOWPATH	453	struct warn_args {
453	struct slowpath_args {
454	const char *fmt;	454	const char *fmt;
455	va_list args;	455	va_list args;
456	};	456	};
457		457
458	static void warn_slowpath_common(const char file, int line, void caller,	458	void __warn(const char file, int line, void caller, unsigned taint,
459	unsigned taint, struct slowpath_args *args)	459	struct pt_regs regs, struct warn_args args)
460	{	460	{
461	disable_trace_on_warning();	461	disable_trace_on_warning();
462		462
463	pr_warn("------------[ cut here ]------------\n");	463	pr_warn("------------[ cut here ]------------\n");
464	pr_warn("WARNING: CPU: %d PID: %d at %s:%d %pS()\n",	464
465	raw_smp_processor_id(), current->pid, file, line, caller);	465	if (file)
		466	pr_warn("WARNING: CPU: %d PID: %d at %s:%d %pS\n",
		467	raw_smp_processor_id(), current->pid, file, line,
		468	caller);
		469	else
		470	pr_warn("WARNING: CPU: %d PID: %d at %pS\n",
		471	raw_smp_processor_id(), current->pid, caller);
466		472
467	if (args)	473	if (args)
468	vprintk(args->fmt, args->args);	474	vprintk(args->fmt, args->args);
@@ -479,20 +485,27 @@ static void warn_slowpath_common(const char file, int line, void caller, · 10 unmodified lines
479	}	485	}
480		486
481	print_modules();	487	print_modules();
482	dump_stack();	488
		489	if (regs)
		490	show_regs(regs);
		491	else
		492	dump_stack();
		493
483	print_oops_end_marker();	494	print_oops_end_marker();
		495
484	/* Just a warning, don't kill lockdep. */	496	/* Just a warning, don't kill lockdep. */
485	add_taint(taint, LOCKDEP_STILL_OK);	497	add_taint(taint, LOCKDEP_STILL_OK);
486	}	498	}
487		499
		500	#ifdef WANT_WARN_ON_SLOWPATH
488	void warn_slowpath_fmt(const char file, int line, const char fmt, ...)	501	void warn_slowpath_fmt(const char file, int line, const char fmt, ...)
489	{	502	{
490	struct slowpath_args args;	503	struct warn_args args;
491		504
492	args.fmt = fmt;	505	args.fmt = fmt;
493	va_start(args.args, fmt);	506	va_start(args.args, fmt);
494	warn_slowpath_common(file, line, __builtin_return_address(0),	507	__warn(file, line, __builtin_return_address(0), TAINT_WARN, NULL,
495	TAINT_WARN, &args);	508	&args);
496	va_end(args.args);	509	va_end(args.args);
497	}	510	}
498	EXPORT_SYMBOL(warn_slowpath_fmt);	511	EXPORT_SYMBOL(warn_slowpath_fmt);
@@ -500,20 +513,18 @@ EXPORT_SYMBOL(warn_slowpath_fmt); · 1 unmodified line
500	void warn_slowpath_fmt_taint(const char *file, int line,	513	void warn_slowpath_fmt_taint(const char *file, int line,
501	unsigned taint, const char *fmt, ...)	514	unsigned taint, const char *fmt, ...)
502	{	515	{
503	struct slowpath_args args;	516	struct warn_args args;
504		517
505	args.fmt = fmt;	518	args.fmt = fmt;
506	va_start(args.args, fmt);	519	va_start(args.args, fmt);
507	warn_slowpath_common(file, line, __builtin_return_address(0),	520	__warn(file, line, __builtin_return_address(0), taint, NULL, &args);
508	taint, &args);
509	va_end(args.args);	521	va_end(args.args);
510	}	522	}
511	EXPORT_SYMBOL(warn_slowpath_fmt_taint);	523	EXPORT_SYMBOL(warn_slowpath_fmt_taint);
512		524
513	void warn_slowpath_null(const char *file, int line)	525	void warn_slowpath_null(const char *file, int line)
514	{	526	{
515	warn_slowpath_common(file, line, __builtin_return_address(0),	527	__warn(file, line, __builtin_return_address(0), TAINT_WARN, NULL, NULL);
516	TAINT_WARN, NULL);
517	}	528	}
518	EXPORT_SYMBOL(warn_slowpath_null);	529	EXPORT_SYMBOL(warn_slowpath_null);
519	#endif	530	#endif

kernel/printk/printk.c+66−74

@@ -367,16 +367,20 @@ static int logbuf_has_space(u32 msg_size, bool empty) · 366 unmodified lines
367		367
368	static int log_make_free_space(u32 msg_size)	368	static int log_make_free_space(u32 msg_size)
369	{	369	{
370	while (log_first_seq < log_next_seq) {	370	while (log_first_seq < log_next_seq &&
371	if (logbuf_has_space(msg_size, false))	371	!logbuf_has_space(msg_size, false)) {
372	return 0;
373	/* drop old messages until we have enough contiguous space */	372	/* drop old messages until we have enough contiguous space */
374	log_first_idx = log_next(log_first_idx);	373	log_first_idx = log_next(log_first_idx);
375	log_first_seq++;	374	log_first_seq++;
376	}	375	}
377		376
		377	if (clear_seq < log_first_seq) {
		378	clear_seq = log_first_seq;
		379	clear_idx = log_first_idx;
		380	}
		381
378	/* sequence numbers are equal, so the log buffer is empty */	382	/* sequence numbers are equal, so the log buffer is empty */
379	if (logbuf_has_space(msg_size, true))	383	if (logbuf_has_space(msg_size, log_first_seq == log_next_seq))
380	return 0;	384	return 0;
381		385
382	return -ENOMEM;	386	return -ENOMEM;
@@ -854,6 +858,7 @@ void log_buf_kexec_setup(void) · 471 unmodified lines
854	VMCOREINFO_SYMBOL(log_buf);	858	VMCOREINFO_SYMBOL(log_buf);
855	VMCOREINFO_SYMBOL(log_buf_len);	859	VMCOREINFO_SYMBOL(log_buf_len);
856	VMCOREINFO_SYMBOL(log_first_idx);	860	VMCOREINFO_SYMBOL(log_first_idx);
		861	VMCOREINFO_SYMBOL(clear_idx);
857	VMCOREINFO_SYMBOL(log_next_idx);	862	VMCOREINFO_SYMBOL(log_next_idx);
858	/*	863	/*
859	* Export struct printk_log size and field offsets. User space tools can	864	* Export struct printk_log size and field offsets. User space tools can
@@ -1216,12 +1221,6 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 356 unmodified lines
1216	u32 idx;	1221	u32 idx;
1217	enum log_flags prev;	1222	enum log_flags prev;
1218		1223
1219	if (clear_seq < log_first_seq) {
1220	/* messages are gone, move to first available one */
1221	clear_seq = log_first_seq;
1222	clear_idx = log_first_idx;
1223	}
1224
1225	/*	1224	/*
1226	* Find first record that fits, including all following records,	1225	* Find first record that fits, including all following records,
1227	* into the user-provided buffer for this dump.	1226	* into the user-provided buffer for this dump.
@@ -1483,58 +1482,6 @@ static void zap_locks(void) · 255 unmodified lines
1483	sema_init(&console_sem, 1);	1482	sema_init(&console_sem, 1);
1484	}	1483	}
1485		1484
1486	/*
1487	* Check if we have any console that is capable of printing while cpu is
1488	* booting or shutting down. Requires console_sem.
1489	*/
1490	static int have_callable_console(void)
1491	{
1492	struct console *con;
1493
1494	for_each_console(con)
1495	if (con->flags & CON_ANYTIME)
1496	return 1;
1497
1498	return 0;
1499	}
1500
1501	/*
1502	* Can we actually use the console at this time on this cpu?
1503	*
1504	* Console drivers may assume that per-cpu resources have been allocated. So
1505	* unless they're explicitly marked as being able to cope (CON_ANYTIME) don't
1506	* call them until this CPU is officially up.
1507	*/
1508	static inline int can_use_console(unsigned int cpu)
1509	{
1510	return cpu_online(cpu) \|\| have_callable_console();
1511	}
1512
1513	/*
1514	* Try to get console ownership to actually show the kernel
1515	* messages from a 'printk'. Return true (and with the
1516	* console_lock held, and 'console_locked' set) if it
1517	* is successful, false otherwise.
1518	*/
1519	static int console_trylock_for_printk(void)
1520	{
1521	unsigned int cpu = smp_processor_id();
1522
1523	if (!console_trylock())
1524	return 0;
1525	/*
1526	* If we can't use the console, we need to release the console
1527	* semaphore by hand to avoid flushing the buffer. We need to hold the
1528	* console semaphore in order to do this test safely.
1529	*/
1530	if (!can_use_console(cpu)) {
1531	console_locked = 0;
1532	up_console_sem();
1533	return 0;
1534	}
1535	return 1;
1536	}
1537
1538	int printk_delay_msec __read_mostly;	1485	int printk_delay_msec __read_mostly;
1539		1486
1540	static inline void printk_delay(void)	1487	static inline void printk_delay(void)
@@ -1681,7 +1628,6 @@ asmlinkage int vprintk_emit(int facility, int level, · 140 unmodified lines
1681	boot_delay_msec(level);	1628	boot_delay_msec(level);
1682	printk_delay();	1629	printk_delay();
1683		1630
1684	/* This stops the holder of console_sem just where we want him */
1685	local_irq_save(flags);	1631	local_irq_save(flags);
1686	this_cpu = smp_processor_id();	1632	this_cpu = smp_processor_id();
1687		1633
@@ -1705,6 +1651,7 @@ asmlinkage int vprintk_emit(int facility, int level, · 17 unmodified lines
1705	}	1651	}
1706		1652
1707	lockdep_off();	1653	lockdep_off();
		1654	/* This stops the holder of console_sem just where we want him */
1708	raw_spin_lock(&logbuf_lock);	1655	raw_spin_lock(&logbuf_lock);
1709	logbuf_cpu = this_cpu;	1656	logbuf_cpu = this_cpu;
1710		1657
@@ -1809,21 +1756,13 @@ asmlinkage int vprintk_emit(int facility, int level, · 98 unmodified lines
1809	/* If called from the scheduler, we can not call up(). */	1756	/* If called from the scheduler, we can not call up(). */
1810	if (!in_sched) {	1757	if (!in_sched) {
1811	lockdep_off();	1758	lockdep_off();
1812	/*
1813	* Disable preemption to avoid being preempted while holding
1814	* console_sem which would prevent anyone from printing to
1815	* console
1816	*/
1817	preempt_disable();
1818
1819	/*	1759	/*
1820	* Try to acquire and then immediately release the console	1760	* Try to acquire and then immediately release the console
1821	* semaphore. The release will print out buffers and wake up	1761	* semaphore. The release will print out buffers and wake up
1822	* /dev/kmsg and syslog() users.	1762	* /dev/kmsg and syslog() users.
1823	*/	1763	*/
1824	if (console_trylock_for_printk())	1764	if (console_trylock())
1825	console_unlock();	1765	console_unlock();
1826	preempt_enable();
1827	lockdep_on();	1766	lockdep_on();
1828	}	1767	}
1829		1768
@@ -2174,7 +2113,20 @@ int console_trylock(void) · 344 unmodified lines
2174	return 0;	2113	return 0;
2175	}	2114	}
2176	console_locked = 1;	2115	console_locked = 1;
2177	console_may_schedule = 0;	2116	/*
		2117	* When PREEMPT_COUNT disabled we can't reliably detect if it's
		2118	* safe to schedule (e.g. calling printk while holding a spin_lock),
		2119	* because preempt_disable()/preempt_enable() are just barriers there
		2120	* and preempt_count() is always 0.
		2121	*
		2122	* RCU read sections have a separate preemption counter when
		2123	* PREEMPT_RCU enabled thus we must take extra care and check
		2124	* rcu_preempt_depth(), otherwise RCU read sections modify
		2125	* preempt_count().
		2126	*/
		2127	console_may_schedule = !oops_in_progress &&
		2128	preemptible() &&
		2129	!rcu_preempt_depth();
2178	return 1;	2130	return 1;
2179	}	2131	}
2180	EXPORT_SYMBOL(console_trylock);	2132	EXPORT_SYMBOL(console_trylock);
@@ -2184,6 +2136,34 @@ int is_console_locked(void) · 3 unmodified lines
2184	return console_locked;	2136	return console_locked;
2185	}	2137	}
2186		2138
		2139	/*
		2140	* Check if we have any console that is capable of printing while cpu is
		2141	* booting or shutting down. Requires console_sem.
		2142	*/
		2143	static int have_callable_console(void)
		2144	{
		2145	struct console *con;
		2146
		2147	for_each_console(con)
		2148	if ((con->flags & CON_ENABLED) &&
		2149	(con->flags & CON_ANYTIME))
		2150	return 1;
		2151
		2152	return 0;
		2153	}
		2154
		2155	/*
		2156	* Can we actually use the console at this time on this cpu?
		2157	*
		2158	* Console drivers may assume that per-cpu resources have been allocated. So
		2159	* unless they're explicitly marked as being able to cope (CON_ANYTIME) don't
		2160	* call them until this CPU is officially up.
		2161	*/
		2162	static inline int can_use_console(void)
		2163	{
		2164	return cpu_online(raw_smp_processor_id()) \|\| have_callable_console();
		2165	}
		2166
2187	static void console_cont_flush(char *text, size_t size)	2167	static void console_cont_flush(char *text, size_t size)
2188	{	2168	{
2189	unsigned long flags;	2169	unsigned long flags;
@@ -2254,9 +2234,21 @@ void console_unlock(void) · 64 unmodified lines
2254	do_cond_resched = console_may_schedule;	2234	do_cond_resched = console_may_schedule;
2255	console_may_schedule = 0;	2235	console_may_schedule = 0;
2256		2236
		2237	again:
		2238	/*
		2239	* We released the console_sem lock, so we need to recheck if
		2240	* cpu is online and (if not) is there at least one CON_ANYTIME
		2241	* console.
		2242	*/
		2243	if (!can_use_console()) {
		2244	console_locked = 0;
		2245	up_console_sem();
		2246	return;
		2247	}
		2248
2257	/* flush buffered message fragment immediately to console */	2249	/* flush buffered message fragment immediately to console */
2258	console_cont_flush(text, sizeof(text));	2250	console_cont_flush(text, sizeof(text));
2259	again:	2251
2260	for (;;) {	2252	for (;;) {
2261	struct printk_log *msg;	2253	struct printk_log *msg;
2262	size_t ext_len = 0;	2254	size_t ext_len = 0;

kernel/sys.c+4−1

 		error = perf_event_task_enable();
 		break;
 	case PR_GET_TIMERSLACK:
-		error = current->timer_slack_ns;
+		if (current->timer_slack_ns > ULONG_MAX)
+			error = ULONG_MAX;
+		else
+			error = current->timer_slack_ns;
 		break;
 	case PR_SET_TIMERSLACK:
 		if (arg2 <= 0)

kernel/sysctl.c+10−0

 static int __maybe_unused four = 4;
 static unsigned long one_ul = 1;
 static int one_hundred = 100;
+static int one_thousand = 1000;
 #ifdef CONFIG_PRINTK
 static int ten_thousand = 10000;
 #endif
 		.proc_handler	= min_free_kbytes_sysctl_handler,
 		.extra1		= &zero,
 	},
+	{
+		.procname	= "watermark_scale_factor",
+		.data		= &watermark_scale_factor,
+		.maxlen		= sizeof(watermark_scale_factor),
+		.mode		= 0644,
+		.proc_handler	= watermark_scale_factor_sysctl_handler,
+		.extra1		= &one,
+		.extra2		= &one_thousand,
+	},
 	{
 		.procname	= "percpu_pagelist_fraction",
 		.data		= &percpu_pagelist_fraction,

kernel/time/hrtimer.c+6−12

 /*
  * High resolution timer enabled ?
  */
-static int hrtimer_hres_enabled __read_mostly  = 1;
+static bool hrtimer_hres_enabled __read_mostly  = true;
 unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
 EXPORT_SYMBOL_GPL(hrtimer_resolution);
  */
 static int __init setup_hrtimer_hres(char *str)
 {
-	if (!strcmp(str, "off"))
+	return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
-		hrtimer_hres_enabled = 0;
-	else if (!strcmp(str, "on"))
-		hrtimer_hres_enabled = 1;
-	else
-		return 0;
-	return 1;
 }
 __setup("highres=", setup_hrtimer_hres);
  *		relative (HRTIMER_MODE_REL)
  */
 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
-			    unsigned long delta_ns, const enum hrtimer_mode mode)
+			    u64 delta_ns, const enum hrtimer_mode mode)
 {
 	struct hrtimer_clock_base *base, *new_base;
 	unsigned long flags;
 	struct restart_block *restart;
 	struct hrtimer_sleeper t;
 	int ret = 0;
-	unsigned long slack;
+	u64 slack;
 	slack = current->timer_slack_ns;
 	if (dl_task(current) || rt_task(current))
  * @clock:	timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME
  */
 int __sched
-schedule_hrtimeout_range_clock(ktime_t *expires, unsigned long delta,
+schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
 			       const enum hrtimer_mode mode, int clock)
 {
 	struct hrtimer_sleeper t;
  *
  * Returns 0 when the timer has expired otherwise -EINTR
  */
-int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
+int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
 				     const enum hrtimer_mode mode)
 {
 	return schedule_hrtimeout_range_clock(expires, delta, mode,

kernel/time/tick-sched.c+2−8

 /*
  * NO HZ enabled ?
  */
-int tick_nohz_enabled __read_mostly = 1;
+bool tick_nohz_enabled __read_mostly  = true;
 unsigned long tick_nohz_active  __read_mostly;
 /*
  * Enable / Disable tickless mode
  */
 static int __init setup_tick_nohz(char *str)
 {
-	if (!strcmp(str, "off"))
+	return (kstrtobool(str, &tick_nohz_enabled) == 0);
-		tick_nohz_enabled = 0;
-	else if (!strcmp(str, "on"))
-		tick_nohz_enabled = 1;
-	else
-		return 0;
-	return 1;
 }
 __setup("nohz=", setup_tick_nohz);

kernel/time/timer.c+2−2

@@ -1698,10 +1698,10 @@ EXPORT_SYMBOL(msleep_interruptible); · 1697 unmodified lines
1698	static void __sched do_usleep_range(unsigned long min, unsigned long max)	1698	static void __sched do_usleep_range(unsigned long min, unsigned long max)
1699	{	1699	{
1700	ktime_t kmin;	1700	ktime_t kmin;
1701	unsigned long delta;	1701	u64 delta;
1702		1702
1703	kmin = ktime_set(0, min * NSEC_PER_USEC);	1703	kmin = ktime_set(0, min * NSEC_PER_USEC);
1704	delta = (max - min) * NSEC_PER_USEC;	1704	delta = (u64)(max - min) * NSEC_PER_USEC;
1705	schedule_hrtimeout_range(&kmin, delta, HRTIMER_MODE_REL);	1705	schedule_hrtimeout_range(&kmin, delta, HRTIMER_MODE_REL);
1706	}	1706	}
1707		1707

kernel/watchdog.c+8−1

 		 * both lockup detectors are disabled if proc_watchdog_update()
 		 * returns an error.
 		 */
+		if (old == new)
+			goto out;
 		err = proc_watchdog_update();
 	}
 out:
 int proc_watchdog_thresh(struct ctl_table *table, int write,
 			 void __user *buffer, size_t *lenp, loff_t *ppos)
 {
-	int err, old;
+	int err, old, new;
 	get_online_cpus();
 	mutex_lock(&watchdog_proc_mutex);
 	/*
 	 * Update the sample period. Restore on failure.
 	 */
+	new = ACCESS_ONCE(watchdog_thresh);
+	if (old == new)
+		goto out;
 	set_sample_period();
 	err = proc_watchdog_update();
 	if (err) {

lib/bug.c+2−13

@@ -167,19 +167,8 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) · 166 unmodified lines
167		167
168	if (warning) {	168	if (warning) {
169	/* this is a WARN_ON rather than BUG/BUG_ON */	169	/* this is a WARN_ON rather than BUG/BUG_ON */
170	pr_warn("------------[ cut here ]------------\n");	170	__warn(file, line, (void *)bugaddr, BUG_GET_TAINT(bug), regs,
171		171	NULL);
172	if (file)
173	pr_warn("WARNING: at %s:%u\n", file, line);
174	else
175	pr_warn("WARNING: at %p [verbose debug info unavailable]\n",
176	(void *)bugaddr);
177
178	print_modules();
179	show_regs(regs);
180	print_oops_end_marker();
181	/* Just a warning, don't kill lockdep. */
182	add_taint(BUG_GET_TAINT(bug), LOCKDEP_STILL_OK);
183	return BUG_TRAP_TYPE_WARN;	172	return BUG_TRAP_TYPE_WARN;
184	}	173	}
185		174

lib/kstrtox.c+64−0

@@ -321,6 +321,70 @@ int kstrtos8(const char s, unsigned int base, s8 res) · 320 unmodified lines
321	}	321	}
322	EXPORT_SYMBOL(kstrtos8);	322	EXPORT_SYMBOL(kstrtos8);
323		323
		324	/**
		325	* kstrtobool - convert common user inputs into boolean values
		326	* @s: input string
		327	* @res: result
		328	*
		329	* This routine returns 0 iff the first character is one of 'Yy1Nn0', or
		330	* [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value
		331	* pointed to by res is updated upon finding a match.
		332	*/
		333	int kstrtobool(const char s, bool res)
		334	{
		335	if (!s)
		336	return -EINVAL;
		337
		338	switch (s[0]) {
		339	case 'y':
		340	case 'Y':
		341	case '1':
		342	*res = true;
		343	return 0;
		344	case 'n':
		345	case 'N':
		346	case '0':
		347	*res = false;
		348	return 0;
		349	case 'o':
		350	case 'O':
		351	switch (s[1]) {
		352	case 'n':
		353	case 'N':
		354	*res = true;
		355	return 0;
		356	case 'f':
		357	case 'F':
		358	*res = false;
		359	return 0;
		360	default:
		361	break;
		362	}
		363	default:
		364	break;
		365	}
		366
		367	return -EINVAL;
		368	}
		369	EXPORT_SYMBOL(kstrtobool);
		370
		371	/*
		372	* Since "base" would be a nonsense argument, this open-codes the
		373	* _from_user helper instead of using the helper macro below.
		374	*/
		375	int kstrtobool_from_user(const char __user s, size_t count, bool res)
		376	{
		377	/* Longest string needed to differentiate, newline, terminator */
		378	char buf[4];
		379
		380	count = min(count, sizeof(buf) - 1);
		381	if (copy_from_user(buf, s, count))
		382	return -EFAULT;
		383	buf[count] = '\0';
		384	return kstrtobool(buf, res);
		385	}
		386	EXPORT_SYMBOL(kstrtobool_from_user);
		387
324	#define kstrto_from_user(f, g, type) \	388	#define kstrto_from_user(f, g, type) \
325	int f(const char __user s, size_t count, unsigned int base, type res) \	389	int f(const char __user s, size_t count, unsigned int base, type res) \
326	{ \	390	{ \

lib/radix-tree.c+148−34

@@ -173,6 +173,41 @@ radix_tree_find_next_bit(const unsigned long *addr, · 172 unmodified lines
173	return size;	173	return size;
174	}	174	}
175		175
		176	#if 0
		177	static void dump_node(void *slot, int height, int offset)
		178	{
		179	struct radix_tree_node *node;
		180	int i;
		181
		182	if (!slot)
		183	return;
		184
		185	if (height == 0) {
		186	pr_debug("radix entry %p offset %d\n", slot, offset);
		187	return;
		188	}
		189
		190	node = indirect_to_ptr(slot);
		191	pr_debug("radix node: %p offset %d tags %lx %lx %lx path %x count %d parent %p\n",
		192	slot, offset, node->tags[0][0], node->tags[1][0],
		193	node->tags[2][0], node->path, node->count, node->parent);
		194
		195	for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
		196	dump_node(node->slots[i], height - 1, i);
		197	}
		198
		199	/* For debug */
		200	static void radix_tree_dump(struct radix_tree_root *root)
		201	{
		202	pr_debug("radix root: %p height %d rnode %p tags %x\n",
		203	root, root->height, root->rnode,
		204	root->gfp_mask >> __GFP_BITS_SHIFT);
		205	if (!radix_tree_is_indirect_ptr(root->rnode))
		206	return;
		207	dump_node(root->rnode, root->height, 0);
		208	}
		209	#endif
		210
176	/*	211	/*
177	* This assumes that the caller has performed appropriate preallocation, and	212	* This assumes that the caller has performed appropriate preallocation, and
178	* that the caller has pinned this thread of control to the current CPU.	213	* that the caller has pinned this thread of control to the current CPU.
@@ -191,6 +226,15 @@ radix_tree_node_alloc(struct radix_tree_root *root) · 12 unmodified lines
191	if (!gfpflags_allow_blocking(gfp_mask) && !in_interrupt()) {	226	if (!gfpflags_allow_blocking(gfp_mask) && !in_interrupt()) {
192	struct radix_tree_preload *rtp;	227	struct radix_tree_preload *rtp;
193		228
		229	/*
		230	* Even if the caller has preloaded, try to allocate from the
		231	* cache first for the new node to get accounted.
		232	*/
		233	ret = kmem_cache_alloc(radix_tree_node_cachep,
		234	gfp_mask \| __GFP_ACCOUNT \| __GFP_NOWARN);
		235	if (ret)
		236	goto out;
		237
194	/*	238	/*
195	* Provided the caller has preloaded here, we will always	239	* Provided the caller has preloaded here, we will always
196	* succeed in getting a node here (and never reach	240	* succeed in getting a node here (and never reach
@@ -208,10 +252,11 @@ radix_tree_node_alloc(struct radix_tree_root *root) · 11 unmodified lines
208	* for debugging.	252	* for debugging.
209	*/	253	*/
210	kmemleak_update_trace(ret);	254	kmemleak_update_trace(ret);
		255	goto out;
211	}	256	}
212	if (ret == NULL)	257	ret = kmem_cache_alloc(radix_tree_node_cachep,
213	ret = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);	258	gfp_mask \| __GFP_ACCOUNT);
214		259	out:
215	BUG_ON(radix_tree_is_indirect_ptr(ret));	260	BUG_ON(radix_tree_is_indirect_ptr(ret));
216	return ret;	261	return ret;
217	}	262	}
@@ -323,7 +368,8 @@ static inline unsigned long radix_tree_maxindex(unsigned int height) · 105 unmodified lines
323	/*	368	/*
324	* Extend a radix tree so it can store key @index.	369	* Extend a radix tree so it can store key @index.
325	*/	370	*/
326	static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)	371	static int radix_tree_extend(struct radix_tree_root *root,
		372	unsigned long index, unsigned order)
327	{	373	{
328	struct radix_tree_node *node;	374	struct radix_tree_node *node;
329	struct radix_tree_node *slot;	375	struct radix_tree_node *slot;
@@ -335,7 +381,7 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index) · 5 unmodified lines
335	while (index > radix_tree_maxindex(height))	381	while (index > radix_tree_maxindex(height))
336	height++;	382	height++;
337		383
338	if (root->rnode == NULL) {	384	if ((root->rnode == NULL) && (order == 0)) {
339	root->height = height;	385	root->height = height;
340	goto out;	386	goto out;
341	}	387	}
@@ -358,9 +404,10 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index) · 16 unmodified lines
358	node->count = 1;	404	node->count = 1;
359	node->parent = NULL;	405	node->parent = NULL;
360	slot = root->rnode;	406	slot = root->rnode;
361	if (newheight > 1) {	407	if (radix_tree_is_indirect_ptr(slot) && newheight > 1) {
362	slot = indirect_to_ptr(slot);	408	slot = indirect_to_ptr(slot);
363	slot->parent = node;	409	slot->parent = node;
		410	slot = ptr_to_indirect(slot);
364	}	411	}
365	node->slots[0] = slot;	412	node->slots[0] = slot;
366	node = ptr_to_indirect(node);	413	node = ptr_to_indirect(node);
@@ -375,6 +422,7 @@ out: · 8 unmodified lines
375	* __radix_tree_create - create a slot in a radix tree	422	* __radix_tree_create - create a slot in a radix tree
376	* @root: radix tree root	423	* @root: radix tree root
377	* @index: index key	424	* @index: index key
		425	* @order: index occupies 2^order aligned slots
378	* @nodep: returns node	426	* @nodep: returns node
379	* @slotp: returns slot	427	* @slotp: returns slot
380	*	428	*
@@ -388,26 +436,29 @@ out: · 7 unmodified lines
388	* Returns -ENOMEM, or 0 for success.	436	* Returns -ENOMEM, or 0 for success.
389	*/	437	*/
390	int __radix_tree_create(struct radix_tree_root *root, unsigned long index,	438	int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
391	struct radix_tree_node nodep, void *slotp)	439	unsigned order, struct radix_tree_node **nodep,
		440	void ***slotp)
392	{	441	{
393	struct radix_tree_node node = NULL, slot;	442	struct radix_tree_node node = NULL, slot;
394	unsigned int height, shift, offset;	443	unsigned int height, shift, offset;
395	int error;	444	int error;
396		445
		446	BUG_ON((0 < order) && (order < RADIX_TREE_MAP_SHIFT));
		447
397	/* Make sure the tree is high enough. */	448	/* Make sure the tree is high enough. */
398	if (index > radix_tree_maxindex(root->height)) {	449	if (index > radix_tree_maxindex(root->height)) {
399	error = radix_tree_extend(root, index);	450	error = radix_tree_extend(root, index, order);
400	if (error)	451	if (error)
401	return error;	452	return error;
402	}	453	}
403		454
404	slot = indirect_to_ptr(root->rnode);	455	slot = root->rnode;
405		456
406	height = root->height;	457	height = root->height;
407	shift = (height-1) * RADIX_TREE_MAP_SHIFT;	458	shift = height * RADIX_TREE_MAP_SHIFT;
408		459
409	offset = 0; /* uninitialised var warning */	460	offset = 0; /* uninitialised var warning */
410	while (height > 0) {	461	while (shift > order) {
411	if (slot == NULL) {	462	if (slot == NULL) {
412	/* Have to add a child node. */	463	/* Have to add a child node. */
413	if (!(slot = radix_tree_node_alloc(root)))	464	if (!(slot = radix_tree_node_alloc(root)))
@@ -415,19 +466,38 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index, · 1 unmodified line
415	slot->path = height;	466	slot->path = height;
416	slot->parent = node;	467	slot->parent = node;
417	if (node) {	468	if (node) {
418	rcu_assign_pointer(node->slots[offset], slot);	469	rcu_assign_pointer(node->slots[offset],
		470	ptr_to_indirect(slot));
419	node->count++;	471	node->count++;
420	slot->path \|= offset << RADIX_TREE_HEIGHT_SHIFT;	472	slot->path \|= offset << RADIX_TREE_HEIGHT_SHIFT;
421	} else	473	} else
422	rcu_assign_pointer(root->rnode, ptr_to_indirect(slot));	474	rcu_assign_pointer(root->rnode,
423	}	475	ptr_to_indirect(slot));
		476	} else if (!radix_tree_is_indirect_ptr(slot))
		477	break;
424		478
425	/* Go a level down */	479	/* Go a level down */
		480	height--;
		481	shift -= RADIX_TREE_MAP_SHIFT;
426	offset = (index >> shift) & RADIX_TREE_MAP_MASK;	482	offset = (index >> shift) & RADIX_TREE_MAP_MASK;
427	node = slot;	483	node = indirect_to_ptr(slot);
428	slot = node->slots[offset];	484	slot = node->slots[offset];
429	shift -= RADIX_TREE_MAP_SHIFT;	485	}
430	height--;	486
		487	/* Insert pointers to the canonical entry */
		488	if ((shift - order) > 0) {
		489	int i, n = 1 << (shift - order);
		490	offset = offset & ~(n - 1);
		491	slot = ptr_to_indirect(&node->slots[offset]);
		492	for (i = 0; i < n; i++) {
		493	if (node->slots[offset + i])
		494	return -EEXIST;
		495	}
		496
		497	for (i = 1; i < n; i++) {
		498	rcu_assign_pointer(node->slots[offset + i], slot);
		499	node->count++;
		500	}
431	}	501	}
432		502
433	if (nodep)	503	if (nodep)
@@ -438,15 +508,16 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index, · 4 unmodified lines
438	}	508	}
439		509
440	/**	510	/**
441	* radix_tree_insert - insert into a radix tree	511	* __radix_tree_insert - insert into a radix tree
442	* @root: radix tree root	512	* @root: radix tree root
443	* @index: index key	513	* @index: index key
		514	* @order: key covers the 2^order indices around index
444	* @item: item to insert	515	* @item: item to insert
445	*	516	*
446	* Insert an item into the radix tree at position @index.	517	* Insert an item into the radix tree at position @index.
447	*/	518	*/
448	int radix_tree_insert(struct radix_tree_root *root,	519	int __radix_tree_insert(struct radix_tree_root *root, unsigned long index,
449	unsigned long index, void *item)	520	unsigned order, void *item)
450	{	521	{
451	struct radix_tree_node *node;	522	struct radix_tree_node *node;
452	void **slot;	523	void **slot;
@@ -454,7 +525,7 @@ int radix_tree_insert(struct radix_tree_root *root, · 1 unmodified line
454		525
455	BUG_ON(radix_tree_is_indirect_ptr(item));	526	BUG_ON(radix_tree_is_indirect_ptr(item));
456		527
457	error = __radix_tree_create(root, index, &node, &slot);	528	error = __radix_tree_create(root, index, order, &node, &slot);
458	if (error)	529	if (error)
459	return error;	530	return error;
460	if (*slot != NULL)	531	if (*slot != NULL)
@@ -472,7 +543,7 @@ int radix_tree_insert(struct radix_tree_root *root, · 11 unmodified lines
472		543
473	return 0;	544	return 0;
474	}	545	}
475	EXPORT_SYMBOL(radix_tree_insert);	546	EXPORT_SYMBOL(__radix_tree_insert);
476		547
477	/**	548	/**
478	* __radix_tree_lookup - lookup an item in a radix tree	549	* __radix_tree_lookup - lookup an item in a radix tree
@@ -523,6 +594,9 @@ void __radix_tree_lookup(struct radix_tree_root root, unsigned long index, · 44 unmodified lines
523	node = rcu_dereference_raw(*slot);	594	node = rcu_dereference_raw(*slot);
524	if (node == NULL)	595	if (node == NULL)
525	return NULL;	596	return NULL;
		597	if (!radix_tree_is_indirect_ptr(node))
		598	break;
		599	node = indirect_to_ptr(node);
526		600
527	shift -= RADIX_TREE_MAP_SHIFT;	601	shift -= RADIX_TREE_MAP_SHIFT;
528	height--;	602	height--;
@@ -609,6 +683,9 @@ void radix_tree_tag_set(struct radix_tree_root root, · 80 unmodified lines
609	tag_set(slot, tag, offset);	683	tag_set(slot, tag, offset);
610	slot = slot->slots[offset];	684	slot = slot->slots[offset];
611	BUG_ON(slot == NULL);	685	BUG_ON(slot == NULL);
		686	if (!radix_tree_is_indirect_ptr(slot))
		687	break;
		688	slot = indirect_to_ptr(slot);
612	shift -= RADIX_TREE_MAP_SHIFT;	689	shift -= RADIX_TREE_MAP_SHIFT;
613	height--;	690	height--;
614	}	691	}
@@ -648,11 +725,14 @@ void radix_tree_tag_clear(struct radix_tree_root root, · 33 unmodified lines
648	goto out;	725	goto out;
649		726
650	shift = height * RADIX_TREE_MAP_SHIFT;	727	shift = height * RADIX_TREE_MAP_SHIFT;
651	slot = indirect_to_ptr(root->rnode);	728	slot = root->rnode;
652		729
653	while (shift) {	730	while (shift) {
654	if (slot == NULL)	731	if (slot == NULL)
655	goto out;	732	goto out;
		733	if (!radix_tree_is_indirect_ptr(slot))
		734	break;
		735	slot = indirect_to_ptr(slot);
656		736
657	shift -= RADIX_TREE_MAP_SHIFT;	737	shift -= RADIX_TREE_MAP_SHIFT;
658	offset = (index >> shift) & RADIX_TREE_MAP_MASK;	738	offset = (index >> shift) & RADIX_TREE_MAP_MASK;
@@ -728,6 +808,7 @@ int radix_tree_tag_get(struct radix_tree_root *root, · 69 unmodified lines
728		808
729	if (node == NULL)	809	if (node == NULL)
730	return 0;	810	return 0;
		811	node = indirect_to_ptr(node);
731		812
732	offset = (index >> shift) & RADIX_TREE_MAP_MASK;	813	offset = (index >> shift) & RADIX_TREE_MAP_MASK;
733	if (!tag_get(node, tag, offset))	814	if (!tag_get(node, tag, offset))
@@ -735,6 +816,8 @@ int radix_tree_tag_get(struct radix_tree_root *root, · 1 unmodified line
735	if (height == 1)	816	if (height == 1)
736	return 1;	817	return 1;
737	node = rcu_dereference_raw(node->slots[offset]);	818	node = rcu_dereference_raw(node->slots[offset]);
		819	if (!radix_tree_is_indirect_ptr(node))
		820	return 1;
738	shift -= RADIX_TREE_MAP_SHIFT;	821	shift -= RADIX_TREE_MAP_SHIFT;
739	height--;	822	height--;
740	}	823	}
@@ -795,6 +878,7 @@ restart: · 54 unmodified lines
795		878
796	node = rnode;	879	node = rnode;
797	while (1) {	880	while (1) {
		881	struct radix_tree_node *slot;
798	if ((flags & RADIX_TREE_ITER_TAGGED) ?	882	if ((flags & RADIX_TREE_ITER_TAGGED) ?
799	!test_bit(offset, node->tags[tag]) :	883	!test_bit(offset, node->tags[tag]) :
800	!node->slots[offset]) {	884	!node->slots[offset]) {
@@ -825,9 +909,12 @@ restart: · 24 unmodified lines
825	if (!shift)	909	if (!shift)
826	break;	910	break;
827		911
828	node = rcu_dereference_raw(node->slots[offset]);	912	slot = rcu_dereference_raw(node->slots[offset]);
829	if (node == NULL)	913	if (slot == NULL)
830	goto restart;	914	goto restart;
		915	if (!radix_tree_is_indirect_ptr(slot))
		916	break;
		917	node = indirect_to_ptr(slot);
831	shift -= RADIX_TREE_MAP_SHIFT;	918	shift -= RADIX_TREE_MAP_SHIFT;
832	offset = (index >> shift) & RADIX_TREE_MAP_MASK;	919	offset = (index >> shift) & RADIX_TREE_MAP_MASK;
833	}	920	}
@@ -925,15 +1012,20 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root, · 91 unmodified lines
925	if (!tag_get(slot, iftag, offset))	1012	if (!tag_get(slot, iftag, offset))
926	goto next;	1013	goto next;
927	if (shift) {	1014	if (shift) {
928	/* Go down one level */
929	shift -= RADIX_TREE_MAP_SHIFT;
930	node = slot;	1015	node = slot;
931	slot = slot->slots[offset];	1016	slot = slot->slots[offset];
932	continue;	1017	if (radix_tree_is_indirect_ptr(slot)) {
		1018	slot = indirect_to_ptr(slot);
		1019	shift -= RADIX_TREE_MAP_SHIFT;
		1020	continue;
		1021	} else {
		1022	slot = node;
		1023	node = node->parent;
		1024	}
933	}	1025	}
934		1026
935	/* tag the leaf */	1027	/* tag the leaf */
936	tagged++;	1028	tagged += 1 << shift;
937	tag_set(slot, settag, offset);	1029	tag_set(slot, settag, offset);
938		1030
939	/* walk back up the path tagging interior nodes */	1031	/* walk back up the path tagging interior nodes */
@@ -1181,10 +1273,20 @@ static unsigned long __locate(struct radix_tree_node slot, void item, · 241 unmodified lines
1181	goto out;	1273	goto out;
1182	}	1274	}
1183		1275
1184	shift -= RADIX_TREE_MAP_SHIFT;
1185	slot = rcu_dereference_raw(slot->slots[i]);	1276	slot = rcu_dereference_raw(slot->slots[i]);
1186	if (slot == NULL)	1277	if (slot == NULL)
1187	goto out;	1278	goto out;
		1279	if (!radix_tree_is_indirect_ptr(slot)) {
		1280	if (slot == item) {
		1281	*found_index = index + i;
		1282	index = 0;
		1283	} else {
		1284	index += shift;
		1285	}
		1286	goto out;
		1287	}
		1288	slot = indirect_to_ptr(slot);
		1289	shift -= RADIX_TREE_MAP_SHIFT;
1188	}	1290	}
1189		1291
1190	/* Bottom level: check items */	1292	/* Bottom level: check items */
@@ -1264,11 +1366,13 @@ static inline void radix_tree_shrink(struct radix_tree_root *root) · 73 unmodified lines
1264		1366
1265	/*	1367	/*
1266	* The candidate node has more than one child, or its child	1368	* The candidate node has more than one child, or its child
1267	* is not at the leftmost slot, we cannot shrink.	1369	* is not at the leftmost slot, or it is a multiorder entry,
		1370	* we cannot shrink.
1268	*/	1371	*/
1269	if (to_free->count != 1)	1372	if (to_free->count != 1)
1270	break;	1373	break;
1271	if (!to_free->slots[0])	1374	slot = to_free->slots[0];
		1375	if (!slot)
1272	break;	1376	break;
1273		1377
1274	/*	1378	/*
@@ -1278,8 +1382,11 @@ static inline void radix_tree_shrink(struct radix_tree_root *root) · 3 unmodified lines
1278	* (to_free->slots[0]), it will be safe to dereference the new	1382	* (to_free->slots[0]), it will be safe to dereference the new
1279	* one (root->rnode) as far as dependent read barriers go.	1383	* one (root->rnode) as far as dependent read barriers go.
1280	*/	1384	*/
1281	slot = to_free->slots[0];
1282	if (root->height > 1) {	1385	if (root->height > 1) {
		1386	if (!radix_tree_is_indirect_ptr(slot))
		1387	break;
		1388
		1389	slot = indirect_to_ptr(slot);
1283	slot->parent = NULL;	1390	slot->parent = NULL;
1284	slot = ptr_to_indirect(slot);	1391	slot = ptr_to_indirect(slot);
1285	}	1392	}
@@ -1377,7 +1484,7 @@ void radix_tree_delete_item(struct radix_tree_root root, · 91 unmodified lines
1377	unsigned long index, void *item)	1484	unsigned long index, void *item)
1378	{	1485	{
1379	struct radix_tree_node *node;	1486	struct radix_tree_node *node;
1380	unsigned int offset;	1487	unsigned int offset, i;
1381	void **slot;	1488	void **slot;
1382	void *entry;	1489	void *entry;
1383	int tag;	1490	int tag;
@@ -1406,6 +1513,13 @@ void radix_tree_delete_item(struct radix_tree_root root, · 22 unmodified lines
1406	radix_tree_tag_clear(root, index, tag);	1513	radix_tree_tag_clear(root, index, tag);
1407	}	1514	}
1408		1515
		1516	/* Delete any sibling slots pointing to this slot */
		1517	for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
		1518	if (node->slots[offset + i] != ptr_to_indirect(slot))
		1519	break;
		1520	node->slots[offset + i] = NULL;
		1521	node->count--;
		1522	}
1409	node->slots[offset] = NULL;	1523	node->slots[offset] = NULL;
1410	node->count--;	1524	node->count--;
1411		1525

lib/string.c+21−24

@@ -631,33 +631,30 @@ bool sysfs_streq(const char s1, const char s2) · 630 unmodified lines
631	EXPORT_SYMBOL(sysfs_streq);	631	EXPORT_SYMBOL(sysfs_streq);
632		632
633	/**	633	/**
634	* strtobool - convert common user inputs into boolean values	634	* match_string - matches given string in an array
635	* @s: input string	635	* @array: array of strings
636	* @res: result	636	* @n: number of strings in the array or -1 for NULL terminated arrays
		637	* @string: string to match with
637	*	638	*
638	* This routine returns 0 iff the first character is one of 'Yy1Nn0'.	639	* Return:
639	* Otherwise it will return -EINVAL. Value pointed to by res is	640	* index of a @string in the @array if matches, or %-EINVAL otherwise.
640	* updated upon finding a match.	641	*/
641	*/	642	int match_string(const char * const array, size_t n, const char string)
642	int strtobool(const char s, bool res)	643	{
643	{	644	int index;
644	switch (s[0]) {	645	const char *item;
645	case 'y':	646
646	case 'Y':	647	for (index = 0; index < n; index++) {
647	case '1':	648	item = array[index];
648	*res = true;	649	if (!item)
649	break;	650	break;
650	case 'n':	651	if (!strcmp(item, string))
651	case 'N':	652	return index;
652	case '0':
653	*res = false;
654	break;
655	default:
656	return -EINVAL;
657	}	653	}
658	return 0;	654
		655	return -EINVAL;
659	}	656	}
660	EXPORT_SYMBOL(strtobool);	657	EXPORT_SYMBOL(match_string);
661		658
662	#ifndef __HAVE_ARCH_MEMSET	659	#ifndef __HAVE_ARCH_MEMSET
663	/**	660	/**

lib/vsprintf.c+58−1

@@ -2640,8 +2640,12 @@ int vsscanf(const char buf, const char fmt, va_list args) · 2639 unmodified lines
2640	if (fmt == '') {	2640	if (fmt == '') {
2641	if (!*str)	2641	if (!*str)
2642	break;	2642	break;
2643	while (!isspace(fmt) && fmt != '%' && *fmt)	2643	while (!isspace(fmt) && fmt != '%' && *fmt) {
		2644	/* '%[' not yet supported, invalid format /
		2645	if (*fmt == '[')
		2646	return num;
2644	fmt++;	2647	fmt++;
		2648	}
2645	while (!isspace(str) && str)	2649	while (!isspace(str) && str)
2646	str++;	2650	str++;
2647	continue;	2651	continue;
@@ -2714,6 +2718,59 @@ int vsscanf(const char buf, const char fmt, va_list args) · 66 unmodified lines
2714	num++;	2718	num++;
2715	}	2719	}
2716	continue;	2720	continue;
		2721	/*
		2722	* Warning: This implementation of the '[' conversion specifier
		2723	* deviates from its glibc counterpart in the following ways:
		2724	* (1) It does NOT support ranges i.e. '-' is NOT a special
		2725	* character
		2726	* (2) It cannot match the closing bracket ']' itself
		2727	* (3) A field width is required
		2728	* (4) '%*[' (discard matching input) is currently not supported
		2729	*
		2730	* Example usage:
		2731	* ret = sscanf("00:0a:95","%2[^:]:%2[^:]:%2[^:]",
		2732	* buf1, buf2, buf3);
		2733	* if (ret < 3)
		2734	* // etc..
		2735	*/
		2736	case '[':
		2737	{
		2738	char s = (char )va_arg(args, char *);
		2739	DECLARE_BITMAP(set, 256) = {0};
		2740	unsigned int len = 0;
		2741	bool negate = (*fmt == '^');
		2742
		2743	/* field width is required */
		2744	if (field_width == -1)
		2745	return num;
		2746
		2747	if (negate)
		2748	++fmt;
		2749
		2750	for ( ; fmt && fmt != ']'; ++fmt, ++len)
		2751	set_bit((u8)*fmt, set);
		2752
		2753	/* no ']' or no character set found */
		2754	if (!*fmt \|\| !len)
		2755	return num;
		2756	++fmt;
		2757
		2758	if (negate) {
		2759	bitmap_complement(set, set, 256);
		2760	/* exclude null '\0' byte */
		2761	clear_bit(0, set);
		2762	}
		2763
		2764	/* match must be non-empty */
		2765	if (!test_bit((u8)*str, set))
		2766	return num;
		2767
		2768	while (test_bit((u8)*str, set) && field_width--)
		2769	s++ = str++;
		2770	*s = '\0';
		2771	++num;
		2772	}
		2773	continue;
2717	case 'o':	2774	case 'o':
2718	base = 8;	2775	base = 8;
2719	break;	2776	break;

mm/Kconfig+1−3

 	bool "Allow for memory hot-add"
 	depends on SPARSEMEM || X86_64_ACPI_NUMA
 	depends on ARCH_ENABLE_MEMORY_HOTPLUG
-	depends on (IA64 || X86 || PPC_BOOK3S_64 || SUPERH || S390)
 config MEMORY_HOTPLUG_SPARSE
 	def_bool y
 config ZONE_DEVICE
 	bool "Device memory (pmem, etc...) hotplug support" if EXPERT
-	default !ZONE_DMA
-	depends on !ZONE_DMA
 	depends on MEMORY_HOTPLUG
 	depends on MEMORY_HOTREMOVE
+	depends on SPARSEMEM_VMEMMAP
 	depends on X86_64 #arch_add_memory() comprehends device memory
 	help

mm/Kconfig.debug+13−0

@@ -79,3 +79,16 @@ config PAGE_POISONING_ZERO · 78 unmodified lines
79	Enabling page poisoning with this option will disable hibernation	79	Enabling page poisoning with this option will disable hibernation
80		80
81	If unsure, say N	81	If unsure, say N
		82	bool
		83
		84	config DEBUG_PAGE_REF
		85	bool "Enable tracepoint to track down page reference manipulation"
		86	depends on DEBUG_KERNEL
		87	depends on TRACEPOINTS
		88	---help---
		89	This is a feature to add tracepoint for tracking down page reference
		90	manipulation. This tracking is useful to diagnose functional failure
		91	due to migration failures caused by page reference mismatches. Be
		92	careful when enabling this feature because it adds about 30 KB to the
		93	kernel code. However the runtime performance overhead is virtually
		94	nil until the tracepoints are actually enabled.

mm/Makefile+1−0

@@ -81,3 +81,4 @@ obj-$(CONFIG_CMA_DEBUGFS) += cma_debug.o · 80 unmodified lines
81	obj-$(CONFIG_USERFAULTFD) += userfaultfd.o	81	obj-$(CONFIG_USERFAULTFD) += userfaultfd.o
82	obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o	82	obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o
83	obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o	83	obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o
		84	obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o

mm/backing-dev.c+2−2

@@ -1026,8 +1026,8 @@ int pdflush_proc_obsolete(struct ctl_table *table, int write, · 1025 unmodified lines
1026		1026
1027	if (copy_to_user(buffer, kbuf, sizeof(kbuf)))	1027	if (copy_to_user(buffer, kbuf, sizeof(kbuf)))
1028	return -EFAULT;	1028	return -EFAULT;
1029	printk_once(KERN_WARNING "%s exported in /proc is scheduled for removal\n",	1029	pr_warn_once("%s exported in /proc is scheduled for removal\n",
1030	table->procname);	1030	table->procname);
1031		1031
1032	*lenp = 2;	1032	*lenp = 2;
1033	ppos += lenp;	1033	ppos += lenp;

mm/bootmem.c+3−4

 #define bdebug(fmt, args...) ({				\
 	if (unlikely(bootmem_debug))			\
-		printk(KERN_INFO			\
+		pr_info("bootmem::%s " fmt,		\
-			"bootmem::%s " fmt,		\
 			__func__, ## args);		\
 })
 	/*
 	 * Whoops, we cannot satisfy the allocation request.
 	 */
-	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
+	pr_alert("bootmem alloc of %lu bytes failed!\n", size);
 	panic("Out of memory");
 	return NULL;
 }
 	if (ptr)
 		return ptr;
-	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
+	pr_alert("bootmem alloc of %lu bytes failed!\n", size);
 	panic("Out of memory");
 	return NULL;
 }

mm/compaction.c+227−5

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	*	7	*
8	* Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>	8	* Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
9	*/	9	*/
		10	#include <linux/cpu.h>
10	#include <linux/swap.h>	11	#include <linux/swap.h>
11	#include <linux/migrate.h>	12	#include <linux/migrate.h>
12	#include <linux/compaction.h>	13	#include <linux/compaction.h>
@@ -17,6 +18,8 @@ · 4 unmodified lines
17	#include <linux/balloon_compaction.h>	18	#include <linux/balloon_compaction.h>
18	#include <linux/page-isolation.h>	19	#include <linux/page-isolation.h>
19	#include <linux/kasan.h>	20	#include <linux/kasan.h>
		21	#include <linux/kthread.h>
		22	#include <linux/freezer.h>
20	#include "internal.h"	23	#include "internal.h"
21		24
22	#ifdef CONFIG_COMPACTION	25	#ifdef CONFIG_COMPACTION
@@ -1188,11 +1191,11 @@ static int __compact_finished(struct zone zone, struct compact_control cc, · 1165 unmodified lines
1188		1191
1189	/*	1192	/*
1190	* Mark that the PG_migrate_skip information should be cleared	1193	* Mark that the PG_migrate_skip information should be cleared
1191	* by kswapd when it goes to sleep. kswapd does not set the	1194	* by kswapd when it goes to sleep. kcompactd does not set the
1192	* flag itself as the decision to be clear should be directly	1195	* flag itself as the decision to be clear should be directly
1193	* based on an allocation request.	1196	* based on an allocation request.
1194	*/	1197	*/
1195	if (!current_is_kswapd())	1198	if (cc->direct_compaction)
1196	zone->compact_blockskip_flush = true;	1199	zone->compact_blockskip_flush = true;
1197		1200
1198	return COMPACT_COMPLETE;	1201	return COMPACT_COMPLETE;
@@ -1335,10 +1338,9 @@ static int compact_zone(struct zone zone, struct compact_control cc) · 136 unmodified lines
1335		1338
1336	/*	1339	/*
1337	* Clear pageblock skip if there were failures recently and compaction	1340	* Clear pageblock skip if there were failures recently and compaction
1338	* is about to be retried after being deferred. kswapd does not do	1341	* is about to be retried after being deferred.
1339	* this reset as it'll reset the cached information when going to sleep.
1340	*/	1342	*/
1341	if (compaction_restarting(zone, cc->order) && !current_is_kswapd())	1343	if (compaction_restarting(zone, cc->order))
1342	__reset_isolation_suitable(zone);	1344	__reset_isolation_suitable(zone);
1343		1345
1344	/*	1346	/*
@@ -1474,6 +1476,7 @@ static unsigned long compact_zone_order(struct zone *zone, int order, · 129 unmodified lines
1474	.mode = mode,	1476	.mode = mode,
1475	.alloc_flags = alloc_flags,	1477	.alloc_flags = alloc_flags,
1476	.classzone_idx = classzone_idx,	1478	.classzone_idx = classzone_idx,
		1479	.direct_compaction = true,
1477	};	1480	};
1478	INIT_LIST_HEAD(&cc.freepages);	1481	INIT_LIST_HEAD(&cc.freepages);
1479	INIT_LIST_HEAD(&cc.migratepages);	1482	INIT_LIST_HEAD(&cc.migratepages);
@@ -1736,4 +1739,223 @@ void compaction_unregister_node(struct node *node) · 256 unmodified lines
1736	}	1739	}
1737	#endif /* CONFIG_SYSFS && CONFIG_NUMA */	1740	#endif /* CONFIG_SYSFS && CONFIG_NUMA */
1738		1741
		1742	static inline bool kcompactd_work_requested(pg_data_t *pgdat)
		1743	{
		1744	return pgdat->kcompactd_max_order > 0;
		1745	}
		1746
		1747	static bool kcompactd_node_suitable(pg_data_t *pgdat)
		1748	{
		1749	int zoneid;
		1750	struct zone *zone;
		1751	enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx;
		1752
		1753	for (zoneid = 0; zoneid < classzone_idx; zoneid++) {
		1754	zone = &pgdat->node_zones[zoneid];
		1755
		1756	if (!populated_zone(zone))
		1757	continue;
		1758
		1759	if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
		1760	classzone_idx) == COMPACT_CONTINUE)
		1761	return true;
		1762	}
		1763
		1764	return false;
		1765	}
		1766
		1767	static void kcompactd_do_work(pg_data_t *pgdat)
		1768	{
		1769	/*
		1770	* With no special task, compact all zones so that a page of requested
		1771	* order is allocatable.
		1772	*/
		1773	int zoneid;
		1774	struct zone *zone;
		1775	struct compact_control cc = {
		1776	.order = pgdat->kcompactd_max_order,
		1777	.classzone_idx = pgdat->kcompactd_classzone_idx,
		1778	.mode = MIGRATE_SYNC_LIGHT,
		1779	.ignore_skip_hint = true,
		1780
		1781	};
		1782	bool success = false;
		1783
		1784	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
		1785	cc.classzone_idx);
		1786	count_vm_event(KCOMPACTD_WAKE);
		1787
		1788	for (zoneid = 0; zoneid < cc.classzone_idx; zoneid++) {
		1789	int status;
		1790
		1791	zone = &pgdat->node_zones[zoneid];
		1792	if (!populated_zone(zone))
		1793	continue;
		1794
		1795	if (compaction_deferred(zone, cc.order))
		1796	continue;
		1797
		1798	if (compaction_suitable(zone, cc.order, 0, zoneid) !=
		1799	COMPACT_CONTINUE)
		1800	continue;
		1801
		1802	cc.nr_freepages = 0;
		1803	cc.nr_migratepages = 0;
		1804	cc.zone = zone;
		1805	INIT_LIST_HEAD(&cc.freepages);
		1806	INIT_LIST_HEAD(&cc.migratepages);
		1807
		1808	status = compact_zone(zone, &cc);
		1809
		1810	if (zone_watermark_ok(zone, cc.order, low_wmark_pages(zone),
		1811	cc.classzone_idx, 0)) {
		1812	success = true;
		1813	compaction_defer_reset(zone, cc.order, false);
		1814	} else if (status == COMPACT_COMPLETE) {
		1815	/*
		1816	* We use sync migration mode here, so we defer like
		1817	* sync direct compaction does.
		1818	*/
		1819	defer_compaction(zone, cc.order);
		1820	}
		1821
		1822	VM_BUG_ON(!list_empty(&cc.freepages));
		1823	VM_BUG_ON(!list_empty(&cc.migratepages));
		1824	}
		1825
		1826	/*
		1827	* Regardless of success, we are done until woken up next. But remember
		1828	* the requested order/classzone_idx in case it was higher/tighter than
		1829	* our current ones
		1830	*/
		1831	if (pgdat->kcompactd_max_order <= cc.order)
		1832	pgdat->kcompactd_max_order = 0;
		1833	if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
		1834	pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
		1835	}
		1836
		1837	void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
		1838	{
		1839	if (!order)
		1840	return;
		1841
		1842	if (pgdat->kcompactd_max_order < order)
		1843	pgdat->kcompactd_max_order = order;
		1844
		1845	if (pgdat->kcompactd_classzone_idx > classzone_idx)
		1846	pgdat->kcompactd_classzone_idx = classzone_idx;
		1847
		1848	if (!waitqueue_active(&pgdat->kcompactd_wait))
		1849	return;
		1850
		1851	if (!kcompactd_node_suitable(pgdat))
		1852	return;
		1853
		1854	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
		1855	classzone_idx);
		1856	wake_up_interruptible(&pgdat->kcompactd_wait);
		1857	}
		1858
		1859	/*
		1860	* The background compaction daemon, started as a kernel thread
		1861	* from the init process.
		1862	*/
		1863	static int kcompactd(void *p)
		1864	{
		1865	pg_data_t pgdat = (pg_data_t)p;
		1866	struct task_struct *tsk = current;
		1867
		1868	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
		1869
		1870	if (!cpumask_empty(cpumask))
		1871	set_cpus_allowed_ptr(tsk, cpumask);
		1872
		1873	set_freezable();
		1874
		1875	pgdat->kcompactd_max_order = 0;
		1876	pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
		1877
		1878	while (!kthread_should_stop()) {
		1879	trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
		1880	wait_event_freezable(pgdat->kcompactd_wait,
		1881	kcompactd_work_requested(pgdat));
		1882
		1883	kcompactd_do_work(pgdat);
		1884	}
		1885
		1886	return 0;
		1887	}
		1888
		1889	/*
		1890	* This kcompactd start function will be called by init and node-hot-add.
		1891	* On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
		1892	*/
		1893	int kcompactd_run(int nid)
		1894	{
		1895	pg_data_t *pgdat = NODE_DATA(nid);
		1896	int ret = 0;
		1897
		1898	if (pgdat->kcompactd)
		1899	return 0;
		1900
		1901	pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
		1902	if (IS_ERR(pgdat->kcompactd)) {
		1903	pr_err("Failed to start kcompactd on node %d\n", nid);
		1904	ret = PTR_ERR(pgdat->kcompactd);
		1905	pgdat->kcompactd = NULL;
		1906	}
		1907	return ret;
		1908	}
		1909
		1910	/*
		1911	* Called by memory hotplug when all memory in a node is offlined. Caller must
		1912	* hold mem_hotplug_begin/end().
		1913	*/
		1914	void kcompactd_stop(int nid)
		1915	{
		1916	struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
		1917
		1918	if (kcompactd) {
		1919	kthread_stop(kcompactd);
		1920	NODE_DATA(nid)->kcompactd = NULL;
		1921	}
		1922	}
		1923
		1924	/*
		1925	* It's optimal to keep kcompactd on the same CPUs as their memory, but
		1926	* not required for correctness. So if the last cpu in a node goes
		1927	* away, we get changed to run anywhere: as the first one comes back,
		1928	* restore their cpu bindings.
		1929	*/
		1930	static int cpu_callback(struct notifier_block *nfb, unsigned long action,
		1931	void *hcpu)
		1932	{
		1933	int nid;
		1934
		1935	if (action == CPU_ONLINE \|\| action == CPU_ONLINE_FROZEN) {
		1936	for_each_node_state(nid, N_MEMORY) {
		1937	pg_data_t *pgdat = NODE_DATA(nid);
		1938	const struct cpumask *mask;
		1939
		1940	mask = cpumask_of_node(pgdat->node_id);
		1941
		1942	if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
		1943	/* One of our CPUs online: restore mask */
		1944	set_cpus_allowed_ptr(pgdat->kcompactd, mask);
		1945	}
		1946	}
		1947	return NOTIFY_OK;
		1948	}
		1949
		1950	static int __init kcompactd_init(void)
		1951	{
		1952	int nid;
		1953
		1954	for_each_node_state(nid, N_MEMORY)
		1955	kcompactd_run(nid);
		1956	hotcpu_notifier(cpu_callback, 0);
		1957	return 0;
		1958	}
		1959	subsys_initcall(kcompactd_init)
		1960
1739	#endif /* CONFIG_COMPACTION */	1961	#endif /* CONFIG_COMPACTION */

mm/debug.c+1−1

@@ -43,7 +43,7 @@ const struct trace_print_flags vmaflag_names[] = { · 42 unmodified lines
43	void __dump_page(struct page page, const char reason)	43	void __dump_page(struct page page, const char reason)
44	{	44	{
45	pr_emerg("page:%p count:%d mapcount:%d mapping:%p index:%#lx",	45	pr_emerg("page:%p count:%d mapcount:%d mapping:%p index:%#lx",
46	page, atomic_read(&page->_count), page_mapcount(page),	46	page, page_ref_count(page), page_mapcount(page),
47	page->mapping, page->index);	47	page->mapping, page->index);
48	if (PageCompound(page))	48	if (PageCompound(page))
49	pr_cont(" compound_mapcount: %d", compound_mapcount(page));	49	pr_cont(" compound_mapcount: %d", compound_mapcount(page));

mm/debug_page_ref.c+54−0

@@ -0,0 +1,54 @@
	1	#include <linux/mm_types.h>
	2	#include <linux/tracepoint.h>
	3
	4	#define CREATE_TRACE_POINTS
	5	#include <trace/events/page_ref.h>
	6
	7	void __page_ref_set(struct page *page, int v)
	8	{
	9	trace_page_ref_set(page, v);
	10	}
	11	EXPORT_SYMBOL(__page_ref_set);
	12	EXPORT_TRACEPOINT_SYMBOL(page_ref_set);
	13
	14	void __page_ref_mod(struct page *page, int v)
	15	{
	16	trace_page_ref_mod(page, v);
	17	}
	18	EXPORT_SYMBOL(__page_ref_mod);
	19	EXPORT_TRACEPOINT_SYMBOL(page_ref_mod);
	20
	21	void __page_ref_mod_and_test(struct page *page, int v, int ret)
	22	{
	23	trace_page_ref_mod_and_test(page, v, ret);
	24	}
	25	EXPORT_SYMBOL(__page_ref_mod_and_test);
	26	EXPORT_TRACEPOINT_SYMBOL(page_ref_mod_and_test);
	27
	28	void __page_ref_mod_and_return(struct page *page, int v, int ret)
	29	{
	30	trace_page_ref_mod_and_return(page, v, ret);
	31	}
	32	EXPORT_SYMBOL(__page_ref_mod_and_return);
	33	EXPORT_TRACEPOINT_SYMBOL(page_ref_mod_and_return);
	34
	35	void __page_ref_mod_unless(struct page *page, int v, int u)
	36	{
	37	trace_page_ref_mod_unless(page, v, u);
	38	}
	39	EXPORT_SYMBOL(__page_ref_mod_unless);
	40	EXPORT_TRACEPOINT_SYMBOL(page_ref_mod_unless);
	41
	42	void __page_ref_freeze(struct page *page, int v, int ret)
	43	{
	44	trace_page_ref_freeze(page, v, ret);
	45	}
	46	EXPORT_SYMBOL(__page_ref_freeze);
	47	EXPORT_TRACEPOINT_SYMBOL(page_ref_freeze);
	48
	49	void __page_ref_unfreeze(struct page *page, int v)
	50	{
	51	trace_page_ref_unfreeze(page, v);
	52	}
	53	EXPORT_SYMBOL(__page_ref_unfreeze);
	54	EXPORT_TRACEPOINT_SYMBOL(page_ref_unfreeze);

mm/dmapool.c+7−11

 					"dma_pool_destroy %s, %p busy\n",
 					pool->name, page->vaddr);
 			else
-				printk(KERN_ERR
+				pr_err("dma_pool_destroy %s, %p busy\n",
-				       "dma_pool_destroy %s, %p busy\n",
 				       pool->name, page->vaddr);
 			/* leak the still-in-use consistent memory */
 			list_del(&page->page_list);
 				"dma_pool_free %s, %p/%lx (bad dma)\n",
 				pool->name, vaddr, (unsigned long)dma);
 		else
-			printk(KERN_ERR "dma_pool_free %s, %p/%lx (bad dma)\n",
+			pr_err("dma_pool_free %s, %p/%lx (bad dma)\n",
 			       pool->name, vaddr, (unsigned long)dma);
 		return;
 	}
 				"dma_pool_free %s, %p (bad vaddr)/%Lx\n",
 				pool->name, vaddr, (unsigned long long)dma);
 		else
-			printk(KERN_ERR
+			pr_err("dma_pool_free %s, %p (bad vaddr)/%Lx\n",
-			       "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
 			       pool->name, vaddr, (unsigned long long)dma);
 		return;
 	}
 			}
 			spin_unlock_irqrestore(&pool->lock, flags);
 			if (pool->dev)
-				dev_err(pool->dev, "dma_pool_free %s, dma %Lx "
+				dev_err(pool->dev, "dma_pool_free %s, dma %Lx already free\n",
-					"already free\n", pool->name,
+					pool->name, (unsigned long long)dma);
-					(unsigned long long)dma);
 			else
-				printk(KERN_ERR "dma_pool_free %s, dma %Lx "
+				pr_err("dma_pool_free %s, dma %Lx already free\n",
-					"already free\n", pool->name,
+				       pool->name, (unsigned long long)dma);
-					(unsigned long long)dma);
 			return;
 		}
 	}

mm/filemap.c+18−37

 	void **slot;
 	int error;
-	error = __radix_tree_create(&mapping->page_tree, page->index,
+	error = __radix_tree_create(&mapping->page_tree, page->index, 0,
 				    &node, &slot);
 	if (error)
 		return error;
 		return 0;
 	rcu_read_lock();
-restart:
 	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
 		struct page *page;
 repeat:
 		if (unlikely(!page))
 			continue;
 		if (radix_tree_exception(page)) {
-			if (radix_tree_deref_retry(page))
+			if (radix_tree_deref_retry(page)) {
-				goto restart;
+				slot = radix_tree_iter_retry(&iter);
+				continue;
+			}
 			/*
 			 * A shadow entry of a recently evicted page, a swap
 			 * entry from shmem/tmpfs or a DAX entry.  Return it
 		return 0;
 	rcu_read_lock();
-restart:
 	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
 		struct page *page;
 repeat:
 		if (radix_tree_exception(page)) {
 			if (radix_tree_deref_retry(page)) {
-				/*
+				slot = radix_tree_iter_retry(&iter);
-				 * Transient condition which can only trigger
+				continue;
-				 * when entry at index 0 moves out of or back
-				 * to root: none yet gotten, safe to restart.
-				 */
-				WARN_ON(iter.index);
-				goto restart;
 			}
 			/*
 			 * A shadow entry of a recently evicted page,
 		return 0;
 	rcu_read_lock();
-restart:
 	radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
 		struct page *page;
 repeat:
 		if (radix_tree_exception(page)) {
 			if (radix_tree_deref_retry(page)) {
-				/*
+				slot = radix_tree_iter_retry(&iter);
-				 * Transient condition which can only trigger
+				continue;
-				 * when entry at index 0 moves out of or back
-				 * to root: none yet gotten, safe to restart.
-				 */
-				goto restart;
 			}
 			/*
 			 * A shadow entry of a recently evicted page,
 		return 0;
 	rcu_read_lock();
-restart:
 	radix_tree_for_each_tagged(slot, &mapping->page_tree,
 				   &iter, *index, tag) {
 		struct page *page;
 		if (radix_tree_exception(page)) {
 			if (radix_tree_deref_retry(page)) {
-				/*
+				slot = radix_tree_iter_retry(&iter);
-				 * Transient condition which can only trigger
+				continue;
-				 * when entry at index 0 moves out of or back
-				 * to root: none yet gotten, safe to restart.
-				 */
-				goto restart;
 			}
 			/*
 			 * A shadow entry of a recently evicted page.
 		return 0;
 	rcu_read_lock();
-restart:
 	radix_tree_for_each_tagged(slot, &mapping->page_tree,
 				   &iter, start, tag) {
 		struct page *page;
 			continue;
 		if (radix_tree_exception(page)) {
 			if (radix_tree_deref_retry(page)) {
-				/*
+				slot = radix_tree_iter_retry(&iter);
-				 * Transient condition which can only trigger
+				continue;
-				 * when entry at index 0 moves out of or back
-				 * to root: none yet gotten, safe to restart.
-				 */
-				goto restart;
 			}
 			/*
 		if (unlikely(!page))
 			goto next;
 		if (radix_tree_exception(page)) {
-			if (radix_tree_deref_retry(page))
+			if (radix_tree_deref_retry(page)) {
-				break;
+				slot = radix_tree_iter_retry(&iter);
-			else
+				continue;
-				goto next;
+			}
+			goto next;
 		}
 		if (!page_cache_get_speculative(page))

mm/huge_memory.c+109−228

@@ -78,7 +78,7 @@ unsigned long transparent_hugepage_flags __read_mostly = · 77 unmodified lines
78	#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE	78	#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
79	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)\|	79	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)\|
80	#endif	80	#endif
81	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)\|	81	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)\|
82	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)\|	82	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)\|
83	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);	83	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
84		84
@@ -168,8 +168,7 @@ static void set_recommended_min_free_kbytes(void) · 83 unmodified lines
168		168
169	if (recommended_min > min_free_kbytes) {	169	if (recommended_min > min_free_kbytes) {
170	if (user_min_free_kbytes >= 0)	170	if (user_min_free_kbytes >= 0)
171	pr_info("raising min_free_kbytes from %d to %lu "	171	pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",
172	"to help transparent hugepage allocations\n",
173	min_free_kbytes, recommended_min);	172	min_free_kbytes, recommended_min);
174		173
175	min_free_kbytes = recommended_min;	174	min_free_kbytes = recommended_min;
@@ -270,37 +269,35 @@ static struct shrinker huge_zero_page_shrinker = { · 94 unmodified lines
270		269
271	#ifdef CONFIG_SYSFS	270	#ifdef CONFIG_SYSFS
272		271
273	static ssize_t double_flag_show(struct kobject *kobj,	272	static ssize_t triple_flag_store(struct kobject *kobj,
274	struct kobj_attribute attr, char buf,
275	enum transparent_hugepage_flag enabled,
276	enum transparent_hugepage_flag req_madv)
277	{
278	if (test_bit(enabled, &transparent_hugepage_flags)) {
279	VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
280	return sprintf(buf, "[always] madvise never\n");
281	} else if (test_bit(req_madv, &transparent_hugepage_flags))
282	return sprintf(buf, "always [madvise] never\n");
283	else
284	return sprintf(buf, "always madvise [never]\n");
285	}
286	static ssize_t double_flag_store(struct kobject *kobj,
287	struct kobj_attribute *attr,	273	struct kobj_attribute *attr,
288	const char *buf, size_t count,	274	const char *buf, size_t count,
289	enum transparent_hugepage_flag enabled,	275	enum transparent_hugepage_flag enabled,
		276	enum transparent_hugepage_flag deferred,
290	enum transparent_hugepage_flag req_madv)	277	enum transparent_hugepage_flag req_madv)
291	{	278	{
292	if (!memcmp("always", buf,	279	if (!memcmp("defer", buf,
		280	min(sizeof("defer")-1, count))) {
		281	if (enabled == deferred)
		282	return -EINVAL;
		283	clear_bit(enabled, &transparent_hugepage_flags);
		284	clear_bit(req_madv, &transparent_hugepage_flags);
		285	set_bit(deferred, &transparent_hugepage_flags);
		286	} else if (!memcmp("always", buf,
293	min(sizeof("always")-1, count))) {	287	min(sizeof("always")-1, count))) {
294	set_bit(enabled, &transparent_hugepage_flags);	288	clear_bit(deferred, &transparent_hugepage_flags);
295	clear_bit(req_madv, &transparent_hugepage_flags);	289	clear_bit(req_madv, &transparent_hugepage_flags);
		290	set_bit(enabled, &transparent_hugepage_flags);
296	} else if (!memcmp("madvise", buf,	291	} else if (!memcmp("madvise", buf,
297	min(sizeof("madvise")-1, count))) {	292	min(sizeof("madvise")-1, count))) {
298	clear_bit(enabled, &transparent_hugepage_flags);	293	clear_bit(enabled, &transparent_hugepage_flags);
		294	clear_bit(deferred, &transparent_hugepage_flags);
299	set_bit(req_madv, &transparent_hugepage_flags);	295	set_bit(req_madv, &transparent_hugepage_flags);
300	} else if (!memcmp("never", buf,	296	} else if (!memcmp("never", buf,
301	min(sizeof("never")-1, count))) {	297	min(sizeof("never")-1, count))) {
302	clear_bit(enabled, &transparent_hugepage_flags);	298	clear_bit(enabled, &transparent_hugepage_flags);
303	clear_bit(req_madv, &transparent_hugepage_flags);	299	clear_bit(req_madv, &transparent_hugepage_flags);
		300	clear_bit(deferred, &transparent_hugepage_flags);
304	} else	301	} else
305	return -EINVAL;	302	return -EINVAL;
306		303
@@ -310,17 +307,22 @@ static ssize_t double_flag_store(struct kobject *kobj, · 3 unmodified lines
310	static ssize_t enabled_show(struct kobject *kobj,	307	static ssize_t enabled_show(struct kobject *kobj,
311	struct kobj_attribute attr, char buf)	308	struct kobj_attribute attr, char buf)
312	{	309	{
313	return double_flag_show(kobj, attr, buf,	310	if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
314	TRANSPARENT_HUGEPAGE_FLAG,	311	return sprintf(buf, "[always] madvise never\n");
315	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);	312	else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
		313	return sprintf(buf, "always [madvise] never\n");
		314	else
		315	return sprintf(buf, "always madvise [never]\n");
316	}	316	}
		317
317	static ssize_t enabled_store(struct kobject *kobj,	318	static ssize_t enabled_store(struct kobject *kobj,
318	struct kobj_attribute *attr,	319	struct kobj_attribute *attr,
319	const char *buf, size_t count)	320	const char *buf, size_t count)
320	{	321	{
321	ssize_t ret;	322	ssize_t ret;
322		323
323	ret = double_flag_store(kobj, attr, buf, count,	324	ret = triple_flag_store(kobj, attr, buf, count,
		325	TRANSPARENT_HUGEPAGE_FLAG,
324	TRANSPARENT_HUGEPAGE_FLAG,	326	TRANSPARENT_HUGEPAGE_FLAG,
325	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);	327	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
326		328
@@ -378,16 +380,23 @@ static ssize_t single_flag_store(struct kobject *kobj, · 51 unmodified lines
378	static ssize_t defrag_show(struct kobject *kobj,	380	static ssize_t defrag_show(struct kobject *kobj,
379	struct kobj_attribute attr, char buf)	381	struct kobj_attribute attr, char buf)
380	{	382	{
381	return double_flag_show(kobj, attr, buf,	383	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
382	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,	384	return sprintf(buf, "[always] defer madvise never\n");
383	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);	385	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
		386	return sprintf(buf, "always [defer] madvise never\n");
		387	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
		388	return sprintf(buf, "always defer [madvise] never\n");
		389	else
		390	return sprintf(buf, "always defer madvise [never]\n");
		391
384	}	392	}
385	static ssize_t defrag_store(struct kobject *kobj,	393	static ssize_t defrag_store(struct kobject *kobj,
386	struct kobj_attribute *attr,	394	struct kobj_attribute *attr,
387	const char *buf, size_t count)	395	const char *buf, size_t count)
388	{	396	{
389	return double_flag_store(kobj, attr, buf, count,	397	return triple_flag_store(kobj, attr, buf, count,
390	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,	398	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
		399	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
391	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);	400	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
392	}	401	}
393	static struct kobj_attribute defrag_attr =	402	static struct kobj_attribute defrag_attr =
@@ -843,9 +852,30 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, · 449 unmodified lines
843	return 0;	852	return 0;
844	}	853	}
845		854
846	static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp)	855	/*
		856	* If THP is set to always then directly reclaim/compact as necessary
		857	* If set to defer then do no reclaim and defer to khugepaged
		858	* If set to madvise and the VMA is flagged then directly reclaim/compact
		859	*/
		860	static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
		861	{
		862	gfp_t reclaim_flags = 0;
		863
		864	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags) &&
		865	(vma->vm_flags & VM_HUGEPAGE))
		866	reclaim_flags = __GFP_DIRECT_RECLAIM;
		867	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
		868	reclaim_flags = __GFP_KSWAPD_RECLAIM;
		869	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
		870	reclaim_flags = __GFP_DIRECT_RECLAIM;
		871
		872	return GFP_TRANSHUGE \| reclaim_flags;
		873	}
		874
		875	/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
		876	static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
847	{	877	{
848	return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_RECLAIM)) \| extra_gfp;	878	return GFP_TRANSHUGE \| (khugepaged_defrag() ? __GFP_DIRECT_RECLAIM : 0);
849	}	879	}
850		880
851	/* Caller must hold page table lock. */	881	/* Caller must hold page table lock. */
@@ -919,7 +949,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct mm, struct vm_area_struct vma, · 67 unmodified lines
919	}	949	}
920	return ret;	950	return ret;
921	}	951	}
922	gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);	952	gfp = alloc_hugepage_direct_gfpmask(vma);
923	page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);	953	page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
924	if (unlikely(!page)) {	954	if (unlikely(!page)) {
925	count_vm_event(THP_FAULT_FALLBACK);	955	count_vm_event(THP_FAULT_FALLBACK);
@@ -1279,7 +1309,7 @@ int do_huge_pmd_wp_page(struct mm_struct mm, struct vm_area_struct vma, · 353 unmodified lines
1279	alloc:	1309	alloc:
1280	if (transparent_hugepage_enabled(vma) &&	1310	if (transparent_hugepage_enabled(vma) &&
1281	!transparent_hugepage_debug_cow()) {	1311	!transparent_hugepage_debug_cow()) {
1282	huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);	1312	huge_gfp = alloc_hugepage_direct_gfpmask(vma);
1283	new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);	1313	new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1284	} else	1314	} else
1285	new_page = NULL;	1315	new_page = NULL;
@@ -2249,11 +2279,12 @@ static int khugepaged_find_target_node(void) · 963 unmodified lines
2249	return 0;	2279	return 0;
2250	}	2280	}
2251		2281
2252	static inline struct page *alloc_hugepage(int defrag)	2282	static inline struct page *alloc_khugepaged_hugepage(void)
2253	{	2283	{
2254	struct page *page;	2284	struct page *page;
2255		2285
2256	page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER);	2286	page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
		2287	HPAGE_PMD_ORDER);
2257	if (page)	2288	if (page)
2258	prep_transhuge_page(page);	2289	prep_transhuge_page(page);
2259	return page;	2290	return page;
@@ -2264,7 +2295,7 @@ static struct page khugepaged_alloc_hugepage(bool wait) · 4 unmodified lines
2264	struct page *hpage;	2295	struct page *hpage;
2265		2296
2266	do {	2297	do {
2267	hpage = alloc_hugepage(khugepaged_defrag());	2298	hpage = alloc_khugepaged_hugepage();
2268	if (!hpage) {	2299	if (!hpage) {
2269	count_vm_event(THP_COLLAPSE_ALLOC_FAILED);	2300	count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2270	if (!*wait)	2301	if (!*wait)
@@ -2335,8 +2366,7 @@ static void collapse_huge_page(struct mm_struct *mm, · 64 unmodified lines
2335	VM_BUG_ON(address & ~HPAGE_PMD_MASK);	2366	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
2336		2367
2337	/* Only allocate from the target node */	2368	/* Only allocate from the target node */
2338	gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) \|	2369	gfp = alloc_hugepage_khugepaged_gfpmask() \| __GFP_OTHER_NODE \| __GFP_THISNODE;
2339	__GFP_THISNODE;
2340		2370
2341	/* release the mmap_sem read lock. */	2371	/* release the mmap_sem read lock. */
2342	new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node);	2372	new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node);
@@ -2857,7 +2887,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct vma, pmd_t pmd, · 514 unmodified lines
2857		2887
2858	page = pmd_page(*pmd);	2888	page = pmd_page(*pmd);
2859	VM_BUG_ON_PAGE(!page_count(page), page);	2889	VM_BUG_ON_PAGE(!page_count(page), page);
2860	atomic_add(HPAGE_PMD_NR - 1, &page->_count);	2890	page_ref_add(page, HPAGE_PMD_NR - 1);
2861	write = pmd_write(*pmd);	2891	write = pmd_write(*pmd);
2862	young = pmd_young(*pmd);	2892	young = pmd_young(*pmd);
2863	dirty = pmd_dirty(*pmd);	2893	dirty = pmd_dirty(*pmd);
@@ -2947,44 +2977,33 @@ static void __split_huge_pmd_locked(struct vm_area_struct vma, pmd_t pmd, · 83 unmodified lines
2947	}	2977	}
2948		2978
2949	void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,	2979	void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
2950	unsigned long address)	2980	unsigned long address, bool freeze)
2951	{	2981	{
2952	spinlock_t *ptl;	2982	spinlock_t *ptl;
2953	struct mm_struct *mm = vma->vm_mm;	2983	struct mm_struct *mm = vma->vm_mm;
2954	struct page *page = NULL;
2955	unsigned long haddr = address & HPAGE_PMD_MASK;	2984	unsigned long haddr = address & HPAGE_PMD_MASK;
2956		2985
2957	mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);	2986	mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
2958	ptl = pmd_lock(mm, pmd);	2987	ptl = pmd_lock(mm, pmd);
2959	if (pmd_trans_huge(*pmd)) {	2988	if (pmd_trans_huge(*pmd)) {
2960	page = pmd_page(*pmd);	2989	struct page page = pmd_page(pmd);
2961	if (PageMlocked(page))	2990	if (PageMlocked(page))
2962	get_page(page);	2991	clear_page_mlock(page);
2963	else
2964	page = NULL;
2965	} else if (!pmd_devmap(*pmd))	2992	} else if (!pmd_devmap(*pmd))
2966	goto out;	2993	goto out;
2967	__split_huge_pmd_locked(vma, pmd, haddr, false);	2994	__split_huge_pmd_locked(vma, pmd, haddr, freeze);
2968	out:	2995	out:
2969	spin_unlock(ptl);	2996	spin_unlock(ptl);
2970	mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);	2997	mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
2971	if (page) {
2972	lock_page(page);
2973	munlock_vma_page(page);
2974	unlock_page(page);
2975	put_page(page);
2976	}
2977	}	2998	}
2978		2999
2979	static void split_huge_pmd_address(struct vm_area_struct *vma,	3000	void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
2980	unsigned long address)	3001	bool freeze, struct page *page)
2981	{	3002	{
2982	pgd_t *pgd;	3003	pgd_t *pgd;
2983	pud_t *pud;	3004	pud_t *pud;
2984	pmd_t *pmd;	3005	pmd_t *pmd;
2985		3006
2986	VM_BUG_ON(!(address & ~HPAGE_PMD_MASK));
2987
2988	pgd = pgd_offset(vma->vm_mm, address);	3007	pgd = pgd_offset(vma->vm_mm, address);
2989	if (!pgd_present(*pgd))	3008	if (!pgd_present(*pgd))
2990	return;	3009	return;
@@ -2996,11 +3015,20 @@ static void split_huge_pmd_address(struct vm_area_struct *vma, · 5 unmodified lines
2996	pmd = pmd_offset(pud, address);	3015	pmd = pmd_offset(pud, address);
2997	if (!pmd_present(pmd) \|\| (!pmd_trans_huge(pmd) && !pmd_devmap(*pmd)))	3016	if (!pmd_present(pmd) \|\| (!pmd_trans_huge(pmd) && !pmd_devmap(*pmd)))
2998	return;	3017	return;
		3018
		3019	/*
		3020	* If caller asks to setup a migration entries, we need a page to check
		3021	* pmd against. Otherwise we can end up replacing wrong page.
		3022	*/
		3023	VM_BUG_ON(freeze && !page);
		3024	if (page && page != pmd_page(*pmd))
		3025	return;
		3026
2999	/*	3027	/*
3000	* Caller holds the mmap_sem write mode, so a huge pmd cannot	3028	* Caller holds the mmap_sem write mode, so a huge pmd cannot
3001	* materialize from under us.	3029	* materialize from under us.
3002	*/	3030	*/
3003	split_huge_pmd(vma, pmd, address);	3031	__split_huge_pmd(vma, pmd, address, freeze);
3004	}	3032	}
3005		3033
3006	void vma_adjust_trans_huge(struct vm_area_struct *vma,	3034	void vma_adjust_trans_huge(struct vm_area_struct *vma,
@@ -3016,7 +3044,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 9 unmodified lines
3016	if (start & ~HPAGE_PMD_MASK &&	3044	if (start & ~HPAGE_PMD_MASK &&
3017	(start & HPAGE_PMD_MASK) >= vma->vm_start &&	3045	(start & HPAGE_PMD_MASK) >= vma->vm_start &&
3018	(start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)	3046	(start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
3019	split_huge_pmd_address(vma, start);	3047	split_huge_pmd_address(vma, start, false, NULL);
3020		3048
3021	/*	3049	/*
3022	* If the new end address isn't hpage aligned and it could	3050	* If the new end address isn't hpage aligned and it could
@@ -3026,7 +3054,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 3 unmodified lines
3026	if (end & ~HPAGE_PMD_MASK &&	3054	if (end & ~HPAGE_PMD_MASK &&
3027	(end & HPAGE_PMD_MASK) >= vma->vm_start &&	3055	(end & HPAGE_PMD_MASK) >= vma->vm_start &&
3028	(end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)	3056	(end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
3029	split_huge_pmd_address(vma, end);	3057	split_huge_pmd_address(vma, end, false, NULL);
3030		3058
3031	/*	3059	/*
3032	* If we're also updating the vma->vm_next->vm_start, if the new	3060	* If we're also updating the vma->vm_next->vm_start, if the new
@@ -3040,184 +3068,36 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 7 unmodified lines
3040	if (nstart & ~HPAGE_PMD_MASK &&	3068	if (nstart & ~HPAGE_PMD_MASK &&
3041	(nstart & HPAGE_PMD_MASK) >= next->vm_start &&	3069	(nstart & HPAGE_PMD_MASK) >= next->vm_start &&
3042	(nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)	3070	(nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
3043	split_huge_pmd_address(next, nstart);	3071	split_huge_pmd_address(next, nstart, false, NULL);
3044	}	3072	}
3045	}	3073	}
3046		3074
3047	static void freeze_page_vma(struct vm_area_struct vma, struct page page,	3075	static void freeze_page(struct page *page)
3048	unsigned long address)
3049	{	3076	{
3050	unsigned long haddr = address & HPAGE_PMD_MASK;	3077	enum ttu_flags ttu_flags = TTU_MIGRATION \| TTU_IGNORE_MLOCK \|
3051	spinlock_t *ptl;	3078	TTU_IGNORE_ACCESS \| TTU_RMAP_LOCKED;
3052	pgd_t *pgd;	3079	int i, ret;
3053	pud_t *pud;
3054	pmd_t *pmd;
3055	pte_t *pte;
3056	int i, nr = HPAGE_PMD_NR;
3057
3058	/* Skip pages which doesn't belong to the VMA */
3059	if (address < vma->vm_start) {
3060	int off = (vma->vm_start - address) >> PAGE_SHIFT;
3061	page += off;
3062	nr -= off;
3063	address = vma->vm_start;
3064	}
3065
3066	pgd = pgd_offset(vma->vm_mm, address);
3067	if (!pgd_present(*pgd))
3068	return;
3069	pud = pud_offset(pgd, address);
3070	if (!pud_present(*pud))
3071	return;
3072	pmd = pmd_offset(pud, address);
3073	ptl = pmd_lock(vma->vm_mm, pmd);
3074	if (!pmd_present(*pmd)) {
3075	spin_unlock(ptl);
3076	return;
3077	}
3078	if (pmd_trans_huge(*pmd)) {
3079	if (page == pmd_page(*pmd))
3080	__split_huge_pmd_locked(vma, pmd, haddr, true);
3081	spin_unlock(ptl);
3082	return;
3083	}
3084	spin_unlock(ptl);
3085
3086	pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
3087	for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
3088	pte_t entry, swp_pte;
3089	swp_entry_t swp_entry;
3090
3091	/*
3092	* We've just crossed page table boundary: need to map next one.
3093	* It can happen if THP was mremaped to non PMD-aligned address.
3094	*/
3095	if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3096	pte_unmap_unlock(pte - 1, ptl);
3097	pmd = mm_find_pmd(vma->vm_mm, address);
3098	if (!pmd)
3099	return;
3100	pte = pte_offset_map_lock(vma->vm_mm, pmd,
3101	address, &ptl);
3102	}
3103
3104	if (!pte_present(*pte))
3105	continue;
3106	if (page_to_pfn(page) != pte_pfn(*pte))
3107	continue;
3108	flush_cache_page(vma, address, page_to_pfn(page));
3109	entry = ptep_clear_flush(vma, address, pte);
3110	if (pte_dirty(entry))
3111	SetPageDirty(page);
3112	swp_entry = make_migration_entry(page, pte_write(entry));
3113	swp_pte = swp_entry_to_pte(swp_entry);
3114	if (pte_soft_dirty(entry))
3115	swp_pte = pte_swp_mksoft_dirty(swp_pte);
3116	set_pte_at(vma->vm_mm, address, pte, swp_pte);
3117	page_remove_rmap(page, false);
3118	put_page(page);
3119	}
3120	pte_unmap_unlock(pte - 1, ptl);
3121	}
3122
3123	static void freeze_page(struct anon_vma anon_vma, struct page page)
3124	{
3125	struct anon_vma_chain *avc;
3126	pgoff_t pgoff = page_to_pgoff(page);
3127		3080
3128	VM_BUG_ON_PAGE(!PageHead(page), page);	3081	VM_BUG_ON_PAGE(!PageHead(page), page);
3129		3082
3130	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff,	3083	/* We only need TTU_SPLIT_HUGE_PMD once */
3131	pgoff + HPAGE_PMD_NR - 1) {	3084	ret = try_to_unmap(page, ttu_flags \| TTU_SPLIT_HUGE_PMD);
3132	unsigned long address = __vma_address(page, avc->vma);	3085	for (i = 1; !ret && i < HPAGE_PMD_NR; i++) {
3133		3086	/* Cut short if the page is unmapped */
3134	mmu_notifier_invalidate_range_start(avc->vma->vm_mm,	3087	if (page_count(page) == 1)
3135	address, address + HPAGE_PMD_SIZE);	3088	return;
3136	freeze_page_vma(avc->vma, page, address);
3137	mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
3138	address, address + HPAGE_PMD_SIZE);
3139	}
3140	}
3141
3142	static void unfreeze_page_vma(struct vm_area_struct vma, struct page page,
3143	unsigned long address)
3144	{
3145	spinlock_t *ptl;
3146	pmd_t *pmd;
3147	pte_t *pte, entry;
3148	swp_entry_t swp_entry;
3149	unsigned long haddr = address & HPAGE_PMD_MASK;
3150	int i, nr = HPAGE_PMD_NR;
3151
3152	/* Skip pages which doesn't belong to the VMA */
3153	if (address < vma->vm_start) {
3154	int off = (vma->vm_start - address) >> PAGE_SHIFT;
3155	page += off;
3156	nr -= off;
3157	address = vma->vm_start;
3158	}
3159
3160	pmd = mm_find_pmd(vma->vm_mm, address);
3161	if (!pmd)
3162	return;
3163
3164	pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
3165	for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
3166	/*
3167	* We've just crossed page table boundary: need to map next one.
3168	* It can happen if THP was mremaped to non-PMD aligned address.
3169	*/
3170	if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3171	pte_unmap_unlock(pte - 1, ptl);
3172	pmd = mm_find_pmd(vma->vm_mm, address);
3173	if (!pmd)
3174	return;
3175	pte = pte_offset_map_lock(vma->vm_mm, pmd,
3176	address, &ptl);
3177	}
3178
3179	if (!is_swap_pte(*pte))
3180	continue;
3181
3182	swp_entry = pte_to_swp_entry(*pte);
3183	if (!is_migration_entry(swp_entry))
3184	continue;
3185	if (migration_entry_to_page(swp_entry) != page)
3186	continue;
3187
3188	get_page(page);
3189	page_add_anon_rmap(page, vma, address, false);
3190
3191	entry = pte_mkold(mk_pte(page, vma->vm_page_prot));
3192	if (PageDirty(page))
3193	entry = pte_mkdirty(entry);
3194	if (is_write_migration_entry(swp_entry))
3195	entry = maybe_mkwrite(entry, vma);
3196
3197	flush_dcache_page(page);
3198	set_pte_at(vma->vm_mm, address, pte, entry);
3199		3089
3200	/* No need to invalidate - it was non-present before */	3090	ret = try_to_unmap(page + i, ttu_flags);
3201	update_mmu_cache(vma, address, pte);
3202	}	3091	}
3203	pte_unmap_unlock(pte - 1, ptl);	3092	VM_BUG_ON(ret);
3204	}	3093	}
3205		3094
3206	static void unfreeze_page(struct anon_vma anon_vma, struct page page)	3095	static void unfreeze_page(struct page *page)
3207	{	3096	{
3208	struct anon_vma_chain *avc;	3097	int i;
3209	pgoff_t pgoff = page_to_pgoff(page);
3210
3211	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
3212	pgoff, pgoff + HPAGE_PMD_NR - 1) {
3213	unsigned long address = __vma_address(page, avc->vma);
3214		3098
3215	mmu_notifier_invalidate_range_start(avc->vma->vm_mm,	3099	for (i = 0; i < HPAGE_PMD_NR; i++)
3216	address, address + HPAGE_PMD_SIZE);	3100	remove_migration_ptes(page + i, page + i, true);
3217	unfreeze_page_vma(avc->vma, page, address);
3218	mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
3219	address, address + HPAGE_PMD_SIZE);
3220	}
3221	}	3101	}
3222		3102
3223	static void __split_huge_page_tail(struct page *head, int tail,	3103	static void __split_huge_page_tail(struct page *head, int tail,
@@ -3226,7 +3106,7 @@ static void __split_huge_page_tail(struct page *head, int tail, · 2 unmodified lines
3226	struct page *page_tail = head + tail;	3106	struct page *page_tail = head + tail;
3227		3107
3228	VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);	3108	VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
3229	VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail);	3109	VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail);
3230		3110
3231	/*	3111	/*
3232	* tail_page->_count is zero and not changing from under us. But	3112	* tail_page->_count is zero and not changing from under us. But
@@ -3239,7 +3119,7 @@ static void __split_huge_page_tail(struct page *head, int tail, · 6 unmodified lines
3239	* atomic_set() here would be safe on all archs (and not only on x86),	3119	* atomic_set() here would be safe on all archs (and not only on x86),
3240	* it's safer to use atomic_inc().	3120	* it's safer to use atomic_inc().
3241	*/	3121	*/
3242	atomic_inc(&page_tail->_count);	3122	page_ref_inc(page_tail);
3243		3123
3244	page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;	3124	page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
3245	page_tail->flags \|= (head->flags &	3125	page_tail->flags \|= (head->flags &
@@ -3295,7 +3175,7 @@ static void __split_huge_page(struct page page, struct list_head list) · 49 unmodified lines
3295	ClearPageCompound(head);	3175	ClearPageCompound(head);
3296	spin_unlock_irq(&zone->lru_lock);	3176	spin_unlock_irq(&zone->lru_lock);
3297		3177
3298	unfreeze_page(page_anon_vma(head), head);	3178	unfreeze_page(head);
3299		3179
3300	for (i = 0; i < HPAGE_PMD_NR; i++) {	3180	for (i = 0; i < HPAGE_PMD_NR; i++) {
3301	struct page *subpage = head + i;	3181	struct page *subpage = head + i;
@@ -3391,7 +3271,7 @@ int split_huge_page_to_list(struct page page, struct list_head list) · 89 unmodified lines
3391	}	3271	}
3392		3272
3393	mlocked = PageMlocked(page);	3273	mlocked = PageMlocked(page);
3394	freeze_page(anon_vma, head);	3274	freeze_page(head);
3395	VM_BUG_ON_PAGE(compound_mapcount(head), head);	3275	VM_BUG_ON_PAGE(compound_mapcount(head), head);
3396		3276
3397	/* Make sure the page is not on per-CPU pagevec as it takes pin */	3277	/* Make sure the page is not on per-CPU pagevec as it takes pin */
@@ -3420,7 +3300,7 @@ int split_huge_page_to_list(struct page page, struct list_head list) · 22 unmodified lines
3420	BUG();	3300	BUG();
3421	} else {	3301	} else {
3422	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);	3302	spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
3423	unfreeze_page(anon_vma, head);	3303	unfreeze_page(head);
3424	ret = -EBUSY;	3304	ret = -EBUSY;
3425	}	3305	}
3426		3306
@@ -3455,6 +3335,7 @@ void deferred_split_huge_page(struct page *page) · 28 unmodified lines
3455		3335
3456	spin_lock_irqsave(&pgdata->split_queue_lock, flags);	3336	spin_lock_irqsave(&pgdata->split_queue_lock, flags);
3457	if (list_empty(page_deferred_list(page))) {	3337	if (list_empty(page_deferred_list(page))) {
		3338	count_vm_event(THP_DEFERRED_SPLIT_PAGE);
3458	list_add_tail(page_deferred_list(page), &pgdata->split_queue);	3339	list_add_tail(page_deferred_list(page), &pgdata->split_queue);
3459	pgdata->split_queue_len++;	3340	pgdata->split_queue_len++;
3460	}	3341	}

mm/hugetlb.c+2−3

 	unsigned long i;
 	if (size_to_hstate(PAGE_SIZE << order)) {
-		pr_warning("hugepagesz= specified twice, ignoring\n");
+		pr_warn("hugepagesz= specified twice, ignoring\n");
 		return;
 	}
 	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
 		mhp = &parsed_hstate->max_huge_pages;
 	if (mhp == last_mhp) {
-		pr_warning("hugepages= specified twice without "
+		pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
-			   "interleaving hugepagesz=, ignoring\n");
 		return 1;
 	}

mm/internal.h+3−10

 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
 		unsigned long floor, unsigned long ceiling);
-static inline void set_page_count(struct page *page, int v)
-{
-	atomic_set(&page->_count, v);
-}
 extern int __do_page_cache_readahead(struct address_space *mapping,
 		struct file *filp, pgoff_t offset, unsigned long nr_to_read,
 		unsigned long lookahead_size);
 static inline void set_page_refcounted(struct page *page)
 {
 	VM_BUG_ON_PAGE(PageTail(page), page);
-	VM_BUG_ON_PAGE(atomic_read(&page->_count), page);
+	VM_BUG_ON_PAGE(page_ref_count(page), page);
 	set_page_count(page, 1);
 }
 extern void __free_pages_bootmem(struct page *page, unsigned long pfn,
 					unsigned int order);
 extern void prep_compound_page(struct page *page, unsigned int order);
-#ifdef CONFIG_MEMORY_FAILURE
-extern bool is_free_buddy_page(struct page *page);
-#endif
 extern int user_min_free_kbytes;
 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
 	unsigned long last_migrated_pfn;/* Not yet flushed page being freed */
 	enum migrate_mode mode;		/* Async or sync migration mode */
 	bool ignore_skip_hint;		/* Scan blocks even if marked skip */
+	bool direct_compaction;		/* False from kcompactd or /proc/... */
 	int order;			/* order a direct compactor needs */
 	const gfp_t gfp_mask;		/* gfp mask of a direct compactor */
 	const int alloc_flags;		/* alloc flags of a direct compactor */
 do { \
 	if (level < mminit_loglevel) { \
 		if (level <= MMINIT_WARNING) \
-			printk(KERN_WARNING "mminit::" prefix " " fmt, ##arg); \
+			pr_warn("mminit::" prefix " " fmt, ##arg);	\
 		else \
 			printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
 	} \

mm/kasan/report.c+2−4

 	 */
 	kasan_disable_current();
 	spin_lock_irqsave(&report_lock, flags);
-	pr_err("================================="
+	pr_err("==================================================================\n");
-		"=================================\n");
 	if (info->access_addr <
 			kasan_shadow_to_mem((void *)KASAN_SHADOW_START)) {
 		if ((unsigned long)info->access_addr < PAGE_SIZE)
 		print_address_description(info);
 		print_shadow_for_address(info->first_bad_addr);
 	}
-	pr_err("================================="
+	pr_err("==================================================================\n");
-		"=================================\n");
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 	spin_unlock_irqrestore(&report_lock, flags);
 	kasan_enable_current();

mm/kmemcheck.c+1−2

@@ -20,8 +20,7 @@ void kmemcheck_alloc_shadow(struct page *page, int order, gfp_t flags, int node) · 19 unmodified lines
20	shadow = alloc_pages_node(node, flags \| __GFP_NOTRACK, order);	20	shadow = alloc_pages_node(node, flags \| __GFP_NOTRACK, order);
21	if (!shadow) {	21	if (!shadow) {
22	if (printk_ratelimit())	22	if (printk_ratelimit())
23	printk(KERN_ERR "kmemcheck: failed to allocate "	23	pr_err("kmemcheck: failed to allocate shadow bitmap\n");
24	"shadow bitmap\n");
25	return;	24	return;
26	}	25	}
27		26

mm/kmemleak-test.c+1−1

@@ -49,7 +49,7 @@ static int __init kmemleak_test_init(void) · 48 unmodified lines
49	struct test_node *elem;	49	struct test_node *elem;
50	int i;	50	int i;
51		51
52	printk(KERN_INFO "Kmemleak testing\n");	52	pr_info("Kmemleak testing\n");
53		53
54	/* make some orphan objects */	54	/* make some orphan objects */
55	pr_info("kmalloc(32) = %p\n", kmalloc(32, GFP_KERNEL));	55	pr_info("kmalloc(32) = %p\n", kmalloc(32, GFP_KERNEL));

mm/kmemleak.c+15−17

  * Print a warning and dump the stack trace.
  */
 #define kmemleak_warn(x...)	do {		\
-	pr_warning(x);				\
+	pr_warn(x);				\
 	dump_stack();				\
 	kmemleak_warning = 1;			\
 } while (0)
 	object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
 	if (!object) {
-		pr_warning("Cannot allocate a kmemleak_object structure\n");
+		pr_warn("Cannot allocate a kmemleak_object structure\n");
 		kmemleak_disable();
 		return NULL;
 	}
 		else if (parent->pointer + parent->size <= ptr)
 			link = &parent->rb_node.rb_right;
 		else {
-			kmemleak_stop("Cannot insert 0x%lx into the object "
+			kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
-				      "search tree (overlaps existing)\n",
 				      ptr);
 			/*
 			 * No need for parent->lock here since "parent" cannot
 	object = find_and_remove_object(ptr, 1);
 	if (!object) {
 #ifdef DEBUG
-		kmemleak_warn("Partially freeing unknown object at 0x%08lx "
+		kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
-			      "(size %zu)\n", ptr, size);
+			      ptr, size);
 #endif
 		return;
 	}
 	object = find_and_get_object(ptr, 0);
 	if (!object) {
-		kmemleak_warn("Trying to color unknown object "
+		kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
-			      "at 0x%08lx as %s\n", ptr,
+			      ptr,
 			      (color == KMEMLEAK_GREY) ? "Grey" :
 			      (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
 		return;
 	area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
 	if (!area) {
-		pr_warning("Cannot allocate a scan area\n");
+		pr_warn("Cannot allocate a scan area\n");
 		goto out;
 	}
 	if (new_leaks) {
 		kmemleak_found_leaks = true;
-		pr_info("%d new suspected memory leaks (see "
+		pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
-			"/sys/kernel/debug/kmemleak)\n", new_leaks);
+			new_leaks);
 	}
 }
 		return;
 	scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
 	if (IS_ERR(scan_thread)) {
-		pr_warning("Failed to create the scan thread\n");
+		pr_warn("Failed to create the scan thread\n");
 		scan_thread = NULL;
 	}
 }
 	if (!kmemleak_found_leaks)
 		__kmemleak_do_cleanup();
 	else
-		pr_info("Kmemleak disabled without freeing internal data. "
+		pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
-			"Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\"\n");
 }
 static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
 	scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
 	if (crt_early_log > ARRAY_SIZE(early_log))
-		pr_warning("Early log buffer exceeded (%d), please increase "
+		pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
-			   "DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n", crt_early_log);
+			crt_early_log);
 	/* the kernel is still in UP mode, so disabling the IRQs is enough */
 	local_irq_save(flags);
 	dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
 				     &kmemleak_fops);
 	if (!dentry)
-		pr_warning("Failed to create the debugfs kmemleak file\n");
+		pr_warn("Failed to create the debugfs kmemleak file\n");
 	mutex_lock(&scan_mutex);
 	start_scan_thread();
 	mutex_unlock(&scan_mutex);

mm/memblock.c+1−2

@@ -238,8 +238,7 @@ phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, · 237 unmodified lines
238	* so we use WARN_ONCE() here to see the stack trace if	238	* so we use WARN_ONCE() here to see the stack trace if
239	* fail happens.	239	* fail happens.
240	*/	240	*/
241	WARN_ONCE(1, "memblock: bottom-up allocation failed, "	241	WARN_ONCE(1, "memblock: bottom-up allocation failed, memory hotunplug may be affected\n");
242	"memory hotunplug may be affected\n");
243	}	242	}
244		243
245	return __memblock_find_range_top_down(start, end, size, align, nid,	244	return __memblock_find_range_top_down(start, end, size, align, nid,

mm/memcontrol.c+108−83

@@ -638,9 +638,8 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, · 637 unmodified lines
638	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);	638	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
639	}	639	}
640		640
641	static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,	641	unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
642	int nid,	642	int nid, unsigned int lru_mask)
643	unsigned int lru_mask)
644	{	643	{
645	unsigned long nr = 0;	644	unsigned long nr = 0;
646	int zid;	645	int zid;
@@ -1151,12 +1150,9 @@ static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) · 504 unmodified lines
1151	*/	1150	*/
1152	void mem_cgroup_print_oom_info(struct mem_cgroup memcg, struct task_struct p)	1151	void mem_cgroup_print_oom_info(struct mem_cgroup memcg, struct task_struct p)
1153	{	1152	{
1154	/* oom_info_lock ensures that parallel ooms do not interleave */
1155	static DEFINE_MUTEX(oom_info_lock);
1156	struct mem_cgroup *iter;	1153	struct mem_cgroup *iter;
1157	unsigned int i;	1154	unsigned int i;
1158		1155
1159	mutex_lock(&oom_info_lock);
1160	rcu_read_lock();	1156	rcu_read_lock();
1161		1157
1162	if (p) {	1158	if (p) {
@@ -1200,7 +1196,6 @@ void mem_cgroup_print_oom_info(struct mem_cgroup memcg, struct task_struct p) · 37 unmodified lines
1200		1196
1201	pr_cont("\n");	1197	pr_cont("\n");
1202	}	1198	}
1203	mutex_unlock(&oom_info_lock);
1204	}	1199	}
1205		1200
1206	/*	1201	/*
@@ -1237,7 +1232,7 @@ static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg) · 30 unmodified lines
1237	return limit;	1232	return limit;
1238	}	1233	}
1239		1234
1240	static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,	1235	static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1241	int order)	1236	int order)
1242	{	1237	{
1243	struct oom_control oc = {	1238	struct oom_control oc = {
@@ -1315,6 +1310,7 @@ static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, · 71 unmodified lines
1315	}	1310	}
1316	unlock:	1311	unlock:
1317	mutex_unlock(&oom_lock);	1312	mutex_unlock(&oom_lock);
		1313	return chosen;
1318	}	1314	}
1319		1315
1320	#if MAX_NUMNODES > 1	1316	#if MAX_NUMNODES > 1
@@ -2325,9 +2321,6 @@ int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, · 1004 unmodified lines
2325	struct page_counter *counter;	2321	struct page_counter *counter;
2326	int ret;	2322	int ret;
2327		2323
2328	if (!memcg_kmem_online(memcg))
2329	return 0;
2330
2331	ret = try_charge(memcg, gfp, nr_pages);	2324	ret = try_charge(memcg, gfp, nr_pages);
2332	if (ret)	2325	if (ret)
2333	return ret;	2326	return ret;
@@ -2346,10 +2339,11 @@ int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, · 12 unmodified lines
2346	int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)	2339	int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2347	{	2340	{
2348	struct mem_cgroup *memcg;	2341	struct mem_cgroup *memcg;
2349	int ret;	2342	int ret = 0;
2350		2343
2351	memcg = get_mem_cgroup_from_mm(current->mm);	2344	memcg = get_mem_cgroup_from_mm(current->mm);
2352	ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);	2345	if (!mem_cgroup_is_root(memcg))
		2346	ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);
2353	css_put(&memcg->css);	2347	css_put(&memcg->css);
2354	return ret;	2348	return ret;
2355	}	2349	}
@@ -2719,39 +2713,48 @@ static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, · 363 unmodified lines
2719	return retval;	2713	return retval;
2720	}	2714	}
2721		2715
2722	static unsigned long tree_stat(struct mem_cgroup *memcg,	2716	static void tree_stat(struct mem_cgroup memcg, unsigned long stat)
2723	enum mem_cgroup_stat_index idx)
2724	{	2717	{
2725	struct mem_cgroup *iter;	2718	struct mem_cgroup *iter;
2726	unsigned long val = 0;	2719	int i;
2727		2720
2728	for_each_mem_cgroup_tree(iter, memcg)	2721	memset(stat, 0, sizeof(stat) MEMCG_NR_STAT);
2729	val += mem_cgroup_read_stat(iter, idx);
2730		2722
2731	return val;	2723	for_each_mem_cgroup_tree(iter, memcg) {
		2724	for (i = 0; i < MEMCG_NR_STAT; i++)
		2725	stat[i] += mem_cgroup_read_stat(iter, i);
		2726	}
2732	}	2727	}
2733		2728
2734	static unsigned long tree_events(struct mem_cgroup *memcg,	2729	static void tree_events(struct mem_cgroup memcg, unsigned long events)
2735	enum mem_cgroup_events_index idx)
2736	{	2730	{
2737	struct mem_cgroup *iter;	2731	struct mem_cgroup *iter;
2738	unsigned long val = 0;	2732	int i;
2739		2733
2740	for_each_mem_cgroup_tree(iter, memcg)	2734	memset(events, 0, sizeof(events) MEMCG_NR_EVENTS);
2741	val += mem_cgroup_read_events(iter, idx);
2742		2735
2743	return val;	2736	for_each_mem_cgroup_tree(iter, memcg) {
		2737	for (i = 0; i < MEMCG_NR_EVENTS; i++)
		2738	events[i] += mem_cgroup_read_events(iter, i);
		2739	}
2744	}	2740	}
2745		2741
2746	static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)	2742	static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2747	{	2743	{
2748	unsigned long val;	2744	unsigned long val = 0;
2749		2745
2750	if (mem_cgroup_is_root(memcg)) {	2746	if (mem_cgroup_is_root(memcg)) {
2751	val = tree_stat(memcg, MEM_CGROUP_STAT_CACHE);	2747	struct mem_cgroup *iter;
2752	val += tree_stat(memcg, MEM_CGROUP_STAT_RSS);	2748
2753	if (swap)	2749	for_each_mem_cgroup_tree(iter, memcg) {
2754	val += tree_stat(memcg, MEM_CGROUP_STAT_SWAP);	2750	val += mem_cgroup_read_stat(iter,
		2751	MEM_CGROUP_STAT_CACHE);
		2752	val += mem_cgroup_read_stat(iter,
		2753	MEM_CGROUP_STAT_RSS);
		2754	if (swap)
		2755	val += mem_cgroup_read_stat(iter,
		2756	MEM_CGROUP_STAT_SWAP);
		2757	}
2755	} else {	2758	} else {
2756	if (!swap)	2759	if (!swap)
2757	val = page_counter_read(&memcg->memory);	2760	val = page_counter_read(&memcg->memory);
@@ -2817,6 +2820,9 @@ static int memcg_online_kmem(struct mem_cgroup *memcg) · 59 unmodified lines
2817	{	2820	{
2818	int memcg_id;	2821	int memcg_id;
2819		2822
		2823	if (cgroup_memory_nokmem)
		2824	return 0;
		2825
2820	BUG_ON(memcg->kmemcg_id >= 0);	2826	BUG_ON(memcg->kmemcg_id >= 0);
2821	BUG_ON(memcg->kmem_state);	2827	BUG_ON(memcg->kmem_state);
2822		2828
@@ -2837,24 +2843,6 @@ static int memcg_online_kmem(struct mem_cgroup *memcg) · 14 unmodified lines
2837	return 0;	2843	return 0;
2838	}	2844	}
2839		2845
2840	static int memcg_propagate_kmem(struct mem_cgroup *parent,
2841	struct mem_cgroup *memcg)
2842	{
2843	int ret = 0;
2844
2845	mutex_lock(&memcg_limit_mutex);
2846	/*
2847	* If the parent cgroup is not kmem-online now, it cannot be
2848	* onlined after this point, because it has at least one child
2849	* already.
2850	*/
2851	if (memcg_kmem_online(parent) \|\|
2852	(cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nokmem))
2853	ret = memcg_online_kmem(memcg);
2854	mutex_unlock(&memcg_limit_mutex);
2855	return ret;
2856	}
2857
2858	static void memcg_offline_kmem(struct mem_cgroup *memcg)	2846	static void memcg_offline_kmem(struct mem_cgroup *memcg)
2859	{	2847	{
2860	struct cgroup_subsys_state *css;	2848	struct cgroup_subsys_state *css;
@@ -2913,10 +2901,6 @@ static void memcg_free_kmem(struct mem_cgroup *memcg) · 52 unmodified lines
2913	}	2901	}
2914	}	2902	}
2915	#else	2903	#else
2916	static int memcg_propagate_kmem(struct mem_cgroup parent, struct mem_cgroup memcg)
2917	{
2918	return 0;
2919	}
2920	static int memcg_online_kmem(struct mem_cgroup *memcg)	2904	static int memcg_online_kmem(struct mem_cgroup *memcg)
2921	{	2905	{
2922	return 0;	2906	return 0;
@@ -2932,22 +2916,10 @@ static void memcg_free_kmem(struct mem_cgroup *memcg) · 9 unmodified lines
2932	static int memcg_update_kmem_limit(struct mem_cgroup *memcg,	2916	static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2933	unsigned long limit)	2917	unsigned long limit)
2934	{	2918	{
2935	int ret = 0;	2919	int ret;
2936		2920
2937	mutex_lock(&memcg_limit_mutex);	2921	mutex_lock(&memcg_limit_mutex);
2938	/* Top-level cgroup doesn't propagate from root */
2939	if (!memcg_kmem_online(memcg)) {
2940	if (cgroup_is_populated(memcg->css.cgroup) \|\|
2941	(memcg->use_hierarchy && memcg_has_children(memcg)))
2942	ret = -EBUSY;
2943	if (ret)
2944	goto out;
2945	ret = memcg_online_kmem(memcg);
2946	if (ret)
2947	goto out;
2948	}
2949	ret = page_counter_limit(&memcg->kmem, limit);	2922	ret = page_counter_limit(&memcg->kmem, limit);
2950	out:
2951	mutex_unlock(&memcg_limit_mutex);	2923	mutex_unlock(&memcg_limit_mutex);
2952	return ret;	2924	return ret;
2953	}	2925	}
@@ -4198,7 +4170,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) · 1244 unmodified lines
4198	return &memcg->css;	4170	return &memcg->css;
4199	}	4171	}
4200		4172
4201	error = memcg_propagate_kmem(parent, memcg);	4173	error = memcg_online_kmem(memcg);
4202	if (error)	4174	if (error)
4203	goto fail;	4175	goto fail;
4204		4176
@@ -4282,9 +4254,11 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) · 77 unmodified lines
4282	{	4254	{
4283	struct mem_cgroup *memcg = mem_cgroup_from_css(css);	4255	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4284		4256
4285	mem_cgroup_resize_limit(memcg, PAGE_COUNTER_MAX);	4257	page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4286	mem_cgroup_resize_memsw_limit(memcg, PAGE_COUNTER_MAX);	4258	page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4287	memcg_update_kmem_limit(memcg, PAGE_COUNTER_MAX);	4259	page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
		4260	page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
		4261	page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4288	memcg->low = 0;	4262	memcg->low = 0;
4289	memcg->high = PAGE_COUNTER_MAX;	4263	memcg->high = PAGE_COUNTER_MAX;
4290	memcg->soft_limit = PAGE_COUNTER_MAX;	4264	memcg->soft_limit = PAGE_COUNTER_MAX;
@@ -5015,6 +4989,7 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 724 unmodified lines
5015	char *buf, size_t nbytes, loff_t off)	4989	char *buf, size_t nbytes, loff_t off)
5016	{	4990	{
5017	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));	4991	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
		4992	unsigned long nr_pages;
5018	unsigned long high;	4993	unsigned long high;
5019	int err;	4994	int err;
5020		4995
@@ -5025,6 +5000,11 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 4 unmodified lines
5025		5000
5026	memcg->high = high;	5001	memcg->high = high;
5027		5002
		5003	nr_pages = page_counter_read(&memcg->memory);
		5004	if (nr_pages > high)
		5005	try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
		5006	GFP_KERNEL, true);
		5007
5028	memcg_wb_domain_size_changed(memcg);	5008	memcg_wb_domain_size_changed(memcg);
5029	return nbytes;	5009	return nbytes;
5030	}	5010	}
@@ -5046,6 +5026,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, · 15 unmodified lines
5046	char *buf, size_t nbytes, loff_t off)	5026	char *buf, size_t nbytes, loff_t off)
5047	{	5027	{
5048	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));	5028	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
		5029	unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
		5030	bool drained = false;
5049	unsigned long max;	5031	unsigned long max;
5050	int err;	5032	int err;
5051		5033
@@ -5054,9 +5036,36 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, · 2 unmodified lines
5054	if (err)	5036	if (err)
5055	return err;	5037	return err;
5056		5038
5057	err = mem_cgroup_resize_limit(memcg, max);	5039	xchg(&memcg->memory.limit, max);
5058	if (err)	5040
5059	return err;	5041	for (;;) {
		5042	unsigned long nr_pages = page_counter_read(&memcg->memory);
		5043
		5044	if (nr_pages <= max)
		5045	break;
		5046
		5047	if (signal_pending(current)) {
		5048	err = -EINTR;
		5049	break;
		5050	}
		5051
		5052	if (!drained) {
		5053	drain_all_stock(memcg);
		5054	drained = true;
		5055	continue;
		5056	}
		5057
		5058	if (nr_reclaims) {
		5059	if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
		5060	GFP_KERNEL, true))
		5061	nr_reclaims--;
		5062	continue;
		5063	}
		5064
		5065	mem_cgroup_events(memcg, MEMCG_OOM, 1);
		5066	if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
		5067	break;
		5068	}
5060		5069
5061	memcg_wb_domain_size_changed(memcg);	5070	memcg_wb_domain_size_changed(memcg);
5062	return nbytes;	5071	return nbytes;
@@ -5077,6 +5086,8 @@ static int memory_events_show(struct seq_file m, void v) · 14 unmodified lines
5077	static int memory_stat_show(struct seq_file m, void v)	5086	static int memory_stat_show(struct seq_file m, void v)
5078	{	5087	{
5079	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));	5088	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
		5089	unsigned long stat[MEMCG_NR_STAT];
		5090	unsigned long events[MEMCG_NR_EVENTS];
5080	int i;	5091	int i;
5081		5092
5082	/*	5093	/*
@@ -5090,22 +5101,27 @@ static int memory_stat_show(struct seq_file m, void v) · 7 unmodified lines
5090	* Current memory state:	5101	* Current memory state:
5091	*/	5102	*/
5092		5103
		5104	tree_stat(memcg, stat);
		5105	tree_events(memcg, events);
		5106
5093	seq_printf(m, "anon %llu\n",	5107	seq_printf(m, "anon %llu\n",
5094	(u64)tree_stat(memcg, MEM_CGROUP_STAT_RSS) * PAGE_SIZE);	5108	(u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE);
5095	seq_printf(m, "file %llu\n",	5109	seq_printf(m, "file %llu\n",
5096	(u64)tree_stat(memcg, MEM_CGROUP_STAT_CACHE) * PAGE_SIZE);	5110	(u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE);
		5111	seq_printf(m, "kernel_stack %llu\n",
		5112	(u64)stat[MEMCG_KERNEL_STACK] * PAGE_SIZE);
		5113	seq_printf(m, "slab %llu\n",
		5114	(u64)(stat[MEMCG_SLAB_RECLAIMABLE] +
		5115	stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5097	seq_printf(m, "sock %llu\n",	5116	seq_printf(m, "sock %llu\n",
5098	(u64)tree_stat(memcg, MEMCG_SOCK) * PAGE_SIZE);	5117	(u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5099		5118
5100	seq_printf(m, "file_mapped %llu\n",	5119	seq_printf(m, "file_mapped %llu\n",
5101	(u64)tree_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED) *	5120	(u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE);
5102	PAGE_SIZE);
5103	seq_printf(m, "file_dirty %llu\n",	5121	seq_printf(m, "file_dirty %llu\n",
5104	(u64)tree_stat(memcg, MEM_CGROUP_STAT_DIRTY) *	5122	(u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE);
5105	PAGE_SIZE);
5106	seq_printf(m, "file_writeback %llu\n",	5123	seq_printf(m, "file_writeback %llu\n",
5107	(u64)tree_stat(memcg, MEM_CGROUP_STAT_WRITEBACK) *	5124	(u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE);
5108	PAGE_SIZE);
5109		5125
5110	for (i = 0; i < NR_LRU_LISTS; i++) {	5126	for (i = 0; i < NR_LRU_LISTS; i++) {
5111	struct mem_cgroup *mi;	5127	struct mem_cgroup *mi;
@@ -5117,12 +5133,17 @@ static int memory_stat_show(struct seq_file m, void v) · 5 unmodified lines
5117	mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);	5133	mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5118	}	5134	}
5119		5135
		5136	seq_printf(m, "slab_reclaimable %llu\n",
		5137	(u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE);
		5138	seq_printf(m, "slab_unreclaimable %llu\n",
		5139	(u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
		5140
5120	/* Accumulated memory events */	5141	/* Accumulated memory events */
5121		5142
5122	seq_printf(m, "pgfault %lu\n",	5143	seq_printf(m, "pgfault %lu\n",
5123	tree_events(memcg, MEM_CGROUP_EVENTS_PGFAULT));	5144	events[MEM_CGROUP_EVENTS_PGFAULT]);
5124	seq_printf(m, "pgmajfault %lu\n",	5145	seq_printf(m, "pgmajfault %lu\n",
5125	tree_events(memcg, MEM_CGROUP_EVENTS_PGMAJFAULT));	5146	events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
5126		5147
5127	return 0;	5148	return 0;
5128	}	5149	}
@@ -5395,6 +5416,10 @@ static void uncharge_list(struct list_head *page_list) · 266 unmodified lines
5395	struct list_head *next;	5416	struct list_head *next;
5396	struct page *page;	5417	struct page *page;
5397		5418
		5419	/*
		5420	* Note that the list can be a single page->lru; hence the
		5421	* do-while loop instead of a simple list_for_each_entry().
		5422	*/
5398	next = page_list->next;	5423	next = page_list->next;
5399	do {	5424	do {
5400	unsigned int nr_pages = 1;	5425	unsigned int nr_pages = 1;

mm/memory-failure.c+21−31

 	struct siginfo si;
 	int ret;
-	printk(KERN_ERR
+	pr_err("MCE %#lx: Killing %s:%d due to hardware memory corruption\n",
-		"MCE %#lx: Killing %s:%d due to hardware memory corruption\n",
+	       pfn, t->comm, t->pid);
-		pfn, t->comm, t->pid);
 	si.si_signo = SIGBUS;
 	si.si_errno = 0;
 	si.si_addr = (void *)addr;
 		ret = send_sig_info(SIGBUS, &si, t);  /* synchronous? */
 	}
 	if (ret < 0)
-		printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
+		pr_info("MCE: Error sending signal to %s:%d: %d\n",
-		       t->comm, t->pid, ret);
+			t->comm, t->pid, ret);
 	return ret;
 }
 	} else {
 		tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
 		if (!tk) {
-			printk(KERN_ERR
+			pr_err("MCE: Out of memory while machine check handling\n");
-		"MCE: Out of memory while machine check handling\n");
 			return;
 		}
 	}
 			 * signal and then access the memory. Just kill it.
 			 */
 			if (fail || tk->addr_valid == 0) {
-				printk(KERN_ERR
+				pr_err("MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
-		"MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
+				       pfn, tk->tsk->comm, tk->tsk->pid);
-					pfn, tk->tsk->comm, tk->tsk->pid);
 				force_sig(SIGKILL, tk->tsk);
 			}
 			 */
 			else if (kill_proc(tk->tsk, tk->addr, trapno,
 					      pfn, page, flags) < 0)
-				printk(KERN_ERR
+				pr_err("MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
-		"MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
+				       pfn, tk->tsk->comm, tk->tsk->pid);
-					pfn, tk->tsk->comm, tk->tsk->pid);
 		}
 		put_task_struct(tk->tsk);
 		kfree(tk);
  */
 static int me_unknown(struct page *p, unsigned long pfn)
 {
-	printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
+	pr_err("MCE %#lx: Unknown page state\n", pfn);
 	return MF_FAILED;
 }
 	if (mapping->a_ops->error_remove_page) {
 		err = mapping->a_ops->error_remove_page(mapping, p);
 		if (err != 0) {
-			printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
+			pr_info("MCE %#lx: Failed to punch page: %d\n",
-					pfn, err);
+				pfn, err);
 		} else if (page_has_private(p) &&
 				!try_to_release_page(p, GFP_NOIO)) {
 			pr_info("MCE %#lx: failed to release buffers\n", pfn);
 		if (invalidate_inode_page(p))
 			ret = MF_RECOVERED;
 		else
-			printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
+			pr_info("MCE %#lx: Failed to invalidate\n", pfn);
-				pfn);
 	}
 	return ret;
 }
 	if (ps->action == me_swapcache_dirty && result == MF_DELAYED)
 		count--;
 	if (count != 0) {
-		printk(KERN_ERR
+		pr_err("MCE %#lx: %s still referenced by %d users\n",
-		       "MCE %#lx: %s still referenced by %d users\n",
 		       pfn, action_page_types[ps->type], count);
 		result = MF_FAILED;
 	}
 	}
 	if (PageSwapCache(p)) {
-		printk(KERN_ERR
+		pr_err("MCE %#lx: keeping poisoned page in swap cache\n", pfn);
-		       "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
 		ttu |= TTU_IGNORE_HWPOISON;
 	}
 		} else {
 			kill = 0;
 			ttu |= TTU_IGNORE_HWPOISON;
-			printk(KERN_INFO
+			pr_info("MCE %#lx: corrupted page was clean: dropped without side effects\n",
-	"MCE %#lx: corrupted page was clean: dropped without side effects\n",
 				pfn);
 		}
 	}
 	ret = try_to_unmap(hpage, ttu);
 	if (ret != SWAP_SUCCESS)
-		printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
+		pr_err("MCE %#lx: failed to unmap page (mapcount=%d)\n",
-				pfn, page_mapcount(hpage));
+		       pfn, page_mapcount(hpage));
 	/*
 	 * Now that the dirty bit has been propagated to the
 		panic("Memory failure from trap %d on page %lx", trapno, pfn);
 	if (!pfn_valid(pfn)) {
-		printk(KERN_ERR
+		pr_err("MCE %#lx: memory outside kernel control\n", pfn);
-		       "MCE %#lx: memory outside kernel control\n",
-		       pfn);
 		return -ENXIO;
 	}
 	p = pfn_to_page(pfn);
 	orig_head = hpage = compound_head(p);
 	if (TestSetPageHWPoison(p)) {
-		printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
+		pr_err("MCE %#lx: already hardware poisoned\n", pfn);
 		return 0;
 	}
 	 * unpoison always clear PG_hwpoison inside page lock
 	 */
 	if (!PageHWPoison(p)) {
-		printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
+		pr_err("MCE %#lx: just unpoisoned\n", pfn);
 		num_poisoned_pages_sub(nr_pages);
 		unlock_page(hpage);
 		put_hwpoison_page(hpage);

mm/memory.c+10−15

 	}
 }
-int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
+int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
-		pmd_t *pmd, unsigned long address)
 {
 	spinlock_t *ptl;
 	pgtable_t new = pte_alloc_one(mm, address);
 			return;
 		}
 		if (nr_unshown) {
-			printk(KERN_ALERT
+			pr_alert("BUG: Bad page map: %lu messages suppressed\n",
-				"BUG: Bad page map: %lu messages suppressed\n",
+				 nr_unshown);
-				nr_unshown);
 			nr_unshown = 0;
 		}
 		nr_shown = 0;
 	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
 	index = linear_page_index(vma, addr);
-	printk(KERN_ALERT
+	pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
-		"BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
+		 current->comm,
-		current->comm,
+		 (long long)pte_val(pte), (long long)pmd_val(*pmd));
-		(long long)pte_val(pte), (long long)pmd_val(*pmd));
 	if (page)
 		dump_page(page, "bad pte");
-	printk(KERN_ALERT
+	pr_alert("addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
-		"addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
+		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
-		(void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
 	/*
 	 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y
 	 */
 	}
 	/*
-	 * Use __pte_alloc instead of pte_alloc_map, because we can't
+	 * Use pte_alloc() instead of pte_alloc_map, because we can't
 	 * run pte_offset_map on the pmd, if an huge pmd could
 	 * materialize from under us from a different thread.
 	 */
-	if (unlikely(pmd_none(*pmd)) &&
+	if (unlikely(pte_alloc(mm, pmd, address)))
-	    unlikely(__pte_alloc(mm, vma, pmd, address)))
 		return VM_FAULT_OOM;
 	/*
 	 * If a huge pmd materialized under us just retry later.  Use

mm/memory_hotplug.c+29−25

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	#include <linux/hugetlb.h>	33	#include <linux/hugetlb.h>
34	#include <linux/memblock.h>	34	#include <linux/memblock.h>
35	#include <linux/bootmem.h>	35	#include <linux/bootmem.h>
		36	#include <linux/compaction.h>
36		37
37	#include <asm/tlbflush.h>	38	#include <asm/tlbflush.h>
38		39
@@ -166,7 +167,7 @@ void get_page_bootmem(unsigned long info, struct page *page, · 127 unmodified lines
166	page->lru.next = (struct list_head *) type;	167	page->lru.next = (struct list_head *) type;
167	SetPagePrivate(page);	168	SetPagePrivate(page);
168	set_page_private(page, info);	169	set_page_private(page, info);
169	atomic_inc(&page->_count);	170	page_ref_inc(page);
170	}	171	}
171		172
172	void put_page_bootmem(struct page *page)	173	void put_page_bootmem(struct page *page)
@@ -177,7 +178,7 @@ void put_page_bootmem(struct page *page) · 4 unmodified lines
177	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE \|\|	178	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE \|\|
178	type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);	179	type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
179		180
180	if (atomic_dec_return(&page->_count) == 1) {	181	if (page_ref_dec_return(page) == 1) {
181	ClearPagePrivate(page);	182	ClearPagePrivate(page);
182	set_page_private(page, 0);	183	set_page_private(page, 0);
183	INIT_LIST_HEAD(&page->lru);	184	INIT_LIST_HEAD(&page->lru);
@@ -1054,14 +1055,13 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 870 unmodified lines
1054	arg.nr_pages = nr_pages;	1055	arg.nr_pages = nr_pages;
1055	node_states_check_changes_online(nr_pages, zone, &arg);	1056	node_states_check_changes_online(nr_pages, zone, &arg);
1056		1057
1057	nid = pfn_to_nid(pfn);	1058	nid = zone_to_nid(zone);
1058		1059
1059	ret = memory_notify(MEM_GOING_ONLINE, &arg);	1060	ret = memory_notify(MEM_GOING_ONLINE, &arg);
1060	ret = notifier_to_errno(ret);	1061	ret = notifier_to_errno(ret);
1061	if (ret) {	1062	if (ret)
1062	memory_notify(MEM_CANCEL_ONLINE, &arg);	1063	goto failed_addition;
1063	return ret;	1064
1064	}
1065	/*	1065	/*
1066	* If this zone is not populated, then it is not in zonelist.	1066	* If this zone is not populated, then it is not in zonelist.
1067	* This means the page allocator ignores this zone.	1067	* This means the page allocator ignores this zone.
@@ -1079,12 +1079,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 11 unmodified lines
1079	if (need_zonelists_rebuild)	1079	if (need_zonelists_rebuild)
1080	zone_pcp_reset(zone);	1080	zone_pcp_reset(zone);
1081	mutex_unlock(&zonelists_mutex);	1081	mutex_unlock(&zonelists_mutex);
1082	printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n",	1082	goto failed_addition;
1083	(unsigned long long) pfn << PAGE_SHIFT,
1084	(((unsigned long long) pfn + nr_pages)
1085	<< PAGE_SHIFT) - 1);
1086	memory_notify(MEM_CANCEL_ONLINE, &arg);
1087	return ret;
1088	}	1083	}
1089		1084
1090	zone->present_pages += onlined_pages;	1085	zone->present_pages += onlined_pages;
@@ -1094,7 +1089,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 3 unmodified lines
1094	pgdat_resize_unlock(zone->zone_pgdat, &flags);	1089	pgdat_resize_unlock(zone->zone_pgdat, &flags);
1095		1090
1096	if (onlined_pages) {	1091	if (onlined_pages) {
1097	node_states_set_node(zone_to_nid(zone), &arg);	1092	node_states_set_node(nid, &arg);
1098	if (need_zonelists_rebuild)	1093	if (need_zonelists_rebuild)
1099	build_all_zonelists(NULL, NULL);	1094	build_all_zonelists(NULL, NULL);
1100	else	1095	else
@@ -1105,8 +1100,10 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 4 unmodified lines
1105		1100
1106	init_per_zone_wmark_min();	1101	init_per_zone_wmark_min();
1107		1102
1108	if (onlined_pages)	1103	if (onlined_pages) {
1109	kswapd_run(zone_to_nid(zone));	1104	kswapd_run(nid);
		1105	kcompactd_run(nid);
		1106	}
1110		1107
1111	vm_total_pages = nr_free_pagecache_pages();	1108	vm_total_pages = nr_free_pagecache_pages();
1112		1109
@@ -1115,6 +1112,13 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 2 unmodified lines
1115	if (onlined_pages)	1112	if (onlined_pages)
1116	memory_notify(MEM_ONLINE, &arg);	1113	memory_notify(MEM_ONLINE, &arg);
1117	return 0;	1114	return 0;
		1115
		1116	failed_addition:
		1117	pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
		1118	(unsigned long long) pfn << PAGE_SHIFT,
		1119	(((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
		1120	memory_notify(MEM_CANCEL_ONLINE, &arg);
		1121	return ret;
1118	}	1122	}
1119	#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */	1123	#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
1120		1124
@@ -1526,8 +1530,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · 405 unmodified lines
1526		1530
1527	} else {	1531	} else {
1528	#ifdef CONFIG_DEBUG_VM	1532	#ifdef CONFIG_DEBUG_VM
1529	printk(KERN_ALERT "removing pfn %lx from LRU failed\n",	1533	pr_alert("removing pfn %lx from LRU failed\n", pfn);
1530	pfn);
1531	dump_page(page, "failed to remove from LRU");	1534	dump_page(page, "failed to remove from LRU");
1532	#endif	1535	#endif
1533	put_page(page);	1536	put_page(page);
@@ -1855,7 +1858,7 @@ repeat: · 321 unmodified lines
1855	ret = -EBUSY;	1858	ret = -EBUSY;
1856	goto failed_removal;	1859	goto failed_removal;
1857	}	1860	}
1858	printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages);	1861	pr_info("Offlined Pages %ld\n", offlined_pages);
1859	/* Ok, all of our target is isolated.	1862	/* Ok, all of our target is isolated.
1860	We cannot do rollback at this point. */	1863	We cannot do rollback at this point. */
1861	offline_isolated_pages(start_pfn, end_pfn);	1864	offline_isolated_pages(start_pfn, end_pfn);
@@ -1880,8 +1883,10 @@ repeat: · 18 unmodified lines
1880	zone_pcp_update(zone);	1883	zone_pcp_update(zone);
1881		1884
1882	node_states_clear_node(node, &arg);	1885	node_states_clear_node(node, &arg);
1883	if (arg.status_change_nid >= 0)	1886	if (arg.status_change_nid >= 0) {
1884	kswapd_stop(node);	1887	kswapd_stop(node);
		1888	kcompactd_stop(node);
		1889	}
1885		1890
1886	vm_total_pages = nr_free_pagecache_pages();	1891	vm_total_pages = nr_free_pagecache_pages();
1887	writeback_set_ratelimit();	1892	writeback_set_ratelimit();
@@ -1890,9 +1895,9 @@ repeat: · 2 unmodified lines
1890	return 0;	1895	return 0;
1891		1896
1892	failed_removal:	1897	failed_removal:
1893	printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n",	1898	pr_debug("memory offlining [mem %#010llx-%#010llx] failed\n",
1894	(unsigned long long) start_pfn << PAGE_SHIFT,	1899	(unsigned long long) start_pfn << PAGE_SHIFT,
1895	((unsigned long long) end_pfn << PAGE_SHIFT) - 1);	1900	((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1896	memory_notify(MEM_CANCEL_OFFLINE, &arg);	1901	memory_notify(MEM_CANCEL_OFFLINE, &arg);
1897	/* pushback to free area */	1902	/* pushback to free area */
1898	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);	1903	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
@@ -1965,8 +1970,7 @@ static int check_memblock_offlined_cb(struct memory_block mem, void arg) · 66 unmodified lines
1965		1970
1966	beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));	1971	beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
1967	endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;	1972	endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
1968	pr_warn("removing memory fails, because memory "	1973	pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
1969	"[%pa-%pa] is onlined\n",
1970	&beginpa, &endpa);	1974	&beginpa, &endpa);
1971	}	1975	}
1972		1976

mm/mempolicy.c+1−3

 		set_numabalancing_state(numabalancing_override == 1);
 	if (num_online_nodes() > 1 && !numabalancing_override) {
-		pr_info("%s automatic NUMA balancing. "
+		pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
-			"Configure with numa_balancing= or the "
-			"kernel.numa_balancing sysctl",
 			numabalancing_default ? "Enabling" : "Disabling");
 		set_numabalancing_state(numabalancing_default);
 	}

mm/mempool.c+16−4

  * returns NULL. Note that due to preallocation, this function
  * *never* fails when called from process contexts. (it might
  * fail if called from an IRQ context.)
- * Note: using __GFP_ZERO is not supported.
+ * Note: neither __GFP_NOMEMALLOC nor __GFP_ZERO are supported.
  */
-void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
+void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
 {
 	void *element;
 	unsigned long flags;
 	wait_queue_t wait;
 	gfp_t gfp_temp;
+	/* If oom killed, memory reserves are essential to prevent livelock */
+	VM_WARN_ON_ONCE(gfp_mask & __GFP_NOMEMALLOC);
+	/* No element size to zero on allocation */
 	VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
 	might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
-	gfp_mask |= __GFP_NOMEMALLOC;	/* don't allocate emergency reserves */
 	gfp_mask |= __GFP_NORETRY;	/* don't loop in __alloc_pages */
 	gfp_mask |= __GFP_NOWARN;	/* failures are OK */
 	gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO);
 repeat_alloc:
+	if (likely(pool->curr_nr)) {
+		/*
+		 * Don't allocate from emergency reserves if there are
+		 * elements available.  This check is racy, but it will
+		 * be rechecked each loop.
+		 */
+		gfp_temp |= __GFP_NOMEMALLOC;
+	}
 	element = pool->alloc(gfp_temp, pool->pool_data);
 	if (likely(element != NULL))
 	 * We use gfp mask w/o direct reclaim or IO for the first round.  If
 	 * alloc failed with that and @pool was empty, retry immediately.
 	 */
-	if (gfp_temp != gfp_mask) {
+	if ((gfp_temp & ~__GFP_NOMEMALLOC) != gfp_mask) {
 		spin_unlock_irqrestore(&pool->lock, flags);
 		gfp_temp = gfp_mask;
 		goto repeat_alloc;
 	}
+	gfp_temp = gfp_mask;
 	/* We must not sleep if !__GFP_DIRECT_RECLAIM */
 	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {

mm/migrate.c+19−14

@@ -172,7 +172,7 @@ static int remove_migration_pte(struct page new, struct vm_area_struct vma, · 171 unmodified lines
172	else	172	else
173	page_add_file_rmap(new);	173	page_add_file_rmap(new);
174		174
175	if (vma->vm_flags & VM_LOCKED)	175	if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
176	mlock_vma_page(new);	176	mlock_vma_page(new);
177		177
178	/* No need to invalidate - it was non-present before */	178	/* No need to invalidate - it was non-present before */
@@ -187,14 +187,17 @@ out: · 8 unmodified lines
187	* Get rid of all migration entries and replace them by	187	* Get rid of all migration entries and replace them by
188	* references to the indicated page.	188	* references to the indicated page.
189	*/	189	*/
190	static void remove_migration_ptes(struct page old, struct page new)	190	void remove_migration_ptes(struct page old, struct page new, bool locked)
191	{	191	{
192	struct rmap_walk_control rwc = {	192	struct rmap_walk_control rwc = {
193	.rmap_one = remove_migration_pte,	193	.rmap_one = remove_migration_pte,
194	.arg = old,	194	.arg = old,
195	};	195	};
196		196
197	rmap_walk(new, &rwc);	197	if (locked)
		198	rmap_walk_locked(new, &rwc);
		199	else
		200	rmap_walk(new, &rwc);
198	}	201	}
199		202
200	/*	203	/*
@@ -349,7 +352,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 148 unmodified lines
349	return -EAGAIN;	352	return -EAGAIN;
350	}	353	}
351		354
352	if (!page_freeze_refs(page, expected_count)) {	355	if (!page_ref_freeze(page, expected_count)) {
353	spin_unlock_irq(&mapping->tree_lock);	356	spin_unlock_irq(&mapping->tree_lock);
354	return -EAGAIN;	357	return -EAGAIN;
355	}	358	}
@@ -363,7 +366,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 7 unmodified lines
363	*/	366	*/
364	if (mode == MIGRATE_ASYNC && head &&	367	if (mode == MIGRATE_ASYNC && head &&
365	!buffer_migrate_lock_buffers(head, mode)) {	368	!buffer_migrate_lock_buffers(head, mode)) {
366	page_unfreeze_refs(page, expected_count);	369	page_ref_unfreeze(page, expected_count);
367	spin_unlock_irq(&mapping->tree_lock);	370	spin_unlock_irq(&mapping->tree_lock);
368	return -EAGAIN;	371	return -EAGAIN;
369	}	372	}
@@ -397,7 +400,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 27 unmodified lines
397	* to one less reference.	400	* to one less reference.
398	* We know this isn't the last reference.	401	* We know this isn't the last reference.
399	*/	402	*/
400	page_unfreeze_refs(page, expected_count - 1);	403	page_ref_unfreeze(page, expected_count - 1);
401		404
402	spin_unlock(&mapping->tree_lock);	405	spin_unlock(&mapping->tree_lock);
403	/* Leave irq disabled to prevent preemption while updating stats */	406	/* Leave irq disabled to prevent preemption while updating stats */
@@ -451,7 +454,7 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, · 47 unmodified lines
451	return -EAGAIN;	454	return -EAGAIN;
452	}	455	}
453		456
454	if (!page_freeze_refs(page, expected_count)) {	457	if (!page_ref_freeze(page, expected_count)) {
455	spin_unlock_irq(&mapping->tree_lock);	458	spin_unlock_irq(&mapping->tree_lock);
456	return -EAGAIN;	459	return -EAGAIN;
457	}	460	}
@@ -463,7 +466,7 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, · 5 unmodified lines
463		466
464	radix_tree_replace_slot(pslot, newpage);	467	radix_tree_replace_slot(pslot, newpage);
465		468
466	page_unfreeze_refs(page, expected_count - 1);	469	page_ref_unfreeze(page, expected_count - 1);
467		470
468	spin_unlock_irq(&mapping->tree_lock);	471	spin_unlock_irq(&mapping->tree_lock);
469		472
@@ -702,7 +705,7 @@ static int writeout(struct address_space mapping, struct page page) · 232 unmodified lines
702	* At this point we know that the migration attempt cannot	705	* At this point we know that the migration attempt cannot
703	* be successful.	706	* be successful.
704	*/	707	*/
705	remove_migration_ptes(page, page);	708	remove_migration_ptes(page, page, false);
706		709
707	rc = mapping->a_ops->writepage(page, &wbc);	710	rc = mapping->a_ops->writepage(page, &wbc);
708		711
@@ -900,7 +903,7 @@ static int __unmap_and_move(struct page page, struct page newpage, · 191 unmodified lines
900		903
901	if (page_was_mapped)	904	if (page_was_mapped)
902	remove_migration_ptes(page,	905	remove_migration_ptes(page,
903	rc == MIGRATEPAGE_SUCCESS ? newpage : page);	906	rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
904		907
905	out_unlock_both:	908	out_unlock_both:
906	unlock_page(newpage);	909	unlock_page(newpage);
@@ -1070,7 +1073,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 163 unmodified lines
1070		1073
1071	if (page_was_mapped)	1074	if (page_was_mapped)
1072	remove_migration_ptes(hpage,	1075	remove_migration_ptes(hpage,
1073	rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage);	1076	rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1074		1077
1075	unlock_page(new_hpage);	1078	unlock_page(new_hpage);
1076		1079
@@ -1773,7 +1776,10 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, · 696 unmodified lines
1773	put_page(new_page);	1776	put_page(new_page);
1774	goto out_fail;	1777	goto out_fail;
1775	}	1778	}
1776		1779	/*
		1780	* We are not sure a pending tlb flush here is for a huge page
		1781	* mapping or not. Hence use the tlb range variant
		1782	*/
1777	if (mm_tlb_flush_pending(mm))	1783	if (mm_tlb_flush_pending(mm))
1778	flush_tlb_range(vma, mmun_start, mmun_end);	1784	flush_tlb_range(vma, mmun_start, mmun_end);
1779		1785
@@ -1829,12 +1835,11 @@ fail_putback: · 49 unmodified lines
1829	page_add_anon_rmap(new_page, vma, mmun_start, true);	1835	page_add_anon_rmap(new_page, vma, mmun_start, true);
1830	pmdp_huge_clear_flush_notify(vma, mmun_start, pmd);	1836	pmdp_huge_clear_flush_notify(vma, mmun_start, pmd);
1831	set_pmd_at(mm, mmun_start, pmd, entry);	1837	set_pmd_at(mm, mmun_start, pmd, entry);
1832	flush_tlb_range(vma, mmun_start, mmun_end);
1833	update_mmu_cache_pmd(vma, address, &entry);	1838	update_mmu_cache_pmd(vma, address, &entry);
1834		1839
1835	if (page_count(page) != 2) {	1840	if (page_count(page) != 2) {
1836	set_pmd_at(mm, mmun_start, pmd, orig_entry);	1841	set_pmd_at(mm, mmun_start, pmd, orig_entry);
1837	flush_tlb_range(vma, mmun_start, mmun_end);	1842	flush_pmd_tlb_range(vma, mmun_start, mmun_end);
1838	mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);	1843	mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
1839	update_mmu_cache_pmd(vma, address, &entry);	1844	update_mmu_cache_pmd(vma, address, &entry);
1840	page_remove_rmap(new_page, true);	1845	page_remove_rmap(new_page, true);

mm/mm_init.c+3−4

 			/* Iterate the zonelist */
 			for_each_zone_zonelist(zone, z, zonelist, zoneid) {
 #ifdef CONFIG_NUMA
-				printk(KERN_CONT "%d:%s ",
+				pr_cont("%d:%s ", zone->node, zone->name);
-					zone->node, zone->name);
 #else
-				printk(KERN_CONT "0:%s ", zone->name);
+				pr_cont("0:%s ", zone->name);
 #endif /* CONFIG_NUMA */
 			}
-			printk(KERN_CONT "\n");
+			pr_cont("\n");
 		}
 	}
 }

mm/mmap.c+3−130

@@ -37,7 +37,6 @@ · 36 unmodified lines
37	#include <linux/khugepaged.h>	37	#include <linux/khugepaged.h>
38	#include <linux/uprobes.h>	38	#include <linux/uprobes.h>
39	#include <linux/rbtree_augmented.h>	39	#include <linux/rbtree_augmented.h>
40	#include <linux/sched/sysctl.h>
41	#include <linux/notifier.h>	40	#include <linux/notifier.h>
42	#include <linux/memory.h>	41	#include <linux/memory.h>
43	#include <linux/printk.h>	42	#include <linux/printk.h>
@@ -123,130 +122,6 @@ void vma_set_page_prot(struct vm_area_struct *vma) · 79 unmodified lines
123	}	122	}
124	}	123	}
125		124
126
127	int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS; /* heuristic overcommit */
128	int sysctl_overcommit_ratio __read_mostly = 50; /* default is 50% */
129	unsigned long sysctl_overcommit_kbytes __read_mostly;
130	int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
131	unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
132	unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
133	/*
134	* Make sure vm_committed_as in one cacheline and not cacheline shared with
135	* other variables. It can be updated by several CPUs frequently.
136	*/
137	struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
138
139	/*
140	* The global memory commitment made in the system can be a metric
141	* that can be used to drive ballooning decisions when Linux is hosted
142	* as a guest. On Hyper-V, the host implements a policy engine for dynamically
143	* balancing memory across competing virtual machines that are hosted.
144	* Several metrics drive this policy engine including the guest reported
145	* memory commitment.
146	*/
147	unsigned long vm_memory_committed(void)
148	{
149	return percpu_counter_read_positive(&vm_committed_as);
150	}
151	EXPORT_SYMBOL_GPL(vm_memory_committed);
152
153	/*
154	* Check that a process has enough memory to allocate a new virtual
155	* mapping. 0 means there is enough memory for the allocation to
156	* succeed and -ENOMEM implies there is not.
157	*
158	* We currently support three overcommit policies, which are set via the
159	* vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
160	*
161	* Strict overcommit modes added 2002 Feb 26 by Alan Cox.
162	* Additional code 2002 Jul 20 by Robert Love.
163	*
164	* cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
165	*
166	* Note this is a helper function intended to be used by LSMs which
167	* wish to use this logic.
168	*/
169	int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
170	{
171	long free, allowed, reserve;
172
173	VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) <
174	-(s64)vm_committed_as_batch * num_online_cpus(),
175	"memory commitment underflow");
176
177	vm_acct_memory(pages);
178
179	/*
180	* Sometimes we want to use more memory than we have
181	*/
182	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
183	return 0;
184
185	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
186	free = global_page_state(NR_FREE_PAGES);
187	free += global_page_state(NR_FILE_PAGES);
188
189	/*
190	* shmem pages shouldn't be counted as free in this
191	* case, they can't be purged, only swapped out, and
192	* that won't affect the overall amount of available
193	* memory in the system.
194	*/
195	free -= global_page_state(NR_SHMEM);
196
197	free += get_nr_swap_pages();
198
199	/*
200	* Any slabs which are created with the
201	* SLAB_RECLAIM_ACCOUNT flag claim to have contents
202	* which are reclaimable, under pressure. The dentry
203	* cache and most inode caches should fall into this
204	*/
205	free += global_page_state(NR_SLAB_RECLAIMABLE);
206
207	/*
208	* Leave reserved pages. The pages are not for anonymous pages.
209	*/
210	if (free <= totalreserve_pages)
211	goto error;
212	else
213	free -= totalreserve_pages;
214
215	/*
216	* Reserve some for root
217	*/
218	if (!cap_sys_admin)
219	free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
220
221	if (free > pages)
222	return 0;
223
224	goto error;
225	}
226
227	allowed = vm_commit_limit();
228	/*
229	* Reserve some for root
230	*/
231	if (!cap_sys_admin)
232	allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
233
234	/*
235	* Don't let a single process grow so big a user can't recover
236	*/
237	if (mm) {
238	reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
239	allowed -= min_t(long, mm->total_vm / 32, reserve);
240	}
241
242	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
243	return 0;
244	error:
245	vm_unacct_memory(pages);
246
247	return -ENOMEM;
248	}
249
250	/*	125	/*
251	* Requires inode->i_mapping->i_mmap_rwsem	126	* Requires inode->i_mapping->i_mmap_rwsem
252	*/	127	*/
@@ -2642,9 +2517,8 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, · 2389 unmodified lines
2642	unsigned long ret = -EINVAL;	2517	unsigned long ret = -EINVAL;
2643	struct file *file;	2518	struct file *file;
2644		2519
2645	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. "	2520	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n",
2646	"See Documentation/vm/remap_file_pages.txt.\n",	2521	current->comm, current->pid);
2647	current->comm, current->pid);
2648		2522
2649	if (prot)	2523	if (prot)
2650	return ret;	2524	return ret;
@@ -3010,8 +2884,7 @@ bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) · 359 unmodified lines
3010	if (is_data_mapping(flags) &&	2884	if (is_data_mapping(flags) &&
3011	mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {	2885	mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
3012	if (ignore_rlimit_data)	2886	if (ignore_rlimit_data)
3013	pr_warn_once("%s (%d): VmData %lu exceed data ulimit "	2887	pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Will be forbidden soon.\n",
3014	"%lu. Will be forbidden soon.\n",
3015	current->comm, current->pid,	2888	current->comm, current->pid,
3016	(mm->data_vm + npages) << PAGE_SHIFT,	2889	(mm->data_vm + npages) << PAGE_SHIFT,
3017	rlimit(RLIMIT_DATA));	2890	rlimit(RLIMIT_DATA));

mm/mmu_notifier.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3	*	3	*
4	* Copyright (C) 2008 Qumranet, Inc.	4	* Copyright (C) 2008 Qumranet, Inc.
5	* Copyright (C) 2008 SGI	5	* Copyright (C) 2008 SGI
6	* Christoph Lameter <clameter@sgi.com>	6	* Christoph Lameter <cl@linux.com>
7	*	7	*
8	* This work is licensed under the terms of the GNU GPL, version 2. See	8	* This work is licensed under the terms of the GNU GPL, version 2. See
9	* the COPYING file in the top-level directory.	9	* the COPYING file in the top-level directory.

mm/mremap.c+1−3

 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/mmu_notifier.h>
-#include <linux/sched/sysctl.h>
 #include <linux/uaccess.h>
 #include <linux/mm-arch-hooks.h>
 				continue;
 			VM_BUG_ON(pmd_trans_huge(*old_pmd));
 		}
-		if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma,
+		if (pte_alloc(new_vma->vm_mm, new_pmd, new_addr))
-						      new_pmd, new_addr))
 			break;
 		next = (new_addr + PMD_SIZE) & PMD_MASK;
 		if (extent > next - new_addr)

mm/nobootmem.c+2−2

@@ -288,7 +288,7 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, · 287 unmodified lines
288	/*	288	/*
289	* Whoops, we cannot satisfy the allocation request.	289	* Whoops, we cannot satisfy the allocation request.
290	*/	290	*/
291	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);	291	pr_alert("bootmem alloc of %lu bytes failed!\n", size);
292	panic("Out of memory");	292	panic("Out of memory");
293	return NULL;	293	return NULL;
294	}	294	}
@@ -360,7 +360,7 @@ static void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, · 65 unmodified lines
360	if (ptr)	360	if (ptr)
361	return ptr;	361	return ptr;
362		362
363	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);	363	pr_alert("bootmem alloc of %lu bytes failed!\n", size);
364	panic("Out of memory");	364	panic("Out of memory");
365	return NULL;	365	return NULL;
366	}	366	}

mm/nommu.c+0−117

@@ -33,7 +33,6 @@ · 32 unmodified lines
33	#include <linux/security.h>	33	#include <linux/security.h>
34	#include <linux/syscalls.h>	34	#include <linux/syscalls.h>
35	#include <linux/audit.h>	35	#include <linux/audit.h>
36	#include <linux/sched/sysctl.h>
37	#include <linux/printk.h>	36	#include <linux/printk.h>
38		37
39	#include <asm/uaccess.h>	38	#include <asm/uaccess.h>
@@ -48,33 +47,11 @@ struct page *mem_map; · 8 unmodified lines
48	unsigned long max_mapnr;	47	unsigned long max_mapnr;
49	EXPORT_SYMBOL(max_mapnr);	48	EXPORT_SYMBOL(max_mapnr);
50	unsigned long highest_memmap_pfn;	49	unsigned long highest_memmap_pfn;
51	struct percpu_counter vm_committed_as;
52	int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
53	int sysctl_overcommit_ratio = 50; /* default is 50% */
54	unsigned long sysctl_overcommit_kbytes __read_mostly;
55	int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
56	int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;	50	int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
57	unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
58	unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
59	int heap_stack_gap = 0;	51	int heap_stack_gap = 0;
60		52
61	atomic_long_t mmap_pages_allocated;	53	atomic_long_t mmap_pages_allocated;
62		54
63	/*
64	* The global memory commitment made in the system can be a metric
65	* that can be used to drive ballooning decisions when Linux is hosted
66	* as a guest. On Hyper-V, the host implements a policy engine for dynamically
67	* balancing memory across competing virtual machines that are hosted.
68	* Several metrics drive this policy engine including the guest reported
69	* memory commitment.
70	*/
71	unsigned long vm_memory_committed(void)
72	{
73	return percpu_counter_read_positive(&vm_committed_as);
74	}
75
76	EXPORT_SYMBOL_GPL(vm_memory_committed);
77
78	EXPORT_SYMBOL(mem_map);	55	EXPORT_SYMBOL(mem_map);
79		56
80	/* list of mapped, potentially shareable regions */	57	/* list of mapped, potentially shareable regions */
@@ -1829,100 +1806,6 @@ void unmap_mapping_range(struct address_space *mapping, · 1748 unmodified lines
1829	}	1806	}
1830	EXPORT_SYMBOL(unmap_mapping_range);	1807	EXPORT_SYMBOL(unmap_mapping_range);
1831		1808
1832	/*
1833	* Check that a process has enough memory to allocate a new virtual
1834	* mapping. 0 means there is enough memory for the allocation to
1835	* succeed and -ENOMEM implies there is not.
1836	*
1837	* We currently support three overcommit policies, which are set via the
1838	* vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
1839	*
1840	* Strict overcommit modes added 2002 Feb 26 by Alan Cox.
1841	* Additional code 2002 Jul 20 by Robert Love.
1842	*
1843	* cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
1844	*
1845	* Note this is a helper function intended to be used by LSMs which
1846	* wish to use this logic.
1847	*/
1848	int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
1849	{
1850	long free, allowed, reserve;
1851
1852	vm_acct_memory(pages);
1853
1854	/*
1855	* Sometimes we want to use more memory than we have
1856	*/
1857	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
1858	return 0;
1859
1860	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
1861	free = global_page_state(NR_FREE_PAGES);
1862	free += global_page_state(NR_FILE_PAGES);
1863
1864	/*
1865	* shmem pages shouldn't be counted as free in this
1866	* case, they can't be purged, only swapped out, and
1867	* that won't affect the overall amount of available
1868	* memory in the system.
1869	*/
1870	free -= global_page_state(NR_SHMEM);
1871
1872	free += get_nr_swap_pages();
1873
1874	/*
1875	* Any slabs which are created with the
1876	* SLAB_RECLAIM_ACCOUNT flag claim to have contents
1877	* which are reclaimable, under pressure. The dentry
1878	* cache and most inode caches should fall into this
1879	*/
1880	free += global_page_state(NR_SLAB_RECLAIMABLE);
1881
1882	/*
1883	* Leave reserved pages. The pages are not for anonymous pages.
1884	*/
1885	if (free <= totalreserve_pages)
1886	goto error;
1887	else
1888	free -= totalreserve_pages;
1889
1890	/*
1891	* Reserve some for root
1892	*/
1893	if (!cap_sys_admin)
1894	free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
1895
1896	if (free > pages)
1897	return 0;
1898
1899	goto error;
1900	}
1901
1902	allowed = vm_commit_limit();
1903	/*
1904	* Reserve some 3% for root
1905	*/
1906	if (!cap_sys_admin)
1907	allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
1908
1909	/*
1910	* Don't let a single process grow so big a user can't recover
1911	*/
1912	if (mm) {
1913	reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
1914	allowed -= min_t(long, mm->total_vm / 32, reserve);
1915	}
1916
1917	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
1918	return 0;
1919
1920	error:
1921	vm_unacct_memory(pages);
1922
1923	return -ENOMEM;
1924	}
1925
1926	int filemap_fault(struct vm_area_struct vma, struct vm_fault vmf)	1809	int filemap_fault(struct vm_area_struct vma, struct vm_fault vmf)
1927	{	1810	{
1928	BUG();	1811	BUG();

mm/oom_kill.c+4−12

 	if (oom_task_origin(task))
 		return OOM_SCAN_SELECT;
-	if (task_will_free_mem(task) && !is_sysrq_oom(oc))
-		return OOM_SCAN_ABORT;
 	return OOM_SCAN_OK;
 }
 static void dump_header(struct oom_control *oc, struct task_struct *p,
 			struct mem_cgroup *memcg)
 {
-	pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), order=%d, "
+	pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), order=%d, oom_score_adj=%hd\n",
-			"oom_score_adj=%hd\n",
 		current->comm, oc->gfp_mask, &oc->gfp_mask, oc->order,
 		current->signal->oom_score_adj);
 bool oom_killer_disable(void)
 {
 	/*
-	 * Make sure to not race with an ongoing OOM killer
+	 * Make sure to not race with an ongoing OOM killer. Check that the
-	 * and that the current is not the victim.
+	 * current is not killed (possibly due to sharing the victim's memory).
 	 */
-	mutex_lock(&oom_lock);
+	if (mutex_lock_killable(&oom_lock))
-	if (test_thread_flag(TIF_MEMDIE)) {
-		mutex_unlock(&oom_lock);
 		return false;
-	}
 	oom_killer_disabled = true;
 	mutex_unlock(&oom_lock);

mm/page_alloc.c+124−68

 int min_free_kbytes = 1024;
 int user_min_free_kbytes = -1;
+int watermark_scale_factor = 10;
 static unsigned long __meminitdata nr_kernel_pages;
 static unsigned long __meminitdata nr_all_pages;
 				unsigned long pfn, unsigned long zone_end,
 				unsigned long *nr_initialised)
 {
+	unsigned long max_initialise;
 	/* Always populate low zones for address-contrained allocations */
 	if (zone_end < pgdat_end_pfn(pgdat))
 		return true;
+	/*
+	 * Initialise at least 2G of a node but also take into account that
+	 * two large system hashes that can take up 1GB for 0.25TB/node.
+	 */
+	max_initialise = max(2UL << (30 - PAGE_SHIFT),
+		(pgdat->node_spanned_pages >> 8));
-	/* Initialise at least 2G of the highest zone */
 	(*nr_initialised)++;
-	if (*nr_initialised > (2UL << (30 - PAGE_SHIFT)) &&
+	if ((*nr_initialised > max_initialise) &&
 	    (pfn & (PAGES_PER_SECTION - 1)) == 0) {
 		pgdat->first_deferred_pfn = pfn;
 		return false;
 unsigned int _debug_guardpage_minorder;
 bool _debug_pagealloc_enabled __read_mostly
 			= IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
+EXPORT_SYMBOL(_debug_pagealloc_enabled);
 bool _debug_guardpage_enabled __read_mostly;
 static int __init early_debug_pagealloc(char *buf)
 	unsigned long res;
 	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
-		printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
+		pr_err("Bad debug_guardpage_minorder value\n");
 		return 0;
 	}
 	_debug_guardpage_minorder = res;
-	printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
+	pr_info("Setting debug_guardpage_minorder to %lu\n", res);
 	return 0;
 }
 __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
 		bad_reason = "nonzero mapcount";
 	if (unlikely(page->mapping != NULL))
 		bad_reason = "non-NULL mapping";
-	if (unlikely(atomic_read(&page->_count) != 0))
+	if (unlikely(page_ref_count(page) != 0))
 		bad_reason = "nonzero _count";
 	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
 		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
 		bad_reason = "nonzero mapcount";
 	if (unlikely(page->mapping != NULL))
 		bad_reason = "non-NULL mapping";
-	if (unlikely(atomic_read(&page->_count) != 0))
+	if (unlikely(page_ref_count(page) != 0))
 		bad_reason = "nonzero _count";
 	if (unlikely(page->flags & __PG_HWPOISON)) {
 		bad_reason = "HWPoisoned (hardware-corrupted)";
 		list_del(&page->lru);
 		pcp->count--;
 	} else {
-		if (unlikely(gfp_flags & __GFP_NOFAIL)) {
+		/*
-			/*
+		 * We most definitely don't want callers attempting to
-			 * __GFP_NOFAIL is not to be used in new code.
+		 * allocate greater than order-1 page units with __GFP_NOFAIL.
-			 *
+		 */
-			 * All __GFP_NOFAIL callers should be fixed so that they
+		WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
-			 * properly detect and handle allocation failures.
-			 *
-			 * We most definitely don't want callers attempting to
-			 * allocate greater than order-1 page units with
-			 * __GFP_NOFAIL.
-			 */
-			WARN_ON_ONCE(order > 1);
-		}
 		spin_lock_irqsave(&zone->lock, flags);
 		page = NULL;
 			 * XXX: Page reclaim didn't yield anything,
 			 * and the OOM killer can't be invoked, but
 			 * keep looping as per tradition.
+			 *
+			 * But do not keep looping if oom_killer_disable()
+			 * was already called, for the system is trying to
+			 * enter a quiescent state during suspend.
 			 */
-			*did_some_progress = 1;
+			*did_some_progress = !oom_killer_disabled;
 			goto out;
 		}
 		if (pm_suspended_storage())
 				(__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
 		gfp_mask &= ~__GFP_ATOMIC;
-	/*
-	 * If this allocation cannot block and it is for a specific node, then
-	 * fail early.  There's no need to wakeup kswapd or retry for a
-	 * speculative node-specific allocation.
-	 */
-	if (IS_ENABLED(CONFIG_NUMA) && (gfp_mask & __GFP_THISNODE) && !can_direct_reclaim)
-		goto nopage;
 retry:
 	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
 		wake_all_kswapds(order, ac);
 		/* Even if we own the page, we do not use atomic_set().
 		 * This would break get_page_unless_zero() users.
 		 */
-		atomic_add(size - 1, &page->_count);
+		page_ref_add(page, size - 1);
 		/* reset page count bias and offset to start of new frag */
 		nc->pfmemalloc = page_is_pfmemalloc(page);
 	if (unlikely(offset < 0)) {
 		page = virt_to_page(nc->va);
-		if (!atomic_sub_and_test(nc->pagecnt_bias, &page->_count))
+		if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
 			goto refill;
 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
 		size = nc->size;
 #endif
 		/* OK, page count is 0, we can safely set it */
-		atomic_set(&page->_count, size);
+		set_page_count(page, size);
 		/* reset page count bias and offset to start of new frag */
 		nc->pagecnt_bias = size;
 		printk("Node %d ", zone_to_nid(zone));
 }
+long si_mem_available(void)
+{
+	long available;
+	unsigned long pagecache;
+	unsigned long wmark_low = 0;
+	unsigned long pages[NR_LRU_LISTS];
+	struct zone *zone;
+	int lru;
+	for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
+		pages[lru] = global_page_state(NR_LRU_BASE + lru);
+	for_each_zone(zone)
+		wmark_low += zone->watermark[WMARK_LOW];
+	/*
+	 * Estimate the amount of memory available for userspace allocations,
+	 * without causing swapping.
+	 */
+	available = global_page_state(NR_FREE_PAGES) - totalreserve_pages;
+	/*
+	 * Not all the page cache can be freed, otherwise the system will
+	 * start swapping. Assume at least half of the page cache, or the
+	 * low watermark worth of cache, needs to stay.
+	 */
+	pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
+	pagecache -= min(pagecache / 2, wmark_low);
+	available += pagecache;
+	/*
+	 * Part of the reclaimable slab consists of items that are in use,
+	 * and cannot be freed. Cap this estimate at the low watermark.
+	 */
+	available += global_page_state(NR_SLAB_RECLAIMABLE) -
+		     min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
+	if (available < 0)
+		available = 0;
+	return available;
+}
+EXPORT_SYMBOL_GPL(si_mem_available);
 void si_meminfo(struct sysinfo *val)
 {
 	val->totalram = totalram_pages;
 	} else if (*s == 'z' || *s == 'Z') {
 		user_zonelist_order = ZONELIST_ORDER_ZONE;
 	} else {
-		printk(KERN_WARNING
+		pr_warn("Ignoring invalid numa_zonelist_order value:  %s\n", s);
-			"Ignoring invalid numa_zonelist_order value:  "
-			"%s\n", s);
 		return -EINVAL;
 	}
 	return 0;
 	else
 		page_group_by_mobility_disabled = 0;
-	pr_info("Built %i zonelists in %s order, mobility grouping %s.  "
+	pr_info("Built %i zonelists in %s order, mobility grouping %s.  Total pages: %ld\n",
-		"Total pages: %ld\n",
+		nr_online_nodes,
-			nr_online_nodes,
+		zonelist_order_name[current_zonelist_order],
-			zonelist_order_name[current_zonelist_order],
+		page_group_by_mobility_disabled ? "off" : "on",
-			page_group_by_mobility_disabled ? "off" : "on",
+		vm_total_pages);
-			vm_total_pages);
 #ifdef CONFIG_NUMA
 	pr_info("Policy zone: %s\n", zone_names[policy_zone]);
 #endif
 #endif
 	init_waitqueue_head(&pgdat->kswapd_wait);
 	init_waitqueue_head(&pgdat->pfmemalloc_wait);
+#ifdef CONFIG_COMPACTION
+	init_waitqueue_head(&pgdat->kcompactd_wait);
+#endif
 	pgdat_page_ext_init(pgdat);
 	for (j = 0; j < MAX_NR_ZONES; j++) {
 					       "  %s zone: %lu pages used for memmap\n",
 					       zone_names[j], memmap_pages);
 			} else
-				printk(KERN_WARNING
+				pr_warn("  %s zone: %lu pages exceeds freesize %lu\n",
-					"  %s zone: %lu pages exceeds freesize %lu\n",
 					zone_names[j], memmap_pages, freesize);
 		}
 		min_pfn = min(min_pfn, start_pfn);
 	if (min_pfn == ULONG_MAX) {
-		printk(KERN_WARNING
+		pr_warn("Could not find start_pfn for node %d\n", nid);
-			"Could not find start_pfn for node %d\n", nid);
 		return 0;
 	}
 #undef	adj_init_size
-	pr_info("Memory: %luK/%luK available "
+	pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
-	       "(%luK kernel code, %luK rwdata, %luK rodata, "
-	       "%luK init, %luK bss, %luK reserved, %luK cma-reserved"
 #ifdef	CONFIG_HIGHMEM
-	       ", %luK highmem"
+		", %luK highmem"
 #endif
-	       "%s%s)\n",
+		"%s%s)\n",
-	       nr_free_pages() << (PAGE_SHIFT-10), physpages << (PAGE_SHIFT-10),
+		nr_free_pages() << (PAGE_SHIFT - 10),
-	       codesize >> 10, datasize >> 10, rosize >> 10,
+		physpages << (PAGE_SHIFT - 10),
-	       (init_data_size + init_code_size) >> 10, bss_size >> 10,
+		codesize >> 10, datasize >> 10, rosize >> 10,
-	       (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT-10),
+		(init_data_size + init_code_size) >> 10, bss_size >> 10,
-	       totalcma_pages << (PAGE_SHIFT-10),
+		(physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
+		totalcma_pages << (PAGE_SHIFT - 10),
 #ifdef	CONFIG_HIGHMEM
-	       totalhigh_pages << (PAGE_SHIFT-10),
+		totalhigh_pages << (PAGE_SHIFT - 10),
 #endif
-	       str ? ", " : "", str ? str : "");
+		str ? ", " : "", str ? str : "");
 }
 /**
 			zone->watermark[WMARK_MIN] = tmp;
 		}
-		zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + (tmp >> 2);
+		/*
-		zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
+		 * Set the kswapd watermarks distance according to the
+		 * scale factor in proportion to available memory, but
+		 * ensure a minimum size on small systems.
+		 */
+		tmp = max_t(u64, tmp >> 2,
+			    mult_frac(zone->managed_pages,
+				      watermark_scale_factor, 10000));
+		zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;
+		zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
 		__mod_zone_page_state(zone, NR_ALLOC_BATCH,
 			high_wmark_pages(zone) - low_wmark_pages(zone) -
 	return 0;
 }
+int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
+	void __user *buffer, size_t *length, loff_t *ppos)
+{
+	int rc;
+	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
+	if (rc)
+		return rc;
+	if (write)
+		setup_per_zone_wmarks();
+	return 0;
+}
 #ifdef CONFIG_NUMA
 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
 	void __user *buffer, size_t *length, loff_t *ppos)
 	if (!table)
 		panic("Failed to allocate %s hash table\n", tablename);
-	printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
+	pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
-	       tablename,
+		tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
-	       (1UL << log2qty),
-	       ilog2(size) - PAGE_SHIFT,
-	       size);
 	if (_hash_shift)
 		*_hash_shift = log2qty;
 		 * This check already skips compound tails of THP
 		 * because their page->_count is zero at all time.
 		 */
-		if (!atomic_read(&page->_count)) {
+		if (!page_ref_count(page)) {
 			if (PageBuddy(page))
 				iter += (1 << page_order(page)) - 1;
 			continue;
 		BUG_ON(!PageBuddy(page));
 		order = page_order(page);
 #ifdef CONFIG_DEBUG_VM
-		printk(KERN_INFO "remove from free list %lx %d %lx\n",
+		pr_info("remove from free list %lx %d %lx\n",
-		       pfn, 1 << order, end_pfn);
+			pfn, 1 << order, end_pfn);
 #endif
 		list_del(&page->lru);
 		rmv_page_order(page);
 }
 #endif
-#ifdef CONFIG_MEMORY_FAILURE
 bool is_free_buddy_page(struct page *page)
 {
 	struct zone *zone = page_zone(page);
 	return order < MAX_ORDER;
 }
-#endif

mm/page_io.c+11−11

@@ -56,10 +56,10 @@ void end_swap_bio_write(struct bio *bio) · 55 unmodified lines
56	* Also clear PG_reclaim to avoid rotate_reclaimable_page()	56	* Also clear PG_reclaim to avoid rotate_reclaimable_page()
57	*/	57	*/
58	set_page_dirty(page);	58	set_page_dirty(page);
59	printk(KERN_ALERT "Write-error on swap-device (%u:%u:%Lu)\n",	59	pr_alert("Write-error on swap-device (%u:%u:%llu)\n",
60	imajor(bio->bi_bdev->bd_inode),	60	imajor(bio->bi_bdev->bd_inode),
61	iminor(bio->bi_bdev->bd_inode),	61	iminor(bio->bi_bdev->bd_inode),
62	(unsigned long long)bio->bi_iter.bi_sector);	62	(unsigned long long)bio->bi_iter.bi_sector);
63	ClearPageReclaim(page);	63	ClearPageReclaim(page);
64	}	64	}
65	end_page_writeback(page);	65	end_page_writeback(page);
@@ -73,10 +73,10 @@ static void end_swap_bio_read(struct bio *bio) · 7 unmodified lines
73	if (bio->bi_error) {	73	if (bio->bi_error) {
74	SetPageError(page);	74	SetPageError(page);
75	ClearPageUptodate(page);	75	ClearPageUptodate(page);
76	printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n",	76	pr_alert("Read-error on swap-device (%u:%u:%llu)\n",
77	imajor(bio->bi_bdev->bd_inode),	77	imajor(bio->bi_bdev->bd_inode),
78	iminor(bio->bi_bdev->bd_inode),	78	iminor(bio->bi_bdev->bd_inode),
79	(unsigned long long)bio->bi_iter.bi_sector);	79	(unsigned long long)bio->bi_iter.bi_sector);
80	goto out;	80	goto out;
81	}	81	}
82		82
@@ -216,7 +216,7 @@ reprobe: · 133 unmodified lines
216	out:	216	out:
217	return ret;	217	return ret;
218	bad_bmap:	218	bad_bmap:
219	printk(KERN_ERR "swapon: swapfile has holes\n");	219	pr_err("swapon: swapfile has holes\n");
220	ret = -EINVAL;	220	ret = -EINVAL;
221	goto out;	221	goto out;
222	}	222	}
@@ -290,8 +290,8 @@ int __swap_writepage(struct page page, struct writeback_control wbc, · 67 unmodified lines
290	*/	290	*/
291	set_page_dirty(page);	291	set_page_dirty(page);
292	ClearPageReclaim(page);	292	ClearPageReclaim(page);
293	pr_err_ratelimited("Write error on dio swapfile (%Lu)\n",	293	pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
294	page_file_offset(page));	294	page_file_offset(page));
295	}	295	}
296	end_page_writeback(page);	296	end_page_writeback(page);
297	return ret;	297	return ret;

mm/page_owner.c+2−3

@@ -198,9 +198,8 @@ void __dump_page_owner(struct page *page) · 197 unmodified lines
198	return;	198	return;
199	}	199	}
200		200
201	pr_alert("page allocated via order %u, migratetype %s, "	201	pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n",
202	"gfp_mask %#x(%pGg)\n", page_ext->order,	202	page_ext->order, migratetype_names[mt], gfp_mask, &gfp_mask);
203	migratetype_names[mt], gfp_mask, &gfp_mask);
204	print_stack_trace(&trace, 0);	203	print_stack_trace(&trace, 0);
205		204
206	if (page_ext->last_migrate_reason != -1)	205	if (page_ext->last_migrate_reason != -1)

mm/percpu-km.c+3−3

@@ -95,7 +95,7 @@ static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai) · 94 unmodified lines
95		95
96	/* all units must be in a single group */	96	/* all units must be in a single group */
97	if (ai->nr_groups != 1) {	97	if (ai->nr_groups != 1) {
98	printk(KERN_CRIT "percpu: can't handle more than one groups\n");	98	pr_crit("can't handle more than one group\n");
99	return -EINVAL;	99	return -EINVAL;
100	}	100	}
101		101
@@ -103,8 +103,8 @@ static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai) · 1 unmodified line
103	alloc_pages = roundup_pow_of_two(nr_pages);	103	alloc_pages = roundup_pow_of_two(nr_pages);
104		104
105	if (alloc_pages > nr_pages)	105	if (alloc_pages > nr_pages)
106	printk(KERN_WARNING "percpu: wasting %zu pages per chunk\n",	106	pr_warn("wasting %zu pages per chunk\n",
107	alloc_pages - nr_pages);	107	alloc_pages - nr_pages);
108		108
109	return 0;	109	return 0;
110	}	110	}

mm/percpu.c+22−21

  *   setup the first chunk containing the kernel static percpu area
  */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/bitmap.h>
 #include <linux/bootmem.h>
 #include <linux/err.h>
 	size = ALIGN(size, 2);
 	if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
-		WARN(true, "illegal size (%zu) or align (%zu) for "
+		WARN(true, "illegal size (%zu) or align (%zu) for percpu allocation\n",
-		     "percpu allocation\n", size, align);
+		     size, align);
 		return NULL;
 	}
 	spin_unlock_irqrestore(&pcpu_lock, flags);
 fail:
 	if (!is_atomic && warn_limit) {
-		pr_warning("PERCPU: allocation failed, size=%zu align=%zu atomic=%d, %s\n",
+		pr_warn("allocation failed, size=%zu align=%zu atomic=%d, %s\n",
-			   size, align, is_atomic, err);
+			size, align, is_atomic, err);
 		dump_stack();
 		if (!--warn_limit)
-			pr_info("PERCPU: limit reached, disable warning\n");
+			pr_info("limit reached, disable warning\n");
 	}
 	if (is_atomic) {
 		/* see the flag handling in pcpu_blance_workfn() */
 		for (alloc_end += gi->nr_units / upa;
 		     alloc < alloc_end; alloc++) {
 			if (!(alloc % apl)) {
-				printk(KERN_CONT "\n");
+				pr_cont("\n");
 				printk("%spcpu-alloc: ", lvl);
 			}
-			printk(KERN_CONT "[%0*d] ", group_width, group);
+			pr_cont("[%0*d] ", group_width, group);
 			for (unit_end += upa; unit < unit_end; unit++)
 				if (gi->cpu_map[unit] != NR_CPUS)
-					printk(KERN_CONT "%0*d ", cpu_width,
+					pr_cont("%0*d ",
-					       gi->cpu_map[unit]);
+						cpu_width, gi->cpu_map[unit]);
 				else
-					printk(KERN_CONT "%s ", empty_str);
+					pr_cont("%s ", empty_str);
 		}
 	}
-	printk(KERN_CONT "\n");
+	pr_cont("\n");
 }
 /**
 #define PCPU_SETUP_BUG_ON(cond)	do {					\
 	if (unlikely(cond)) {						\
-		pr_emerg("PERCPU: failed to initialize, %s", #cond);	\
+		pr_emerg("failed to initialize, %s\n", #cond);		\
-		pr_emerg("PERCPU: cpu_possible_mask=%*pb\n",		\
+		pr_emerg("cpu_possible_mask=%*pb\n",			\
 			 cpumask_pr_args(cpu_possible_mask));		\
 		pcpu_dump_alloc_info(KERN_EMERG, ai);			\
 		BUG();							\
 		pcpu_chosen_fc = PCPU_FC_PAGE;
 #endif
 	else
-		pr_warning("PERCPU: unknown allocator %s specified\n", str);
+		pr_warn("unknown allocator %s specified\n", str);
 	return 0;
 }
 	/* warn if maximum distance is further than 75% of vmalloc space */
 	if (max_distance > VMALLOC_TOTAL * 3 / 4) {
-		pr_warning("PERCPU: max_distance=0x%zx too large for vmalloc "
+		pr_warn("max_distance=0x%zx too large for vmalloc space 0x%lx\n",
-			   "space 0x%lx\n", max_distance,
+			max_distance, VMALLOC_TOTAL);
-			   VMALLOC_TOTAL);
 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
 		/* and fail if we have fallback */
 		rc = -EINVAL;
 #endif
 	}
-	pr_info("PERCPU: Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n",
+	pr_info("Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n",
 		PFN_DOWN(size_sum), base, ai->static_size, ai->reserved_size,
 		ai->dyn_size, ai->unit_size);
 			ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE);
 			if (!ptr) {
-				pr_warning("PERCPU: failed to allocate %s page "
+				pr_warn("failed to allocate %s page for cpu%u\n",
-					   "for cpu%u\n", psize_str, cpu);
+					psize_str, cpu);
 				goto enomem;
 			}
 			/* kmemleak tracks the percpu allocations separately */
 	}
 	/* we're ready, commit */
-	pr_info("PERCPU: %d %s pages/cpu @%p s%zu r%zu d%zu\n",
+	pr_info("%d %s pages/cpu @%p s%zu r%zu d%zu\n",
 		unit_pages, psize_str, vm.addr, ai->static_size,
 		ai->reserved_size, ai->dyn_size);

mm/pgtable-generic.c+0−14

@@ -84,20 +84,6 @@ pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, · 83 unmodified lines
84		84
85	#ifdef CONFIG_TRANSPARENT_HUGEPAGE	85	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
86		86
87	#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
88
89	/*
90	* ARCHes with special requirements for evicting THP backing TLB entries can
91	* implement this. Otherwise also, it can help optimize normal TLB flush in
92	* THP regime. stock flush_tlb_range() typically has optimization to nuke the
93	* entire TLB if flush span is greater than a threshold, which will
94	* likely be true for a single huge page. Thus a single thp flush will
95	* invalidate the entire TLB which is not desirable.
96	* e.g. see arch/arc: flush_pmd_tlb_range
97	*/
98	#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
99	#endif
100
101	#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS	87	#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
102	int pmdp_set_access_flags(struct vm_area_struct *vma,	88	int pmdp_set_access_flags(struct vm_area_struct *vma,
103	unsigned long address, pmd_t *pmdp,	89	unsigned long address, pmd_t *pmdp,

mm/quicklist.c+1−1

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	* improved on it.	8	* improved on it.
9	*	9	*
10	* Copyright (C) 2007 SGI,	10	* Copyright (C) 2007 SGI,
11	* Christoph Lameter <clameter@sgi.com>	11	* Christoph Lameter <cl@linux.com>
12	* Generalized, added support for multiple lists and	12	* Generalized, added support for multiple lists and
13	* constructors / destructors.	13	* constructors / destructors.
14	*/	14	*/

mm/rmap.c+54−17

@@ -1431,6 +1431,14 @@ static int try_to_unmap_one(struct page page, struct vm_area_struct vma, · 1430 unmodified lines
1431	if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))	1431	if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
1432	goto out;	1432	goto out;
1433		1433
		1434	if (flags & TTU_SPLIT_HUGE_PMD) {
		1435	split_huge_pmd_address(vma, address,
		1436	flags & TTU_MIGRATION, page);
		1437	/* check if we have anything to do after split */
		1438	if (page_mapcount(page) == 0)
		1439	goto out;
		1440	}
		1441
1434	pte = page_check_address(page, mm, address, &ptl, 0);	1442	pte = page_check_address(page, mm, address, &ptl, 0);
1435	if (!pte)	1443	if (!pte)
1436	goto out;	1444	goto out;
@@ -1576,10 +1584,10 @@ static bool invalid_migration_vma(struct vm_area_struct vma, void arg) · 139 unmodified lines
1576	return is_vma_temporary_stack(vma);	1584	return is_vma_temporary_stack(vma);
1577	}	1585	}
1578		1586
1579	static int page_not_mapped(struct page *page)	1587	static int page_mapcount_is_zero(struct page *page)
1580	{	1588	{
1581	return !page_mapped(page);	1589	return !page_mapcount(page);
1582	};	1590	}
1583		1591
1584	/**	1592	/**
1585	* try_to_unmap - try to remove all page table mappings to a page	1593	* try_to_unmap - try to remove all page table mappings to a page
@@ -1606,12 +1614,10 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) · 20 unmodified lines
1606	struct rmap_walk_control rwc = {	1614	struct rmap_walk_control rwc = {
1607	.rmap_one = try_to_unmap_one,	1615	.rmap_one = try_to_unmap_one,
1608	.arg = &rp,	1616	.arg = &rp,
1609	.done = page_not_mapped,	1617	.done = page_mapcount_is_zero,
1610	.anon_lock = page_lock_anon_vma_read,	1618	.anon_lock = page_lock_anon_vma_read,
1611	};	1619	};
1612		1620
1613	VM_BUG_ON_PAGE(!PageHuge(page) && PageTransHuge(page), page);
1614
1615	/*	1621	/*
1616	* During exec, a temporary VMA is setup and later moved.	1622	* During exec, a temporary VMA is setup and later moved.
1617	* The VMA is moved under the anon_vma lock but not the	1623	* The VMA is moved under the anon_vma lock but not the
@@ -1623,9 +1629,12 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) · 5 unmodified lines
1623	if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page))	1629	if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page))
1624	rwc.invalid_vma = invalid_migration_vma;	1630	rwc.invalid_vma = invalid_migration_vma;
1625		1631
1626	ret = rmap_walk(page, &rwc);	1632	if (flags & TTU_RMAP_LOCKED)
		1633	ret = rmap_walk_locked(page, &rwc);
		1634	else
		1635	ret = rmap_walk(page, &rwc);
1627		1636
1628	if (ret != SWAP_MLOCK && !page_mapped(page)) {	1637	if (ret != SWAP_MLOCK && !page_mapcount(page)) {
1629	ret = SWAP_SUCCESS;	1638	ret = SWAP_SUCCESS;
1630	if (rp.lazyfreed && !PageDirty(page))	1639	if (rp.lazyfreed && !PageDirty(page))
1631	ret = SWAP_LZFREE;	1640	ret = SWAP_LZFREE;
@@ -1633,6 +1642,11 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) · 1 unmodified line
1633	return ret;	1642	return ret;
1634	}	1643	}
1635		1644
		1645	static int page_not_mapped(struct page *page)
		1646	{
		1647	return !page_mapped(page);
		1648	};
		1649
1636	/**	1650	/**
1637	* try_to_munlock - try to munlock a page	1651	* try_to_munlock - try to munlock a page
1638	* @page: the page to be munlocked	1652	* @page: the page to be munlocked
@@ -1715,14 +1729,21 @@ static struct anon_vma rmap_walk_anon_lock(struct page page, · 76 unmodified lines
1715	* vm_flags for that VMA. That should be OK, because that vma shouldn't be	1729	* vm_flags for that VMA. That should be OK, because that vma shouldn't be
1716	* LOCKED.	1730	* LOCKED.
1717	*/	1731	*/
1718	static int rmap_walk_anon(struct page page, struct rmap_walk_control rwc)	1732	static int rmap_walk_anon(struct page page, struct rmap_walk_control rwc,
		1733	bool locked)
1719	{	1734	{
1720	struct anon_vma *anon_vma;	1735	struct anon_vma *anon_vma;
1721	pgoff_t pgoff;	1736	pgoff_t pgoff;
1722	struct anon_vma_chain *avc;	1737	struct anon_vma_chain *avc;
1723	int ret = SWAP_AGAIN;	1738	int ret = SWAP_AGAIN;
1724		1739
1725	anon_vma = rmap_walk_anon_lock(page, rwc);	1740	if (locked) {
		1741	anon_vma = page_anon_vma(page);
		1742	/* anon_vma disappear under us? */
		1743	VM_BUG_ON_PAGE(!anon_vma, page);
		1744	} else {
		1745	anon_vma = rmap_walk_anon_lock(page, rwc);
		1746	}
1726	if (!anon_vma)	1747	if (!anon_vma)
1727	return ret;	1748	return ret;
1728		1749
@@ -1742,7 +1763,9 @@ static int rmap_walk_anon(struct page page, struct rmap_walk_control rwc) · 13 unmodified lines
1742	if (rwc->done && rwc->done(page))	1763	if (rwc->done && rwc->done(page))
1743	break;	1764	break;
1744	}	1765	}
1745	anon_vma_unlock_read(anon_vma);	1766
		1767	if (!locked)
		1768	anon_vma_unlock_read(anon_vma);
1746	return ret;	1769	return ret;
1747	}	1770	}
1748		1771
@@ -1759,9 +1782,10 @@ static int rmap_walk_anon(struct page page, struct rmap_walk_control rwc) · 10 unmodified lines
1759	* vm_flags for that VMA. That should be OK, because that vma shouldn't be	1782	* vm_flags for that VMA. That should be OK, because that vma shouldn't be
1760	* LOCKED.	1783	* LOCKED.
1761	*/	1784	*/
1762	static int rmap_walk_file(struct page page, struct rmap_walk_control rwc)	1785	static int rmap_walk_file(struct page page, struct rmap_walk_control rwc,
		1786	bool locked)
1763	{	1787	{
1764	struct address_space *mapping = page->mapping;	1788	struct address_space *mapping = page_mapping(page);
1765	pgoff_t pgoff;	1789	pgoff_t pgoff;
1766	struct vm_area_struct *vma;	1790	struct vm_area_struct *vma;
1767	int ret = SWAP_AGAIN;	1791	int ret = SWAP_AGAIN;
@@ -1778,7 +1802,8 @@ static int rmap_walk_file(struct page page, struct rmap_walk_control rwc) · 10 unmodified lines
1778	return ret;	1802	return ret;
1779		1803
1780	pgoff = page_to_pgoff(page);	1804	pgoff = page_to_pgoff(page);
1781	i_mmap_lock_read(mapping);	1805	if (!locked)
		1806	i_mmap_lock_read(mapping);
1782	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {	1807	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1783	unsigned long address = vma_address(page, vma);	1808	unsigned long address = vma_address(page, vma);
1784		1809
@@ -1795,7 +1820,8 @@ static int rmap_walk_file(struct page page, struct rmap_walk_control rwc) · 10 unmodified lines
1795	}	1820	}
1796		1821
1797	done:	1822	done:
1798	i_mmap_unlock_read(mapping);	1823	if (!locked)
		1824	i_mmap_unlock_read(mapping);
1799	return ret;	1825	return ret;
1800	}	1826	}
1801		1827
@@ -1804,9 +1830,20 @@ int rmap_walk(struct page page, struct rmap_walk_control rwc) · 2 unmodified lines
1804	if (unlikely(PageKsm(page)))	1830	if (unlikely(PageKsm(page)))
1805	return rmap_walk_ksm(page, rwc);	1831	return rmap_walk_ksm(page, rwc);
1806	else if (PageAnon(page))	1832	else if (PageAnon(page))
1807	return rmap_walk_anon(page, rwc);	1833	return rmap_walk_anon(page, rwc, false);
		1834	else
		1835	return rmap_walk_file(page, rwc, false);
		1836	}
		1837
		1838	/* Like rmap_walk, but caller holds relevant rmap lock */
		1839	int rmap_walk_locked(struct page page, struct rmap_walk_control rwc)
		1840	{
		1841	/* no ksm support for now */
		1842	VM_BUG_ON_PAGE(PageKsm(page), page);
		1843	if (PageAnon(page))
		1844	return rmap_walk_anon(page, rwc, true);
1808	else	1845	else
1809	return rmap_walk_file(page, rwc);	1846	return rmap_walk_file(page, rwc, true);
1810	}	1847	}
1811		1848
1812	#ifdef CONFIG_HUGETLB_PAGE	1849	#ifdef CONFIG_HUGETLB_PAGE

mm/shmem.c+21−28

 	rcu_read_lock();
-restart:
 	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
 		if (iter.index >= end)
 			break;
 		page = radix_tree_deref_slot(slot);
-		/*
+		if (radix_tree_deref_retry(page)) {
-		 * This should only be possible to happen at index 0, so we
+			slot = radix_tree_iter_retry(&iter);
-		 * don't need to reset the counter, nor do we risk infinite
+			continue;
-		 * restarts.
+		}
-		 */
-		if (radix_tree_deref_retry(page))
-			goto restart;
 		if (radix_tree_exceptional_entry(page))
 			swapped++;
 		if (need_resched()) {
 			cond_resched_rcu();
-			start = iter.index + 1;
+			slot = radix_tree_iter_next(&iter);
-			goto restart;
 		}
 	}
 	start = 0;
 	rcu_read_lock();
-restart:
 	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
 		page = radix_tree_deref_slot(slot);
 		if (!page || radix_tree_exception(page)) {
-			if (radix_tree_deref_retry(page))
+			if (radix_tree_deref_retry(page)) {
-				goto restart;
+				slot = radix_tree_iter_retry(&iter);
+				continue;
+			}
 		} else if (page_count(page) - page_mapcount(page) > 1) {
 			spin_lock_irq(&mapping->tree_lock);
 			radix_tree_tag_set(&mapping->page_tree, iter.index,
 		if (need_resched()) {
 			cond_resched_rcu();
-			start = iter.index + 1;
+			slot = radix_tree_iter_next(&iter);
-			goto restart;
 		}
 	}
 	rcu_read_unlock();
 		start = 0;
 		rcu_read_lock();
-restart:
 		radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter,
 					   start, SHMEM_TAG_PINNED) {
 			page = radix_tree_deref_slot(slot);
 			if (radix_tree_exception(page)) {
-				if (radix_tree_deref_retry(page))
+				if (radix_tree_deref_retry(page)) {
-					goto restart;
+					slot = radix_tree_iter_retry(&iter);
+					continue;
+				}
 				page = NULL;
 			}
 continue_resched:
 			if (need_resched()) {
 				cond_resched_rcu();
-				start = iter.index + 1;
+				slot = radix_tree_iter_next(&iter);
-				goto restart;
 			}
 		}
 		rcu_read_unlock();
 		if ((value = strchr(this_char,'=')) != NULL) {
 			*value++ = 0;
 		} else {
-			printk(KERN_ERR
+			pr_err("tmpfs: No value for mount option '%s'\n",
-			    "tmpfs: No value for mount option '%s'\n",
+			       this_char);
-			    this_char);
 			goto error;
 		}
 			if (mpol_parse_str(value, &mpol))
 				goto bad_val;
 		} else {
-			printk(KERN_ERR "tmpfs: Bad mount option %s\n",
+			pr_err("tmpfs: Bad mount option %s\n", this_char);
-			       this_char);
 			goto error;
 		}
 	}
 	return 0;
 bad_val:
-	printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
+	pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
 	       value, this_char);
 error:
 	mpol_put(mpol);
 	error = register_filesystem(&shmem_fs_type);
 	if (error) {
-		printk(KERN_ERR "Could not register tmpfs\n");
+		pr_err("Could not register tmpfs\n");
 		goto out2;
 	}
 	shm_mnt = kern_mount(&shmem_fs_type);
 	if (IS_ERR(shm_mnt)) {
 		error = PTR_ERR(shm_mnt);
-		printk(KERN_ERR "Could not kern_mount tmpfs\n");
+		pr_err("Could not kern_mount tmpfs\n");
 		goto out1;
 	}
 	return 0;

mm/slab.c+40−49

 static void __slab_error(const char *function, struct kmem_cache *cachep,
 			char *msg)
 {
-	printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
+	pr_err("slab error in %s(): cache `%s': %s\n",
 	       function, cachep->name, msg);
 	dump_stack();
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 static inline gfp_t gfp_exact_node(gfp_t flags)
 {
-	return flags;
+	return flags & ~__GFP_NOFAIL;
 }
 #else	/* CONFIG_NUMA */
 }
 /*
- * Construct gfp mask to allocate from a specific node but do not direct reclaim
+ * Construct gfp mask to allocate from a specific node but do not reclaim or
- * or warn about failures. kswapd may still wake to reclaim in the background.
+ * warn about failures.
  */
 static inline gfp_t gfp_exact_node(gfp_t flags)
 {
-	return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~__GFP_DIRECT_RECLAIM;
+	return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
 }
 #endif
  */
 static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
 {
-	const unsigned long nr_freed = (1 << cachep->gfporder);
+	int order = cachep->gfporder;
+	unsigned long nr_freed = (1 << order);
-	kmemcheck_free_shadow(page, cachep->gfporder);
+	kmemcheck_free_shadow(page, order);
 	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
 		sub_zone_page_state(page_zone(page),
 	if (current->reclaim_state)
 		current->reclaim_state->reclaimed_slab += nr_freed;
-	__free_kmem_pages(page, cachep->gfporder);
+	memcg_uncharge_slab(page, order, cachep);
+	__free_pages(page, order);
 }
 static void kmem_rcu_free(struct rcu_head *head)
 	unsigned char error = 0;
 	int bad_count = 0;
-	printk(KERN_ERR "%03x: ", offset);
+	pr_err("%03x: ", offset);
 	for (i = 0; i < limit; i++) {
 		if (data[offset + i] != POISON_FREE) {
 			error = data[offset + i];
 	if (bad_count == 1) {
 		error ^= POISON_FREE;
 		if (!(error & (error - 1))) {
-			printk(KERN_ERR "Single bit error detected. Probably "
+			pr_err("Single bit error detected. Probably bad RAM.\n");
-					"bad RAM.\n");
 #ifdef CONFIG_X86
-			printk(KERN_ERR "Run memtest86+ or a similar memory "
+			pr_err("Run memtest86+ or a similar memory test tool.\n");
-					"test tool.\n");
 #else
-			printk(KERN_ERR "Run a memory test tool.\n");
+			pr_err("Run a memory test tool.\n");
 #endif
 		}
 	}
 	char *realobj;
 	if (cachep->flags & SLAB_RED_ZONE) {
-		printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
+		pr_err("Redzone: 0x%llx/0x%llx\n",
-			*dbg_redzone1(cachep, objp),
+		       *dbg_redzone1(cachep, objp),
-			*dbg_redzone2(cachep, objp));
+		       *dbg_redzone2(cachep, objp));
 	}
 	if (cachep->flags & SLAB_STORE_USER) {
-		printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
+		pr_err("Last user: [<%p>](%pSR)\n",
 		       *dbg_userword(cachep, objp),
 		       *dbg_userword(cachep, objp));
 	}
 			/* Mismatch ! */
 			/* Print header */
 			if (lines == 0) {
-				printk(KERN_ERR
+				pr_err("Slab corruption (%s): %s start=%p, len=%d\n",
-					"Slab corruption (%s): %s start=%p, len=%d\n",
+				       print_tainted(), cachep->name,
-					print_tainted(), cachep->name, realobj, size);
+				       realobj, size);
 				print_objinfo(cachep, objp, 0);
 			}
 			/* Hexdump the affected line */
 		if (objnr) {
 			objp = index_to_obj(cachep, page, objnr - 1);
 			realobj = (char *)objp + obj_offset(cachep);
-			printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
+			pr_err("Prev obj: start=%p, len=%d\n", realobj, size);
-			       realobj, size);
 			print_objinfo(cachep, objp, 2);
 		}
 		if (objnr + 1 < cachep->num) {
 			objp = index_to_obj(cachep, page, objnr + 1);
 			realobj = (char *)objp + obj_offset(cachep);
-			printk(KERN_ERR "Next obj: start=%p, len=%d\n",
+			pr_err("Next obj: start=%p, len=%d\n", realobj, size);
-			       realobj, size);
 			print_objinfo(cachep, objp, 2);
 		}
 	}
 		}
 		if (cachep->flags & SLAB_RED_ZONE) {
 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
-				slab_error(cachep, "start of a freed object "
+				slab_error(cachep, "start of a freed object was overwritten");
-					   "was overwritten");
 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
-				slab_error(cachep, "end of a freed object "
+				slab_error(cachep, "end of a freed object was overwritten");
-					   "was overwritten");
 		}
 	}
 }
 		if (cachep->flags & SLAB_RED_ZONE) {
 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
-				slab_error(cachep, "constructor overwrote the"
+				slab_error(cachep, "constructor overwrote the end of an object");
-					   " end of an object");
 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
-				slab_error(cachep, "constructor overwrote the"
+				slab_error(cachep, "constructor overwrote the start of an object");
-					   " start of an object");
 		}
 		/* need to poison the objs? */
 		if (cachep->flags & SLAB_POISON) {
 	/* Verify double free bug */
 	for (i = page->active; i < cachep->num; i++) {
 		if (get_free_obj(page, i) == objnr) {
-			printk(KERN_ERR "slab: double free detected in cache "
+			pr_err("slab: double free detected in cache '%s', objp %p\n",
-					"'%s', objp %p\n", cachep->name, objp);
+			       cachep->name, objp);
 			BUG();
 		}
 	}
 static void kfree_debugcheck(const void *objp)
 {
 	if (!virt_addr_valid(objp)) {
-		printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
+		pr_err("kfree_debugcheck: out of range ptr %lxh\n",
 		       (unsigned long)objp);
 		BUG();
 	}
 	else
 		slab_error(cache, "memory outside object was overwritten");
-	printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
+	pr_err("%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
-			obj, redzone1, redzone2);
+	       obj, redzone1, redzone2);
 }
 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
 	if (cachep->flags & SLAB_RED_ZONE) {
 		if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
 				*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
-			slab_error(cachep, "double free, or memory outside"
+			slab_error(cachep, "double free, or memory outside object was overwritten");
-						" object was overwritten");
+			pr_err("%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
-			printk(KERN_ERR
+			       objp, *dbg_redzone1(cachep, objp),
-				"%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
+			       *dbg_redzone2(cachep, objp));
-				objp, *dbg_redzone1(cachep, objp),
-				*dbg_redzone2(cachep, objp));
 		}
 		*dbg_redzone1(cachep, objp) = RED_ACTIVE;
 		*dbg_redzone2(cachep, objp) = RED_ACTIVE;
 		cachep->ctor(objp);
 	if (ARCH_SLAB_MINALIGN &&
 	    ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
-		printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
+		pr_err("0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
 		       objp, (int)ARCH_SLAB_MINALIGN);
 	}
 	return objp;
 skip_setup:
 	err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
 	if (err)
-		printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
+		pr_err("enable_cpucache failed for %s, error %d\n",
 		       cachep->name, -err);
 	return err;
 }
 	name = cachep->name;
 	if (error)
-		printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
+		pr_err("slab: cache %s error: %s\n", name, error);
 	sinfo->active_objs = active_objs;
 	sinfo->num_objs = num_objs;
 		unsigned long node_frees = cachep->node_frees;
 		unsigned long overflows = cachep->node_overflow;
-		seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
+		seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu %4lu",
-			   "%4lu %4lu %4lu %4lu %4lu",
 			   allocs, high, grown,
 			   reaped, errors, max_freeable, node_allocs,
 			   node_frees, overflows);

mm/slab.h+28−2

@@ -246,12 +246,33 @@ static __always_inline int memcg_charge_slab(struct page *page, · 245 unmodified lines
246	gfp_t gfp, int order,	246	gfp_t gfp, int order,
247	struct kmem_cache *s)	247	struct kmem_cache *s)
248	{	248	{
		249	int ret;
		250
249	if (!memcg_kmem_enabled())	251	if (!memcg_kmem_enabled())
250	return 0;	252	return 0;
251	if (is_root_cache(s))	253	if (is_root_cache(s))
252	return 0;	254	return 0;
253	return __memcg_kmem_charge_memcg(page, gfp, order,	255
254	s->memcg_params.memcg);	256	ret = __memcg_kmem_charge_memcg(page, gfp, order,
		257	s->memcg_params.memcg);
		258	if (ret)
		259	return ret;
		260
		261	memcg_kmem_update_page_stat(page,
		262	(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		263	MEMCG_SLAB_RECLAIMABLE : MEMCG_SLAB_UNRECLAIMABLE,
		264	1 << order);
		265	return 0;
		266	}
		267
		268	static __always_inline void memcg_uncharge_slab(struct page *page, int order,
		269	struct kmem_cache *s)
		270	{
		271	memcg_kmem_update_page_stat(page,
		272	(s->flags & SLAB_RECLAIM_ACCOUNT) ?
		273	MEMCG_SLAB_RECLAIMABLE : MEMCG_SLAB_UNRECLAIMABLE,
		274	-(1 << order));
		275	memcg_kmem_uncharge(page, order);
255	}	276	}
256		277
257	extern void slab_init_memcg_params(struct kmem_cache *);	278	extern void slab_init_memcg_params(struct kmem_cache *);
@@ -294,6 +315,11 @@ static inline int memcg_charge_slab(struct page *page, gfp_t gfp, int order, · 36 unmodified lines
294	return 0;	315	return 0;
295	}	316	}
296		317
		318	static inline void memcg_uncharge_slab(struct page *page, int order,
		319	struct kmem_cache *s)
		320	{
		321	}
		322
297	static inline void slab_init_memcg_params(struct kmem_cache *s)	323	static inline void slab_init_memcg_params(struct kmem_cache *s)
298	{	324	{
299	}	325	}

mm/slab_common.c+6−8

 			panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n",
 				name, err);
 		else {
-			printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d",
+			pr_warn("kmem_cache_create(%s) failed with error %d\n",
 				name, err);
 			dump_stack();
 		}
 	 * The memory cgroup could have been offlined while the cache
 	 * creation work was pending.
 	 */
-	if (!memcg_kmem_online(memcg))
+	if (memcg->kmem_state != KMEM_ONLINE)
 		goto out_unlock;
 	idx = memcg_cache_id(memcg);
 		err = shutdown_cache(s, &release, &need_rcu_barrier);
 	if (err) {
-		pr_err("kmem_cache_destroy %s: "
+		pr_err("kmem_cache_destroy %s: Slab cache still has objects\n",
-		       "Slab cache still has objects\n", s->name);
+		       s->name);
 		dump_stack();
 	}
 out_unlock:
 #else
 	seq_puts(m, "slabinfo - version: 2.1\n");
 #endif
-	seq_puts(m, "# name            <active_objs> <num_objs> <objsize> "
+	seq_puts(m, "# name            <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
-		 "<objperslab> <pagesperslab>");
 	seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
 	seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
 #ifdef CONFIG_DEBUG_SLAB
-	seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
+	seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
-		 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
 	seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
 #endif
 	seq_putc(m, '\n');

mm/slub.c+15−16

 {
 	void *p;
-#ifdef CONFIG_DEBUG_PAGEALLOC
+	if (!debug_pagealloc_enabled())
+		return get_freepointer(s, object);
 	probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
-#else
-	p = get_freepointer(s, object);
-#endif
 	return p;
 }
 		max_objects = MAX_OBJS_PER_PAGE;
 	if (page->objects != max_objects) {
-		slab_err(s, page, "Wrong number of objects. Found %d but "
+		slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
-			"should be %d", page->objects, max_objects);
+			 page->objects, max_objects);
 		page->objects = max_objects;
 		slab_fix(s, "Number of objects adjusted.");
 	}
 	if (page->inuse != page->objects - nr) {
-		slab_err(s, page, "Wrong object count. Counter is %d but "
+		slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
-			"counted were %d", page->inuse, page->objects - nr);
+			 page->inuse, page->objects - nr);
 		page->inuse = page->objects - nr;
 		slab_fix(s, "Object count adjusted.");
 	}
 	if (unlikely(s != page->slab_cache)) {
 		if (!PageSlab(page)) {
-			slab_err(s, page, "Attempt to free object(0x%p) "
+			slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
-				"outside of slab", object);
+				 object);
 		} else if (!page->slab_cache) {
 			pr_err("SLUB <none>: no slab for object 0x%p.\n",
 			       object);
 	 */
 	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
 	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
-		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_DIRECT_RECLAIM;
+		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
 	page = alloc_slab_page(s, alloc_gfp, node, oo);
 	if (unlikely(!page)) {
 	page_mapcount_reset(page);
 	if (current->reclaim_state)
 		current->reclaim_state->reclaimed_slab += pages;
-	__free_kmem_pages(page, order);
+	memcg_uncharge_slab(page, order, s);
+	__free_pages(page, order);
 }
 #define need_reserve_slab_rcu						\
 	free_kmem_cache_nodes(s);
 error:
 	if (flags & SLAB_PANIC)
-		panic("Cannot create slab %s size=%lu realsize=%u "
+		panic("Cannot create slab %s size=%lu realsize=%u order=%u offset=%u flags=%lx\n",
-			"order=%u offset=%u flags=%lx\n",
+		      s->name, (unsigned long)s->size, s->size,
-			s->name, (unsigned long)s->size, s->size,
+		      oo_order(s->oo), s->offset, flags);
-			oo_order(s->oo), s->offset, flags);
 	return -EINVAL;
 }

mm/sparse-vmemmap.c+4−4

@@ -166,8 +166,8 @@ void __meminit vmemmap_verify(pte_t *pte, int node, · 165 unmodified lines
166	int actual_node = early_pfn_to_nid(pfn);	166	int actual_node = early_pfn_to_nid(pfn);
167		167
168	if (node_distance(actual_node, node) > LOCAL_DISTANCE)	168	if (node_distance(actual_node, node) > LOCAL_DISTANCE)
169	printk(KERN_WARNING "[%lx-%lx] potential offnode "	169	pr_warn("[%lx-%lx] potential offnode page_structs\n",
170	"page_structs\n", start, end - 1);	170	start, end - 1);
171	}	171	}
172		172
173	pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node)	173	pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node)
@@ -292,8 +292,8 @@ void __init sparse_mem_maps_populate_node(struct page **map_map, · 118 unmodified lines
292	if (map_map[pnum])	292	if (map_map[pnum])
293	continue;	293	continue;
294	ms = __nr_to_section(pnum);	294	ms = __nr_to_section(pnum);
295	printk(KERN_ERR "%s: sparsemem memory map backing failed "	295	pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
296	"some memory will not be available.\n", __func__);	296	__func__);
297	ms->section_mem_map = 0;	297	ms->section_mem_map = 0;
298	}	298	}
299		299

mm/sparse.c+9−12

 	usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
 	if (usemap_nid != nid) {
-		printk(KERN_INFO
+		pr_info("node %d must be removed before remove section %ld\n",
-		       "node %d must be removed before remove section %ld\n",
+			nid, usemap_snr);
-		       nid, usemap_snr);
 		return;
 	}
 	/*
 	 * gather other removable sections for dynamic partitioning.
 	 * Just notify un-removable section's number here.
 	 */
-	printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr,
+	pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
-	       pgdat_snr, nid);
+		usemap_snr, pgdat_snr, nid);
-	printk(KERN_CONT
-	       " have a circular dependency on usemap and pgdat allocations\n");
 }
 #else
 static unsigned long * __init
 	usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
 							  size * usemap_count);
 	if (!usemap) {
-		printk(KERN_WARNING "%s: allocation failed\n", __func__);
+		pr_warn("%s: allocation failed\n", __func__);
 		return;
 	}
 		if (map_map[pnum])
 			continue;
 		ms = __nr_to_section(pnum);
-		printk(KERN_ERR "%s: sparsemem memory map backing failed "
+		pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
-			"some memory will not be available.\n", __func__);
+		       __func__);
 		ms->section_mem_map = 0;
 	}
 }
 	if (map)
 		return map;
-	printk(KERN_ERR "%s: sparsemem memory map backing failed "
+	pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
-			"some memory will not be available.\n", __func__);
+	       __func__);
 	ms->section_mem_map = 0;
 	return NULL;
 }

mm/swap_cgroup.c+2−3

@@ -174,9 +174,8 @@ int swap_cgroup_swapon(int type, unsigned long max_pages) · 173 unmodified lines
174		174
175	return 0;	175	return 0;
176	nomem:	176	nomem:
177	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");	177	pr_info("couldn't allocate enough memory for swap_cgroup\n");
178	printk(KERN_INFO	178	pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
179	"swap_cgroup can be disabled by swapaccount=0 boot option\n");
180	return -ENOMEM;	179	return -ENOMEM;
181	}	180	}
182		181

mm/swapfile.c+1−2

@@ -2526,8 +2526,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) · 2525 unmodified lines
2526	(swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;	2526	(swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
2527	enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);	2527	enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
2528		2528
2529	pr_info("Adding %uk swap on %s. "	2529	pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2530	"Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2531	p->pages<<(PAGE_SHIFT-10), name->name, p->prio,	2530	p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
2532	nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),	2531	nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
2533	(p->flags & SWP_SOLIDSTATE) ? "SS" : "",	2532	(p->flags & SWP_SOLIDSTATE) ? "SS" : "",

mm/userfaultfd.c+1−2

 			break;
 		}
 		if (unlikely(pmd_none(dst_pmdval)) &&
-		    unlikely(__pte_alloc(dst_mm, dst_vma, dst_pmd,
+		    unlikely(__pte_alloc(dst_mm, dst_pmd, dst_addr))) {
-					 dst_addr))) {
 			err = -ENOMEM;
 			break;
 		}

mm/util.c+124−0

@@ -396,6 +396,13 @@ int __page_mapcount(struct page *page) · 395 unmodified lines
396	}	396	}
397	EXPORT_SYMBOL_GPL(__page_mapcount);	397	EXPORT_SYMBOL_GPL(__page_mapcount);
398		398
		399	int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;
		400	int sysctl_overcommit_ratio __read_mostly = 50;
		401	unsigned long sysctl_overcommit_kbytes __read_mostly;
		402	int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
		403	unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
		404	unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
		405
399	int overcommit_ratio_handler(struct ctl_table *table, int write,	406	int overcommit_ratio_handler(struct ctl_table *table, int write,
400	void __user buffer, size_t lenp,	407	void __user buffer, size_t lenp,
401	loff_t *ppos)	408	loff_t *ppos)
@@ -437,6 +444,123 @@ unsigned long vm_commit_limit(void) · 35 unmodified lines
437	return allowed;	444	return allowed;
438	}	445	}
439		446
		447	/*
		448	* Make sure vm_committed_as in one cacheline and not cacheline shared with
		449	* other variables. It can be updated by several CPUs frequently.
		450	*/
		451	struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
		452
		453	/*
		454	* The global memory commitment made in the system can be a metric
		455	* that can be used to drive ballooning decisions when Linux is hosted
		456	* as a guest. On Hyper-V, the host implements a policy engine for dynamically
		457	* balancing memory across competing virtual machines that are hosted.
		458	* Several metrics drive this policy engine including the guest reported
		459	* memory commitment.
		460	*/
		461	unsigned long vm_memory_committed(void)
		462	{
		463	return percpu_counter_read_positive(&vm_committed_as);
		464	}
		465	EXPORT_SYMBOL_GPL(vm_memory_committed);
		466
		467	/*
		468	* Check that a process has enough memory to allocate a new virtual
		469	* mapping. 0 means there is enough memory for the allocation to
		470	* succeed and -ENOMEM implies there is not.
		471	*
		472	* We currently support three overcommit policies, which are set via the
		473	* vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
		474	*
		475	* Strict overcommit modes added 2002 Feb 26 by Alan Cox.
		476	* Additional code 2002 Jul 20 by Robert Love.
		477	*
		478	* cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
		479	*
		480	* Note this is a helper function intended to be used by LSMs which
		481	* wish to use this logic.
		482	*/
		483	int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
		484	{
		485	long free, allowed, reserve;
		486
		487	VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) <
		488	-(s64)vm_committed_as_batch * num_online_cpus(),
		489	"memory commitment underflow");
		490
		491	vm_acct_memory(pages);
		492
		493	/*
		494	* Sometimes we want to use more memory than we have
		495	*/
		496	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
		497	return 0;
		498
		499	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
		500	free = global_page_state(NR_FREE_PAGES);
		501	free += global_page_state(NR_FILE_PAGES);
		502
		503	/*
		504	* shmem pages shouldn't be counted as free in this
		505	* case, they can't be purged, only swapped out, and
		506	* that won't affect the overall amount of available
		507	* memory in the system.
		508	*/
		509	free -= global_page_state(NR_SHMEM);
		510
		511	free += get_nr_swap_pages();
		512
		513	/*
		514	* Any slabs which are created with the
		515	* SLAB_RECLAIM_ACCOUNT flag claim to have contents
		516	* which are reclaimable, under pressure. The dentry
		517	* cache and most inode caches should fall into this
		518	*/
		519	free += global_page_state(NR_SLAB_RECLAIMABLE);
		520
		521	/*
		522	* Leave reserved pages. The pages are not for anonymous pages.
		523	*/
		524	if (free <= totalreserve_pages)
		525	goto error;
		526	else
		527	free -= totalreserve_pages;
		528
		529	/*
		530	* Reserve some for root
		531	*/
		532	if (!cap_sys_admin)
		533	free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
		534
		535	if (free > pages)
		536	return 0;
		537
		538	goto error;
		539	}
		540
		541	allowed = vm_commit_limit();
		542	/*
		543	* Reserve some for root
		544	*/
		545	if (!cap_sys_admin)
		546	allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
		547
		548	/*
		549	* Don't let a single process grow so big a user can't recover
		550	*/
		551	if (mm) {
		552	reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
		553	allowed -= min_t(long, mm->total_vm / 32, reserve);
		554	}
		555
		556	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
		557	return 0;
		558	error:
		559	vm_unacct_memory(pages);
		560
		561	return -ENOMEM;
		562	}
		563
440	/**	564	/**
441	* get_cmdline() - copy the cmdline value to a buffer.	565	* get_cmdline() - copy the cmdline value to a buffer.
442	* @task: the task whose cmdline value to copy.	566	* @task: the task whose cmdline value to copy.

mm/vmalloc.c+15−16

@@ -469,8 +469,8 @@ overflow: · 468 unmodified lines
469	goto retry;	469	goto retry;
470	}	470	}
471	if (printk_ratelimit())	471	if (printk_ratelimit())
472	pr_warn("vmap allocation for size %lu failed: "	472	pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
473	"use vmalloc=<size> to increase size.\n", size);	473	size);
474	kfree(va);	474	kfree(va);
475	return ERR_PTR(-EBUSY);	475	return ERR_PTR(-EBUSY);
476	}	476	}
@@ -531,22 +531,21 @@ static void unmap_vmap_area(struct vmap_area *va) · 54 unmodified lines
531	static void vmap_debug_free_range(unsigned long start, unsigned long end)	531	static void vmap_debug_free_range(unsigned long start, unsigned long end)
532	{	532	{
533	/*	533	/*
534	* Unmap page tables and force a TLB flush immediately if	534	* Unmap page tables and force a TLB flush immediately if pagealloc
535	* CONFIG_DEBUG_PAGEALLOC is set. This catches use after free	535	* debugging is enabled. This catches use after free bugs similarly to
536	* bugs similarly to those in linear kernel virtual address	536	* those in linear kernel virtual address space after a page has been
537	* space after a page has been freed.	537	* freed.
538	*	538	*
539	* All the lazy freeing logic is still retained, in order to	539	* All the lazy freeing logic is still retained, in order to minimise
540	* minimise intrusiveness of this debugging feature.	540	* intrusiveness of this debugging feature.
541	*	541	*
542	* This is going to be slow (linear kernel virtual address	542	* This is going to be slow (linear kernel virtual address debugging
543	* debugging doesn't do a broadcast TLB flush so it is a lot	543	* doesn't do a broadcast TLB flush so it is a lot faster).
544	* faster).
545	*/	544	*/
546	#ifdef CONFIG_DEBUG_PAGEALLOC	545	if (debug_pagealloc_enabled()) {
547	vunmap_page_range(start, end);	546	vunmap_page_range(start, end);
548	flush_tlb_kernel_range(start, end);	547	flush_tlb_kernel_range(start, end);
549	#endif	548	}
550	}	549	}
551		550
552	/*	551	/*
@@ -1086,7 +1085,7 @@ void vm_unmap_ram(const void *mem, unsigned int count) · 533 unmodified lines
1086	BUG_ON(!addr);	1085	BUG_ON(!addr);
1087	BUG_ON(addr < VMALLOC_START);	1086	BUG_ON(addr < VMALLOC_START);
1088	BUG_ON(addr > VMALLOC_END);	1087	BUG_ON(addr > VMALLOC_END);
1089	BUG_ON(!IS_ALIGNED(addr, PAGE_SIZE));	1088	BUG_ON(!PAGE_ALIGNED(addr));
1090		1089
1091	debug_check_no_locks_freed(mem, size);	1090	debug_check_no_locks_freed(mem, size);
1092	vmap_debug_free_range(addr, addr+size);	1091	vmap_debug_free_range(addr, addr+size);

mm/vmscan.c+61−108

  *
  * @memcg specifies the memory cgroup to target. If it is not NULL,
  * only shrinkers with SHRINKER_MEMCG_AWARE set will be called to scan
- * objects from the memory cgroup specified. Otherwise all shrinkers
+ * objects from the memory cgroup specified. Otherwise, only unaware
- * are called, and memcg aware shrinkers are supposed to scan the
+ * shrinkers are called.
- * global list then.
  *
  * @nr_scanned and @nr_eligible form a ratio that indicate how much of
  * the available objects should be scanned.  Page reclaim for example
 	struct shrinker *shrinker;
 	unsigned long freed = 0;
-	if (memcg && !memcg_kmem_online(memcg))
+	if (memcg && (!memcg_kmem_enabled() || !mem_cgroup_online(memcg)))
 		return 0;
 	if (nr_scanned == 0)
 			.memcg = memcg,
 		};
-		if (memcg && !(shrinker->flags & SHRINKER_MEMCG_AWARE))
+		/*
+		 * If kernel memory accounting is disabled, we ignore
+		 * SHRINKER_MEMCG_AWARE flag and call all shrinkers
+		 * passing NULL for memcg.
+		 */
+		if (memcg_kmem_enabled() &&
+		    !!memcg != !!(shrinker->flags & SHRINKER_MEMCG_AWARE))
 			continue;
 		if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
 	 * Note that if SetPageDirty is always performed via set_page_dirty,
 	 * and thus under tree_lock, then this ordering is not required.
 	 */
-	if (!page_freeze_refs(page, 2))
+	if (!page_ref_freeze(page, 2))
 		goto cannot_free;
 	/* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
 	if (unlikely(PageDirty(page))) {
-		page_unfreeze_refs(page, 2);
+		page_ref_unfreeze(page, 2);
 		goto cannot_free;
 	}
 		 * drops the pagecache ref for us without requiring another
 		 * atomic operation.
 		 */
-		page_unfreeze_refs(page, 1);
+		page_ref_unfreeze(page, 1);
 		return 1;
 	}
 	return 0;
 	} while (memcg);
 }
-static bool zone_balanced(struct zone *zone, int order,
+static bool zone_balanced(struct zone *zone, int order, bool highorder,
-			  unsigned long balance_gap, int classzone_idx)
+			unsigned long balance_gap, int classzone_idx)
 {
-	if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone) +
+	unsigned long mark = high_wmark_pages(zone) + balance_gap;
-				    balance_gap, classzone_idx))
-		return false;
-	if (IS_ENABLED(CONFIG_COMPACTION) && order && compaction_suitable(zone,
+	/*
-				order, 0, classzone_idx) == COMPACT_SKIPPED)
+	 * When checking from pgdat_balanced(), kswapd should stop and sleep
-		return false;
+	 * when it reaches the high order-0 watermark and let kcompactd take
+	 * over. Other callers such as wakeup_kswapd() want to determine the
+	 * true high-order watermark.
+	 */
+	if (IS_ENABLED(CONFIG_COMPACTION) && !highorder) {
+		mark += (1UL << order);
+		order = 0;
+	}
-	return true;
+	return zone_watermark_ok_safe(zone, order, mark, classzone_idx);
 }
 /*
 			continue;
 		}
-		if (zone_balanced(zone, order, 0, i))
+		if (zone_balanced(zone, order, false, 0, i))
 			balanced_pages += zone->managed_pages;
 		else if (!order)
 			return false;
  */
 static bool kswapd_shrink_zone(struct zone *zone,
 			       int classzone_idx,
-			       struct scan_control *sc,
+			       struct scan_control *sc)
-			       unsigned long *nr_attempted)
 {
-	int testorder = sc->order;
 	unsigned long balance_gap;
 	bool lowmem_pressure;
 	/* Reclaim above the high watermark. */
 	sc->nr_to_reclaim = max(SWAP_CLUSTER_MAX, high_wmark_pages(zone));
-	/*
-	 * Kswapd reclaims only single pages with compaction enabled. Trying
-	 * too hard to reclaim until contiguous free pages have become
-	 * available can hurt performance by evicting too much useful data
-	 * from memory. Do not reclaim more than needed for compaction.
-	 */
-	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
-			compaction_suitable(zone, sc->order, 0, classzone_idx)
-							!= COMPACT_SKIPPED)
-		testorder = 0;
 	/*
 	 * We put equal pressure on every zone, unless one zone has way too
 	 * many pages free already. The "too many pages" is defined as the
 	 * reclaim is necessary
 	 */
 	lowmem_pressure = (buffer_heads_over_limit && is_highmem(zone));
-	if (!lowmem_pressure && zone_balanced(zone, testorder,
+	if (!lowmem_pressure && zone_balanced(zone, sc->order, false,
 						balance_gap, classzone_idx))
 		return true;
 	shrink_zone(zone, sc, zone_idx(zone) == classzone_idx);
-	/* Account for the number of pages attempted to reclaim */
-	*nr_attempted += sc->nr_to_reclaim;
 	clear_bit(ZONE_WRITEBACK, &zone->flags);
 	/*
 	 * waits.
 	 */
 	if (zone_reclaimable(zone) &&
-	    zone_balanced(zone, testorder, 0, classzone_idx)) {
+	    zone_balanced(zone, sc->order, false, 0, classzone_idx)) {
 		clear_bit(ZONE_CONGESTED, &zone->flags);
 		clear_bit(ZONE_DIRTY, &zone->flags);
 	}
  * For kswapd, balance_pgdat() will work across all this node's zones until
  * they are all at high_wmark_pages(zone).
  *
- * Returns the final order kswapd was reclaiming at
+ * Returns the highest zone idx kswapd was reclaiming at
  *
  * There is special handling here for zones which are full of pinned pages.
  * This can happen if the pages are all mlocked, or if they are all used by
  * interoperates with the page allocator fallback scheme to ensure that aging
  * of pages is balanced across the zones.
  */
-static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
+static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
-							int *classzone_idx)
 {
 	int i;
 	int end_zone = 0;	/* Inclusive.  0 = ZONE_DMA */
 	count_vm_event(PAGEOUTRUN);
 	do {
-		unsigned long nr_attempted = 0;
 		bool raise_priority = true;
-		bool pgdat_needs_compaction = (order > 0);
 		sc.nr_reclaimed = 0;
 				break;
 			}
-			if (!zone_balanced(zone, order, 0, 0)) {
+			if (!zone_balanced(zone, order, false, 0, 0)) {
 				end_zone = i;
 				break;
 			} else {
 		if (i < 0)
 			goto out;
-		for (i = 0; i <= end_zone; i++) {
-			struct zone *zone = pgdat->node_zones + i;
-			if (!populated_zone(zone))
-				continue;
-			/*
-			 * If any zone is currently balanced then kswapd will
-			 * not call compaction as it is expected that the
-			 * necessary pages are already available.
-			 */
-			if (pgdat_needs_compaction &&
-					zone_watermark_ok(zone, order,
-						low_wmark_pages(zone),
-						*classzone_idx, 0))
-				pgdat_needs_compaction = false;
-		}
 		/*
 		 * If we're getting trouble reclaiming, start doing writepage
 		 * even in laptop mode.
 			 * that that high watermark would be met at 100%
 			 * efficiency.
 			 */
-			if (kswapd_shrink_zone(zone, end_zone,
+			if (kswapd_shrink_zone(zone, end_zone, &sc))
-					       &sc, &nr_attempted))
 				raise_priority = false;
 		}
 				pfmemalloc_watermark_ok(pgdat))
 			wake_up_all(&pgdat->pfmemalloc_wait);
-		/*
-		 * Fragmentation may mean that the system cannot be rebalanced
-		 * for high-order allocations in all zones. If twice the
-		 * allocation size has been reclaimed and the zones are still
-		 * not balanced then recheck the watermarks at order-0 to
-		 * prevent kswapd reclaiming excessively. Assume that a
-		 * process requested a high-order can direct reclaim/compact.
-		 */
-		if (order && sc.nr_reclaimed >= 2UL << order)
-			order = sc.order = 0;
 		/* Check if kswapd should be suspending */
 		if (try_to_freeze() || kthread_should_stop())
 			break;
-		/*
-		 * Compact if necessary and kswapd is reclaiming at least the
-		 * high watermark number of pages as requsted
-		 */
-		if (pgdat_needs_compaction && sc.nr_reclaimed > nr_attempted)
-			compact_pgdat(pgdat, order);
 		/*
 		 * Raise priority if scanning rate is too low or there was no
 		 * progress in reclaiming pages
 		if (raise_priority || !sc.nr_reclaimed)
 			sc.priority--;
 	} while (sc.priority >= 1 &&
-		 !pgdat_balanced(pgdat, order, *classzone_idx));
+			!pgdat_balanced(pgdat, order, classzone_idx));
 out:
 	/*
-	 * Return the order we were reclaiming at so prepare_kswapd_sleep()
+	 * Return the highest zone idx we were reclaiming at so
-	 * makes a decision on the order we were last reclaiming at. However,
+	 * prepare_kswapd_sleep() makes the same decisions as here.
-	 * if another caller entered the allocator slow path while kswapd
-	 * was awake, order will remain at the higher level
 	 */
-	*classzone_idx = end_zone;
+	return end_zone;
-	return order;
 }
-static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx)
+static void kswapd_try_to_sleep(pg_data_t *pgdat, int order,
+				int classzone_idx, int balanced_classzone_idx)
 {
 	long remaining = 0;
 	DEFINE_WAIT(wait);
 	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
 	/* Try to sleep for a short interval */
-	if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
+	if (prepare_kswapd_sleep(pgdat, order, remaining,
+						balanced_classzone_idx)) {
 		remaining = schedule_timeout(HZ/10);
 		finish_wait(&pgdat->kswapd_wait, &wait);
 		prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
 	 * After a short sleep, check if it was a premature sleep. If not, then
 	 * go fully to sleep until explicitly woken up.
 	 */
-	if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
+	if (prepare_kswapd_sleep(pgdat, order, remaining,
+						balanced_classzone_idx)) {
 		trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
 		/*
 		 */
 		reset_isolation_suitable(pgdat);
+		/*
+		 * We have freed the memory, now we should compact it to make
+		 * allocation of the requested order possible.
+		 */
+		wakeup_kcompactd(pgdat, order, classzone_idx);
 		if (!kthread_should_stop())
 			schedule();
 static int kswapd(void *p)
 {
 	unsigned long order, new_order;
-	unsigned balanced_order;
 	int classzone_idx, new_classzone_idx;
 	int balanced_classzone_idx;
 	pg_data_t *pgdat = (pg_data_t*)p;
 	set_freezable();
 	order = new_order = 0;
-	balanced_order = 0;
 	classzone_idx = new_classzone_idx = pgdat->nr_zones - 1;
 	balanced_classzone_idx = classzone_idx;
 	for ( ; ; ) {
 		bool ret;
 		/*
-		 * If the last balance_pgdat was unsuccessful it's unlikely a
+		 * While we were reclaiming, there might have been another
-		 * new request of a similar or harder type will succeed soon
+		 * wakeup, so check the values.
-		 * so consider going to sleep on the basis we reclaimed at
 		 */
-		if (balanced_classzone_idx >= new_classzone_idx &&
+		new_order = pgdat->kswapd_max_order;
-					balanced_order == new_order) {
+		new_classzone_idx = pgdat->classzone_idx;
-			new_order = pgdat->kswapd_max_order;
+		pgdat->kswapd_max_order =  0;
-			new_classzone_idx = pgdat->classzone_idx;
+		pgdat->classzone_idx = pgdat->nr_zones - 1;
-			pgdat->kswapd_max_order =  0;
-			pgdat->classzone_idx = pgdat->nr_zones - 1;
-		}
 		if (order < new_order || classzone_idx > new_classzone_idx) {
 			/*
 			order = new_order;
 			classzone_idx = new_classzone_idx;
 		} else {
-			kswapd_try_to_sleep(pgdat, balanced_order,
+			kswapd_try_to_sleep(pgdat, order, classzone_idx,
 						balanced_classzone_idx);
 			order = pgdat->kswapd_max_order;
 			classzone_idx = pgdat->classzone_idx;
 		 */
 		if (!ret) {
 			trace_mm_vmscan_kswapd_wake(pgdat->node_id, order);
-			balanced_classzone_idx = classzone_idx;
+			balanced_classzone_idx = balance_pgdat(pgdat, order,
-			balanced_order = balance_pgdat(pgdat, order,
+								classzone_idx);
-						&balanced_classzone_idx);
 		}
 	}
 	}
 	if (!waitqueue_active(&pgdat->kswapd_wait))
 		return;
-	if (zone_balanced(zone, order, 0, 0))
+	if (zone_balanced(zone, order, true, 0, 0))
 		return;
 	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);

mm/vmstat.c+2−0

 	"compact_stall",
 	"compact_fail",
 	"compact_success",
+	"compact_daemon_wake",
 #endif
 #ifdef CONFIG_HUGETLB_PAGE
 	"thp_collapse_alloc_failed",
 	"thp_split_page",
 	"thp_split_page_failed",
+	"thp_deferred_split_page",
 	"thp_split_pmd",
 	"thp_zero_page_alloc",
 	"thp_zero_page_alloc_failed",

mm/workingset.c+8−2

 	shadow_nodes = list_lru_shrink_count(&workingset_shadow_nodes, sc);
 	local_irq_enable();
-	pages = node_present_pages(sc->nid);
+	if (memcg_kmem_enabled())
+		pages = mem_cgroup_node_nr_lru_pages(sc->memcg, sc->nid,
+						     LRU_ALL_FILE);
+	else
+		pages = node_page_state(sc->nid, NR_ACTIVE_FILE) +
+			node_page_state(sc->nid, NR_INACTIVE_FILE);
 	/*
 	 * Active cache pages are limited to 50% of memory, and shadow
 	 * entries that represent a refault distance bigger than that
 	.count_objects = count_shadow_nodes,
 	.scan_objects = scan_shadow_nodes,
 	.seeks = DEFAULT_SEEKS,
-	.flags = SHRINKER_NUMA_AWARE,
+	.flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE,
 };
 /*

mm/zsmalloc.c+16−13

 #endif
 	char *vm_addr; /* address of kmap_atomic()'ed pages */
 	enum zs_mapmode vm_mm; /* mapping mode */
-	bool huge;
 };
 static int create_handle_cache(struct zs_pool *pool)
 	debugfs_remove_recursive(zs_stat_root);
 }
+static unsigned long zs_can_compact(struct size_class *class);
 static int zs_stats_size_show(struct seq_file *s, void *v)
 {
 	int i;
 	struct size_class *class;
 	int objs_per_zspage;
 	unsigned long class_almost_full, class_almost_empty;
-	unsigned long obj_allocated, obj_used, pages_used;
+	unsigned long obj_allocated, obj_used, pages_used, freeable;
 	unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
 	unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
+	unsigned long total_freeable = 0;
-	seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s\n",
+	seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
 			"class", "size", "almost_full", "almost_empty",
 			"obj_allocated", "obj_used", "pages_used",
-			"pages_per_zspage");
+			"pages_per_zspage", "freeable");
 	for (i = 0; i < zs_size_classes; i++) {
 		class = pool->size_class[i];
 		class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
 		obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
 		obj_used = zs_stat_get(class, OBJ_USED);
+		freeable = zs_can_compact(class);
 		spin_unlock(&class->lock);
 		objs_per_zspage = get_maxobj_per_zspage(class->size,
 		pages_used = obj_allocated / objs_per_zspage *
 				class->pages_per_zspage;
-		seq_printf(s, " %5u %5u %11lu %12lu %13lu %10lu %10lu %16d\n",
+		seq_printf(s, " %5u %5u %11lu %12lu %13lu"
+				" %10lu %10lu %16d %8lu\n",
 			i, class->size, class_almost_full, class_almost_empty,
 			obj_allocated, obj_used, pages_used,
-			class->pages_per_zspage);
+			class->pages_per_zspage, freeable);
 		total_class_almost_full += class_almost_full;
 		total_class_almost_empty += class_almost_empty;
 		total_objs += obj_allocated;
 		total_used_objs += obj_used;
 		total_pages += pages_used;
+		total_freeable += freeable;
 	}
 	seq_puts(s, "\n");
-	seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu\n",
+	seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
 			"Total", "", total_class_almost_full,
 			total_class_almost_empty, total_objs,
-			total_used_objs, total_pages);
+			total_used_objs, total_pages, "", total_freeable);
 	return 0;
 }
 		goto out;
 	buf = area->vm_buf;
-	if (!area->huge) {
+	buf = buf + ZS_HANDLE_SIZE;
-		buf = buf + ZS_HANDLE_SIZE;
+	size -= ZS_HANDLE_SIZE;
-		size -= ZS_HANDLE_SIZE;
+	off += ZS_HANDLE_SIZE;
-		off += ZS_HANDLE_SIZE;
-	}
 	sizes[0] = PAGE_SIZE - off;
 	sizes[1] = size - sizes[0];

net/core/sock.c+1−1

@@ -1903,7 +1903,7 @@ EXPORT_SYMBOL(sock_cmsg_send); · 1902 unmodified lines
1903	bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)	1903	bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
1904	{	1904	{
1905	if (pfrag->page) {	1905	if (pfrag->page) {
1906	if (atomic_read(&pfrag->page->_count) == 1) {	1906	if (page_ref_count(pfrag->page) == 1) {
1907	pfrag->offset = 0;	1907	pfrag->offset = 0;
1908	return true;	1908	return true;
1909	}	1909	}

sound/drivers/pcsp/pcsp.c+5−4

 #include <linux/input.h>
 #include <linux/delay.h>
 #include <linux/bitops.h>
+#include <linux/mm.h>
 #include "pcsp_input.h"
 #include "pcsp.h"
 		return err;
 	}
-#ifdef CONFIG_DEBUG_PAGEALLOC
 	/* Well, CONFIG_DEBUG_PAGEALLOC makes the sound horrible. Lets alert */
-	printk(KERN_WARNING "PCSP: CONFIG_DEBUG_PAGEALLOC is enabled, "
+	if (debug_pagealloc_enabled()) {
-	       "which may make the sound noisy.\n");
+		printk(KERN_WARNING "PCSP: CONFIG_DEBUG_PAGEALLOC is enabled, "
-#endif
+		       "which may make the sound noisy.\n");
+	}
 	return 0;
 }

tools/testing/radix-tree/.gitignore+2−0

@@ -0,0 +1,2 @@
	1	main
	2	radix-tree.c

tools/testing/radix-tree/Makefile+19−0

@@ -0,0 +1,19 @@
	1
	2	CFLAGS += -I. -g -Wall -D_LGPL_SOURCE
	3	LDFLAGS += -lpthread -lurcu
	4	TARGETS = main
	5	OFILES = main.o radix-tree.o linux.o test.o tag_check.o find_next_bit.o \
	6	regression1.o regression2.o regression3.o
	7
	8	targets: $(TARGETS)
	9
	10	main: $(OFILES)
	11	$(CC) $(CFLAGS) $(LDFLAGS) $(OFILES) -o main
	12
	13	clean:
	14	$(RM) -f $(TARGETS) *.o radix-tree.c
	15
	16	$(OFILES): .h /*.h
	17
	18	radix-tree.c: ../../../lib/radix-tree.c
	19	sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@

tools/testing/radix-tree/find_next_bit.c+57−0

@@ -0,0 +1,57 @@
	1	/* find_next_bit.c: fallback find next bit implementation
	2	*
	3	* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
	4	* Written by David Howells (dhowells@redhat.com)
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; either version
	9	* 2 of the License, or (at your option) any later version.
	10	*/
	11
	12	#include <linux/types.h>
	13	#include <linux/bitops.h>
	14
	15	#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
	16
	17	/*
	18	* Find the next set bit in a memory region.
	19	*/
	20	unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
	21	unsigned long offset)
	22	{
	23	const unsigned long *p = addr + BITOP_WORD(offset);
	24	unsigned long result = offset & ~(BITS_PER_LONG-1);
	25	unsigned long tmp;
	26
	27	if (offset >= size)
	28	return size;
	29	size -= result;
	30	offset %= BITS_PER_LONG;
	31	if (offset) {
	32	tmp = *(p++);
	33	tmp &= (~0UL << offset);
	34	if (size < BITS_PER_LONG)
	35	goto found_first;
	36	if (tmp)
	37	goto found_middle;
	38	size -= BITS_PER_LONG;
	39	result += BITS_PER_LONG;
	40	}
	41	while (size & ~(BITS_PER_LONG-1)) {
	42	if ((tmp = *(p++)))
	43	goto found_middle;
	44	result += BITS_PER_LONG;
	45	size -= BITS_PER_LONG;
	46	}
	47	if (!size)
	48	return result;
	49	tmp = *p;
	50
	51	found_first:
	52	tmp &= (~0UL >> (BITS_PER_LONG - size));
	53	if (tmp == 0UL) /* Are any bits set? */
	54	return result + size; /* Nope. */
	55	found_middle:
	56	return result + __ffs(tmp);
	57	}

tools/testing/radix-tree/linux.c+60−0

@@ -0,0 +1,60 @@
	1	#include <stdlib.h>
	2	#include <string.h>
	3	#include <malloc.h>
	4	#include <unistd.h>
	5	#include <assert.h>
	6
	7	#include <linux/mempool.h>
	8	#include <linux/slab.h>
	9	#include <urcu/uatomic.h>
	10
	11	int nr_allocated;
	12
	13	void mempool_alloc(mempool_t pool, int gfp_mask)
	14	{
	15	return pool->alloc(gfp_mask, pool->data);
	16	}
	17
	18	void mempool_free(void element, mempool_t pool)
	19	{
	20	pool->free(element, pool->data);
	21	}
	22
	23	mempool_t mempool_create(int min_nr, mempool_alloc_t alloc_fn,
	24	mempool_free_t free_fn, void pool_data)
	25	{
	26	mempool_t ret = malloc(sizeof(ret));
	27
	28	ret->alloc = alloc_fn;
	29	ret->free = free_fn;
	30	ret->data = pool_data;
	31	return ret;
	32	}
	33
	34	void kmem_cache_alloc(struct kmem_cache cachep, int flags)
	35	{
	36	void *ret = malloc(cachep->size);
	37	if (cachep->ctor)
	38	cachep->ctor(ret);
	39	uatomic_inc(&nr_allocated);
	40	return ret;
	41	}
	42
	43	void kmem_cache_free(struct kmem_cache cachep, void objp)
	44	{
	45	assert(objp);
	46	uatomic_dec(&nr_allocated);
	47	memset(objp, 0, cachep->size);
	48	free(objp);
	49	}
	50
	51	struct kmem_cache *
	52	kmem_cache_create(const char *name, size_t size, size_t offset,
	53	unsigned long flags, void (ctor)(void ))
	54	{
	55	struct kmem_cache ret = malloc(sizeof(ret));
	56
	57	ret->size = size;
	58	ret->ctor = ctor;
	59	return ret;
	60	}

tools/testing/radix-tree/linux/bitops.h+150−0

@@ -0,0 +1,150 @@
	1	#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
	2	#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
	3
	4	#include <linux/types.h>
	5
	6	#define BITOP_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
	7	#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
	8
	9	/**
	10	* __set_bit - Set a bit in memory
	11	* @nr: the bit to set
	12	* @addr: the address to start counting from
	13	*
	14	* Unlike set_bit(), this function is non-atomic and may be reordered.
	15	* If it's called on the same region of memory simultaneously, the effect
	16	* may be that only one operation succeeds.
	17	*/
	18	static inline void __set_bit(int nr, volatile unsigned long *addr)
	19	{
	20	unsigned long mask = BITOP_MASK(nr);
	21	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	22
	23	*p \|= mask;
	24	}
	25
	26	static inline void __clear_bit(int nr, volatile unsigned long *addr)
	27	{
	28	unsigned long mask = BITOP_MASK(nr);
	29	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	30
	31	*p &= ~mask;
	32	}
	33
	34	/**
	35	* __change_bit - Toggle a bit in memory
	36	* @nr: the bit to change
	37	* @addr: the address to start counting from
	38	*
	39	* Unlike change_bit(), this function is non-atomic and may be reordered.
	40	* If it's called on the same region of memory simultaneously, the effect
	41	* may be that only one operation succeeds.
	42	*/
	43	static inline void __change_bit(int nr, volatile unsigned long *addr)
	44	{
	45	unsigned long mask = BITOP_MASK(nr);
	46	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	47
	48	*p ^= mask;
	49	}
	50
	51	/**
	52	* __test_and_set_bit - Set a bit and return its old value
	53	* @nr: Bit to set
	54	* @addr: Address to count from
	55	*
	56	* This operation is non-atomic and can be reordered.
	57	* If two examples of this operation race, one can appear to succeed
	58	* but actually fail. You must protect multiple accesses with a lock.
	59	*/
	60	static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
	61	{
	62	unsigned long mask = BITOP_MASK(nr);
	63	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	64	unsigned long old = *p;
	65
	66	*p = old \| mask;
	67	return (old & mask) != 0;
	68	}
	69
	70	/**
	71	* __test_and_clear_bit - Clear a bit and return its old value
	72	* @nr: Bit to clear
	73	* @addr: Address to count from
	74	*
	75	* This operation is non-atomic and can be reordered.
	76	* If two examples of this operation race, one can appear to succeed
	77	* but actually fail. You must protect multiple accesses with a lock.
	78	*/
	79	static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
	80	{
	81	unsigned long mask = BITOP_MASK(nr);
	82	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	83	unsigned long old = *p;
	84
	85	*p = old & ~mask;
	86	return (old & mask) != 0;
	87	}
	88
	89	/* WARNING: non atomic and it can be reordered! */
	90	static inline int __test_and_change_bit(int nr,
	91	volatile unsigned long *addr)
	92	{
	93	unsigned long mask = BITOP_MASK(nr);
	94	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	95	unsigned long old = *p;
	96
	97	*p = old ^ mask;
	98	return (old & mask) != 0;
	99	}
	100
	101	/**
	102	* test_bit - Determine whether a bit is set
	103	* @nr: bit number to test
	104	* @addr: Address to start counting from
	105	*/
	106	static inline int test_bit(int nr, const volatile unsigned long *addr)
	107	{
	108	return 1UL & (addr[BITOP_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
	109	}
	110
	111	/**
	112	* __ffs - find first bit in word.
	113	* @word: The word to search
	114	*
	115	* Undefined if no bit exists, so code should check against 0 first.
	116	*/
	117	static inline unsigned long __ffs(unsigned long word)
	118	{
	119	int num = 0;
	120
	121	if ((word & 0xffffffff) == 0) {
	122	num += 32;
	123	word >>= 32;
	124	}
	125	if ((word & 0xffff) == 0) {
	126	num += 16;
	127	word >>= 16;
	128	}
	129	if ((word & 0xff) == 0) {
	130	num += 8;
	131	word >>= 8;
	132	}
	133	if ((word & 0xf) == 0) {
	134	num += 4;
	135	word >>= 4;
	136	}
	137	if ((word & 0x3) == 0) {
	138	num += 2;
	139	word >>= 2;
	140	}
	141	if ((word & 0x1) == 0)
	142	num += 1;
	143	return num;
	144	}
	145
	146	unsigned long find_next_bit(const unsigned long *addr,
	147	unsigned long size,
	148	unsigned long offset);
	149
	150	#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */

tools/testing/radix-tree/linux/bitops/__ffs.h+43−0

@@ -0,0 +1,43 @@
	1	#ifndef _ASM_GENERIC_BITOPS___FFS_H_
	2	#define _ASM_GENERIC_BITOPS___FFS_H_
	3
	4	#include <asm/types.h>
	5
	6	/**
	7	* __ffs - find first bit in word.
	8	* @word: The word to search
	9	*
	10	* Undefined if no bit exists, so code should check against 0 first.
	11	*/
	12	static inline unsigned long __ffs(unsigned long word)
	13	{
	14	int num = 0;
	15
	16	#if BITS_PER_LONG == 64
	17	if ((word & 0xffffffff) == 0) {
	18	num += 32;
	19	word >>= 32;
	20	}
	21	#endif
	22	if ((word & 0xffff) == 0) {
	23	num += 16;
	24	word >>= 16;
	25	}
	26	if ((word & 0xff) == 0) {
	27	num += 8;
	28	word >>= 8;
	29	}
	30	if ((word & 0xf) == 0) {
	31	num += 4;
	32	word >>= 4;
	33	}
	34	if ((word & 0x3) == 0) {
	35	num += 2;
	36	word >>= 2;
	37	}
	38	if ((word & 0x1) == 0)
	39	num += 1;
	40	return num;
	41	}
	42
	43	#endif /* _ASM_GENERIC_BITOPS___FFS_H_ */

tools/testing/radix-tree/linux/bitops/ffs.h+41−0

@@ -0,0 +1,41 @@
	1	#ifndef _ASM_GENERIC_BITOPS_FFS_H_
	2	#define _ASM_GENERIC_BITOPS_FFS_H_
	3
	4	/**
	5	* ffs - find first bit set
	6	* @x: the word to search
	7	*
	8	* This is defined the same way as
	9	* the libc and compiler builtin ffs routines, therefore
	10	* differs in spirit from the above ffz (man ffs).
	11	*/
	12	static inline int ffs(int x)
	13	{
	14	int r = 1;
	15
	16	if (!x)
	17	return 0;
	18	if (!(x & 0xffff)) {
	19	x >>= 16;
	20	r += 16;
	21	}
	22	if (!(x & 0xff)) {
	23	x >>= 8;
	24	r += 8;
	25	}
	26	if (!(x & 0xf)) {
	27	x >>= 4;
	28	r += 4;
	29	}
	30	if (!(x & 3)) {
	31	x >>= 2;
	32	r += 2;
	33	}
	34	if (!(x & 1)) {
	35	x >>= 1;
	36	r += 1;
	37	}
	38	return r;
	39	}
	40
	41	#endif /* _ASM_GENERIC_BITOPS_FFS_H_ */

tools/testing/radix-tree/linux/bitops/ffz.h+12−0

@@ -0,0 +1,12 @@
	1	#ifndef _ASM_GENERIC_BITOPS_FFZ_H_
	2	#define _ASM_GENERIC_BITOPS_FFZ_H_
	3
	4	/*
	5	* ffz - find first zero in word.
	6	* @word: The word to search
	7	*
	8	* Undefined if no zero exists, so code should check against ~0UL first.
	9	*/
	10	#define ffz(x) __ffs(~(x))
	11
	12	#endif /* _ASM_GENERIC_BITOPS_FFZ_H_ */

tools/testing/radix-tree/linux/bitops/find.h+13−0

@@ -0,0 +1,13 @@
	1	#ifndef _ASM_GENERIC_BITOPS_FIND_H_
	2	#define _ASM_GENERIC_BITOPS_FIND_H_
	3
	4	extern unsigned long find_next_bit(const unsigned long *addr, unsigned long
	5	size, unsigned long offset);
	6
	7	extern unsigned long find_next_zero_bit(const unsigned long *addr, unsigned
	8	long size, unsigned long offset);
	9
	10	#define find_first_bit(addr, size) find_next_bit((addr), (size), 0)
	11	#define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0)
	12
	13	#endif /_ASM_GENERIC_BITOPS_FIND_H_ /

tools/testing/radix-tree/linux/bitops/fls.h+41−0

@@ -0,0 +1,41 @@
	1	#ifndef _ASM_GENERIC_BITOPS_FLS_H_
	2	#define _ASM_GENERIC_BITOPS_FLS_H_
	3
	4	/**
	5	* fls - find last (most-significant) bit set
	6	* @x: the word to search
	7	*
	8	* This is defined the same way as ffs.
	9	* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
	10	*/
	11
	12	static inline int fls(int x)
	13	{
	14	int r = 32;
	15
	16	if (!x)
	17	return 0;
	18	if (!(x & 0xffff0000u)) {
	19	x <<= 16;
	20	r -= 16;
	21	}
	22	if (!(x & 0xff000000u)) {
	23	x <<= 8;
	24	r -= 8;
	25	}
	26	if (!(x & 0xf0000000u)) {
	27	x <<= 4;
	28	r -= 4;
	29	}
	30	if (!(x & 0xc0000000u)) {
	31	x <<= 2;
	32	r -= 2;
	33	}
	34	if (!(x & 0x80000000u)) {
	35	x <<= 1;
	36	r -= 1;
	37	}
	38	return r;
	39	}
	40
	41	#endif /* _ASM_GENERIC_BITOPS_FLS_H_ */

tools/testing/radix-tree/linux/bitops/fls64.h+14−0

@@ -0,0 +1,14 @@
	1	#ifndef _ASM_GENERIC_BITOPS_FLS64_H_
	2	#define _ASM_GENERIC_BITOPS_FLS64_H_
	3
	4	#include <asm/types.h>
	5
	6	static inline int fls64(__u64 x)
	7	{
	8	__u32 h = x >> 32;
	9	if (h)
	10	return fls(h) + 32;
	11	return fls(x);
	12	}
	13
	14	#endif /* _ASM_GENERIC_BITOPS_FLS64_H_ */

tools/testing/radix-tree/linux/bitops/hweight.h+11−0

@@ -0,0 +1,11 @@
	1	#ifndef _ASM_GENERIC_BITOPS_HWEIGHT_H_
	2	#define _ASM_GENERIC_BITOPS_HWEIGHT_H_
	3
	4	#include <asm/types.h>
	5
	6	extern unsigned int hweight32(unsigned int w);
	7	extern unsigned int hweight16(unsigned int w);
	8	extern unsigned int hweight8(unsigned int w);
	9	extern unsigned long hweight64(__u64 w);
	10
	11	#endif /* _ASM_GENERIC_BITOPS_HWEIGHT_H_ */

tools/testing/radix-tree/linux/bitops/le.h+53−0

@@ -0,0 +1,53 @@
	1	#ifndef _ASM_GENERIC_BITOPS_LE_H_
	2	#define _ASM_GENERIC_BITOPS_LE_H_
	3
	4	#include <asm/types.h>
	5	#include <asm/byteorder.h>
	6
	7	#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
	8	#define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7)
	9
	10	#if defined(__LITTLE_ENDIAN)
	11
	12	#define generic_test_le_bit(nr, addr) test_bit(nr, addr)
	13	#define generic___set_le_bit(nr, addr) __set_bit(nr, addr)
	14	#define generic___clear_le_bit(nr, addr) __clear_bit(nr, addr)
	15
	16	#define generic_test_and_set_le_bit(nr, addr) test_and_set_bit(nr, addr)
	17	#define generic_test_and_clear_le_bit(nr, addr) test_and_clear_bit(nr, addr)
	18
	19	#define generic___test_and_set_le_bit(nr, addr) __test_and_set_bit(nr, addr)
	20	#define generic___test_and_clear_le_bit(nr, addr) __test_and_clear_bit(nr, addr)
	21
	22	#define generic_find_next_zero_le_bit(addr, size, offset) find_next_zero_bit(addr, size, offset)
	23
	24	#elif defined(__BIG_ENDIAN)
	25
	26	#define generic_test_le_bit(nr, addr) \
	27	test_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
	28	#define generic___set_le_bit(nr, addr) \
	29	__set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
	30	#define generic___clear_le_bit(nr, addr) \
	31	__clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
	32
	33	#define generic_test_and_set_le_bit(nr, addr) \
	34	test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
	35	#define generic_test_and_clear_le_bit(nr, addr) \
	36	test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
	37
	38	#define generic___test_and_set_le_bit(nr, addr) \
	39	__test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
	40	#define generic___test_and_clear_le_bit(nr, addr) \
	41	__test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
	42
	43	extern unsigned long generic_find_next_zero_le_bit(const unsigned long *addr,
	44	unsigned long size, unsigned long offset);
	45
	46	#else
	47	#error "Please fix <asm/byteorder.h>"
	48	#endif
	49
	50	#define generic_find_first_zero_le_bit(addr, size) \
	51	generic_find_next_zero_le_bit((addr), (size), 0)
	52
	53	#endif /* _ASM_GENERIC_BITOPS_LE_H_ */

tools/testing/radix-tree/linux/bitops/non-atomic.h+111−0

@@ -0,0 +1,111 @@
	1	#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
	2	#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
	3
	4	#include <asm/types.h>
	5
	6	#define BITOP_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
	7	#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
	8
	9	/**
	10	* __set_bit - Set a bit in memory
	11	* @nr: the bit to set
	12	* @addr: the address to start counting from
	13	*
	14	* Unlike set_bit(), this function is non-atomic and may be reordered.
	15	* If it's called on the same region of memory simultaneously, the effect
	16	* may be that only one operation succeeds.
	17	*/
	18	static inline void __set_bit(int nr, volatile unsigned long *addr)
	19	{
	20	unsigned long mask = BITOP_MASK(nr);
	21	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	22
	23	*p \|= mask;
	24	}
	25
	26	static inline void __clear_bit(int nr, volatile unsigned long *addr)
	27	{
	28	unsigned long mask = BITOP_MASK(nr);
	29	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	30
	31	*p &= ~mask;
	32	}
	33
	34	/**
	35	* __change_bit - Toggle a bit in memory
	36	* @nr: the bit to change
	37	* @addr: the address to start counting from
	38	*
	39	* Unlike change_bit(), this function is non-atomic and may be reordered.
	40	* If it's called on the same region of memory simultaneously, the effect
	41	* may be that only one operation succeeds.
	42	*/
	43	static inline void __change_bit(int nr, volatile unsigned long *addr)
	44	{
	45	unsigned long mask = BITOP_MASK(nr);
	46	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	47
	48	*p ^= mask;
	49	}
	50
	51	/**
	52	* __test_and_set_bit - Set a bit and return its old value
	53	* @nr: Bit to set
	54	* @addr: Address to count from
	55	*
	56	* This operation is non-atomic and can be reordered.
	57	* If two examples of this operation race, one can appear to succeed
	58	* but actually fail. You must protect multiple accesses with a lock.
	59	*/
	60	static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
	61	{
	62	unsigned long mask = BITOP_MASK(nr);
	63	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	64	unsigned long old = *p;
	65
	66	*p = old \| mask;
	67	return (old & mask) != 0;
	68	}
	69
	70	/**
	71	* __test_and_clear_bit - Clear a bit and return its old value
	72	* @nr: Bit to clear
	73	* @addr: Address to count from
	74	*
	75	* This operation is non-atomic and can be reordered.
	76	* If two examples of this operation race, one can appear to succeed
	77	* but actually fail. You must protect multiple accesses with a lock.
	78	*/
	79	static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
	80	{
	81	unsigned long mask = BITOP_MASK(nr);
	82	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	83	unsigned long old = *p;
	84
	85	*p = old & ~mask;
	86	return (old & mask) != 0;
	87	}
	88
	89	/* WARNING: non atomic and it can be reordered! */
	90	static inline int __test_and_change_bit(int nr,
	91	volatile unsigned long *addr)
	92	{
	93	unsigned long mask = BITOP_MASK(nr);
	94	unsigned long p = ((unsigned long )addr) + BITOP_WORD(nr);
	95	unsigned long old = *p;
	96
	97	*p = old ^ mask;
	98	return (old & mask) != 0;
	99	}
	100
	101	/**
	102	* test_bit - Determine whether a bit is set
	103	* @nr: bit number to test
	104	* @addr: Address to start counting from
	105	*/
	106	static inline int test_bit(int nr, const volatile unsigned long *addr)
	107	{
	108	return 1UL & (addr[BITOP_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
	109	}
	110
	111	#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */

tools/testing/radix-tree/linux/bug.h+1−0

@@ -0,0 +1 @@
		1	#define WARN_ON_ONCE(x) assert(x)

tools/testing/radix-tree/linux/cpu.h+34−0

@@ -0,0 +1,34 @@
	1
	2	#define hotcpu_notifier(a, b)
	3
	4	#define CPU_ONLINE 0x0002 /* CPU (unsigned)v is up */
	5	#define CPU_UP_PREPARE 0x0003 /* CPU (unsigned)v coming up */
	6	#define CPU_UP_CANCELED 0x0004 /* CPU (unsigned)v NOT coming up */
	7	#define CPU_DOWN_PREPARE 0x0005 /* CPU (unsigned)v going down */
	8	#define CPU_DOWN_FAILED 0x0006 /* CPU (unsigned)v NOT going down */
	9	#define CPU_DEAD 0x0007 /* CPU (unsigned)v dead */
	10	#define CPU_DYING 0x0008 /* CPU (unsigned)v not running any task,
	11	* not handling interrupts, soon dead.
	12	* Called on the dying cpu, interrupts
	13	* are already disabled. Must not
	14	* sleep, must not fail */
	15	#define CPU_POST_DEAD 0x0009 /* CPU (unsigned)v dead, cpu_hotplug
	16	* lock is dropped */
	17	#define CPU_STARTING 0x000A /* CPU (unsigned)v soon running.
	18	* Called on the new cpu, just before
	19	* enabling interrupts. Must not sleep,
	20	* must not fail */
	21	#define CPU_DYING_IDLE 0x000B /* CPU (unsigned)v dying, reached
	22	* idle loop. */
	23	#define CPU_BROKEN 0x000C /* CPU (unsigned)v did not die properly,
	24	* perhaps due to preemption. */
	25	#define CPU_TASKS_FROZEN 0x0010
	26
	27	#define CPU_ONLINE_FROZEN (CPU_ONLINE \| CPU_TASKS_FROZEN)
	28	#define CPU_UP_PREPARE_FROZEN (CPU_UP_PREPARE \| CPU_TASKS_FROZEN)
	29	#define CPU_UP_CANCELED_FROZEN (CPU_UP_CANCELED \| CPU_TASKS_FROZEN)
	30	#define CPU_DOWN_PREPARE_FROZEN (CPU_DOWN_PREPARE \| CPU_TASKS_FROZEN)
	31	#define CPU_DOWN_FAILED_FROZEN (CPU_DOWN_FAILED \| CPU_TASKS_FROZEN)
	32	#define CPU_DEAD_FROZEN (CPU_DEAD \| CPU_TASKS_FROZEN)
	33	#define CPU_DYING_FROZEN (CPU_DYING \| CPU_TASKS_FROZEN)
	34	#define CPU_STARTING_FROZEN (CPU_STARTING \| CPU_TASKS_FROZEN)

tools/testing/radix-tree/linux/export.h+2−0

@@ -0,0 +1,2 @@
	1
	2	#define EXPORT_SYMBOL(sym)

tools/testing/radix-tree/linux/gfp.h+10−0

@@ -0,0 +1,10 @@
	1	#ifndef _GFP_H
	2	#define _GFP_H
	3
	4	#define __GFP_BITS_SHIFT 22
	5	#define __GFP_BITS_MASK ((gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
	6	#define __GFP_WAIT 1
	7	#define __GFP_ACCOUNT 0
	8	#define __GFP_NOWARN 0
	9
	10	#endif

tools/testing/radix-tree/linux/kernel.h+35−0

@@ -0,0 +1,35 @@
	1	#ifndef _KERNEL_H
	2	#define _KERNEL_H
	3
	4	#include <assert.h>
	5	#include <string.h>
	6	#include <stdio.h>
	7	#include <stddef.h>
	8	#include <limits.h>
	9
	10	#ifndef NULL
	11	#define NULL 0
	12	#endif
	13
	14	#define BUG_ON(expr) assert(!(expr))
	15	#define __init
	16	#define __must_check
	17	#define panic(expr)
	18	#define printk printf
	19	#define __force
	20	#define likely(c) (c)
	21	#define unlikely(c) (c)
	22	#define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
	23
	24	#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
	25
	26	#define container_of(ptr, type, member) ({ \
	27	const typeof( ((type )0)->member ) __mptr = (ptr); \
	28	(type )( (char )__mptr - offsetof(type, member) );})
	29	#define min(a, b) ((a) < (b) ? (a) : (b))
	30
	31	static inline int in_interrupt(void)
	32	{
	33	return 0;
	34	}
	35	#endif /* _KERNEL_H */

tools/testing/radix-tree/linux/kmemleak.h+1−0

@@ -0,0 +1 @@
		1	static inline void kmemleak_update_trace(const void *ptr) { }

tools/testing/radix-tree/linux/mempool.h+16−0

@@ -0,0 +1,16 @@
	1
	2	#include <linux/slab.h>
	3
	4	typedef void (mempool_alloc_t)(int gfp_mask, void pool_data);
	5	typedef void (mempool_free_t)(void element, void pool_data);
	6
	7	typedef struct {
	8	mempool_alloc_t *alloc;
	9	mempool_free_t *free;
	10	void *data;
	11	} mempool_t;
	12
	13	void mempool_alloc(mempool_t pool, int gfp_mask);
	14	void mempool_free(void element, mempool_t pool);
	15	mempool_t mempool_create(int min_nr, mempool_alloc_t alloc_fn,
	16	mempool_free_t free_fn, void pool_data);

tools/testing/radix-tree/linux/notifier.h+8−0

@@ -0,0 +1,8 @@
	1	#ifndef _NOTIFIER_H
	2	#define _NOTIFIER_H
	3
	4	struct notifier_block;
	5
	6	#define NOTIFY_OK 0x0001 /* Suits me */
	7
	8	#endif

tools/testing/radix-tree/linux/percpu.h+7−0

@@ -0,0 +1,7 @@
	1
	2	#define DEFINE_PER_CPU(type, val) type val
	3
	4	#define __get_cpu_var(var) var
	5	#define this_cpu_ptr(var) var
	6	#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
	7	#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))

tools/testing/radix-tree/linux/preempt.h+4−0

@@ -0,0 +1,4 @@
	1	/* */
	2
	3	#define preempt_disable() do { } while (0)
	4	#define preempt_enable() do { } while (0)

tools/testing/radix-tree/linux/radix-tree.h+1−0

@@ -0,0 +1 @@
		1	#include "../../../../include/linux/radix-tree.h"

tools/testing/radix-tree/linux/rcupdate.h+9−0

@@ -0,0 +1,9 @@
	1	#ifndef _RCUPDATE_H
	2	#define _RCUPDATE_H
	3
	4	#include <urcu.h>
	5
	6	#define rcu_dereference_raw(p) rcu_dereference(p)
	7	#define rcu_dereference_protected(p, cond) rcu_dereference(p)
	8
	9	#endif

tools/testing/radix-tree/linux/slab.h+28−0

@@ -0,0 +1,28 @@
	1	#ifndef SLAB_H
	2	#define SLAB_H
	3
	4	#include <linux/types.h>
	5
	6	#define GFP_KERNEL 1
	7	#define SLAB_HWCACHE_ALIGN 1
	8	#define SLAB_PANIC 2
	9	#define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */
	10
	11	static inline int gfpflags_allow_blocking(gfp_t mask)
	12	{
	13	return 1;
	14	}
	15
	16	struct kmem_cache {
	17	int size;
	18	void (ctor)(void );
	19	};
	20
	21	void kmem_cache_alloc(struct kmem_cache cachep, int flags);
	22	void kmem_cache_free(struct kmem_cache cachep, void objp);
	23
	24	struct kmem_cache *
	25	kmem_cache_create(const char *name, size_t size, size_t offset,
	26	unsigned long flags, void (ctor)(void ));
	27
	28	#endif /* SLAB_H */

tools/testing/radix-tree/linux/types.h+28−0

@@ -0,0 +1,28 @@
	1	#ifndef _TYPES_H
	2	#define _TYPES_H
	3
	4	#define __rcu
	5	#define __read_mostly
	6
	7	#define BITS_PER_LONG (sizeof(long) * 8)
	8
	9	struct list_head {
	10	struct list_head next, prev;
	11	};
	12
	13	static inline void INIT_LIST_HEAD(struct list_head *list)
	14	{
	15	list->next = list;
	16	list->prev = list;
	17	}
	18
	19	typedef struct {
	20	unsigned int x;
	21	} spinlock_t;
	22
	23	#define uninitialized_var(x) x = x
	24
	25	typedef unsigned gfp_t;
	26	#include <linux/gfp.h>
	27
	28	#endif

tools/testing/radix-tree/main.c+272−0

@@ -0,0 +1,272 @@
	1	#include <stdio.h>
	2	#include <stdlib.h>
	3	#include <unistd.h>
	4	#include <time.h>
	5	#include <assert.h>
	6
	7	#include <linux/slab.h>
	8	#include <linux/radix-tree.h>
	9
	10	#include "test.h"
	11	#include "regression.h"
	12
	13	void __gang_check(unsigned long middle, long down, long up, int chunk, int hop)
	14	{
	15	long idx;
	16	RADIX_TREE(tree, GFP_KERNEL);
	17
	18	middle = 1 << 30;
	19
	20	for (idx = -down; idx < up; idx++)
	21	item_insert(&tree, middle + idx);
	22
	23	item_check_absent(&tree, middle - down - 1);
	24	for (idx = -down; idx < up; idx++)
	25	item_check_present(&tree, middle + idx);
	26	item_check_absent(&tree, middle + up);
	27
	28	item_gang_check_present(&tree, middle - down,
	29	up + down, chunk, hop);
	30	item_full_scan(&tree, middle - down, down + up, chunk);
	31	item_kill_tree(&tree);
	32	}
	33
	34	void gang_check(void)
	35	{
	36	__gang_check(1 << 30, 128, 128, 35, 2);
	37	__gang_check(1 << 31, 128, 128, 32, 32);
	38	__gang_check(1 << 31, 128, 128, 32, 100);
	39	__gang_check(1 << 31, 128, 128, 17, 7);
	40	__gang_check(0xffff0000, 0, 65536, 17, 7);
	41	__gang_check(0xfffffffe, 1, 1, 17, 7);
	42	}
	43
	44	void __big_gang_check(void)
	45	{
	46	unsigned long start;
	47	int wrapped = 0;
	48
	49	start = 0;
	50	do {
	51	unsigned long old_start;
	52
	53	// printf("0x%08lx\n", start);
	54	__gang_check(start, rand() % 113 + 1, rand() % 71,
	55	rand() % 157, rand() % 91 + 1);
	56	old_start = start;
	57	start += rand() % 1000000;
	58	start %= 1ULL << 33;
	59	if (start < old_start)
	60	wrapped = 1;
	61	} while (!wrapped);
	62	}
	63
	64	void big_gang_check(void)
	65	{
	66	int i;
	67
	68	for (i = 0; i < 1000; i++) {
	69	__big_gang_check();
	70	srand(time(0));
	71	printf("%d ", i);
	72	fflush(stdout);
	73	}
	74	}
	75
	76	void add_and_check(void)
	77	{
	78	RADIX_TREE(tree, GFP_KERNEL);
	79
	80	item_insert(&tree, 44);
	81	item_check_present(&tree, 44);
	82	item_check_absent(&tree, 43);
	83	item_kill_tree(&tree);
	84	}
	85
	86	void dynamic_height_check(void)
	87	{
	88	int i;
	89	RADIX_TREE(tree, GFP_KERNEL);
	90	tree_verify_min_height(&tree, 0);
	91
	92	item_insert(&tree, 42);
	93	tree_verify_min_height(&tree, 42);
	94
	95	item_insert(&tree, 1000000);
	96	tree_verify_min_height(&tree, 1000000);
	97
	98	assert(item_delete(&tree, 1000000));
	99	tree_verify_min_height(&tree, 42);
	100
	101	assert(item_delete(&tree, 42));
	102	tree_verify_min_height(&tree, 0);
	103
	104	for (i = 0; i < 1000; i++) {
	105	item_insert(&tree, i);
	106	tree_verify_min_height(&tree, i);
	107	}
	108
	109	i--;
	110	for (;;) {
	111	assert(item_delete(&tree, i));
	112	if (i == 0) {
	113	tree_verify_min_height(&tree, 0);
	114	break;
	115	}
	116	i--;
	117	tree_verify_min_height(&tree, i);
	118	}
	119
	120	item_kill_tree(&tree);
	121	}
	122
	123	void check_copied_tags(struct radix_tree_root tree, unsigned long start, unsigned long end, unsigned long idx, int count, int fromtag, int totag)
	124	{
	125	int i;
	126
	127	for (i = 0; i < count; i++) {
	128	/* if (i % 1000 == 0)
	129	putchar('.'); */
	130	if (idx[i] < start \|\| idx[i] > end) {
	131	if (item_tag_get(tree, idx[i], totag)) {
	132	printf("%lu-%lu: %lu, tags %d-%d\n", start, end, idx[i], item_tag_get(tree, idx[i], fromtag), item_tag_get(tree, idx[i], totag));
	133	}
	134	assert(!item_tag_get(tree, idx[i], totag));
	135	continue;
	136	}
	137	if (item_tag_get(tree, idx[i], fromtag) ^
	138	item_tag_get(tree, idx[i], totag)) {
	139	printf("%lu-%lu: %lu, tags %d-%d\n", start, end, idx[i], item_tag_get(tree, idx[i], fromtag), item_tag_get(tree, idx[i], totag));
	140	}
	141	assert(!(item_tag_get(tree, idx[i], fromtag) ^
	142	item_tag_get(tree, idx[i], totag)));
	143	}
	144	}
	145
	146	#define ITEMS 50000
	147
	148	void copy_tag_check(void)
	149	{
	150	RADIX_TREE(tree, GFP_KERNEL);
	151	unsigned long idx[ITEMS];
	152	unsigned long start, end, count = 0, tagged, cur, tmp;
	153	int i;
	154
	155	// printf("generating radix tree indices...\n");
	156	start = rand();
	157	end = rand();
	158	if (start > end && (rand() % 10)) {
	159	cur = start;
	160	start = end;
	161	end = cur;
	162	}
	163	/* Specifically create items around the start and the end of the range
	164	* with high probability to check for off by one errors */
	165	cur = rand();
	166	if (cur & 1) {
	167	item_insert(&tree, start);
	168	if (cur & 2) {
	169	if (start <= end)
	170	count++;
	171	item_tag_set(&tree, start, 0);
	172	}
	173	}
	174	if (cur & 4) {
	175	item_insert(&tree, start-1);
	176	if (cur & 8)
	177	item_tag_set(&tree, start-1, 0);
	178	}
	179	if (cur & 16) {
	180	item_insert(&tree, end);
	181	if (cur & 32) {
	182	if (start <= end)
	183	count++;
	184	item_tag_set(&tree, end, 0);
	185	}
	186	}
	187	if (cur & 64) {
	188	item_insert(&tree, end+1);
	189	if (cur & 128)
	190	item_tag_set(&tree, end+1, 0);
	191	}
	192
	193	for (i = 0; i < ITEMS; i++) {
	194	do {
	195	idx[i] = rand();
	196	} while (item_lookup(&tree, idx[i]));
	197
	198	item_insert(&tree, idx[i]);
	199	if (rand() & 1) {
	200	item_tag_set(&tree, idx[i], 0);
	201	if (idx[i] >= start && idx[i] <= end)
	202	count++;
	203	}
	204	/* if (i % 1000 == 0)
	205	putchar('.'); */
	206	}
	207
	208	// printf("\ncopying tags...\n");
	209	cur = start;
	210	tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, ITEMS, 0, 1);
	211
	212	// printf("checking copied tags\n");
	213	assert(tagged == count);
	214	check_copied_tags(&tree, start, end, idx, ITEMS, 0, 1);
	215
	216	/* Copy tags in several rounds */
	217	// printf("\ncopying tags...\n");
	218	cur = start;
	219	do {
	220	tmp = rand() % (count/10+2);
	221	tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, tmp, 0, 2);
	222	} while (tmp == tagged);
	223
	224	// printf("%lu %lu %lu\n", tagged, tmp, count);
	225	// printf("checking copied tags\n");
	226	check_copied_tags(&tree, start, end, idx, ITEMS, 0, 2);
	227	assert(tagged < tmp);
	228	verify_tag_consistency(&tree, 0);
	229	verify_tag_consistency(&tree, 1);
	230	verify_tag_consistency(&tree, 2);
	231	// printf("\n");
	232	item_kill_tree(&tree);
	233	}
	234
	235	static void single_thread_tests(void)
	236	{
	237	int i;
	238
	239	tag_check();
	240	printf("after tag_check: %d allocated\n", nr_allocated);
	241	gang_check();
	242	printf("after gang_check: %d allocated\n", nr_allocated);
	243	add_and_check();
	244	printf("after add_and_check: %d allocated\n", nr_allocated);
	245	dynamic_height_check();
	246	printf("after dynamic_height_check: %d allocated\n", nr_allocated);
	247	big_gang_check();
	248	printf("after big_gang_check: %d allocated\n", nr_allocated);
	249	for (i = 0; i < 2000; i++) {
	250	copy_tag_check();
	251	printf("%d ", i);
	252	fflush(stdout);
	253	}
	254	printf("after copy_tag_check: %d allocated\n", nr_allocated);
	255	}
	256
	257	int main(void)
	258	{
	259	rcu_register_thread();
	260	radix_tree_init();
	261
	262	regression1_test();
	263	regression2_test();
	264	regression3_test();
	265	single_thread_tests();
	266
	267	sleep(1);
	268	printf("after sleep(1): %d allocated\n", nr_allocated);
	269	rcu_unregister_thread();
	270
	271	exit(0);
	272	}

tools/testing/radix-tree/rcupdate.c+86−0

@@ -0,0 +1,86 @@
	1	#include <linux/rcupdate.h>
	2	#include <pthread.h>
	3	#include <stdio.h>
	4	#include <assert.h>
	5
	6	static pthread_mutex_t rculock = PTHREAD_MUTEX_INITIALIZER;
	7	static struct rcu_head *rcuhead_global = NULL;
	8	static __thread int nr_rcuhead = 0;
	9	static __thread struct rcu_head *rcuhead = NULL;
	10	static __thread struct rcu_head *rcutail = NULL;
	11
	12	static pthread_cond_t rcu_worker_cond = PTHREAD_COND_INITIALIZER;
	13
	14	/* switch to urcu implementation when it is merged. */
	15	void call_rcu(struct rcu_head head, void (func)(struct rcu_head *head))
	16	{
	17	head->func = func;
	18	head->next = rcuhead;
	19	rcuhead = head;
	20	if (!rcutail)
	21	rcutail = head;
	22	nr_rcuhead++;
	23	if (nr_rcuhead >= 1000) {
	24	int signal = 0;
	25
	26	pthread_mutex_lock(&rculock);
	27	if (!rcuhead_global)
	28	signal = 1;
	29	rcutail->next = rcuhead_global;
	30	rcuhead_global = head;
	31	pthread_mutex_unlock(&rculock);
	32
	33	nr_rcuhead = 0;
	34	rcuhead = NULL;
	35	rcutail = NULL;
	36
	37	if (signal) {
	38	pthread_cond_signal(&rcu_worker_cond);
	39	}
	40	}
	41	}
	42
	43	static void rcu_worker(void arg)
	44	{
	45	struct rcu_head *r;
	46
	47	rcupdate_thread_init();
	48
	49	while (1) {
	50	pthread_mutex_lock(&rculock);
	51	while (!rcuhead_global) {
	52	pthread_cond_wait(&rcu_worker_cond, &rculock);
	53	}
	54	r = rcuhead_global;
	55	rcuhead_global = NULL;
	56
	57	pthread_mutex_unlock(&rculock);
	58
	59	synchronize_rcu();
	60
	61	while (r) {
	62	struct rcu_head *tmp = r->next;
	63	r->func(r);
	64	r = tmp;
	65	}
	66	}
	67
	68	rcupdate_thread_exit();
	69
	70	return NULL;
	71	}
	72
	73	static pthread_t worker_thread;
	74	void rcupdate_init(void)
	75	{
	76	pthread_create(&worker_thread, NULL, rcu_worker, NULL);
	77	}
	78
	79	void rcupdate_thread_init(void)
	80	{
	81	rcu_register_thread();
	82	}
	83	void rcupdate_thread_exit(void)
	84	{
	85	rcu_unregister_thread();
	86	}

tools/testing/radix-tree/regression.h+8−0

@@ -0,0 +1,8 @@
	1	#ifndef __REGRESSION_H__
	2	#define __REGRESSION_H__
	3
	4	void regression1_test(void);
	5	void regression2_test(void);
	6	void regression3_test(void);
	7
	8	#endif

tools/testing/radix-tree/regression1.c+220−0

@@ -0,0 +1,220 @@
	1	/*
	2	* Regression1
	3	* Description:
	4	* Salman Qazi describes the following radix-tree bug:
	5	*
	6	* In the following case, we get can get a deadlock:
	7	*
	8	* 0. The radix tree contains two items, one has the index 0.
	9	* 1. The reader (in this case find_get_pages) takes the rcu_read_lock.
	10	* 2. The reader acquires slot(s) for item(s) including the index 0 item.
	11	* 3. The non-zero index item is deleted, and as a consequence the other item
	12	* is moved to the root of the tree. The place where it used to be is queued
	13	* for deletion after the readers finish.
	14	* 3b. The zero item is deleted, removing it from the direct slot, it remains in
	15	* the rcu-delayed indirect node.
	16	* 4. The reader looks at the index 0 slot, and finds that the page has 0 ref
	17	* count
	18	* 5. The reader looks at it again, hoping that the item will either be freed
	19	* or the ref count will increase. This never happens, as the slot it is
	20	* looking at will never be updated. Also, this slot can never be reclaimed
	21	* because the reader is holding rcu_read_lock and is in an infinite loop.
	22	*
	23	* The fix is to re-use the same "indirect" pointer case that requires a slot
	24	* lookup retry into a general "retry the lookup" bit.
	25	*
	26	* Running:
	27	* This test should run to completion in a few seconds. The above bug would
	28	* cause it to hang indefinitely.
	29	*
	30	* Upstream commit:
	31	* Not yet
	32	*/
	33	#include <linux/kernel.h>
	34	#include <linux/gfp.h>
	35	#include <linux/slab.h>
	36	#include <linux/radix-tree.h>
	37	#include <linux/rcupdate.h>
	38	#include <stdlib.h>
	39	#include <pthread.h>
	40	#include <stdio.h>
	41	#include <assert.h>
	42
	43	#include "regression.h"
	44
	45	static RADIX_TREE(mt_tree, GFP_KERNEL);
	46	static pthread_mutex_t mt_lock;
	47
	48	struct page {
	49	pthread_mutex_t lock;
	50	struct rcu_head rcu;
	51	int count;
	52	unsigned long index;
	53	};
	54
	55	static struct page *page_alloc(void)
	56	{
	57	struct page *p;
	58	p = malloc(sizeof(struct page));
	59	p->count = 1;
	60	p->index = 1;
	61	pthread_mutex_init(&p->lock, NULL);
	62
	63	return p;
	64	}
	65
	66	static void page_rcu_free(struct rcu_head *rcu)
	67	{
	68	struct page *p = container_of(rcu, struct page, rcu);
	69	assert(!p->count);
	70	pthread_mutex_destroy(&p->lock);
	71	free(p);
	72	}
	73
	74	static void page_free(struct page *p)
	75	{
	76	call_rcu(&p->rcu, page_rcu_free);
	77	}
	78
	79	static unsigned find_get_pages(unsigned long start,
	80	unsigned int nr_pages, struct page **pages)
	81	{
	82	unsigned int i;
	83	unsigned int ret;
	84	unsigned int nr_found;
	85
	86	rcu_read_lock();
	87	restart:
	88	nr_found = radix_tree_gang_lookup_slot(&mt_tree,
	89	(void ***)pages, NULL, start, nr_pages);
	90	ret = 0;
	91	for (i = 0; i < nr_found; i++) {
	92	struct page *page;
	93	repeat:
	94	page = radix_tree_deref_slot((void **)pages[i]);
	95	if (unlikely(!page))
	96	continue;
	97
	98	if (radix_tree_exception(page)) {
	99	if (radix_tree_deref_retry(page)) {
	100	/*
	101	* Transient condition which can only trigger
	102	* when entry at index 0 moves out of or back
	103	* to root: none yet gotten, safe to restart.
	104	*/
	105	assert((start \| i) == 0);
	106	goto restart;
	107	}
	108	/*
	109	* No exceptional entries are inserted in this test.
	110	*/
	111	assert(0);
	112	}
	113
	114	pthread_mutex_lock(&page->lock);
	115	if (!page->count) {
	116	pthread_mutex_unlock(&page->lock);
	117	goto repeat;
	118	}
	119	/* don't actually update page refcount */
	120	pthread_mutex_unlock(&page->lock);
	121
	122	/* Has the page moved? */
	123	if (unlikely(page != ((void *)pages[i]))) {
	124	goto repeat;
	125	}
	126
	127	pages[ret] = page;
	128	ret++;
	129	}
	130	rcu_read_unlock();
	131	return ret;
	132	}
	133
	134	static pthread_barrier_t worker_barrier;
	135
	136	static void regression1_fn(void arg)
	137	{
	138	rcu_register_thread();
	139
	140	if (pthread_barrier_wait(&worker_barrier) ==
	141	PTHREAD_BARRIER_SERIAL_THREAD) {
	142	int j;
	143
	144	for (j = 0; j < 1000000; j++) {
	145	struct page *p;
	146
	147	p = page_alloc();
	148	pthread_mutex_lock(&mt_lock);
	149	radix_tree_insert(&mt_tree, 0, p);
	150	pthread_mutex_unlock(&mt_lock);
	151
	152	p = page_alloc();
	153	pthread_mutex_lock(&mt_lock);
	154	radix_tree_insert(&mt_tree, 1, p);
	155	pthread_mutex_unlock(&mt_lock);
	156
	157	pthread_mutex_lock(&mt_lock);
	158	p = radix_tree_delete(&mt_tree, 1);
	159	pthread_mutex_lock(&p->lock);
	160	p->count--;
	161	pthread_mutex_unlock(&p->lock);
	162	pthread_mutex_unlock(&mt_lock);
	163	page_free(p);
	164
	165	pthread_mutex_lock(&mt_lock);
	166	p = radix_tree_delete(&mt_tree, 0);
	167	pthread_mutex_lock(&p->lock);
	168	p->count--;
	169	pthread_mutex_unlock(&p->lock);
	170	pthread_mutex_unlock(&mt_lock);
	171	page_free(p);
	172	}
	173	} else {
	174	int j;
	175
	176	for (j = 0; j < 100000000; j++) {
	177	struct page *pages[10];
	178
	179	find_get_pages(0, 10, pages);
	180	}
	181	}
	182
	183	rcu_unregister_thread();
	184
	185	return NULL;
	186	}
	187
	188	static pthread_t *threads;
	189	void regression1_test(void)
	190	{
	191	int nr_threads;
	192	int i;
	193	long arg;
	194
	195	/* Regression #1 */
	196	printf("running regression test 1, should finish in under a minute\n");
	197	nr_threads = 2;
	198	pthread_barrier_init(&worker_barrier, NULL, nr_threads);
	199
	200	threads = malloc(nr_threads * sizeof(pthread_t *));
	201
	202	for (i = 0; i < nr_threads; i++) {
	203	arg = i;
	204	if (pthread_create(&threads[i], NULL, regression1_fn, (void *)arg)) {
	205	perror("pthread_create");
	206	exit(1);
	207	}
	208	}
	209
	210	for (i = 0; i < nr_threads; i++) {
	211	if (pthread_join(threads[i], NULL)) {
	212	perror("pthread_join");
	213	exit(1);
	214	}
	215	}
	216
	217	free(threads);
	218
	219	printf("regression test 1, done\n");
	220	}

tools/testing/radix-tree/regression2.c+126−0

@@ -0,0 +1,126 @@
	1	/*
	2	* Regression2
	3	* Description:
	4	* Toshiyuki Okajima describes the following radix-tree bug:
	5	*
	6	* In the following case, we can get a hangup on
	7	* radix_radix_tree_gang_lookup_tag_slot.
	8	*
	9	* 0. The radix tree contains RADIX_TREE_MAP_SIZE items. And the tag of
	10	* a certain item has PAGECACHE_TAG_DIRTY.
	11	* 1. radix_tree_range_tag_if_tagged(, start, end, , PAGECACHE_TAG_DIRTY,
	12	* PAGECACHE_TAG_TOWRITE) is called to add PAGECACHE_TAG_TOWRITE tag
	13	* for the tag which has PAGECACHE_TAG_DIRTY. However, there is no tag with
	14	* PAGECACHE_TAG_DIRTY within the range from start to end. As the result,
	15	* There is no tag with PAGECACHE_TAG_TOWRITE but the root tag has
	16	* PAGECACHE_TAG_TOWRITE.
	17	* 2. An item is added into the radix tree and then the level of it is
	18	* extended into 2 from 1. At that time, the new radix tree node succeeds
	19	* the tag status of the root tag. Therefore the tag of the new radix tree
	20	* node has PAGECACHE_TAG_TOWRITE but there is not slot with
	21	* PAGECACHE_TAG_TOWRITE tag in the child node of the new radix tree node.
	22	* 3. The tag of a certain item is cleared with PAGECACHE_TAG_DIRTY.
	23	* 4. All items within the index range from 0 to RADIX_TREE_MAP_SIZE - 1 are
	24	* released. (Only the item which index is RADIX_TREE_MAP_SIZE exist in the
	25	* radix tree.) As the result, the slot of the radix tree node is NULL but
	26	* the tag which corresponds to the slot has PAGECACHE_TAG_TOWRITE.
	27	* 5. radix_tree_gang_lookup_tag_slot(PAGECACHE_TAG_TOWRITE) calls
	28	* __lookup_tag. __lookup_tag returns with 0. And __lookup_tag doesn't
	29	* change the index that is the input and output parameter. Because the 1st
	30	* slot of the radix tree node is NULL, but the tag which corresponds to
	31	* the slot has PAGECACHE_TAG_TOWRITE.
	32	* Therefore radix_tree_gang_lookup_tag_slot tries to get some items by
	33	* calling __lookup_tag, but it cannot get any items forever.
	34	*
	35	* The fix is to change that radix_tree_tag_if_tagged doesn't tag the root tag
	36	* if it doesn't set any tags within the specified range.
	37	*
	38	* Running:
	39	* This test should run to completion immediately. The above bug would cause it
	40	* to hang indefinitely.
	41	*
	42	* Upstream commit:
	43	* Not yet
	44	*/
	45	#include <linux/kernel.h>
	46	#include <linux/gfp.h>
	47	#include <linux/slab.h>
	48	#include <linux/radix-tree.h>
	49	#include <stdlib.h>
	50	#include <stdio.h>
	51
	52	#include "regression.h"
	53
	54	#ifdef __KERNEL__
	55	#define RADIX_TREE_MAP_SHIFT (CONFIG_BASE_SMALL ? 4 : 6)
	56	#else
	57	#define RADIX_TREE_MAP_SHIFT 3 /* For more stressful testing */
	58	#endif
	59
	60	#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
	61	#define PAGECACHE_TAG_DIRTY 0
	62	#define PAGECACHE_TAG_WRITEBACK 1
	63	#define PAGECACHE_TAG_TOWRITE 2
	64
	65	static RADIX_TREE(mt_tree, GFP_KERNEL);
	66	unsigned long page_count = 0;
	67
	68	struct page {
	69	unsigned long index;
	70	};
	71
	72	static struct page *page_alloc(void)
	73	{
	74	struct page *p;
	75	p = malloc(sizeof(struct page));
	76	p->index = page_count++;
	77
	78	return p;
	79	}
	80
	81	void regression2_test(void)
	82	{
	83	int i;
	84	struct page *p;
	85	int max_slots = RADIX_TREE_MAP_SIZE;
	86	unsigned long int start, end;
	87	struct page *pages[1];
	88
	89	printf("running regression test 2 (should take milliseconds)\n");
	90	/* 0. */
	91	for (i = 0; i <= max_slots - 1; i++) {
	92	p = page_alloc();
	93	radix_tree_insert(&mt_tree, i, p);
	94	}
	95	radix_tree_tag_set(&mt_tree, max_slots - 1, PAGECACHE_TAG_DIRTY);
	96
	97	/* 1. */
	98	start = 0;
	99	end = max_slots - 2;
	100	radix_tree_range_tag_if_tagged(&mt_tree, &start, end, 1,
	101	PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
	102
	103	/* 2. */
	104	p = page_alloc();
	105	radix_tree_insert(&mt_tree, max_slots, p);
	106
	107	/* 3. */
	108	radix_tree_tag_clear(&mt_tree, max_slots - 1, PAGECACHE_TAG_DIRTY);
	109
	110	/* 4. */
	111	for (i = max_slots - 1; i >= 0; i--)
	112	radix_tree_delete(&mt_tree, i);
	113
	114	/* 5. */
	115	// NOTE: start should not be 0 because radix_tree_gang_lookup_tag_slot
	116	// can return.
	117	start = 1;
	118	end = max_slots - 2;
	119	radix_tree_gang_lookup_tag_slot(&mt_tree, (void ***)pages, start, end,
	120	PAGECACHE_TAG_TOWRITE);
	121
	122	/* We remove all the remained nodes */
	123	radix_tree_delete(&mt_tree, max_slots);
	124
	125	printf("regression test 2, done\n");
	126	}

tools/testing/radix-tree/regression3.c+117−0

@@ -0,0 +1,117 @@
	1	/*
	2	* Regression3
	3	* Description:
	4	* Helper radix_tree_iter_retry resets next_index to the current index.
	5	* In following radix_tree_next_slot current chunk size becomes zero.
	6	* This isn't checked and it tries to dereference null pointer in slot.
	7	*
	8	* Helper radix_tree_iter_next reset slot to NULL and next_index to index + 1,
	9	* for tagger iteraction it also must reset cached tags in iterator to abort
	10	* next radix_tree_next_slot and go to slow-path into radix_tree_next_chunk.
	11	*
	12	* Running:
	13	* This test should run to completion immediately. The above bug would
	14	* cause it to segfault.
	15	*
	16	* Upstream commit:
	17	* Not yet
	18	*/
	19	#include <linux/kernel.h>
	20	#include <linux/gfp.h>
	21	#include <linux/slab.h>
	22	#include <linux/radix-tree.h>
	23	#include <stdlib.h>
	24	#include <stdio.h>
	25
	26	#include "regression.h"
	27
	28	void regression3_test(void)
	29	{
	30	RADIX_TREE(root, GFP_KERNEL);
	31	void ptr0 = (void )4ul;
	32	void ptr = (void )8ul;
	33	struct radix_tree_iter iter;
	34	void **slot;
	35	bool first;
	36
	37	printf("running regression test 3 (should take milliseconds)\n");
	38
	39	radix_tree_insert(&root, 0, ptr0);
	40	radix_tree_tag_set(&root, 0, 0);
	41
	42	first = true;
	43	radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) {
	44	printf("tagged %ld %p\n", iter.index, *slot);
	45	if (first) {
	46	radix_tree_insert(&root, 1, ptr);
	47	radix_tree_tag_set(&root, 1, 0);
	48	first = false;
	49	}
	50	if (radix_tree_deref_retry(*slot)) {
	51	printf("retry at %ld\n", iter.index);
	52	slot = radix_tree_iter_retry(&iter);
	53	continue;
	54	}
	55	}
	56	radix_tree_delete(&root, 1);
	57
	58	first = true;
	59	radix_tree_for_each_slot(slot, &root, &iter, 0) {
	60	printf("slot %ld %p\n", iter.index, *slot);
	61	if (first) {
	62	radix_tree_insert(&root, 1, ptr);
	63	first = false;
	64	}
	65	if (radix_tree_deref_retry(*slot)) {
	66	printk("retry at %ld\n", iter.index);
	67	slot = radix_tree_iter_retry(&iter);
	68	continue;
	69	}
	70	}
	71	radix_tree_delete(&root, 1);
	72
	73	first = true;
	74	radix_tree_for_each_contig(slot, &root, &iter, 0) {
	75	printk("contig %ld %p\n", iter.index, *slot);
	76	if (first) {
	77	radix_tree_insert(&root, 1, ptr);
	78	first = false;
	79	}
	80	if (radix_tree_deref_retry(*slot)) {
	81	printk("retry at %ld\n", iter.index);
	82	slot = radix_tree_iter_retry(&iter);
	83	continue;
	84	}
	85	}
	86
	87	radix_tree_for_each_slot(slot, &root, &iter, 0) {
	88	printf("slot %ld %p\n", iter.index, *slot);
	89	if (!iter.index) {
	90	printf("next at %ld\n", iter.index);
	91	slot = radix_tree_iter_next(&iter);
	92	}
	93	}
	94
	95	radix_tree_for_each_contig(slot, &root, &iter, 0) {
	96	printf("contig %ld %p\n", iter.index, *slot);
	97	if (!iter.index) {
	98	printf("next at %ld\n", iter.index);
	99	slot = radix_tree_iter_next(&iter);
	100	}
	101	}
	102
	103	radix_tree_tag_set(&root, 0, 0);
	104	radix_tree_tag_set(&root, 1, 0);
	105	radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) {
	106	printf("tagged %ld %p\n", iter.index, *slot);
	107	if (!iter.index) {
	108	printf("next at %ld\n", iter.index);
	109	slot = radix_tree_iter_next(&iter);
	110	}
	111	}
	112
	113	radix_tree_delete(&root, 0);
	114	radix_tree_delete(&root, 1);
	115
	116	printf("regression test 3 passed\n");
	117	}

tools/testing/radix-tree/tag_check.c+332−0

@@ -0,0 +1,332 @@
	1	#include <stdlib.h>
	2	#include <assert.h>
	3	#include <stdio.h>
	4	#include <string.h>
	5
	6	#include <linux/slab.h>
	7	#include <linux/radix-tree.h>
	8
	9	#include "test.h"
	10
	11
	12	static void
	13	__simple_checks(struct radix_tree_root *tree, unsigned long index, int tag)
	14	{
	15	int ret;
	16
	17	item_check_absent(tree, index);
	18	assert(item_tag_get(tree, index, tag) == 0);
	19
	20	item_insert(tree, index);
	21	assert(item_tag_get(tree, index, tag) == 0);
	22	item_tag_set(tree, index, tag);
	23	ret = item_tag_get(tree, index, tag);
	24	assert(ret != 0);
	25	ret = item_delete(tree, index);
	26	assert(ret != 0);
	27	item_insert(tree, index);
	28	ret = item_tag_get(tree, index, tag);
	29	assert(ret == 0);
	30	ret = item_delete(tree, index);
	31	assert(ret != 0);
	32	ret = item_delete(tree, index);
	33	assert(ret == 0);
	34	}
	35
	36	void simple_checks(void)
	37	{
	38	unsigned long index;
	39	RADIX_TREE(tree, GFP_KERNEL);
	40
	41	for (index = 0; index < 10000; index++) {
	42	__simple_checks(&tree, index, 0);
	43	__simple_checks(&tree, index, 1);
	44	}
	45	verify_tag_consistency(&tree, 0);
	46	verify_tag_consistency(&tree, 1);
	47	printf("before item_kill_tree: %d allocated\n", nr_allocated);
	48	item_kill_tree(&tree);
	49	printf("after item_kill_tree: %d allocated\n", nr_allocated);
	50	}
	51
	52	/*
	53	* Check that tags propagate correctly when extending a tree.
	54	*/
	55	static void extend_checks(void)
	56	{
	57	RADIX_TREE(tree, GFP_KERNEL);
	58
	59	item_insert(&tree, 43);
	60	assert(item_tag_get(&tree, 43, 0) == 0);
	61	item_tag_set(&tree, 43, 0);
	62	assert(item_tag_get(&tree, 43, 0) == 1);
	63	item_insert(&tree, 1000000);
	64	assert(item_tag_get(&tree, 43, 0) == 1);
	65
	66	item_insert(&tree, 0);
	67	item_tag_set(&tree, 0, 0);
	68	item_delete(&tree, 1000000);
	69	assert(item_tag_get(&tree, 43, 0) != 0);
	70	item_delete(&tree, 43);
	71	assert(item_tag_get(&tree, 43, 0) == 0); /* crash */
	72	assert(item_tag_get(&tree, 0, 0) == 1);
	73
	74	verify_tag_consistency(&tree, 0);
	75
	76	item_kill_tree(&tree);
	77	}
	78
	79	/*
	80	* Check that tags propagate correctly when contracting a tree.
	81	*/
	82	static void contract_checks(void)
	83	{
	84	struct item *item;
	85	int tmp;
	86	RADIX_TREE(tree, GFP_KERNEL);
	87
	88	tmp = 1<<RADIX_TREE_MAP_SHIFT;
	89	item_insert(&tree, tmp);
	90	item_insert(&tree, tmp+1);
	91	item_tag_set(&tree, tmp, 0);
	92	item_tag_set(&tree, tmp, 1);
	93	item_tag_set(&tree, tmp+1, 0);
	94	item_delete(&tree, tmp+1);
	95	item_tag_clear(&tree, tmp, 1);
	96
	97	assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 0) == 1);
	98	assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 1) == 0);
	99
	100	assert(item_tag_get(&tree, tmp, 0) == 1);
	101	assert(item_tag_get(&tree, tmp, 1) == 0);
	102
	103	verify_tag_consistency(&tree, 0);
	104	item_kill_tree(&tree);
	105	}
	106
	107	/*
	108	* Stupid tag thrasher
	109	*
	110	* Create a large linear array corresponding to the tree. Each element in
	111	* the array is coherent with each node in the tree
	112	*/
	113
	114	enum {
	115	NODE_ABSENT = 0,
	116	NODE_PRESENT = 1,
	117	NODE_TAGGED = 2,
	118	};
	119
	120	#define THRASH_SIZE 1000 * 1000
	121	#define N 127
	122	#define BATCH 33
	123
	124	static void gang_check(struct radix_tree_root *tree,
	125	char *thrash_state, int tag)
	126	{
	127	struct item *items[BATCH];
	128	int nr_found;
	129	unsigned long index = 0;
	130	unsigned long last_index = 0;
	131
	132	while ((nr_found = radix_tree_gang_lookup_tag(tree, (void **)items,
	133	index, BATCH, tag))) {
	134	int i;
	135
	136	for (i = 0; i < nr_found; i++) {
	137	struct item *item = items[i];
	138
	139	while (last_index < item->index) {
	140	assert(thrash_state[last_index] != NODE_TAGGED);
	141	last_index++;
	142	}
	143	assert(thrash_state[last_index] == NODE_TAGGED);
	144	last_index++;
	145	}
	146	index = items[nr_found - 1]->index + 1;
	147	}
	148	}
	149
	150	static void do_thrash(struct radix_tree_root tree, char thrash_state, int tag)
	151	{
	152	int insert_chunk;
	153	int delete_chunk;
	154	int tag_chunk;
	155	int untag_chunk;
	156	int total_tagged = 0;
	157	int total_present = 0;
	158
	159	for (insert_chunk = 1; insert_chunk < THRASH_SIZE; insert_chunk *= N)
	160	for (delete_chunk = 1; delete_chunk < THRASH_SIZE; delete_chunk *= N)
	161	for (tag_chunk = 1; tag_chunk < THRASH_SIZE; tag_chunk *= N)
	162	for (untag_chunk = 1; untag_chunk < THRASH_SIZE; untag_chunk *= N) {
	163	int i;
	164	unsigned long index;
	165	int nr_inserted = 0;
	166	int nr_deleted = 0;
	167	int nr_tagged = 0;
	168	int nr_untagged = 0;
	169	int actual_total_tagged;
	170	int actual_total_present;
	171
	172	for (i = 0; i < insert_chunk; i++) {
	173	index = rand() % THRASH_SIZE;
	174	if (thrash_state[index] != NODE_ABSENT)
	175	continue;
	176	item_check_absent(tree, index);
	177	item_insert(tree, index);
	178	assert(thrash_state[index] != NODE_PRESENT);
	179	thrash_state[index] = NODE_PRESENT;
	180	nr_inserted++;
	181	total_present++;
	182	}
	183
	184	for (i = 0; i < delete_chunk; i++) {
	185	index = rand() % THRASH_SIZE;
	186	if (thrash_state[index] == NODE_ABSENT)
	187	continue;
	188	item_check_present(tree, index);
	189	if (item_tag_get(tree, index, tag)) {
	190	assert(thrash_state[index] == NODE_TAGGED);
	191	total_tagged--;
	192	} else {
	193	assert(thrash_state[index] == NODE_PRESENT);
	194	}
	195	item_delete(tree, index);
	196	assert(thrash_state[index] != NODE_ABSENT);
	197	thrash_state[index] = NODE_ABSENT;
	198	nr_deleted++;
	199	total_present--;
	200	}
	201
	202	for (i = 0; i < tag_chunk; i++) {
	203	index = rand() % THRASH_SIZE;
	204	if (thrash_state[index] != NODE_PRESENT) {
	205	if (item_lookup(tree, index))
	206	assert(item_tag_get(tree, index, tag));
	207	continue;
	208	}
	209	item_tag_set(tree, index, tag);
	210	item_tag_set(tree, index, tag);
	211	assert(thrash_state[index] != NODE_TAGGED);
	212	thrash_state[index] = NODE_TAGGED;
	213	nr_tagged++;
	214	total_tagged++;
	215	}
	216
	217	for (i = 0; i < untag_chunk; i++) {
	218	index = rand() % THRASH_SIZE;
	219	if (thrash_state[index] != NODE_TAGGED)
	220	continue;
	221	item_check_present(tree, index);
	222	assert(item_tag_get(tree, index, tag));
	223	item_tag_clear(tree, index, tag);
	224	item_tag_clear(tree, index, tag);
	225	assert(thrash_state[index] != NODE_PRESENT);
	226	thrash_state[index] = NODE_PRESENT;
	227	nr_untagged++;
	228	total_tagged--;
	229	}
	230
	231	actual_total_tagged = 0;
	232	actual_total_present = 0;
	233	for (index = 0; index < THRASH_SIZE; index++) {
	234	switch (thrash_state[index]) {
	235	case NODE_ABSENT:
	236	item_check_absent(tree, index);
	237	break;
	238	case NODE_PRESENT:
	239	item_check_present(tree, index);
	240	assert(!item_tag_get(tree, index, tag));
	241	actual_total_present++;
	242	break;
	243	case NODE_TAGGED:
	244	item_check_present(tree, index);
	245	assert(item_tag_get(tree, index, tag));
	246	actual_total_present++;
	247	actual_total_tagged++;
	248	break;
	249	}
	250	}
	251
	252	gang_check(tree, thrash_state, tag);
	253
	254	printf("%d(%d) %d(%d) %d(%d) %d(%d) / "
	255	"%d(%d) present, %d(%d) tagged\n",
	256	insert_chunk, nr_inserted,
	257	delete_chunk, nr_deleted,
	258	tag_chunk, nr_tagged,
	259	untag_chunk, nr_untagged,
	260	total_present, actual_total_present,
	261	total_tagged, actual_total_tagged);
	262	}
	263	}
	264
	265	static void thrash_tags(void)
	266	{
	267	RADIX_TREE(tree, GFP_KERNEL);
	268	char *thrash_state;
	269
	270	thrash_state = malloc(THRASH_SIZE);
	271	memset(thrash_state, 0, THRASH_SIZE);
	272
	273	do_thrash(&tree, thrash_state, 0);
	274
	275	verify_tag_consistency(&tree, 0);
	276	item_kill_tree(&tree);
	277	free(thrash_state);
	278	}
	279
	280	static void leak_check(void)
	281	{
	282	RADIX_TREE(tree, GFP_KERNEL);
	283
	284	item_insert(&tree, 1000000);
	285	item_delete(&tree, 1000000);
	286	item_kill_tree(&tree);
	287	}
	288
	289	static void __leak_check(void)
	290	{
	291	RADIX_TREE(tree, GFP_KERNEL);
	292
	293	printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
	294	item_insert(&tree, 1000000);
	295	printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
	296	item_delete(&tree, 1000000);
	297	printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
	298	item_kill_tree(&tree);
	299	printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
	300	}
	301
	302	static void single_check(void)
	303	{
	304	struct item *items[BATCH];
	305	RADIX_TREE(tree, GFP_KERNEL);
	306	int ret;
	307
	308	item_insert(&tree, 0);
	309	item_tag_set(&tree, 0, 0);
	310	ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 0);
	311	assert(ret == 1);
	312	ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 1, BATCH, 0);
	313	assert(ret == 0);
	314	verify_tag_consistency(&tree, 0);
	315	verify_tag_consistency(&tree, 1);
	316	item_kill_tree(&tree);
	317	}
	318
	319	void tag_check(void)
	320	{
	321	single_check();
	322	extend_checks();
	323	contract_checks();
	324	printf("after extend_checks: %d allocated\n", nr_allocated);
	325	__leak_check();
	326	leak_check();
	327	printf("after leak_check: %d allocated\n", nr_allocated);
	328	simple_checks();
	329	printf("after simple_checks: %d allocated\n", nr_allocated);
	330	thrash_tags();
	331	printf("after thrash_tags: %d allocated\n", nr_allocated);
	332	}

tools/testing/radix-tree/test.c+219−0

@@ -0,0 +1,219 @@
	1	#include <stdlib.h>
	2	#include <assert.h>
	3	#include <stdio.h>
	4	#include <linux/types.h>
	5	#include <linux/kernel.h>
	6	#include <linux/bitops.h>
	7
	8	#include "test.h"
	9
	10	struct item *
	11	item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
	12	{
	13	return radix_tree_tag_set(root, index, tag);
	14	}
	15
	16	struct item *
	17	item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
	18	{
	19	return radix_tree_tag_clear(root, index, tag);
	20	}
	21
	22	int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
	23	{
	24	return radix_tree_tag_get(root, index, tag);
	25	}
	26
	27	int __item_insert(struct radix_tree_root root, struct item item)
	28	{
	29	return radix_tree_insert(root, item->index, item);
	30	}
	31
	32	int item_insert(struct radix_tree_root *root, unsigned long index)
	33	{
	34	return __item_insert(root, item_create(index));
	35	}
	36
	37	int item_delete(struct radix_tree_root *root, unsigned long index)
	38	{
	39	struct item *item = radix_tree_delete(root, index);
	40
	41	if (item) {
	42	assert(item->index == index);
	43	free(item);
	44	return 1;
	45	}
	46	return 0;
	47	}
	48
	49	struct item *item_create(unsigned long index)
	50	{
	51	struct item ret = malloc(sizeof(ret));
	52
	53	ret->index = index;
	54	return ret;
	55	}
	56
	57	void item_check_present(struct radix_tree_root *root, unsigned long index)
	58	{
	59	struct item *item;
	60
	61	item = radix_tree_lookup(root, index);
	62	assert(item != 0);
	63	assert(item->index == index);
	64	}
	65
	66	struct item item_lookup(struct radix_tree_root root, unsigned long index)
	67	{
	68	return radix_tree_lookup(root, index);
	69	}
	70
	71	void item_check_absent(struct radix_tree_root *root, unsigned long index)
	72	{
	73	struct item *item;
	74
	75	item = radix_tree_lookup(root, index);
	76	assert(item == 0);
	77	}
	78
	79	/*
	80	* Scan only the passed (start, start+nr] for present items
	81	*/
	82	void item_gang_check_present(struct radix_tree_root *root,
	83	unsigned long start, unsigned long nr,
	84	int chunk, int hop)
	85	{
	86	struct item *items[chunk];
	87	unsigned long into;
	88
	89	for (into = 0; into < nr; ) {
	90	int nfound;
	91	int nr_to_find = chunk;
	92	int i;
	93
	94	if (nr_to_find > (nr - into))
	95	nr_to_find = nr - into;
	96
	97	nfound = radix_tree_gang_lookup(root, (void **)items,
	98	start + into, nr_to_find);
	99	assert(nfound == nr_to_find);
	100	for (i = 0; i < nfound; i++)
	101	assert(items[i]->index == start + into + i);
	102	into += hop;
	103	}
	104	}
	105
	106	/*
	107	* Scan the entire tree, only expecting present items (start, start+nr]
	108	*/
	109	void item_full_scan(struct radix_tree_root *root, unsigned long start,
	110	unsigned long nr, int chunk)
	111	{
	112	struct item *items[chunk];
	113	unsigned long into = 0;
	114	unsigned long this_index = start;
	115	int nfound;
	116	int i;
	117
	118	// printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
	119
	120	while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
	121	chunk))) {
	122	// printf("At 0x%08lx, nfound=%d\n", into, nfound);
	123	for (i = 0; i < nfound; i++) {
	124	assert(items[i]->index == this_index);
	125	this_index++;
	126	}
	127	// printf("Found 0x%08lx->0x%08lx\n",
	128	// items[0]->index, items[nfound-1]->index);
	129	into = this_index;
	130	}
	131	if (chunk)
	132	assert(this_index == start + nr);
	133	nfound = radix_tree_gang_lookup(root, (void **)items,
	134	this_index, chunk);
	135	assert(nfound == 0);
	136	}
	137
	138	static int verify_node(struct radix_tree_node *slot, unsigned int tag,
	139	unsigned int height, int tagged)
	140	{
	141	int anyset = 0;
	142	int i;
	143	int j;
	144
	145	slot = indirect_to_ptr(slot);
	146
	147	/* Verify consistency at this level */
	148	for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
	149	if (slot->tags[tag][i]) {
	150	anyset = 1;
	151	break;
	152	}
	153	}
	154	if (tagged != anyset) {
	155	printf("tag: %u, height %u, tagged: %d, anyset: %d\n", tag, height, tagged, anyset);
	156	for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
	157	printf("tag %d: ", j);
	158	for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
	159	printf("%016lx ", slot->tags[j][i]);
	160	printf("\n");
	161	}
	162	return 1;
	163	}
	164	assert(tagged == anyset);
	165
	166	/* Go for next level */
	167	if (height > 1) {
	168	for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
	169	if (slot->slots[i])
	170	if (verify_node(slot->slots[i], tag, height - 1,
	171	!!test_bit(i, slot->tags[tag]))) {
	172	printf("Failure at off %d\n", i);
	173	for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
	174	printf("tag %d: ", j);
	175	for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
	176	printf("%016lx ", slot->tags[j][i]);
	177	printf("\n");
	178	}
	179	return 1;
	180	}
	181	}
	182	return 0;
	183	}
	184
	185	void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
	186	{
	187	if (!root->height)
	188	return;
	189	verify_node(root->rnode, tag, root->height, !!root_tag_get(root, tag));
	190	}
	191
	192	void item_kill_tree(struct radix_tree_root *root)
	193	{
	194	struct item *items[32];
	195	int nfound;
	196
	197	while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
	198	int i;
	199
	200	for (i = 0; i < nfound; i++) {
	201	void *ret;
	202
	203	ret = radix_tree_delete(root, items[i]->index);
	204	assert(ret == items[i]);
	205	free(items[i]);
	206	}
	207	}
	208	assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
	209	assert(root->rnode == NULL);
	210	}
	211
	212	void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
	213	{
	214	assert(radix_tree_maxindex(root->height) >= maxindex);
	215	if (root->height > 1)
	216	assert(radix_tree_maxindex(root->height-1) < maxindex);
	217	else if (root->height == 1)
	218	assert(radix_tree_maxindex(root->height-1) <= maxindex);
	219	}

tools/testing/radix-tree/test.h+40−0

@@ -0,0 +1,40 @@
	1	#include <linux/gfp.h>
	2	#include <linux/types.h>
	3	#include <linux/radix-tree.h>
	4	#include <linux/rcupdate.h>
	5
	6	struct item {
	7	unsigned long index;
	8	};
	9
	10	struct item *item_create(unsigned long index);
	11	int __item_insert(struct radix_tree_root root, struct item item);
	12	int item_insert(struct radix_tree_root *root, unsigned long index);
	13	int item_delete(struct radix_tree_root *root, unsigned long index);
	14	struct item item_lookup(struct radix_tree_root root, unsigned long index);
	15
	16	void item_check_present(struct radix_tree_root *root, unsigned long index);
	17	void item_check_absent(struct radix_tree_root *root, unsigned long index);
	18	void item_gang_check_present(struct radix_tree_root *root,
	19	unsigned long start, unsigned long nr,
	20	int chunk, int hop);
	21	void item_full_scan(struct radix_tree_root *root, unsigned long start,
	22	unsigned long nr, int chunk);
	23	void item_kill_tree(struct radix_tree_root *root);
	24
	25	void tag_check(void);
	26
	27	struct item *
	28	item_tag_set(struct radix_tree_root *root, unsigned long index, int tag);
	29	struct item *
	30	item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag);
	31	int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag);
	32	void tree_verify_min_height(struct radix_tree_root *root, int maxindex);
	33	void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag);
	34
	35	extern int nr_allocated;
	36
	37	/* Normally private parts of lib/radix-tree.c */
	38	void indirect_to_ptr(void ptr);
	39	int root_tag_get(struct radix_tree_root *root, unsigned int tag);
	40	unsigned long radix_tree_maxindex(unsigned int height);

tools/vm/page-types.c+119−14

 #define PM_PFRAME_BITS		55
 #define PM_PFRAME_MASK		((1LL << PM_PFRAME_BITS) - 1)
 #define PM_PFRAME(x)		((x) & PM_PFRAME_MASK)
+#define MAX_SWAPFILES_SHIFT	5
+#define PM_SWAP_OFFSET(x)	(((x) & PM_PFRAME_MASK) >> MAX_SWAPFILES_SHIFT)
 #define PM_SOFT_DIRTY		(1ULL << 55)
 #define PM_MMAP_EXCLUSIVE	(1ULL << 56)
 #define PM_FILE			(1ULL << 61)
 #define KPF_BYTES		8
 #define PROC_KPAGEFLAGS		"/proc/kpageflags"
+#define PROC_KPAGECGROUP	"/proc/kpagecgroup"
 /* [32-] kernel hacking assistances */
 #define KPF_RESERVED		32
 #define KPF_SLOB_FREE		49
 #define KPF_SLUB_FROZEN		50
 #define KPF_SLUB_DEBUG		51
-#define KPF_FILE		62
+#define KPF_FILE		61
+#define KPF_SWAP		62
 #define KPF_MMAP_EXCLUSIVE	63
 #define KPF_ALL_BITS		((uint64_t)~0ULL)
 	[KPF_SLUB_DEBUG]	= "E:slub_debug",
 	[KPF_FILE]		= "F:file",
+	[KPF_SWAP]		= "w:swap",
 	[KPF_MMAP_EXCLUSIVE]	= "1:mmap_exclusive",
 };
 static int		opt_list;	/* list pages (in ranges) */
 static int		opt_no_summary;	/* don't show summary */
 static pid_t		opt_pid;	/* process to walk */
-const char *		opt_file;
+const char *		opt_file;	/* file or directory path */
+static uint64_t		opt_cgroup;	/* cgroup inode */
+static int		opt_list_cgroup;/* list page cgroup */
 #define MAX_ADDR_RANGES	1024
 static int		nr_addr_ranges;
 static int		pagemap_fd;
 static int		kpageflags_fd;
+static int		kpagecgroup_fd = -1;
 static int		opt_hwpoison;
 static int		opt_unpoison;
 	return do_u64_read(kpageflags_fd, PROC_KPAGEFLAGS, buf, index, pages);
 }
+static unsigned long kpagecgroup_read(uint64_t *buf,
+				      unsigned long index,
+				      unsigned long pages)
+{
+	if (kpagecgroup_fd < 0)
+		return pages;
+	return do_u64_read(kpagecgroup_fd, PROC_KPAGEFLAGS, buf, index, pages);
+}
 static unsigned long pagemap_read(uint64_t *buf,
 				  unsigned long index,
 				  unsigned long pages)
 	return pfn;
 }
+static unsigned long pagemap_swap_offset(uint64_t val)
+{
+	return val & PM_SWAP ? PM_SWAP_OFFSET(val) : 0;
+}
 /*
  * page flag names
  */
 static void show_page_range(unsigned long voffset, unsigned long offset,
-			    unsigned long size, uint64_t flags)
+			    unsigned long size, uint64_t flags, uint64_t cgroup)
 {
 	static uint64_t      flags0;
+	static uint64_t	     cgroup0;
 	static unsigned long voff;
 	static unsigned long index;
 	static unsigned long count;
-	if (flags == flags0 && offset == index + count &&
+	if (flags == flags0 && cgroup == cgroup0 && offset == index + count &&
 	    size && voffset == voff + count) {
 		count += size;
 		return;
 			printf("%lx\t", voff);
 		if (opt_file)
 			printf("%lu\t", voff);
+		if (opt_list_cgroup)
+			printf("@%llu\t", (unsigned long long)cgroup0);
 		printf("%lx\t%lx\t%s\n",
 				index, count, page_flag_name(flags0));
 	}
 	flags0 = flags;
+	cgroup0= cgroup;
 	index  = offset;
 	voff   = voffset;
 	count  = size;
 static void flush_page_range(void)
 {
-	show_page_range(0, 0, 0, 0);
+	show_page_range(0, 0, 0, 0, 0);
 }
-static void show_page(unsigned long voffset,
+static void show_page(unsigned long voffset, unsigned long offset,
-		      unsigned long offset, uint64_t flags)
+		      uint64_t flags, uint64_t cgroup)
 {
 	if (opt_pid)
 		printf("%lx\t", voffset);
 	if (opt_file)
 		printf("%lu\t", voffset);
+	if (opt_list_cgroup)
+		printf("@%llu\t", (unsigned long long)cgroup);
 	printf("%lx\t%s\n", offset, page_flag_name(flags));
 }
 		flags |= BIT(SOFTDIRTY);
 	if (pme & PM_FILE)
 		flags |= BIT(FILE);
+	if (pme & PM_SWAP)
+		flags |= BIT(SWAP);
 	if (pme & PM_MMAP_EXCLUSIVE)
 		flags |= BIT(MMAP_EXCLUSIVE);
 	exit(EXIT_FAILURE);
 }
-static void add_page(unsigned long voffset,
+static void add_page(unsigned long voffset, unsigned long offset,
-		     unsigned long offset, uint64_t flags, uint64_t pme)
+		     uint64_t flags, uint64_t cgroup, uint64_t pme)
 {
 	flags = kpageflags_flags(flags, pme);
 	if (!bit_mask_ok(flags))
 		return;
+	if (opt_cgroup && cgroup != (uint64_t)opt_cgroup)
+		return;
 	if (opt_hwpoison)
 		hwpoison_page(offset);
 	if (opt_unpoison)
 		unpoison_page(offset);
 	if (opt_list == 1)
-		show_page_range(voffset, offset, 1, flags);
+		show_page_range(voffset, offset, 1, flags, cgroup);
 	else if (opt_list == 2)
-		show_page(voffset, offset, flags);
+		show_page(voffset, offset, flags, cgroup);
 	nr_pages[hash_slot(flags)]++;
 	total_pages++;
 		     uint64_t pme)
 {
 	uint64_t buf[KPAGEFLAGS_BATCH];
+	uint64_t cgi[KPAGEFLAGS_BATCH];
 	unsigned long batch;
 	unsigned long pages;
 	unsigned long i;
+	/*
+	 * kpagecgroup_read() reads only if kpagecgroup were opened, but
+	 * /proc/kpagecgroup might even not exist, so it's better to fill
+	 * them with zeros here.
+	 */
+	if (count == 1)
+		cgi[0] = 0;
+	else
+		memset(cgi, 0, sizeof cgi);
 	while (count) {
 		batch = min_t(unsigned long, count, KPAGEFLAGS_BATCH);
 		pages = kpageflags_read(buf, index, batch);
 		if (pages == 0)
 			break;
+		if (kpagecgroup_read(cgi, index, pages) != pages)
+			fatal("kpagecgroup returned fewer pages than expected");
 		for (i = 0; i < pages; i++)
-			add_page(voffset + i, index + i, buf[i], pme);
+			add_page(voffset + i, index + i, buf[i], cgi[i], pme);
 		index += pages;
 		count -= pages;
 	}
 }
+static void walk_swap(unsigned long voffset, uint64_t pme)
+{
+	uint64_t flags = kpageflags_flags(0, pme);
+	if (!bit_mask_ok(flags))
+		return;
+	if (opt_cgroup)
+		return;
+	if (opt_list == 1)
+		show_page_range(voffset, pagemap_swap_offset(pme), 1, flags, 0);
+	else if (opt_list == 2)
+		show_page(voffset, pagemap_swap_offset(pme), flags, 0);
+	nr_pages[hash_slot(flags)]++;
+	total_pages++;
+}
 #define PAGEMAP_BATCH	(64 << 10)
 static void walk_vma(unsigned long index, unsigned long count)
 {
 			pfn = pagemap_pfn(buf[i]);
 			if (pfn)
 				walk_pfn(index + i, pfn, 1, buf[i]);
+			if (buf[i] & PM_SWAP)
+				walk_swap(index + i, buf[i]);
 		}
 		index += pages;
 "            -d|--describe flags        Describe flags\n"
 "            -a|--addr    addr-spec     Walk a range of pages\n"
 "            -b|--bits    bits-spec     Walk pages with specified bits\n"
+"            -c|--cgroup  path|@inode   Walk pages within memory cgroup\n"
 "            -p|--pid     pid           Walk process address space\n"
 "            -f|--file    filename      Walk file address space\n"
 "            -l|--list                  Show page details in ranges\n"
 "            -L|--list-each             Show page details one by one\n"
+"            -C|--list-cgroup           Show cgroup inode for pages\n"
 "            -N|--no-summary            Don't show summary info\n"
 "            -X|--hwpoison              hwpoison pages\n"
 "            -x|--unpoison              unpoison pages\n"
 {
 	uint8_t vec[PAGEMAP_BATCH];
 	uint64_t buf[PAGEMAP_BATCH], flags;
+	uint64_t cgroup = 0;
 	unsigned long nr_pages, pfn, i;
 	off_t off, end = st->st_size;
 	int fd;
 				continue;
 			if (!kpageflags_read(&flags, pfn, 1))
 				continue;
+			if (!kpagecgroup_read(&cgroup, pfn, 1))
+				fatal("kpagecgroup_read failed");
 			if (first && opt_list) {
 				first = 0;
 				flush_page_range();
 				show_file(name, st);
 			}
-			add_page(off / page_size + i, pfn, flags, buf[i]);
+			add_page(off / page_size + i, pfn,
+				 flags, cgroup, buf[i]);
 		}
 	}
 	opt_file = name;
 }
+static void parse_cgroup(const char *path)
+{
+	if (path[0] == '@') {
+		opt_cgroup = parse_number(path + 1);
+		return;
+	}
+	struct stat st;
+	if (stat(path, &st))
+		fatal("stat failed: %s: %m\n", path);
+	if (!S_ISDIR(st.st_mode))
+		fatal("cgroup supposed to be a directory: %s\n", path);
+	opt_cgroup = st.st_ino;
+}
 static void parse_addr_range(const char *optarg)
 {
 	unsigned long offset;
 	{ "file"      , 1, NULL, 'f' },
 	{ "addr"      , 1, NULL, 'a' },
 	{ "bits"      , 1, NULL, 'b' },
+	{ "cgroup"    , 1, NULL, 'c' },
 	{ "describe"  , 1, NULL, 'd' },
 	{ "list"      , 0, NULL, 'l' },
 	{ "list-each" , 0, NULL, 'L' },
+	{ "list-cgroup", 0, NULL, 'C' },
 	{ "no-summary", 0, NULL, 'N' },
 	{ "hwpoison"  , 0, NULL, 'X' },
 	{ "unpoison"  , 0, NULL, 'x' },
 	page_size = getpagesize();
 	while ((c = getopt_long(argc, argv,
-				"rp:f:a:b:d:lLNXxh", opts, NULL)) != -1) {
+				"rp:f:a:b:d:c:ClLNXxh", opts, NULL)) != -1) {
 		switch (c) {
 		case 'r':
 			opt_raw = 1;
 		case 'b':
 			parse_bits_mask(optarg);
 			break;
+		case 'c':
+			parse_cgroup(optarg);
+			break;
+		case 'C':
+			opt_list_cgroup = 1;
+			break;
 		case 'd':
 			describe_flags(optarg);
 			exit(0);
 		}
 	}
+	if (opt_cgroup || opt_list_cgroup)
+		kpagecgroup_fd = checked_open(PROC_KPAGECGROUP, O_RDONLY);
 	if (opt_list && opt_pid)
 		printf("voffset\t");
 	if (opt_list && opt_file)
 		printf("foffset\t");
+	if (opt_list && opt_list_cgroup)
+		printf("cgroup\t");
 	if (opt_list == 1)
 		printf("offset\tlen\tflags\n");
 	if (opt_list == 2)

@@ -82,15 +82,15 @@ struct buffer_head { · 81 unmodified lines
82	* and buffer_foo() functions.	82	* and buffer_foo() functions.
83	*/	83	*/
84	#define BUFFER_FNS(bit, name) \	84	#define BUFFER_FNS(bit, name) \
85	static inline void set_buffer_##name(struct buffer_head *bh) \	85	static __always_inline void set_buffer_##name(struct buffer_head *bh) \
86	{ \	86	{ \
87	set_bit(BH_##bit, &(bh)->b_state); \	87	set_bit(BH_##bit, &(bh)->b_state); \
88	} \	88	} \
89	static inline void clear_buffer_##name(struct buffer_head *bh) \	89	static __always_inline void clear_buffer_##name(struct buffer_head *bh) \
90	{ \	90	{ \
91	clear_bit(BH_##bit, &(bh)->b_state); \	91	clear_bit(BH_##bit, &(bh)->b_state); \
92	} \	92	} \
93	static inline int buffer_##name(const struct buffer_head *bh) \	93	static __always_inline int buffer_##name(const struct buffer_head *bh) \
94	{ \	94	{ \
95	return test_bit(BH_##bit, &(bh)->b_state); \	95	return test_bit(BH_##bit, &(bh)->b_state); \
96	}	96	}
@@ -99,11 +99,11 @@ static inline int buffer_##name(const struct buffer_head *bh) \ · 2 unmodified lines
99	* test_set_buffer_foo() and test_clear_buffer_foo()	99	* test_set_buffer_foo() and test_clear_buffer_foo()
100	*/	100	*/
101	#define TAS_BUFFER_FNS(bit, name) \	101	#define TAS_BUFFER_FNS(bit, name) \
102	static inline int test_set_buffer_##name(struct buffer_head *bh) \	102	static __always_inline int test_set_buffer_##name(struct buffer_head *bh) \
103	{ \	103	{ \
104	return test_and_set_bit(BH_##bit, &(bh)->b_state); \	104	return test_and_set_bit(BH_##bit, &(bh)->b_state); \
105	} \	105	} \
106	static inline int test_clear_buffer_##name(struct buffer_head *bh) \	106	static __always_inline int test_clear_buffer_##name(struct buffer_head *bh) \
107	{ \	107	{ \
108	return test_and_clear_bit(BH_##bit, &(bh)->b_state); \	108	return test_and_clear_bit(BH_##bit, &(bh)->b_state); \
109	} \	109	} \

@@ -4,62 +4,62 @@ · 3 unmodified lines
4	#include <linux/kernel.h>	4	#include <linux/kernel.h>
5	#include <asm/byteorder.h>	5	#include <asm/byteorder.h>
6		6
7	static inline u16 get_unaligned_le16(const void *p)	7	static __always_inline u16 get_unaligned_le16(const void *p)
8	{	8	{
9	return le16_to_cpup((__le16 *)p);	9	return le16_to_cpup((__le16 *)p);
10	}	10	}
11		11
12	static inline u32 get_unaligned_le32(const void *p)	12	static __always_inline u32 get_unaligned_le32(const void *p)
13	{	13	{
14	return le32_to_cpup((__le32 *)p);	14	return le32_to_cpup((__le32 *)p);
15	}	15	}
16		16
17	static inline u64 get_unaligned_le64(const void *p)	17	static __always_inline u64 get_unaligned_le64(const void *p)
18	{	18	{
19	return le64_to_cpup((__le64 *)p);	19	return le64_to_cpup((__le64 *)p);
20	}	20	}
21		21
22	static inline u16 get_unaligned_be16(const void *p)	22	static __always_inline u16 get_unaligned_be16(const void *p)
23	{	23	{
24	return be16_to_cpup((__be16 *)p);	24	return be16_to_cpup((__be16 *)p);
25	}	25	}
26		26
27	static inline u32 get_unaligned_be32(const void *p)	27	static __always_inline u32 get_unaligned_be32(const void *p)
28	{	28	{
29	return be32_to_cpup((__be32 *)p);	29	return be32_to_cpup((__be32 *)p);
30	}	30	}
31		31
32	static inline u64 get_unaligned_be64(const void *p)	32	static __always_inline u64 get_unaligned_be64(const void *p)
33	{	33	{
34	return be64_to_cpup((__be64 *)p);	34	return be64_to_cpup((__be64 *)p);
35	}	35	}
36		36
37	static inline void put_unaligned_le16(u16 val, void *p)	37	static __always_inline void put_unaligned_le16(u16 val, void *p)
38	{	38	{
39	((__le16 )p) = cpu_to_le16(val);	39	((__le16 )p) = cpu_to_le16(val);
40	}	40	}
41		41
42	static inline void put_unaligned_le32(u32 val, void *p)	42	static __always_inline void put_unaligned_le32(u32 val, void *p)
43	{	43	{
44	((__le32 )p) = cpu_to_le32(val);	44	((__le32 )p) = cpu_to_le32(val);
45	}	45	}
46		46
47	static inline void put_unaligned_le64(u64 val, void *p)	47	static __always_inline void put_unaligned_le64(u64 val, void *p)
48	{	48	{
49	((__le64 )p) = cpu_to_le64(val);	49	((__le64 )p) = cpu_to_le64(val);
50	}	50	}
51		51
52	static inline void put_unaligned_be16(u16 val, void *p)	52	static __always_inline void put_unaligned_be16(u16 val, void *p)
53	{	53	{
54	((__be16 )p) = cpu_to_be16(val);	54	((__be16 )p) = cpu_to_be16(val);
55	}	55	}
56		56
57	static inline void put_unaligned_be32(u32 val, void *p)	57	static __always_inline void put_unaligned_be32(u32 val, void *p)
58	{	58	{
59	((__be32 )p) = cpu_to_be32(val);	59	((__be32 )p) = cpu_to_be32(val);
60	}	60	}
61		61
62	static inline void put_unaligned_be64(u64 val, void *p)	62	static __always_inline void put_unaligned_be64(u64 val, void *p)
63	{	63	{
64	((__be64 )p) = cpu_to_be64(val);	64	((__be64 )p) = cpu_to_be64(val);
65	}	65	}

@@ -40,51 +40,51 @@ · 39 unmodified lines
40	#define __cpu_to_be16(x) ((__force __be16)(__u16)(x))	40	#define __cpu_to_be16(x) ((__force __be16)(__u16)(x))
41	#define __be16_to_cpu(x) ((__force __u16)(__be16)(x))	41	#define __be16_to_cpu(x) ((__force __u16)(__be16)(x))
42		42
43	static inline __le64 __cpu_to_le64p(const __u64 *p)	43	static __always_inline __le64 __cpu_to_le64p(const __u64 *p)
44	{	44	{
45	return (__force __le64)__swab64p(p);	45	return (__force __le64)__swab64p(p);
46	}	46	}
47	static inline __u64 __le64_to_cpup(const __le64 *p)	47	static __always_inline __u64 __le64_to_cpup(const __le64 *p)
48	{	48	{
49	return __swab64p((__u64 *)p);	49	return __swab64p((__u64 *)p);
50	}	50	}
51	static inline __le32 __cpu_to_le32p(const __u32 *p)	51	static __always_inline __le32 __cpu_to_le32p(const __u32 *p)
52	{	52	{
53	return (__force __le32)__swab32p(p);	53	return (__force __le32)__swab32p(p);
54	}	54	}
55	static inline __u32 __le32_to_cpup(const __le32 *p)	55	static __always_inline __u32 __le32_to_cpup(const __le32 *p)
56	{	56	{
57	return __swab32p((__u32 *)p);	57	return __swab32p((__u32 *)p);
58	}	58	}
59	static inline __le16 __cpu_to_le16p(const __u16 *p)	59	static __always_inline __le16 __cpu_to_le16p(const __u16 *p)
60	{	60	{
61	return (__force __le16)__swab16p(p);	61	return (__force __le16)__swab16p(p);
62	}	62	}
63	static inline __u16 __le16_to_cpup(const __le16 *p)	63	static __always_inline __u16 __le16_to_cpup(const __le16 *p)
64	{	64	{
65	return __swab16p((__u16 *)p);	65	return __swab16p((__u16 *)p);
66	}	66	}
67	static inline __be64 __cpu_to_be64p(const __u64 *p)	67	static __always_inline __be64 __cpu_to_be64p(const __u64 *p)
68	{	68	{
69	return (__force __be64)*p;	69	return (__force __be64)*p;
70	}	70	}
71	static inline __u64 __be64_to_cpup(const __be64 *p)	71	static __always_inline __u64 __be64_to_cpup(const __be64 *p)
72	{	72	{
73	return (__force __u64)*p;	73	return (__force __u64)*p;
74	}	74	}
75	static inline __be32 __cpu_to_be32p(const __u32 *p)	75	static __always_inline __be32 __cpu_to_be32p(const __u32 *p)
76	{	76	{
77	return (__force __be32)*p;	77	return (__force __be32)*p;
78	}	78	}
79	static inline __u32 __be32_to_cpup(const __be32 *p)	79	static __always_inline __u32 __be32_to_cpup(const __be32 *p)
80	{	80	{
81	return (__force __u32)*p;	81	return (__force __u32)*p;
82	}	82	}
83	static inline __be16 __cpu_to_be16p(const __u16 *p)	83	static __always_inline __be16 __cpu_to_be16p(const __u16 *p)
84	{	84	{
85	return (__force __be16)*p;	85	return (__force __be16)*p;
86	}	86	}
87	static inline __u16 __be16_to_cpup(const __be16 *p)	87	static __always_inline __u16 __be16_to_cpup(const __be16 *p)
88	{	88	{
89	return (__force __u16)*p;	89	return (__force __u16)*p;
90	}	90	}