History

Merge branch 'akpm' (patches from Andrew)

Merge second patch-bomb from Andrew Morton: - a couple of hotfixes - the rest of MM - a new timer slack control in procfs - a couple of procfs fixes - a few misc things - some printk tweaks - lib/ updates, notably to radix-tree. - add my and Nick Piggin's old userspace radix-tree test harness to tools/testing/radix-tree/. Matthew said it was a godsend during the radix-tree work he did. - a few code-size improvements, switching to __always_inline where gcc screwed up. - partially implement character sets in sscanf * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (118 commits) sscanf: implement basic character sets lib/bug.c: use common WARN helper param: convert some "on"/"off" users to strtobool lib: add "on"/"off" support to kstrtobool lib: update single-char callers of strtobool() lib: move strtobool() to kstrtobool() include/linux/unaligned: force inlining of byteswap operations include/uapi/linux/byteorder, swab: force inlining of some byteswap operations include/asm-generic/atomic-long.h: force inlining of some atomic_long operations usb: common: convert to use match_string() helper ide: hpt366: convert to use match_string() helper ata: hpt366: convert to use match_string() helper power: ab8500: convert to use match_string() helper power: charger_manager: convert to use match_string() helper drm/edid: convert to use match_string() helper pinctrl: convert to use match_string() helper device property: convert to use match_string() helper lib/string: introduce match_string() helper radix-tree tests: add test for radix_tree_iter_next radix-tree tests: add regression3 test ...

Linus Torvalds committed 10 years ago

Commit 814a2bf2 parents 237045f f9310b2225 files · +5048 −1915Browse files

Unified · Split

225 files
  1. Documentation/cgroup-v1/cgroups.txt +1 −1
  2. Documentation/cgroup-v1/cpusets.txt +1 −1
  3. Documentation/cgroup-v2.txt +25 −0
  4. Documentation/filesystems/proc.txt +18 −0
  5. Documentation/sysctl/vm.txt +18 −0
  6. Documentation/vm/transhuge.txt +22 −0
  7. MAINTAINERS +1 −2
  8. arch/arc/include/asm/hugepage.h +0 −3
  9. arch/arm/mm/fault.c +1 −1
  10. arch/arm/mm/mmu.c +3 −3
  11. arch/arm/mm/pgd.c +1 −1
  12. arch/arm64/mm/fault.c +1 −1
  13. arch/arm64/mm/hugetlbpage.c +1 −1
  14. arch/c6x/Kconfig +1 −0
  15. arch/ia64/include/asm/io.h +1 −0
  16. arch/ia64/include/asm/rwsem.h +1 −1
  17. arch/ia64/mm/hugetlbpage.c +1 −1
  18. arch/metag/mm/hugetlbpage.c +1 −1
  19. arch/mips/mm/gup.c +1 −1
  20. arch/mn10300/Kconfig +1 −0
  21. arch/mn10300/kernel/fpu-nofpu.c +1 −0
  22. arch/parisc/mm/hugetlbpage.c +1 −1
  23. arch/powerpc/Kconfig +1 −0
  24. arch/powerpc/kernel/rtasd.c +2 −7
  25. arch/powerpc/kernel/traps.c +2 −3
  26. arch/powerpc/mm/hash_utils_64.c +20 −16
  27. arch/powerpc/mm/init_32.c +4 −4
  28. arch/powerpc/mm/mmu_context_hash64.c +1 −2
  29. arch/powerpc/mm/pgtable_64.c +1 −1
  30. arch/powerpc/platforms/512x/mpc512x_shared.c +1 −1
  31. arch/powerpc/platforms/pseries/hotplug-cpu.c +2 −8
  32. arch/s390/kernel/time.c +2 −6
  33. arch/s390/kernel/topology.c +2 −5
  34. arch/sh/mm/hugetlbpage.c +1 −1
  35. arch/sparc/mm/hugetlbpage.c +1 −1
  36. arch/tile/mm/hugetlbpage.c +1 −1
  37. arch/tile/mm/init.c +7 −4
  38. arch/um/kernel/skas/mmu.c +1 −1
  39. arch/unicore32/mm/fault.c +1 −1
  40. arch/unicore32/mm/pgd.c +1 −1
  41. arch/x86/kernel/aperture_64.c +2 −10
  42. arch/x86/kernel/tboot.c +1 −1
  43. arch/x86/mm/gup.c +1 −1
  44. arch/xtensa/mm/fault.c +1 −1
  45. crypto/async_tx/async_pq.c +1 −1
  46. drivers/ata/pata_hpt366.c +5 −8
  47. drivers/base/property.c +4 −8
  48. drivers/block/aoe/aoecmd.c +2 −2
  49. drivers/firmware/broadcom/bcm47xx_nvram.c +2 −3
  50. drivers/gpu/drm/drm_edid_load.c +6 −11
  51. drivers/ide/hpt366.c +2 −7
  52. drivers/net/ethernet/freescale/gianfar.c +1 −1
  53. drivers/net/ethernet/intel/fm10k/fm10k_main.c +1 −1
  54. drivers/net/ethernet/intel/igb/igb_main.c +1 −1
  55. drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +1 −1
  56. drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +1 −1
  57. drivers/net/ethernet/mellanox/mlx4/en_rx.c +4 −5
  58. drivers/net/ethernet/sun/niu.c +1 −1
  59. drivers/net/wireless/marvell/mwifiex/debugfs.c +4 −6
  60. drivers/pinctrl/pinmux.c +3 −10
  61. drivers/power/ab8500_btemp.c +5 −10
  62. drivers/power/ab8500_charger.c +5 −11
  63. drivers/power/ab8500_fg.c +5 −10
  64. drivers/power/abx500_chargalg.c +5 −9
  65. drivers/power/charger-manager.c +4 −23
  66. drivers/usb/common/common.c +8 −14
  67. drivers/virtio/virtio_balloon.c +6 −0
  68. fs/btrfs/tests/btrfs-tests.c +1 −2
  69. fs/cifs/cifs_debug.c +14 −42
  70. fs/cifs/cifs_debug.h +1 −1
  71. fs/cifs/cifsfs.c +3 −3
  72. fs/cifs/cifsglob.h +2 −2
  73. fs/eventpoll.c +1 −1
  74. fs/nilfs2/page.c +1 −1
  75. fs/proc/base.c +70 −1
  76. fs/proc/meminfo.c +1 −30
  77. fs/proc/page.c +6 −2
  78. fs/proc/vmcore.c +5 −2
  79. fs/select.c +4 −4
  80. include/asm-generic/atomic-long.h +3 −3
  81. include/asm-generic/bug.h +18 −9
  82. include/asm-generic/pgtable.h +17 −0
  83. include/linux/buffer_head.h +5 −5
  84. include/linux/compaction.h +16 −0
  85. include/linux/freezer.h +1 −1
  86. include/linux/gfp.h +21 −16
  87. include/linux/hrtimer.h +7 −5
  88. include/linux/huge_mm.h +13 −9
  89. include/linux/kernel.h +2 −0
  90. include/linux/list_bl.h +2 −2
  91. include/linux/memcontrol.h +33 −11
  92. include/linux/mm.h +38 −30
  93. include/linux/mmzone.h +8 −0
  94. include/linux/page-flags-layout.h +2 −0
  95. include/linux/page-flags.h +17 −15
  96. include/linux/page_ref.h +173 −0
  97. include/linux/pagemap.h +3 −16
  98. include/linux/poll.h +1 −1
  99. include/linux/quicklist.h +1 −1
  100. include/linux/radix-tree.h +26 −2
  101. include/linux/rmap.h +6 −0
  102. include/linux/sched.h +2 −2
  103. include/linux/sched/sysctl.h +0 −21
  104. include/linux/string.h +7 −1
  105. include/linux/tick.h +1 −1
  106. include/linux/unaligned/access_ok.h +12 −12
  107. include/linux/vm_event_item.h +2 −0
  108. include/trace/events/compaction.h +55 −0
  109. include/trace/events/mmflags.h +16 −7
  110. include/trace/events/page_ref.h +134 −0
  111. include/uapi/linux/byteorder/big_endian.h +12 −12
  112. include/uapi/linux/byteorder/little_endian.h +12 −12
  113. include/uapi/linux/elf-em.h +0 −3
  114. include/uapi/linux/swab.h +5 −5
  115. include/uapi/linux/virtio_balloon.h +2 −1
  116. kernel/fork.c +9 −1
  117. kernel/panic.c +26 −15
  118. kernel/printk/printk.c +66 −74
  119. kernel/sys.c +4 −1
  120. kernel/sysctl.c +10 −0
  121. kernel/time/hrtimer.c +6 −12
  122. kernel/time/tick-sched.c +2 −8
  123. kernel/time/timer.c +2 −2
  124. kernel/watchdog.c +8 −1
  125. lib/bug.c +2 −13
  126. lib/kstrtox.c +64 −0
  127. lib/radix-tree.c +148 −34
  128. lib/string.c +21 −24
  129. lib/vsprintf.c +58 −1
  130. mm/Kconfig +1 −3
  131. mm/Kconfig.debug +13 −0
  132. mm/Makefile +1 −0
  133. mm/backing-dev.c +2 −2
  134. mm/bootmem.c +3 −4
  135. mm/compaction.c +227 −5
  136. mm/debug.c +1 −1
  137. mm/debug_page_ref.c +54 −0
  138. mm/dmapool.c +7 −11
  139. mm/filemap.c +18 −37
  140. mm/huge_memory.c +109 −228
  141. mm/hugetlb.c +2 −3
  142. mm/internal.h +3 −10
  143. mm/kasan/report.c +2 −4
  144. mm/kmemcheck.c +1 −2
  145. mm/kmemleak-test.c +1 −1
  146. mm/kmemleak.c +15 −17
  147. mm/memblock.c +1 −2
  148. mm/memcontrol.c +108 −83
  149. mm/memory-failure.c +21 −31
  150. mm/memory.c +10 −15
  151. mm/memory_hotplug.c +29 −25
  152. mm/mempolicy.c +1 −3
  153. mm/mempool.c +16 −4
  154. mm/migrate.c +19 −14
  155. mm/mm_init.c +3 −4
  156. mm/mmap.c +3 −130
  157. mm/mmu_notifier.c +1 −1
  158. mm/mremap.c +1 −3
  159. mm/nobootmem.c +2 −2
  160. mm/nommu.c +0 −117
  161. mm/oom_kill.c +4 −12
  162. mm/page_alloc.c +124 −68
  163. mm/page_io.c +11 −11
  164. mm/page_owner.c +2 −3
  165. mm/percpu-km.c +3 −3
  166. mm/percpu.c +22 −21
  167. mm/pgtable-generic.c +0 −14
  168. mm/quicklist.c +1 −1
  169. mm/rmap.c +54 −17
  170. mm/shmem.c +21 −28
  171. mm/slab.c +40 −49
  172. mm/slab.h +28 −2
  173. mm/slab_common.c +6 −8
  174. mm/slub.c +15 −16
  175. mm/sparse-vmemmap.c +4 −4
  176. mm/sparse.c +9 −12
  177. mm/swap_cgroup.c +2 −3
  178. mm/swapfile.c +1 −2
  179. mm/userfaultfd.c +1 −2
  180. mm/util.c +124 −0
  181. mm/vmalloc.c +15 −16
  182. mm/vmscan.c +61 −108
  183. mm/vmstat.c +2 −0
  184. mm/workingset.c +8 −2
  185. mm/zsmalloc.c +16 −13
  186. net/core/sock.c +1 −1
  187. sound/drivers/pcsp/pcsp.c +5 −4
  188. tools/testing/radix-tree/.gitignore +2 −0
  189. tools/testing/radix-tree/Makefile +19 −0
  190. tools/testing/radix-tree/find_next_bit.c +57 −0
  191. tools/testing/radix-tree/linux.c +60 −0
  192. tools/testing/radix-tree/linux/bitops.h +150 −0
  193. tools/testing/radix-tree/linux/bitops/__ffs.h +43 −0
  194. tools/testing/radix-tree/linux/bitops/ffs.h +41 −0
  195. tools/testing/radix-tree/linux/bitops/ffz.h +12 −0
  196. tools/testing/radix-tree/linux/bitops/find.h +13 −0
  197. tools/testing/radix-tree/linux/bitops/fls.h +41 −0
  198. tools/testing/radix-tree/linux/bitops/fls64.h +14 −0
  199. tools/testing/radix-tree/linux/bitops/hweight.h +11 −0
  200. tools/testing/radix-tree/linux/bitops/le.h +53 −0
  201. tools/testing/radix-tree/linux/bitops/non-atomic.h +111 −0
  202. tools/testing/radix-tree/linux/bug.h +1 −0
  203. tools/testing/radix-tree/linux/cpu.h +34 −0
  204. tools/testing/radix-tree/linux/export.h +2 −0
  205. tools/testing/radix-tree/linux/gfp.h +10 −0
  206. tools/testing/radix-tree/linux/kernel.h +35 −0
  207. tools/testing/radix-tree/linux/kmemleak.h +1 −0
  208. tools/testing/radix-tree/linux/mempool.h +16 −0
  209. tools/testing/radix-tree/linux/notifier.h +8 −0
  210. tools/testing/radix-tree/linux/percpu.h +7 −0
  211. tools/testing/radix-tree/linux/preempt.h +4 −0
  212. tools/testing/radix-tree/linux/radix-tree.h +1 −0
  213. tools/testing/radix-tree/linux/rcupdate.h +9 −0
  214. tools/testing/radix-tree/linux/slab.h +28 −0
  215. tools/testing/radix-tree/linux/types.h +28 −0
  216. tools/testing/radix-tree/main.c +272 −0
  217. tools/testing/radix-tree/rcupdate.c +86 −0
  218. tools/testing/radix-tree/regression.h +8 −0
  219. tools/testing/radix-tree/regression1.c +220 −0
  220. tools/testing/radix-tree/regression2.c +126 −0
  221. tools/testing/radix-tree/regression3.c +117 −0
  222. tools/testing/radix-tree/tag_check.c +332 −0
  223. tools/testing/radix-tree/test.c +219 −0
  224. tools/testing/radix-tree/test.h +40 −0
  225. tools/vm/page-types.c +119 −14
Documentation/cgroup-v1/cgroups.txt+1−1
@@ -8,7 +8,7 @@ Original copyright statements from cpusets.txt: · 7 unmodified lines
88 Portions Copyright (C) 2004 BULL SA.
99 Portions Copyright (c) 2004-2006 Silicon Graphics, Inc.
1010 Modified by Paul Jackson <pj@sgi.com>
11−Modified by Christoph Lameter <clameter@sgi.com>
11+Modified by Christoph Lameter <cl@linux.com>
1212
1313 CONTENTS:
1414 =========
Documentation/cgroup-v1/cpusets.txt+1−1
@@ -6,7 +6,7 @@ Written by Simon.Derr@bull.net · 5 unmodified lines
66
77 Portions Copyright (c) 2004-2006 Silicon Graphics, Inc.
88 Modified by Paul Jackson <pj@sgi.com>
9−Modified by Christoph Lameter <clameter@sgi.com>
9+Modified by Christoph Lameter <cl@linux.com>
1010 Modified by Paul Menage <menage@google.com>
1111 Modified by Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
1212
Documentation/cgroup-v2.txt+25−0
@@ -843,6 +843,15 @@ PAGE_SIZE multiple when read back. · 842 unmodified lines
843843 Amount of memory used to cache filesystem data,
844844 including tmpfs and shared memory.
845845
846+ kernel_stack
847+
848+ Amount of memory allocated to kernel stacks.
849+
850+ slab
851+
852+ Amount of memory used for storing in-kernel data
853+ structures.
854+
846855 sock
847856
848857 Amount of memory used in network transmission buffers
@@ -871,6 +880,16 @@ PAGE_SIZE multiple when read back. · 22 unmodified lines
871880 on the internal memory management lists used by the
872881 page reclaim algorithm
873882
883+ slab_reclaimable
884+
885+ Part of "slab" that might be reclaimed, such as
886+ dentries and inodes.
887+
888+ slab_unreclaimable
889+
890+ Part of "slab" that cannot be reclaimed on memory
891+ pressure.
892+
874893 pgfault
875894
876895 Total number of page faults incurred
@@ -1368,6 +1387,12 @@ system than killing the group. Otherwise, memory.max is there to · 491 unmodified lines
13681387 limit this type of spillover and ultimately contain buggy or even
13691388 malicious applications.
13701389
1390+Setting the original memory.limit_in_bytes below the current usage was
1391+subject to a race condition, where concurrent charges could cause the
1392+limit setting to fail. memory.max on the other hand will first set the
1393+limit to prevent new charges, and then reclaim and OOM kill until the
1394+new limit is met - or the task writing to memory.max is killed.
1395+
13711396 The combined memory+swap accounting and limiting is replaced by real
13721397 control over swap space.
13731398
Documentation/filesystems/proc.txt+18−0
@@ -43,6 +43,7 @@ Table of Contents · 42 unmodified lines
4343 3.7 /proc/<pid>/task/<tid>/children - Information about task children
4444 3.8 /proc/<pid>/fdinfo/<fd> - Information about opened file
4545 3.9 /proc/<pid>/map_files - Information about memory mapped files
46+ 3.10 /proc/<pid>/timerslack_ns - Task timerslack value
4647
4748 4 Configuring procfs
4849 4.1 Mount options
@@ -1862,6 +1863,23 @@ time one can open(2) mappings from the listings of two processes and · 1813 unmodified lines
18621863 comparing their inode numbers to figure out which anonymous memory areas
18631864 are actually shared.
18641865
1866+3.10 /proc/<pid>/timerslack_ns - Task timerslack value
1867+---------------------------------------------------------
1868+This file provides the value of the task's timerslack value in nanoseconds.
1869+This value specifies a amount of time that normal timers may be deferred
1870+in order to coalesce timers and avoid unnecessary wakeups.
1871+
1872+This allows a task's interactivity vs power consumption trade off to be
1873+adjusted.
1874+
1875+Writing 0 to the file will set the tasks timerslack to the default value.
1876+
1877+Valid values are from 0 - ULLONG_MAX
1878+
1879+An application setting the value must have PTRACE_MODE_ATTACH_FSCREDS level
1880+permissions on the task specified to change its timerslack_ns value.
1881+
1882+
18651883 ------------------------------------------------------------------------------
18661884 Configuring procfs
18671885 ------------------------------------------------------------------------------
Documentation/sysctl/vm.txt+18−0
@@ -803,6 +803,24 @@ performance impact. Reclaim code needs to take various locks to find freeable · 802 unmodified lines
803803 directory and inode objects. With vfs_cache_pressure=1000, it will look for
804804 ten times more freeable objects than there are.
805805
806+=============================================================
807+
808+watermark_scale_factor:
809+
810+This factor controls the aggressiveness of kswapd. It defines the
811+amount of memory left in a node/system before kswapd is woken up and
812+how much memory needs to be free before kswapd goes back to sleep.
813+
814+The unit is in fractions of 10,000. The default value of 10 means the
815+distances between watermarks are 0.1% of the available memory in the
816+node/system. The maximum value is 1000, or 10% of memory.
817+
818+A high rate of threads entering direct reclaim (allocstall) or kswapd
819+going to sleep prematurely (kswapd_low_wmark_hit_quickly) can indicate
820+that the number of free pages kswapd maintains for latency reasons is
821+too small for the allocation bursts occurring in the system. This knob
822+can then be used to tune kswapd aggressiveness accordingly.
823+
806824 ==============================================================
807825
808826 zone_reclaim_mode:
Documentation/vm/transhuge.txt+22−0
@@ -113,9 +113,26 @@ guaranteed, but it may be more likely in case the allocation is for a · 112 unmodified lines
113113 MADV_HUGEPAGE region.
114114
115115 echo always >/sys/kernel/mm/transparent_hugepage/defrag
116+echo defer >/sys/kernel/mm/transparent_hugepage/defrag
116117 echo madvise >/sys/kernel/mm/transparent_hugepage/defrag
117118 echo never >/sys/kernel/mm/transparent_hugepage/defrag
118119
120+"always" means that an application requesting THP will stall on allocation
121+failure and directly reclaim pages and compact memory in an effort to
122+allocate a THP immediately. This may be desirable for virtual machines
123+that benefit heavily from THP use and are willing to delay the VM start
124+to utilise them.
125+
126+"defer" means that an application will wake kswapd in the background
127+to reclaim pages and wake kcompact to compact memory so that THP is
128+available in the near future. It's the responsibility of khugepaged
129+to then install the THP pages later.
130+
131+"madvise" will enter direct reclaim like "always" but only for regions
132+that are have used madvise(MADV_HUGEPAGE). This is the default behaviour.
133+
134+"never" should be self-explanatory.
135+
119136 By default kernel tries to use huge zero page on read page fault.
120137 It's possible to disable huge zero page by writing 0 or enable it
121138 back by writing 1:
@@ -229,6 +246,11 @@ thp_split_page is incremented every time a huge page is split into base · 107 unmodified lines
229246 thp_split_page_failed is is incremented if kernel fails to split huge
230247 page. This can happen if the page was pinned by somebody.
231248
249+thp_deferred_split_page is incremented when a huge page is put onto split
250+ queue. This happens when a huge page is partially unmapped and
251+ splitting it would free up some memory. Pages on split queue are
252+ going to be split under memory pressure.
253+
232254 thp_split_pmd is incremented every time a PMD split into table of PTEs.
233255 This can happen, for instance, when application calls mprotect() or
234256 munmap() on part of huge page. It doesn't split huge page, only
MAINTAINERS+1−2
@@ -8498,7 +8498,7 @@ F: include/crypto/pcrypt.h · 8497 unmodified lines
84988498
84998499 PER-CPU MEMORY ALLOCATOR
85008500 M: Tejun Heo <tj@kernel.org>
8501−M: Christoph Lameter <cl@linux-foundation.org>
8501+M: Christoph Lameter <cl@linux.com>
85028502 T: git git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git
85038503 S: Maintained
85048504 F: include/linux/percpu*.h
@@ -11296,7 +11296,6 @@ F: include/linux/cdrom.h · 2791 unmodified lines
1129611296 F: include/uapi/linux/cdrom.h
1129711297
1129811298 UNISYS S-PAR DRIVERS
11299−M: Benjamin Romer <benjamin.romer@unisys.com>
1130011299 M: David Kershner <david.kershner@unisys.com>
1130111300 L: sparmaintainer@unisys.com (Unisys internal)
1130211301 S: Supported
arch/arc/include/asm/hugepage.h+0−3
@@ -30,19 +30,16 @@ static inline pmd_t pte_pmd(pte_t pte) · 29 unmodified lines
3030 #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
3131 #define pmd_mkhuge(pmd) pte_pmd(pte_mkhuge(pmd_pte(pmd)))
3232 #define pmd_mknotpresent(pmd) pte_pmd(pte_mknotpresent(pmd_pte(pmd)))
33−#define pmd_mksplitting(pmd) pte_pmd(pte_mkspecial(pmd_pte(pmd)))
3433 #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
3534
3635 #define pmd_write(pmd) pte_write(pmd_pte(pmd))
3736 #define pmd_young(pmd) pte_young(pmd_pte(pmd))
3837 #define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
3938 #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
40−#define pmd_special(pmd) pte_special(pmd_pte(pmd))
4139
4240 #define mk_pmd(page, prot) pte_pmd(mk_pte(page, prot))
4341
4442 #define pmd_trans_huge(pmd) (pmd_val(pmd) & _PAGE_HW_SZ)
45−#define pmd_trans_splitting(pmd) (pmd_trans_huge(pmd) && pmd_special(pmd))
4643
4744 #define pfn_pmd(pfn, prot) (__pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot)))
4845
arch/arm/mm/fault.c+1−1
@@ -346,7 +346,7 @@ retry: · 345 unmodified lines
346346 up_read(&mm->mmap_sem);
347347
348348 /*
349− * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
349+ * Handle the "normal" case first - VM_FAULT_MAJOR
350350 */
351351 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS))))
352352 return 0;
arch/arm/mm/mmu.c+3−3
@@ -732,7 +732,7 @@ static void *__init late_alloc(unsigned long sz) · 731 unmodified lines
732732 return ptr;
733733 }
734734
735−static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr,
735+static pte_t * __init arm_pte_alloc(pmd_t *pmd, unsigned long addr,
736736 unsigned long prot,
737737 void *(*alloc)(unsigned long sz))
738738 {
@@ -747,7 +747,7 @@ static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr, · 8 unmodified lines
747747 static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr,
748748 unsigned long prot)
749749 {
750− return pte_alloc(pmd, addr, prot, early_alloc);
750+ return arm_pte_alloc(pmd, addr, prot, early_alloc);
751751 }
752752
753753 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
@@ -756,7 +756,7 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, · 2 unmodified lines
756756 void *(*alloc)(unsigned long sz),
757757 bool ng)
758758 {
759− pte_t *pte = pte_alloc(pmd, addr, type->prot_l1, alloc);
759+ pte_t *pte = arm_pte_alloc(pmd, addr, type->prot_l1, alloc);
760760 do {
761761 set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)),
762762 ng ? PTE_EXT_NG : 0);
arch/arm/mm/pgd.c+1−1
@@ -80,7 +80,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm) · 79 unmodified lines
8080 if (!new_pmd)
8181 goto no_pmd;
8282
83− new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);
83+ new_pte = pte_alloc_map(mm, new_pmd, 0);
8484 if (!new_pte)
8585 goto no_pte;
8686
arch/arm64/mm/fault.c+1−1
@@ -304,7 +304,7 @@ retry: · 303 unmodified lines
304304 up_read(&mm->mmap_sem);
305305
306306 /*
307− * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
307+ * Handle the "normal" case first - VM_FAULT_MAJOR
308308 */
309309 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
310310 VM_FAULT_BADACCESS))))
arch/arm64/mm/hugetlbpage.c+1−1
@@ -124,7 +124,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, · 123 unmodified lines
124124 * will be no pte_unmap() to correspond with this
125125 * pte_alloc_map().
126126 */
127− pte = pte_alloc_map(mm, NULL, pmd, addr);
127+ pte = pte_alloc_map(mm, pmd, addr);
128128 } else if (sz == PMD_SIZE) {
129129 if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
130130 pud_none(*pud))
arch/c6x/Kconfig+1−0
@@ -36,6 +36,7 @@ config GENERIC_HWEIGHT · 35 unmodified lines
3636
3737 config GENERIC_BUG
3838 def_bool y
39+ depends on BUG
3940
4041 config C6X_BIG_KERNEL
4142 bool "Build a big kernel"
arch/ia64/include/asm/io.h+1−0
@@ -433,6 +433,7 @@ static inline void __iomem * ioremap_cache (unsigned long phys_addr, unsigned lo · 432 unmodified lines
433433 return ioremap(phys_addr, size);
434434 }
435435 #define ioremap_cache ioremap_cache
436+#define ioremap_uc ioremap_nocache
436437
437438
438439 /*
arch/ia64/include/asm/rwsem.h+1−1
@@ -3,7 +3,7 @@ · 2 unmodified lines
33 *
44 * Copyright (C) 2003 Ken Chen <kenneth.w.chen@intel.com>
55 * Copyright (C) 2003 Asit Mallick <asit.k.mallick@intel.com>
6− * Copyright (C) 2005 Christoph Lameter <clameter@sgi.com>
6+ * Copyright (C) 2005 Christoph Lameter <cl@linux.com>
77 *
88 * Based on asm-i386/rwsem.h and other architecture implementation.
99 *
arch/ia64/mm/hugetlbpage.c+1−1
@@ -38,7 +38,7 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) · 37 unmodified lines
3838 if (pud) {
3939 pmd = pmd_alloc(mm, pud, taddr);
4040 if (pmd)
41− pte = pte_alloc_map(mm, NULL, pmd, taddr);
41+ pte = pte_alloc_map(mm, pmd, taddr);
4242 }
4343 return pte;
4444 }
arch/metag/mm/hugetlbpage.c+1−1
@@ -67,7 +67,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, · 66 unmodified lines
6767 pgd = pgd_offset(mm, addr);
6868 pud = pud_offset(pgd, addr);
6969 pmd = pmd_offset(pud, addr);
70− pte = pte_alloc_map(mm, NULL, pmd, addr);
70+ pte = pte_alloc_map(mm, pmd, addr);
7171 pgd->pgd &= ~_PAGE_SZ_MASK;
7272 pgd->pgd |= _PAGE_SZHUGE;
7373
arch/mips/mm/gup.c+1−1
@@ -64,7 +64,7 @@ static inline void get_head_page_multiple(struct page *page, int nr) · 63 unmodified lines
6464 {
6565 VM_BUG_ON(page != compound_head(page));
6666 VM_BUG_ON(page_count(page) == 0);
67− atomic_add(nr, &page->_count);
67+ page_ref_add(page, nr);
6868 SetPageReferenced(page);
6969 }
7070
arch/mn10300/Kconfig+1−0
@@ -53,6 +53,7 @@ config GENERIC_HWEIGHT · 52 unmodified lines
5353
5454 config GENERIC_BUG
5555 def_bool y
56+ depends on BUG
5657
5758 config QUICKLIST
5859 def_bool y
arch/mn10300/kernel/fpu-nofpu.c+1−0
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 * 2 of the Licence, or (at your option) any later version.
1010 */
1111 #include <asm/fpu.h>
12+#include <asm/elf.h>
1213
1314 /*
1415 * handle an FPU operational exception
arch/parisc/mm/hugetlbpage.c+1−1
@@ -63,7 +63,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, · 62 unmodified lines
6363 if (pud) {
6464 pmd = pmd_alloc(mm, pud, addr);
6565 if (pmd)
66− pte = pte_alloc_map(mm, NULL, pmd, addr);
66+ pte = pte_alloc_map(mm, pmd, addr);
6767 }
6868 return pte;
6969 }
arch/powerpc/Kconfig+1−0
@@ -158,6 +158,7 @@ config PPC · 157 unmodified lines
158158 select ARCH_HAS_DEVMEM_IS_ALLOWED
159159 select HAVE_ARCH_SECCOMP_FILTER
160160 select ARCH_HAS_UBSAN_SANITIZE_ALL
161+ select ARCH_SUPPORTS_DEFERRED_STRUCT_PAGE_INIT
161162
162163 config GENERIC_CSUM
163164 def_bool CPU_LITTLE_ENDIAN
arch/powerpc/kernel/rtasd.c+2−7
@@ -49,7 +49,7 @@ static unsigned int rtas_error_log_buffer_max; · 48 unmodified lines
4949 static unsigned int event_scan;
5050 static unsigned int rtas_event_scan_rate;
5151
52−static int full_rtas_msgs = 0;
52+static bool full_rtas_msgs;
5353
5454 /* Stop logging to nvram after first fatal error */
5555 static int logging_enabled; /* Until we initialize everything,
@@ -592,11 +592,6 @@ __setup("surveillance=", surveillance_setup); · 536 unmodified lines
592592
593593 static int __init rtasmsgs_setup(char *str)
594594 {
595− if (strcmp(str, "on") == 0)
596− full_rtas_msgs = 1;
597− else if (strcmp(str, "off") == 0)
598− full_rtas_msgs = 0;
599−
600− return 1;
595+ return (kstrtobool(str, &full_rtas_msgs) == 0);
601596 }
602597 __setup("rtasmsgs=", rtasmsgs_setup);
arch/powerpc/kernel/traps.c+2−3
@@ -203,9 +203,8 @@ static int __kprobes __die(const char *str, struct pt_regs *regs, long err) · 202 unmodified lines
203203 #ifdef CONFIG_SMP
204204 printk("SMP NR_CPUS=%d ", NR_CPUS);
205205 #endif
206−#ifdef CONFIG_DEBUG_PAGEALLOC
207− printk("DEBUG_PAGEALLOC ");
208−#endif
206+ if (debug_pagealloc_enabled())
207+ printk("DEBUG_PAGEALLOC ");
209208 #ifdef CONFIG_NUMA
210209 printk("NUMA ");
211210 #endif
arch/powerpc/mm/hash_utils_64.c+20−16
@@ -255,8 +255,10 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend, · 254 unmodified lines
255255
256256 if (ret < 0)
257257 break;
258+
258259 #ifdef CONFIG_DEBUG_PAGEALLOC
259− if ((paddr >> PAGE_SHIFT) < linear_map_hash_count)
260+ if (debug_pagealloc_enabled() &&
261+ (paddr >> PAGE_SHIFT) < linear_map_hash_count)
260262 linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret | 0x80;
261263 #endif /* CONFIG_DEBUG_PAGEALLOC */
262264 }
@@ -512,17 +514,17 @@ static void __init htab_init_page_sizes(void) · 249 unmodified lines
512514 if (mmu_has_feature(MMU_FTR_16M_PAGE))
513515 memcpy(mmu_psize_defs, mmu_psize_defaults_gp,
514516 sizeof(mmu_psize_defaults_gp));
515− found:
516−#ifndef CONFIG_DEBUG_PAGEALLOC
517− /*
518− * Pick a size for the linear mapping. Currently, we only support
519− * 16M, 1M and 4K which is the default
520− */
521− if (mmu_psize_defs[MMU_PAGE_16M].shift)
522− mmu_linear_psize = MMU_PAGE_16M;
523− else if (mmu_psize_defs[MMU_PAGE_1M].shift)
524− mmu_linear_psize = MMU_PAGE_1M;
525−#endif /* CONFIG_DEBUG_PAGEALLOC */
517+found:
518+ if (!debug_pagealloc_enabled()) {
519+ /*
520+ * Pick a size for the linear mapping. Currently, we only
521+ * support 16M, 1M and 4K which is the default
522+ */
523+ if (mmu_psize_defs[MMU_PAGE_16M].shift)
524+ mmu_linear_psize = MMU_PAGE_16M;
525+ else if (mmu_psize_defs[MMU_PAGE_1M].shift)
526+ mmu_linear_psize = MMU_PAGE_1M;
527+ }
526528
527529 #ifdef CONFIG_PPC_64K_PAGES
528530 /*
@@ -721,10 +723,12 @@ static void __init htab_initialize(void) · 192 unmodified lines
721723 prot = pgprot_val(PAGE_KERNEL);
722724
723725 #ifdef CONFIG_DEBUG_PAGEALLOC
724− linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT;
725− linear_map_hash_slots = __va(memblock_alloc_base(linear_map_hash_count,
726− 1, ppc64_rma_size));
727− memset(linear_map_hash_slots, 0, linear_map_hash_count);
726+ if (debug_pagealloc_enabled()) {
727+ linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT;
728+ linear_map_hash_slots = __va(memblock_alloc_base(
729+ linear_map_hash_count, 1, ppc64_rma_size));
730+ memset(linear_map_hash_slots, 0, linear_map_hash_count);
731+ }
728732 #endif /* CONFIG_DEBUG_PAGEALLOC */
729733
730734 /* On U3 based machines, we need to reserve the DART area and
arch/powerpc/mm/init_32.c+4−4
@@ -112,10 +112,10 @@ void __init MMU_setup(void) · 111 unmodified lines
112112 if (strstr(boot_command_line, "noltlbs")) {
113113 __map_without_ltlbs = 1;
114114 }
115−#ifdef CONFIG_DEBUG_PAGEALLOC
116− __map_without_bats = 1;
117− __map_without_ltlbs = 1;
118−#endif
115+ if (debug_pagealloc_enabled()) {
116+ __map_without_bats = 1;
117+ __map_without_ltlbs = 1;
118+ }
119119 }
120120
121121 /*
arch/powerpc/mm/mmu_context_hash64.c+1−2
@@ -118,8 +118,7 @@ static void destroy_pagetable_page(struct mm_struct *mm) · 117 unmodified lines
118118 /* drop all the pending references */
119119 count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
120120 /* We allow PTE_FRAG_NR fragments from a PTE page */
121− count = atomic_sub_return(PTE_FRAG_NR - count, &page->_count);
122− if (!count) {
121+ if (page_ref_sub_and_test(page, PTE_FRAG_NR - count)) {
123122 pgtable_page_dtor(page);
124123 free_hot_cold_page(page, 0);
125124 }
arch/powerpc/mm/pgtable_64.c+1−1
@@ -403,7 +403,7 @@ static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel) · 402 unmodified lines
403403 * count.
404404 */
405405 if (likely(!mm->context.pte_frag)) {
406− atomic_set(&page->_count, PTE_FRAG_NR);
406+ set_page_count(page, PTE_FRAG_NR);
407407 mm->context.pte_frag = ret + PTE_FRAG_SIZE;
408408 }
409409 spin_unlock(&mm->page_table_lock);
arch/powerpc/platforms/512x/mpc512x_shared.c+1−1
@@ -188,7 +188,7 @@ static struct fsl_diu_shared_fb __attribute__ ((__aligned__(8))) diu_shared_fb; · 187 unmodified lines
188188 static inline void mpc512x_free_bootmem(struct page *page)
189189 {
190190 BUG_ON(PageTail(page));
191− BUG_ON(atomic_read(&page->_count) > 1);
191+ BUG_ON(page_ref_count(page) > 1);
192192 free_reserved_page(page);
193193 }
194194
arch/powerpc/platforms/pseries/hotplug-cpu.c+2−8
@@ -47,20 +47,14 @@ static DEFINE_PER_CPU(enum cpu_state_vals, current_state) = CPU_STATE_OFFLINE; · 46 unmodified lines
4747
4848 static enum cpu_state_vals default_offline_state = CPU_STATE_OFFLINE;
4949
50−static int cede_offline_enabled __read_mostly = 1;
50+static bool cede_offline_enabled __read_mostly = true;
5151
5252 /*
5353 * Enable/disable cede_offline when available.
5454 */
5555 static int __init setup_cede_offline(char *str)
5656 {
57− if (!strcmp(str, "off"))
58− cede_offline_enabled = 0;
59− else if (!strcmp(str, "on"))
60− cede_offline_enabled = 1;
61− else
62− return 0;
63− return 1;
57+ return (kstrtobool(str, &cede_offline_enabled) == 0);
6458 }
6559
6660 __setup("cede_offline=", setup_cede_offline);
arch/s390/kernel/time.c+2−6
@@ -1432,7 +1432,7 @@ device_initcall(etr_init_sysfs); · 1431 unmodified lines
14321432 /*
14331433 * Server Time Protocol (STP) code.
14341434 */
1435−static int stp_online;
1435+static bool stp_online;
14361436 static struct stp_sstpi stp_info;
14371437 static void *stp_page;
14381438
@@ -1443,11 +1443,7 @@ static struct timer_list stp_timer; · 4 unmodified lines
14431443
14441444 static int __init early_parse_stp(char *p)
14451445 {
1446− if (strncmp(p, "off", 3) == 0)
1447− stp_online = 0;
1448− else if (strncmp(p, "on", 2) == 0)
1449− stp_online = 1;
1450− return 0;
1446+ return kstrtobool(p, &stp_online);
14511447 }
14521448 early_param("stp", early_parse_stp);
14531449
arch/s390/kernel/topology.c+2−5
@@ -37,7 +37,7 @@ static void set_topology_timer(void); · 36 unmodified lines
3737 static void topology_work_fn(struct work_struct *work);
3838 static struct sysinfo_15_1_x *tl_info;
3939
40−static int topology_enabled = 1;
40+static bool topology_enabled = true;
4141 static DECLARE_WORK(topology_work, topology_work_fn);
4242
4343 /*
@@ -444,10 +444,7 @@ static const struct cpumask *cpu_book_mask(int cpu) · 400 unmodified lines
444444
445445 static int __init early_parse_topology(char *p)
446446 {
447− if (strncmp(p, "off", 3))
448− return 0;
449− topology_enabled = 0;
450− return 0;
447+ return kstrtobool(p, &topology_enabled);
451448 }
452449 early_param("topology", early_parse_topology);
453450
arch/sh/mm/hugetlbpage.c+1−1
@@ -35,7 +35,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, · 34 unmodified lines
3535 if (pud) {
3636 pmd = pmd_alloc(mm, pud, addr);
3737 if (pmd)
38− pte = pte_alloc_map(mm, NULL, pmd, addr);
38+ pte = pte_alloc_map(mm, pmd, addr);
3939 }
4040 }
4141
arch/sparc/mm/hugetlbpage.c+1−1
@@ -146,7 +146,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, · 145 unmodified lines
146146 if (pud) {
147147 pmd = pmd_alloc(mm, pud, addr);
148148 if (pmd)
149− pte = pte_alloc_map(mm, NULL, pmd, addr);
149+ pte = pte_alloc_map(mm, pmd, addr);
150150 }
151151 return pte;
152152 }
arch/tile/mm/hugetlbpage.c+1−1
@@ -77,7 +77,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, · 76 unmodified lines
7777 else {
7878 if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
7979 panic("Unexpected page size %#lx\n", sz);
80− return pte_alloc_map(mm, NULL, pmd, addr);
80+ return pte_alloc_map(mm, pmd, addr);
8181 }
8282 }
8383 #else
arch/tile/mm/init.c+7−4
@@ -896,17 +896,15 @@ void __init pgtable_cache_init(void) · 895 unmodified lines
896896 panic("pgtable_cache_init(): Cannot create pgd cache");
897897 }
898898
899−#ifdef CONFIG_DEBUG_PAGEALLOC
900−static long __write_once initfree;
901−#else
902899 static long __write_once initfree = 1;
903−#endif
900+static bool __write_once set_initfree_done;
904901
905902 /* Select whether to free (1) or mark unusable (0) the __init pages. */
906903 static int __init set_initfree(char *str)
907904 {
908905 long val;
909906 if (kstrtol(str, 0, &val) == 0) {
907+ set_initfree_done = true;
910908 initfree = val;
911909 pr_info("initfree: %s free init pages\n",
912910 initfree ? "will" : "won't");
@@ -919,6 +917,11 @@ static void free_init_pages(char *what, unsigned long begin, unsigned long end) · 6 unmodified lines
919917 {
920918 unsigned long addr = (unsigned long) begin;
921919
920+ /* Prefer user request first */
921+ if (!set_initfree_done) {
922+ if (debug_pagealloc_enabled())
923+ initfree = 0;
924+ }
922925 if (kdata_huge && !initfree) {
923926 pr_warn("Warning: ignoring initfree=0: incompatible with kdata=huge\n");
924927 initfree = 1;
arch/um/kernel/skas/mmu.c+1−1
@@ -31,7 +31,7 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc, · 30 unmodified lines
3131 if (!pmd)
3232 goto out_pmd;
3333
34− pte = pte_alloc_map(mm, NULL, pmd, proc);
34+ pte = pte_alloc_map(mm, pmd, proc);
3535 if (!pte)
3636 goto out_pte;
3737
arch/unicore32/mm/fault.c+1−1
@@ -276,7 +276,7 @@ retry: · 275 unmodified lines
276276 up_read(&mm->mmap_sem);
277277
278278 /*
279− * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
279+ * Handle the "normal" case first - VM_FAULT_MAJOR
280280 */
281281 if (likely(!(fault &
282282 (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS))))
arch/unicore32/mm/pgd.c+1−1
@@ -54,7 +54,7 @@ pgd_t *get_pgd_slow(struct mm_struct *mm) · 53 unmodified lines
5454 if (!new_pmd)
5555 goto no_pmd;
5656
57− new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);
57+ new_pte = pte_alloc_map(mm, new_pmd, 0);
5858 if (!new_pte)
5959 goto no_pte;
6060
arch/x86/kernel/aperture_64.c+2−10
@@ -227,19 +227,11 @@ static u32 __init search_agp_bridge(u32 *order, int *valid_agp) · 226 unmodified lines
227227 return 0;
228228 }
229229
230−static int gart_fix_e820 __initdata = 1;
230+static bool gart_fix_e820 __initdata = true;
231231
232232 static int __init parse_gart_mem(char *p)
233233 {
234− if (!p)
235− return -EINVAL;
236−
237− if (!strncmp(p, "off", 3))
238− gart_fix_e820 = 0;
239− else if (!strncmp(p, "on", 2))
240− gart_fix_e820 = 1;
241−
242− return 0;
234+ return kstrtobool(p, &gart_fix_e820);
243235 }
244236 early_param("gart_fix_e820", parse_gart_mem);
245237
arch/x86/kernel/tboot.c+1−1
@@ -135,7 +135,7 @@ static int map_tboot_page(unsigned long vaddr, unsigned long pfn, · 134 unmodified lines
135135 pmd = pmd_alloc(&tboot_mm, pud, vaddr);
136136 if (!pmd)
137137 return -1;
138− pte = pte_alloc_map(&tboot_mm, NULL, pmd, vaddr);
138+ pte = pte_alloc_map(&tboot_mm, pmd, vaddr);
139139 if (!pte)
140140 return -1;
141141 set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));
arch/x86/mm/gup.c+1−1
@@ -131,7 +131,7 @@ static inline void get_head_page_multiple(struct page *page, int nr) · 130 unmodified lines
131131 {
132132 VM_BUG_ON_PAGE(page != compound_head(page), page);
133133 VM_BUG_ON_PAGE(page_count(page) == 0, page);
134− atomic_add(nr, &page->_count);
134+ page_ref_add(page, nr);
135135 SetPageReferenced(page);
136136 }
137137
arch/xtensa/mm/fault.c+1−1
@@ -146,7 +146,7 @@ good_area: · 145 unmodified lines
146146 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
147147 if (flags & VM_FAULT_MAJOR)
148148 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
149− else if (flags & VM_FAULT_MINOR)
149+ else
150150 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
151151
152152 return;
crypto/async_tx/async_pq.c+1−1
@@ -444,7 +444,7 @@ static int __init async_pq_init(void) · 443 unmodified lines
444444
445445 static void __exit async_pq_exit(void)
446446 {
447− put_page(pq_scribble_page);
447+ __free_page(pq_scribble_page);
448448 }
449449
450450 module_init(async_pq_init);
drivers/ata/pata_hpt366.c+5−8
@@ -176,17 +176,14 @@ static int hpt_dma_blacklisted(const struct ata_device *dev, char *modestr, · 175 unmodified lines
176176 const char * const list[])
177177 {
178178 unsigned char model_num[ATA_ID_PROD_LEN + 1];
179− int i = 0;
179+ int i;
180180
181181 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
182182
183− while (list[i] != NULL) {
184− if (!strcmp(list[i], model_num)) {
185− pr_warn("%s is not supported for %s\n",
186− modestr, list[i]);
187− return 1;
188− }
189− i++;
183+ i = match_string(list, -1, model_num);
184+ if (i >= 0) {
185+ pr_warn("%s is not supported for %s\n", modestr, list[i]);
186+ return 1;
190187 }
191188 return 0;
192189 }
drivers/base/property.c+4−8
@@ -651,7 +651,7 @@ int fwnode_property_match_string(struct fwnode_handle *fwnode, · 650 unmodified lines
651651 const char *propname, const char *string)
652652 {
653653 const char **values;
654− int nval, ret, i;
654+ int nval, ret;
655655
656656 nval = fwnode_property_read_string_array(fwnode, propname, NULL, 0);
657657 if (nval < 0)
@@ -668,13 +668,9 @@ int fwnode_property_match_string(struct fwnode_handle *fwnode, · 10 unmodified lines
668668 if (ret < 0)
669669 goto out;
670670
671− ret = -ENODATA;
672− for (i = 0; i < nval; i++) {
673− if (!strcmp(values[i], string)) {
674− ret = i;
675− break;
676− }
677− }
671+ ret = match_string(values, nval, string);
672+ if (ret < 0)
673+ ret = -ENODATA;
678674 out:
679675 kfree(values);
680676 return ret;
drivers/block/aoe/aoecmd.c+2−2
@@ -875,7 +875,7 @@ bio_pageinc(struct bio *bio) · 874 unmodified lines
875875 * compound pages is no longer allowed by the kernel.
876876 */
877877 page = compound_head(bv.bv_page);
878− atomic_inc(&page->_count);
878+ page_ref_inc(page);
879879 }
880880 }
881881
@@ -888,7 +888,7 @@ bio_pagedec(struct bio *bio) · 6 unmodified lines
888888
889889 bio_for_each_segment(bv, bio, iter) {
890890 page = compound_head(bv.bv_page);
891− atomic_dec(&page->_count);
891+ page_ref_dec(page);
892892 }
893893 }
894894
drivers/firmware/broadcom/bcm47xx_nvram.c+2−3
@@ -94,15 +94,14 @@ static int nvram_find_and_copy(void __iomem *iobase, u32 lim) · 93 unmodified lines
9494
9595 found:
9696 __ioread32_copy(nvram_buf, header, sizeof(*header) / 4);
97− header = (struct nvram_header *)nvram_buf;
98− nvram_len = header->len;
97+ nvram_len = ((struct nvram_header *)(nvram_buf))->len;
9998 if (nvram_len > size) {
10099 pr_err("The nvram size according to the header seems to be bigger than the partition on flash\n");
101100 nvram_len = size;
102101 }
103102 if (nvram_len >= NVRAM_SPACE) {
104103 pr_err("nvram on flash (%i bytes) is bigger than the reserved space in memory, will just copy the first %i bytes\n",
105− header->len, NVRAM_SPACE - 1);
104+ nvram_len, NVRAM_SPACE - 1);
106105 nvram_len = NVRAM_SPACE - 1;
107106 }
108107 /* proceed reading data after header */
drivers/gpu/drm/drm_edid_load.c+6−11
@@ -170,16 +170,11 @@ static void *edid_load(struct drm_connector *connector, const char *name, · 169 unmodified lines
170170 int i, valid_extensions = 0;
171171 bool print_bad_edid = !connector->bad_edid_counter || (drm_debug & DRM_UT_KMS);
172172
173− builtin = 0;
174− for (i = 0; i < GENERIC_EDIDS; i++) {
175− if (strcmp(name, generic_edid_name[i]) == 0) {
176− fwdata = generic_edid[i];
177− fwsize = sizeof(generic_edid[i]);
178− builtin = 1;
179− break;
180− }
181− }
182− if (!builtin) {
173+ builtin = match_string(generic_edid_name, GENERIC_EDIDS, name);
174+ if (builtin >= 0) {
175+ fwdata = generic_edid[builtin];
176+ fwsize = sizeof(generic_edid[builtin]);
177+ } else {
183178 struct platform_device *pdev;
184179 int err;
185180
@@ -252,7 +247,7 @@ static void *edid_load(struct drm_connector *connector, const char *name, · 66 unmodified lines
252247 }
253248
254249 DRM_INFO("Got %s EDID base block and %d extension%s from "
255− "\"%s\" for connector \"%s\"\n", builtin ? "built-in" :
250+ "\"%s\" for connector \"%s\"\n", (builtin >= 0) ? "built-in" :
256251 "external", valid_extensions, valid_extensions == 1 ? "" : "s",
257252 name, connector_name);
258253
drivers/ide/hpt366.c+2−7
@@ -531,14 +531,9 @@ static const struct hpt_info hpt371n = { · 530 unmodified lines
531531 .timings = &hpt37x_timings
532532 };
533533
534−static int check_in_drive_list(ide_drive_t *drive, const char **list)
534+static bool check_in_drive_list(ide_drive_t *drive, const char **list)
535535 {
536− char *m = (char *)&drive->id[ATA_ID_PROD];
537−
538− while (*list)
539− if (!strcmp(*list++, m))
540− return 1;
541− return 0;
536+ return match_string(list, -1, (char *)&drive->id[ATA_ID_PROD]) >= 0;
542537 }
543538
544539 static struct hpt_info *hpt3xx_get_info(struct device *dev)
drivers/net/ethernet/freescale/gianfar.c+1−1
@@ -2944,7 +2944,7 @@ static bool gfar_add_rx_frag(struct gfar_rx_buff *rxb, u32 lstatus, · 2943 unmodified lines
29442944 /* change offset to the other half */
29452945 rxb->page_offset ^= GFAR_RXB_TRUESIZE;
29462946
2947− atomic_inc(&page->_count);
2947+ page_ref_inc(page);
29482948
29492949 return true;
29502950 }
drivers/net/ethernet/intel/fm10k/fm10k_main.c+1−1
@@ -243,7 +243,7 @@ static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer, · 242 unmodified lines
243243 /* Even if we own the page, we are not allowed to use atomic_set()
244244 * This would break get_page_unless_zero() users.
245245 */
246− atomic_inc(&page->_count);
246+ page_ref_inc(page);
247247
248248 return true;
249249 }
drivers/net/ethernet/intel/igb/igb_main.c+1−1
@@ -6630,7 +6630,7 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer, · 6629 unmodified lines
66306630 /* Even if we own the page, we are not allowed to use atomic_set()
66316631 * This would break get_page_unless_zero() users.
66326632 */
6633− atomic_inc(&page->_count);
6633+ page_ref_inc(page);
66346634
66356635 return true;
66366636 }
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c+1−1
@@ -1942,7 +1942,7 @@ static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, · 1941 unmodified lines
19421942 /* Even if we own the page, we are not allowed to use atomic_set()
19431943 * This would break get_page_unless_zero() users.
19441944 */
1945− atomic_inc(&page->_count);
1945+ page_ref_inc(page);
19461946
19471947 return true;
19481948 }
drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c+1−1
@@ -837,7 +837,7 @@ add_tail_frag: · 836 unmodified lines
837837 /* Even if we own the page, we are not allowed to use atomic_set()
838838 * This would break get_page_unless_zero() users.
839839 */
840− atomic_inc(&page->_count);
840+ page_ref_inc(page);
841841
842842 return true;
843843 }
drivers/net/ethernet/mellanox/mlx4/en_rx.c+4−5
@@ -82,8 +82,7 @@ static int mlx4_alloc_pages(struct mlx4_en_priv *priv, · 81 unmodified lines
8282 /* Not doing get_page() for each frag is a big win
8383 * on asymetric workloads. Note we can not use atomic_set().
8484 */
85− atomic_add(page_alloc->page_size / frag_info->frag_stride - 1,
86− &page->_count);
85+ page_ref_add(page, page_alloc->page_size / frag_info->frag_stride - 1);
8786 return 0;
8887 }
8988
@@ -127,7 +126,7 @@ out: · 37 unmodified lines
127126 dma_unmap_page(priv->ddev, page_alloc[i].dma,
128127 page_alloc[i].page_size, PCI_DMA_FROMDEVICE);
129128 page = page_alloc[i].page;
130− atomic_set(&page->_count, 1);
129+ set_page_count(page, 1);
131130 put_page(page);
132131 }
133132 }
@@ -165,7 +164,7 @@ static int mlx4_en_init_allocator(struct mlx4_en_priv *priv, · 31 unmodified lines
165164
166165 en_dbg(DRV, priv, " frag %d allocator: - size:%d frags:%d\n",
167166 i, ring->page_alloc[i].page_size,
168− atomic_read(&ring->page_alloc[i].page->_count));
167+ page_ref_count(ring->page_alloc[i].page));
169168 }
170169 return 0;
171170
@@ -177,7 +176,7 @@ out: · 5 unmodified lines
177176 dma_unmap_page(priv->ddev, page_alloc->dma,
178177 page_alloc->page_size, PCI_DMA_FROMDEVICE);
179178 page = page_alloc->page;
180− atomic_set(&page->_count, 1);
179+ set_page_count(page, 1);
181180 put_page(page);
182181 page_alloc->page = NULL;
183182 }
drivers/net/ethernet/sun/niu.c+1−1
@@ -3341,7 +3341,7 @@ static int niu_rbr_add_page(struct niu *np, struct rx_ring_info *rp, · 3340 unmodified lines
33413341
33423342 niu_hash_page(rp, page, addr);
33433343 if (rp->rbr_blocks_per_page > 1)
3344− atomic_add(rp->rbr_blocks_per_page - 1, &page->_count);
3344+ page_ref_add(page, rp->rbr_blocks_per_page - 1);
33453345
33463346 for (i = 0; i < rp->rbr_blocks_per_page; i++) {
33473347 __le32 *rbr = &rp->rbr[start_index + i];
drivers/net/wireless/marvell/mwifiex/debugfs.c+4−6
@@ -880,14 +880,12 @@ mwifiex_reset_write(struct file *file, · 879 unmodified lines
880880 {
881881 struct mwifiex_private *priv = file->private_data;
882882 struct mwifiex_adapter *adapter = priv->adapter;
883− char cmd;
884883 bool result;
884+ int rc;
885885
886− if (copy_from_user(&cmd, ubuf, sizeof(cmd)))
887− return -EFAULT;
888−
889− if (strtobool(&cmd, &result))
890− return -EINVAL;
886+ rc = kstrtobool_from_user(ubuf, count, &result);
887+ if (rc)
888+ return rc;
891889
892890 if (!result)
893891 return -EINVAL;
drivers/pinctrl/pinmux.c+3−10
@@ -334,7 +334,6 @@ int pinmux_map_to_setting(struct pinctrl_map const *map, · 333 unmodified lines
334334 unsigned num_groups;
335335 int ret;
336336 const char *group;
337− int i;
338337
339338 if (!pmxops) {
340339 dev_err(pctldev->dev, "does not support mux function\n");
@@ -363,19 +362,13 @@ int pinmux_map_to_setting(struct pinctrl_map const *map, · 22 unmodified lines
363362 return -EINVAL;
364363 }
365364 if (map->data.mux.group) {
366− bool found = false;
367365 group = map->data.mux.group;
368− for (i = 0; i < num_groups; i++) {
369− if (!strcmp(group, groups[i])) {
370− found = true;
371− break;
372− }
373− }
374− if (!found) {
366+ ret = match_string(groups, num_groups, group);
367+ if (ret < 0) {
375368 dev_err(pctldev->dev,
376369 "invalid group \"%s\" for function \"%s\"\n",
377370 group, map->data.mux.function);
378− return -EINVAL;
371+ return ret;
379372 }
380373 } else {
381374 group = groups[0];
drivers/power/ab8500_btemp.c+5−10
@@ -906,26 +906,21 @@ static int ab8500_btemp_get_property(struct power_supply *psy, · 905 unmodified lines
906906 static int ab8500_btemp_get_ext_psy_data(struct device *dev, void *data)
907907 {
908908 struct power_supply *psy;
909− struct power_supply *ext;
909+ struct power_supply *ext = dev_get_drvdata(dev);
910+ const char **supplicants = (const char **)ext->supplied_to;
910911 struct ab8500_btemp *di;
911912 union power_supply_propval ret;
912− int i, j;
913− bool psy_found = false;
913+ int j;
914914
915915 psy = (struct power_supply *)data;
916− ext = dev_get_drvdata(dev);
917916 di = power_supply_get_drvdata(psy);
918917
919918 /*
920919 * For all psy where the name of your driver
921920 * appears in any supplied_to
922921 */
923− for (i = 0; i < ext->num_supplicants; i++) {
924− if (!strcmp(ext->supplied_to[i], psy->desc->name))
925− psy_found = true;
926− }
927−
928− if (!psy_found)
922+ j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
923+ if (j < 0)
929924 return 0;
930925
931926 /* Go through all properties for the psy */
drivers/power/ab8500_charger.c+5−11
@@ -1929,11 +1929,11 @@ static int ab8540_charger_usb_pre_chg_enable(struct ux500_charger *charger, · 1928 unmodified lines
19291929 static int ab8500_charger_get_ext_psy_data(struct device *dev, void *data)
19301930 {
19311931 struct power_supply *psy;
1932− struct power_supply *ext;
1932+ struct power_supply *ext = dev_get_drvdata(dev);
1933+ const char **supplicants = (const char **)ext->supplied_to;
19331934 struct ab8500_charger *di;
19341935 union power_supply_propval ret;
1935− int i, j;
1936− bool psy_found = false;
1936+ int j;
19371937 struct ux500_charger *usb_chg;
19381938
19391939 usb_chg = (struct ux500_charger *)data;
@@ -1941,15 +1941,9 @@ static int ab8500_charger_get_ext_psy_data(struct device *dev, void *data) · 1 unmodified line
19411941
19421942 di = to_ab8500_charger_usb_device_info(usb_chg);
19431943
1944− ext = dev_get_drvdata(dev);
1945−
19461944 /* For all psy where the driver name appears in any supplied_to */
1947− for (i = 0; i < ext->num_supplicants; i++) {
1948− if (!strcmp(ext->supplied_to[i], psy->desc->name))
1949− psy_found = true;
1950− }
1951−
1952− if (!psy_found)
1945+ j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
1946+ if (j < 0)
19531947 return 0;
19541948
19551949 /* Go through all properties for the psy */
drivers/power/ab8500_fg.c+5−10
@@ -2168,26 +2168,21 @@ static int ab8500_fg_get_property(struct power_supply *psy, · 2167 unmodified lines
21682168 static int ab8500_fg_get_ext_psy_data(struct device *dev, void *data)
21692169 {
21702170 struct power_supply *psy;
2171− struct power_supply *ext;
2171+ struct power_supply *ext = dev_get_drvdata(dev);
2172+ const char **supplicants = (const char **)ext->supplied_to;
21722173 struct ab8500_fg *di;
21732174 union power_supply_propval ret;
2174− int i, j;
2175− bool psy_found = false;
2175+ int j;
21762176
21772177 psy = (struct power_supply *)data;
2178− ext = dev_get_drvdata(dev);
21792178 di = power_supply_get_drvdata(psy);
21802179
21812180 /*
21822181 * For all psy where the name of your driver
21832182 * appears in any supplied_to
21842183 */
2185− for (i = 0; i < ext->num_supplicants; i++) {
2186− if (!strcmp(ext->supplied_to[i], psy->desc->name))
2187− psy_found = true;
2188− }
2189−
2190− if (!psy_found)
2184+ j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
2185+ if (j < 0)
21912186 return 0;
21922187
21932188 /* Go through all properties for the psy */
drivers/power/abx500_chargalg.c+5−9
@@ -975,22 +975,18 @@ static void handle_maxim_chg_curr(struct abx500_chargalg *di) · 974 unmodified lines
975975 static int abx500_chargalg_get_ext_psy_data(struct device *dev, void *data)
976976 {
977977 struct power_supply *psy;
978− struct power_supply *ext;
978+ struct power_supply *ext = dev_get_drvdata(dev);
979+ const char **supplicants = (const char **)ext->supplied_to;
979980 struct abx500_chargalg *di;
980981 union power_supply_propval ret;
981− int i, j;
982− bool psy_found = false;
982+ int j;
983983 bool capacity_updated = false;
984984
985985 psy = (struct power_supply *)data;
986− ext = dev_get_drvdata(dev);
987986 di = power_supply_get_drvdata(psy);
988987 /* For all psy where the driver name appears in any supplied_to */
989− for (i = 0; i < ext->num_supplicants; i++) {
990− if (!strcmp(ext->supplied_to[i], psy->desc->name))
991− psy_found = true;
992− }
993− if (!psy_found)
988+ j = match_string(supplicants, ext->num_supplicants, psy->desc->name);
989+ if (j < 0)
994990 return 0;
995991
996992 /*
drivers/power/charger-manager.c+4−23
@@ -2019,27 +2019,6 @@ static void __exit charger_manager_cleanup(void) · 2018 unmodified lines
20192019 }
20202020 module_exit(charger_manager_cleanup);
20212021
2022−/**
2023− * find_power_supply - find the associated power_supply of charger
2024− * @cm: the Charger Manager representing the battery
2025− * @psy: pointer to instance of charger's power_supply
2026− */
2027−static bool find_power_supply(struct charger_manager *cm,
2028− struct power_supply *psy)
2029−{
2030− int i;
2031− bool found = false;
2032−
2033− for (i = 0; cm->desc->psy_charger_stat[i]; i++) {
2034− if (!strcmp(psy->desc->name, cm->desc->psy_charger_stat[i])) {
2035− found = true;
2036− break;
2037− }
2038− }
2039−
2040− return found;
2041−}
2042−
20432022 /**
20442023 * cm_notify_event - charger driver notify Charger Manager of charger event
20452024 * @psy: pointer to instance of charger's power_supply
@@ -2057,9 +2036,11 @@ void cm_notify_event(struct power_supply *psy, enum cm_event_types type, · 11 unmodified lines
20572036
20582037 mutex_lock(&cm_list_mtx);
20592038 list_for_each_entry(cm, &cm_list, entry) {
2060− found_power_supply = find_power_supply(cm, psy);
2061− if (found_power_supply)
2039+ if (match_string(cm->desc->psy_charger_stat, -1,
2040+ psy->desc->name) >= 0) {
2041+ found_power_supply = true;
20622042 break;
2043+ }
20632044 }
20642045 mutex_unlock(&cm_list_mtx);
20652046
drivers/usb/common/common.c+8−14
@@ -65,18 +65,15 @@ EXPORT_SYMBOL_GPL(usb_speed_string); · 64 unmodified lines
6565 enum usb_device_speed usb_get_maximum_speed(struct device *dev)
6666 {
6767 const char *maximum_speed;
68− int err;
69− int i;
68+ int ret;
7069
71− err = device_property_read_string(dev, "maximum-speed", &maximum_speed);
72− if (err < 0)
70+ ret = device_property_read_string(dev, "maximum-speed", &maximum_speed);
71+ if (ret < 0)
7372 return USB_SPEED_UNKNOWN;
7473
75− for (i = 0; i < ARRAY_SIZE(speed_names); i++)
76− if (strcmp(maximum_speed, speed_names[i]) == 0)
77− return i;
74+ ret = match_string(speed_names, ARRAY_SIZE(speed_names), maximum_speed);
7875
79− return USB_SPEED_UNKNOWN;
76+ return (ret < 0) ? USB_SPEED_UNKNOWN : ret;
8077 }
8178 EXPORT_SYMBOL_GPL(usb_get_maximum_speed);
8279
@@ -110,13 +107,10 @@ static const char *const usb_dr_modes[] = { · 27 unmodified lines
110107
111108 static enum usb_dr_mode usb_get_dr_mode_from_string(const char *str)
112109 {
113− int i;
114−
115− for (i = 0; i < ARRAY_SIZE(usb_dr_modes); i++)
116− if (!strcmp(usb_dr_modes[i], str))
117− return i;
110+ int ret;
118111
119− return USB_DR_MODE_UNKNOWN;
112+ ret = match_string(usb_dr_modes, ARRAY_SIZE(usb_dr_modes), str);
113+ return (ret < 0) ? USB_DR_MODE_UNKNOWN : ret;
120114 }
121115
122116 enum usb_dr_mode usb_get_dr_mode(struct device *dev)
drivers/virtio/virtio_balloon.c+6−0
@@ -30,6 +30,7 @@ · 29 unmodified lines
3030 #include <linux/balloon_compaction.h>
3131 #include <linux/oom.h>
3232 #include <linux/wait.h>
33+#include <linux/mm.h>
3334
3435 /*
3536 * Balloon device works in 4K page units. So each page is pointed to by
@@ -229,10 +230,13 @@ static void update_balloon_stats(struct virtio_balloon *vb) · 193 unmodified lines
229230 unsigned long events[NR_VM_EVENT_ITEMS];
230231 struct sysinfo i;
231232 int idx = 0;
233+ long available;
232234
233235 all_vm_events(events);
234236 si_meminfo(&i);
235237
238+ available = si_mem_available();
239+
236240 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
237241 pages_to_bytes(events[PSWPIN]));
238242 update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
@@ -243,6 +247,8 @@ static void update_balloon_stats(struct virtio_balloon *vb) · 4 unmodified lines
243247 pages_to_bytes(i.freeram));
244248 update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
245249 pages_to_bytes(i.totalram));
250+ update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL,
251+ pages_to_bytes(available));
246252 }
247253
248254 /*
fs/btrfs/tests/btrfs-tests.c+1−2
@@ -137,7 +137,6 @@ static void btrfs_free_dummy_fs_info(struct btrfs_fs_info *fs_info) · 136 unmodified lines
137137 void **slot;
138138
139139 spin_lock(&fs_info->buffer_lock);
140−restart:
141140 radix_tree_for_each_slot(slot, &fs_info->buffer_radix, &iter, 0) {
142141 struct extent_buffer *eb;
143142
@@ -147,7 +146,7 @@ restart: · 3 unmodified lines
147146 /* Shouldn't happen but that kind of thinking creates CVE's */
148147 if (radix_tree_exception(eb)) {
149148 if (radix_tree_deref_retry(eb))
150− goto restart;
149+ slot = radix_tree_iter_retry(&iter);
151150 continue;
152151 }
153152 spin_unlock(&fs_info->buffer_lock);
fs/cifs/cifs_debug.c+14−42
@@ -255,7 +255,6 @@ static const struct file_operations cifs_debug_data_proc_fops = { · 254 unmodified lines
255255 static ssize_t cifs_stats_proc_write(struct file *file,
256256 const char __user *buffer, size_t count, loff_t *ppos)
257257 {
258− char c;
259258 bool bv;
260259 int rc;
261260 struct list_head *tmp1, *tmp2, *tmp3;
@@ -263,11 +262,8 @@ static ssize_t cifs_stats_proc_write(struct file *file, · 1 unmodified line
263262 struct cifs_ses *ses;
264263 struct cifs_tcon *tcon;
265264
266− rc = get_user(c, buffer);
267− if (rc)
268− return rc;
269−
270− if (strtobool(&c, &bv) == 0) {
265+ rc = kstrtobool_from_user(buffer, count, &bv);
266+ if (rc == 0) {
271267 #ifdef CONFIG_CIFS_STATS2
272268 atomic_set(&totBufAllocCount, 0);
273269 atomic_set(&totSmBufAllocCount, 0);
@@ -290,6 +286,8 @@ static ssize_t cifs_stats_proc_write(struct file *file, · 16 unmodified lines
290286 }
291287 }
292288 spin_unlock(&cifs_tcp_ses_lock);
289+ } else {
290+ return rc;
293291 }
294292
295293 return count;
@@ -433,17 +431,17 @@ static int cifsFYI_proc_open(struct inode *inode, struct file *file) · 137 unmodified lines
433431 static ssize_t cifsFYI_proc_write(struct file *file, const char __user *buffer,
434432 size_t count, loff_t *ppos)
435433 {
436− char c;
434+ char c[2] = { '\0' };
437435 bool bv;
438436 int rc;
439437
440− rc = get_user(c, buffer);
438+ rc = get_user(c[0], buffer);
441439 if (rc)
442440 return rc;
443− if (strtobool(&c, &bv) == 0)
441+ if (strtobool(c, &bv) == 0)
444442 cifsFYI = bv;
445− else if ((c > '1') && (c <= '9'))
446− cifsFYI = (int) (c - '0'); /* see cifs_debug.h for meanings */
443+ else if ((c[0] > '1') && (c[0] <= '9'))
444+ cifsFYI = (int) (c[0] - '0'); /* see cifs_debug.h for meanings */
447445
448446 return count;
449447 }
@@ -471,20 +469,12 @@ static int cifs_linux_ext_proc_open(struct inode *inode, struct file *file) · 21 unmodified lines
471469 static ssize_t cifs_linux_ext_proc_write(struct file *file,
472470 const char __user *buffer, size_t count, loff_t *ppos)
473471 {
474− char c;
475− bool bv;
476472 int rc;
477473
478− rc = get_user(c, buffer);
474+ rc = kstrtobool_from_user(buffer, count, &linuxExtEnabled);
479475 if (rc)
480476 return rc;
481477
482− rc = strtobool(&c, &bv);
483− if (rc)
484− return rc;
485−
486− linuxExtEnabled = bv;
487−
488478 return count;
489479 }
490480
@@ -511,20 +501,12 @@ static int cifs_lookup_cache_proc_open(struct inode *inode, struct file *file) · 20 unmodified lines
511501 static ssize_t cifs_lookup_cache_proc_write(struct file *file,
512502 const char __user *buffer, size_t count, loff_t *ppos)
513503 {
514− char c;
515− bool bv;
516504 int rc;
517505
518− rc = get_user(c, buffer);
506+ rc = kstrtobool_from_user(buffer, count, &lookupCacheEnabled);
519507 if (rc)
520508 return rc;
521509
522− rc = strtobool(&c, &bv);
523− if (rc)
524− return rc;
525−
526− lookupCacheEnabled = bv;
527−
528510 return count;
529511 }
530512
@@ -551,20 +533,12 @@ static int traceSMB_proc_open(struct inode *inode, struct file *file) · 20 unmodified lines
551533 static ssize_t traceSMB_proc_write(struct file *file, const char __user *buffer,
552534 size_t count, loff_t *ppos)
553535 {
554− char c;
555− bool bv;
556536 int rc;
557537
558− rc = get_user(c, buffer);
538+ rc = kstrtobool_from_user(buffer, count, &traceSMB);
559539 if (rc)
560540 return rc;
561541
562− rc = strtobool(&c, &bv);
563− if (rc)
564− return rc;
565−
566− traceSMB = bv;
567−
568542 return count;
569543 }
570544
@@ -622,7 +596,6 @@ static ssize_t cifs_security_flags_proc_write(struct file *file, · 51 unmodified lines
622596 int rc;
623597 unsigned int flags;
624598 char flags_string[12];
625− char c;
626599 bool bv;
627600
628601 if ((count < 1) || (count > 11))
@@ -635,11 +608,10 @@ static ssize_t cifs_security_flags_proc_write(struct file *file, · 6 unmodified lines
635608
636609 if (count < 3) {
637610 /* single char or single char followed by null */
638− c = flags_string[0];
639− if (strtobool(&c, &bv) == 0) {
611+ if (strtobool(flags_string, &bv) == 0) {
640612 global_secflags = bv ? CIFSSEC_MAX : CIFSSEC_DEF;
641613 return count;
642− } else if (!isdigit(c)) {
614+ } else if (!isdigit(flags_string[0])) {
643615 cifs_dbg(VFS, "Invalid SecurityFlags: %s\n",
644616 flags_string);
645617 return -EINVAL;
fs/cifs/cifs_debug.h+1−1
@@ -25,7 +25,7 @@ · 24 unmodified lines
2525 void cifs_dump_mem(char *label, void *data, int length);
2626 void cifs_dump_detail(void *);
2727 void cifs_dump_mids(struct TCP_Server_Info *);
28−extern int traceSMB; /* flag which enables the function below */
28+extern bool traceSMB; /* flag which enables the function below */
2929 void dump_smb(void *, int);
3030 #define CIFS_INFO 0x01
3131 #define CIFS_RC 0x02
fs/cifs/cifsfs.c+3−3
@@ -54,10 +54,10 @@ · 53 unmodified lines
5454 #endif
5555
5656 int cifsFYI = 0;
57−int traceSMB = 0;
57+bool traceSMB;
5858 bool enable_oplocks = true;
59−unsigned int linuxExtEnabled = 1;
60−unsigned int lookupCacheEnabled = 1;
59+bool linuxExtEnabled = true;
60+bool lookupCacheEnabled = true;
6161 unsigned int global_secflags = CIFSSEC_DEF;
6262 /* unsigned int ntlmv2_support = 0; */
6363 unsigned int sign_CIFS_PDUs = 1;
fs/cifs/cifsglob.h+2−2
@@ -1596,11 +1596,11 @@ GLOBAL_EXTERN atomic_t midCount; · 1595 unmodified lines
15961596
15971597 /* Misc globals */
15981598 GLOBAL_EXTERN bool enable_oplocks; /* enable or disable oplocks */
1599−GLOBAL_EXTERN unsigned int lookupCacheEnabled;
1599+GLOBAL_EXTERN bool lookupCacheEnabled;
16001600 GLOBAL_EXTERN unsigned int global_secflags; /* if on, session setup sent
16011601 with more secure ntlmssp2 challenge/resp */
16021602 GLOBAL_EXTERN unsigned int sign_CIFS_PDUs; /* enable smb packet signing */
1603−GLOBAL_EXTERN unsigned int linuxExtEnabled;/*enable Linux/Unix CIFS extensions*/
1603+GLOBAL_EXTERN bool linuxExtEnabled;/*enable Linux/Unix CIFS extensions*/
16041604 GLOBAL_EXTERN unsigned int CIFSMaxBufSize; /* max size not including hdr */
16051605 GLOBAL_EXTERN unsigned int cifs_min_rcv; /* min size of big ntwrk buf pool */
16061606 GLOBAL_EXTERN unsigned int cifs_min_small; /* min size of small buf pool */
fs/eventpoll.c+1−1
@@ -1616,7 +1616,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, · 1615 unmodified lines
16161616 {
16171617 int res = 0, eavail, timed_out = 0;
16181618 unsigned long flags;
1619− long slack = 0;
1619+ u64 slack = 0;
16201620 wait_queue_t wait;
16211621 ktime_t expires, *to = NULL;
16221622
fs/nilfs2/page.c+1−1
@@ -180,7 +180,7 @@ void nilfs_page_bug(struct page *page) · 179 unmodified lines
180180
181181 printk(KERN_CRIT "NILFS_PAGE_BUG(%p): cnt=%d index#=%llu flags=0x%lx "
182182 "mapping=%p ino=%lu\n",
183− page, atomic_read(&page->_count),
183+ page, page_ref_count(page),
184184 (unsigned long long)page->index, page->flags, m, ino);
185185
186186 if (page_has_buffers(page)) {
fs/proc/base.c+70−1
@@ -434,7 +434,7 @@ static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns, · 433 unmodified lines
434434 && !lookup_symbol_name(wchan, symname))
435435 seq_printf(m, "%s", symname);
436436 else
437− seq_putc(m, '0');
437+ seq_puts(m, "0\n");
438438
439439 return 0;
440440 }
@@ -2158,6 +2158,7 @@ static const struct file_operations proc_map_files_operations = { · 1717 unmodified lines
21582158 .llseek = default_llseek,
21592159 };
21602160
2161+#ifdef CONFIG_CHECKPOINT_RESTORE
21612162 struct timers_private {
21622163 struct pid *pid;
21632164 struct task_struct *task;
@@ -2256,6 +2257,73 @@ static const struct file_operations proc_timers_operations = { · 92 unmodified lines
22562257 .llseek = seq_lseek,
22572258 .release = seq_release_private,
22582259 };
2260+#endif
2261+
2262+static ssize_t timerslack_ns_write(struct file *file, const char __user *buf,
2263+ size_t count, loff_t *offset)
2264+{
2265+ struct inode *inode = file_inode(file);
2266+ struct task_struct *p;
2267+ u64 slack_ns;
2268+ int err;
2269+
2270+ err = kstrtoull_from_user(buf, count, 10, &slack_ns);
2271+ if (err < 0)
2272+ return err;
2273+
2274+ p = get_proc_task(inode);
2275+ if (!p)
2276+ return -ESRCH;
2277+
2278+ if (ptrace_may_access(p, PTRACE_MODE_ATTACH_FSCREDS)) {
2279+ task_lock(p);
2280+ if (slack_ns == 0)
2281+ p->timer_slack_ns = p->default_timer_slack_ns;
2282+ else
2283+ p->timer_slack_ns = slack_ns;
2284+ task_unlock(p);
2285+ } else
2286+ count = -EPERM;
2287+
2288+ put_task_struct(p);
2289+
2290+ return count;
2291+}
2292+
2293+static int timerslack_ns_show(struct seq_file *m, void *v)
2294+{
2295+ struct inode *inode = m->private;
2296+ struct task_struct *p;
2297+ int err = 0;
2298+
2299+ p = get_proc_task(inode);
2300+ if (!p)
2301+ return -ESRCH;
2302+
2303+ if (ptrace_may_access(p, PTRACE_MODE_ATTACH_FSCREDS)) {
2304+ task_lock(p);
2305+ seq_printf(m, "%llu\n", p->timer_slack_ns);
2306+ task_unlock(p);
2307+ } else
2308+ err = -EPERM;
2309+
2310+ put_task_struct(p);
2311+
2312+ return err;
2313+}
2314+
2315+static int timerslack_ns_open(struct inode *inode, struct file *filp)
2316+{
2317+ return single_open(filp, timerslack_ns_show, inode);
2318+}
2319+
2320+static const struct file_operations proc_pid_set_timerslack_ns_operations = {
2321+ .open = timerslack_ns_open,
2322+ .read = seq_read,
2323+ .write = timerslack_ns_write,
2324+ .llseek = seq_lseek,
2325+ .release = single_release,
2326+};
22592327
22602328 static int proc_pident_instantiate(struct inode *dir,
22612329 struct dentry *dentry, struct task_struct *task, const void *ptr)
@@ -2831,6 +2899,7 @@ static const struct pid_entry tgid_base_stuff[] = { · 569 unmodified lines
28312899 #ifdef CONFIG_CHECKPOINT_RESTORE
28322900 REG("timers", S_IRUGO, proc_timers_operations),
28332901 #endif
2902+ REG("timerslack_ns", S_IRUGO|S_IWUGO, proc_pid_set_timerslack_ns_operations),
28342903 };
28352904
28362905 static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx)
fs/proc/meminfo.c+1−30
@@ -29,10 +29,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) · 28 unmodified lines
2929 unsigned long committed;
3030 long cached;
3131 long available;
32− unsigned long pagecache;
33− unsigned long wmark_low = 0;
3432 unsigned long pages[NR_LRU_LISTS];
35− struct zone *zone;
3633 int lru;
3734
3835 /*
@@ -51,33 +48,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) · 12 unmodified lines
5148 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
5249 pages[lru] = global_page_state(NR_LRU_BASE + lru);
5350
54− for_each_zone(zone)
55− wmark_low += zone->watermark[WMARK_LOW];
56−
57− /*
58− * Estimate the amount of memory available for userspace allocations,
59− * without causing swapping.
60− */
61− available = i.freeram - totalreserve_pages;
62−
63− /*
64− * Not all the page cache can be freed, otherwise the system will
65− * start swapping. Assume at least half of the page cache, or the
66− * low watermark worth of cache, needs to stay.
67− */
68− pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
69− pagecache -= min(pagecache / 2, wmark_low);
70− available += pagecache;
71−
72− /*
73− * Part of the reclaimable slab consists of items that are in use,
74− * and cannot be freed. Cap this estimate at the low watermark.
75− */
76− available += global_page_state(NR_SLAB_RECLAIMABLE) -
77− min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
78−
79− if (available < 0)
80− available = 0;
51+ available = si_mem_available();
8152
8253 /*
8354 * Tagged format, for easy grepping and expansion.
fs/proc/page.c+6−2
@@ -103,9 +103,9 @@ u64 stable_page_flags(struct page *page) · 102 unmodified lines
103103 * pseudo flags for the well known (anonymous) memory mapped pages
104104 *
105105 * Note that page->_mapcount is overloaded in SLOB/SLUB/SLQB, so the
106− * simple test in page_mapcount() is not enough.
106+ * simple test in page_mapped() is not enough.
107107 */
108− if (!PageSlab(page) && page_mapcount(page))
108+ if (!PageSlab(page) && page_mapped(page))
109109 u |= 1 << KPF_MMAP;
110110 if (PageAnon(page))
111111 u |= 1 << KPF_ANON;
@@ -148,6 +148,8 @@ u64 stable_page_flags(struct page *page) · 36 unmodified lines
148148 */
149149 if (PageBuddy(page))
150150 u |= 1 << KPF_BUDDY;
151+ else if (page_count(page) == 0 && is_free_buddy_page(page))
152+ u |= 1 << KPF_BUDDY;
151153
152154 if (PageBalloon(page))
153155 u |= 1 << KPF_BALLOON;
@@ -158,6 +160,8 @@ u64 stable_page_flags(struct page *page) · 4 unmodified lines
158160 u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked);
159161
160162 u |= kpf_copy_bit(k, KPF_SLAB, PG_slab);
163+ if (PageTail(page) && PageSlab(compound_head(page)))
164+ u |= 1 << KPF_SLAB;
161165
162166 u |= kpf_copy_bit(k, KPF_ERROR, PG_error);
163167 u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty);
fs/proc/vmcore.c+5−2
@@ -231,7 +231,9 @@ static ssize_t __read_vmcore(char *buffer, size_t buflen, loff_t *fpos, · 230 unmodified lines
231231
232232 list_for_each_entry(m, &vmcore_list, list) {
233233 if (*fpos < m->offset + m->size) {
234− tsz = min_t(size_t, m->offset + m->size - *fpos, buflen);
234+ tsz = (size_t)min_t(unsigned long long,
235+ m->offset + m->size - *fpos,
236+ buflen);
235237 start = m->paddr + *fpos - m->offset;
236238 tmp = read_from_oldmem(buffer, tsz, &start, userbuf);
237239 if (tmp < 0)
@@ -461,7 +463,8 @@ static int mmap_vmcore(struct file *file, struct vm_area_struct *vma) · 223 unmodified lines
461463 if (start < m->offset + m->size) {
462464 u64 paddr = 0;
463465
464− tsz = min_t(size_t, m->offset + m->size - start, size);
466+ tsz = (size_t)min_t(unsigned long long,
467+ m->offset + m->size - start, size);
465468 paddr = m->paddr + start - m->offset;
466469 if (vmcore_remap_oldmem_pfn(vma, vma->vm_start + len,
467470 paddr >> PAGE_SHIFT, tsz,
fs/select.c+4−4
@@ -70,9 +70,9 @@ static long __estimate_accuracy(struct timespec *tv) · 69 unmodified lines
7070 return slack;
7171 }
7272
73−long select_estimate_accuracy(struct timespec *tv)
73+u64 select_estimate_accuracy(struct timespec *tv)
7474 {
75− unsigned long ret;
75+ u64 ret;
7676 struct timespec now;
7777
7878 /*
@@ -402,7 +402,7 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time) · 323 unmodified lines
402402 struct poll_wqueues table;
403403 poll_table *wait;
404404 int retval, i, timed_out = 0;
405− unsigned long slack = 0;
405+ u64 slack = 0;
406406 unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
407407 unsigned long busy_end = 0;
408408
@@ -784,7 +784,7 @@ static int do_poll(struct poll_list *list, struct poll_wqueues *wait, · 375 unmodified lines
784784 poll_table* pt = &wait->pt;
785785 ktime_t expire, *to = NULL;
786786 int timed_out = 0, count = 0;
787− unsigned long slack = 0;
787+ u64 slack = 0;
788788 unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
789789 unsigned long busy_end = 0;
790790
include/asm-generic/atomic-long.h+3−3
@@ -98,14 +98,14 @@ ATOMIC_LONG_ADD_SUB_OP(sub, _release) · 97 unmodified lines
9898 #define atomic_long_xchg(v, new) \
9999 (ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
100100
101−static inline void atomic_long_inc(atomic_long_t *l)
101+static __always_inline void atomic_long_inc(atomic_long_t *l)
102102 {
103103 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
104104
105105 ATOMIC_LONG_PFX(_inc)(v);
106106 }
107107
108−static inline void atomic_long_dec(atomic_long_t *l)
108+static __always_inline void atomic_long_dec(atomic_long_t *l)
109109 {
110110 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
111111
@@ -113,7 +113,7 @@ static inline void atomic_long_dec(atomic_long_t *l) · 1 unmodified line
113113 }
114114
115115 #define ATOMIC_LONG_OP(op) \
116−static inline void \
116+static __always_inline void \
117117 atomic_long_##op(long i, atomic_long_t *l) \
118118 { \
119119 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
include/asm-generic/bug.h+18−9
@@ -81,6 +81,12 @@ extern void warn_slowpath_null(const char *file, const int line); · 80 unmodified lines
8181 do { printk(arg); __WARN_TAINT(taint); } while (0)
8282 #endif
8383
84+/* used internally by panic.c */
85+struct warn_args;
86+
87+void __warn(const char *file, int line, void *caller, unsigned taint,
88+ struct pt_regs *regs, struct warn_args *args);
89+
8490 #ifndef WARN_ON
8591 #define WARN_ON(condition) ({ \
8692 int __ret_warn_on = !!(condition); \
@@ -110,9 +116,10 @@ extern void warn_slowpath_null(const char *file, const int line); · 23 unmodified lines
110116 static bool __section(.data.unlikely) __warned; \
111117 int __ret_warn_once = !!(condition); \
112118 \
113− if (unlikely(__ret_warn_once)) \
114− if (WARN_ON(!__warned)) \
115− __warned = true; \
119+ if (unlikely(__ret_warn_once && !__warned)) { \
120+ __warned = true; \
121+ WARN_ON(1); \
122+ } \
116123 unlikely(__ret_warn_once); \
117124 })
118125
@@ -120,9 +127,10 @@ extern void warn_slowpath_null(const char *file, const int line); · 1 unmodified line
120127 static bool __section(.data.unlikely) __warned; \
121128 int __ret_warn_once = !!(condition); \
122129 \
123− if (unlikely(__ret_warn_once)) \
124− if (WARN(!__warned, format)) \
125− __warned = true; \
130+ if (unlikely(__ret_warn_once && !__warned)) { \
131+ __warned = true; \
132+ WARN(1, format); \
133+ } \
126134 unlikely(__ret_warn_once); \
127135 })
128136
@@ -130,9 +138,10 @@ extern void warn_slowpath_null(const char *file, const int line); · 1 unmodified line
130138 static bool __section(.data.unlikely) __warned; \
131139 int __ret_warn_once = !!(condition); \
132140 \
133− if (unlikely(__ret_warn_once)) \
134− if (WARN_TAINT(!__warned, taint, format)) \
135− __warned = true; \
141+ if (unlikely(__ret_warn_once && !__warned)) { \
142+ __warned = true; \
143+ WARN_TAINT(1, taint, format); \
144+ } \
136145 unlikely(__ret_warn_once); \
137146 })
138147
include/asm-generic/pgtable.h+17−0
@@ -783,6 +783,23 @@ static inline int pmd_clear_huge(pmd_t *pmd) · 782 unmodified lines
783783 }
784784 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
785785
786+#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
787+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
788+/*
789+ * ARCHes with special requirements for evicting THP backing TLB entries can
790+ * implement this. Otherwise also, it can help optimize normal TLB flush in
791+ * THP regime. stock flush_tlb_range() typically has optimization to nuke the
792+ * entire TLB TLB if flush span is greater than a threshold, which will
793+ * likely be true for a single huge page. Thus a single thp flush will
794+ * invalidate the entire TLB which is not desitable.
795+ * e.g. see arch/arc: flush_pmd_tlb_range
796+ */
797+#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
798+#else
799+#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()
800+#endif
801+#endif
802+
786803 #endif /* !__ASSEMBLY__ */
787804
788805 #ifndef io_remap_pfn_range
include/linux/buffer_head.h+5−5
@@ -82,15 +82,15 @@ struct buffer_head { · 81 unmodified lines
8282 * and buffer_foo() functions.
8383 */
8484 #define BUFFER_FNS(bit, name) \
85−static inline void set_buffer_##name(struct buffer_head *bh) \
85+static __always_inline void set_buffer_##name(struct buffer_head *bh) \
8686 { \
8787 set_bit(BH_##bit, &(bh)->b_state); \
8888 } \
89−static inline void clear_buffer_##name(struct buffer_head *bh) \
89+static __always_inline void clear_buffer_##name(struct buffer_head *bh) \
9090 { \
9191 clear_bit(BH_##bit, &(bh)->b_state); \
9292 } \
93−static inline int buffer_##name(const struct buffer_head *bh) \
93+static __always_inline int buffer_##name(const struct buffer_head *bh) \
9494 { \
9595 return test_bit(BH_##bit, &(bh)->b_state); \
9696 }
@@ -99,11 +99,11 @@ static inline int buffer_##name(const struct buffer_head *bh) \ · 2 unmodified lines
9999 * test_set_buffer_foo() and test_clear_buffer_foo()
100100 */
101101 #define TAS_BUFFER_FNS(bit, name) \
102−static inline int test_set_buffer_##name(struct buffer_head *bh) \
102+static __always_inline int test_set_buffer_##name(struct buffer_head *bh) \
103103 { \
104104 return test_and_set_bit(BH_##bit, &(bh)->b_state); \
105105 } \
106−static inline int test_clear_buffer_##name(struct buffer_head *bh) \
106+static __always_inline int test_clear_buffer_##name(struct buffer_head *bh) \
107107 { \
108108 return test_and_clear_bit(BH_##bit, &(bh)->b_state); \
109109 } \
include/linux/compaction.h+16−0
@@ -52,6 +52,10 @@ extern void compaction_defer_reset(struct zone *zone, int order, · 51 unmodified lines
5252 bool alloc_success);
5353 extern bool compaction_restarting(struct zone *zone, int order);
5454
55+extern int kcompactd_run(int nid);
56+extern void kcompactd_stop(int nid);
57+extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx);
58+
5559 #else
5660 static inline unsigned long try_to_compact_pages(gfp_t gfp_mask,
5761 unsigned int order, int alloc_flags,
@@ -84,6 +88,18 @@ static inline bool compaction_deferred(struct zone *zone, int order) · 26 unmodified lines
8488 return true;
8589 }
8690
91+static inline int kcompactd_run(int nid)
92+{
93+ return 0;
94+}
95+static inline void kcompactd_stop(int nid)
96+{
97+}
98+
99+static inline void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
100+{
101+}
102+
87103 #endif /* CONFIG_COMPACTION */
88104
89105 #if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
include/linux/freezer.h+1−1
@@ -231,7 +231,7 @@ static inline long freezable_schedule_timeout_killable_unsafe(long timeout) · 230 unmodified lines
231231 * call this with locks held.
232232 */
233233 static inline int freezable_schedule_hrtimeout_range(ktime_t *expires,
234− unsigned long delta, const enum hrtimer_mode mode)
234+ u64 delta, const enum hrtimer_mode mode)
235235 {
236236 int __retval;
237237 freezer_do_not_count();
include/linux/gfp.h+21−16
@@ -105,8 +105,6 @@ struct vm_area_struct; · 104 unmodified lines
105105 *
106106 * __GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves.
107107 * This takes precedence over the __GFP_MEMALLOC flag if both are set.
108− *
109− * __GFP_NOACCOUNT ignores the accounting for kmemcg limit enforcement.
110108 */
111109 #define __GFP_ATOMIC ((__force gfp_t)___GFP_ATOMIC)
112110 #define __GFP_HIGH ((__force gfp_t)___GFP_HIGH)
@@ -259,7 +257,7 @@ struct vm_area_struct; · 146 unmodified lines
259257 #define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE)
260258 #define GFP_TRANSHUGE ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
261259 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN) & \
262− ~__GFP_KSWAPD_RECLAIM)
260+ ~__GFP_RECLAIM)
263261
264262 /* Convert GFP flags to their corresponding migrate type */
265263 #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
@@ -333,22 +331,29 @@ static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) · 67 unmodified lines
333331 * 0xe => BAD (MOVABLE+DMA32+HIGHMEM)
334332 * 0xf => BAD (MOVABLE+DMA32+HIGHMEM+DMA)
335333 *
336− * ZONES_SHIFT must be <= 2 on 32 bit platforms.
334+ * GFP_ZONES_SHIFT must be <= 2 on 32 bit platforms.
337335 */
338336
339−#if 16 * ZONES_SHIFT > BITS_PER_LONG
340−#error ZONES_SHIFT too large to create GFP_ZONE_TABLE integer
337+#if defined(CONFIG_ZONE_DEVICE) && (MAX_NR_ZONES-1) <= 4
338+/* ZONE_DEVICE is not a valid GFP zone specifier */
339+#define GFP_ZONES_SHIFT 2
340+#else
341+#define GFP_ZONES_SHIFT ZONES_SHIFT
342+#endif
343+
344+#if 16 * GFP_ZONES_SHIFT > BITS_PER_LONG
345+#error GFP_ZONES_SHIFT too large to create GFP_ZONE_TABLE integer
341346 #endif
342347
343348 #define GFP_ZONE_TABLE ( \
344− (ZONE_NORMAL << 0 * ZONES_SHIFT) \
345− | (OPT_ZONE_DMA << ___GFP_DMA * ZONES_SHIFT) \
346− | (OPT_ZONE_HIGHMEM << ___GFP_HIGHMEM * ZONES_SHIFT) \
347− | (OPT_ZONE_DMA32 << ___GFP_DMA32 * ZONES_SHIFT) \
348− | (ZONE_NORMAL << ___GFP_MOVABLE * ZONES_SHIFT) \
349− | (OPT_ZONE_DMA << (___GFP_MOVABLE | ___GFP_DMA) * ZONES_SHIFT) \
350− | (ZONE_MOVABLE << (___GFP_MOVABLE | ___GFP_HIGHMEM) * ZONES_SHIFT) \
351− | (OPT_ZONE_DMA32 << (___GFP_MOVABLE | ___GFP_DMA32) * ZONES_SHIFT) \
349+ (ZONE_NORMAL << 0 * GFP_ZONES_SHIFT) \
350+ | (OPT_ZONE_DMA << ___GFP_DMA * GFP_ZONES_SHIFT) \
351+ | (OPT_ZONE_HIGHMEM << ___GFP_HIGHMEM * GFP_ZONES_SHIFT) \
352+ | (OPT_ZONE_DMA32 << ___GFP_DMA32 * GFP_ZONES_SHIFT) \
353+ | (ZONE_NORMAL << ___GFP_MOVABLE * GFP_ZONES_SHIFT) \
354+ | (OPT_ZONE_DMA << (___GFP_MOVABLE | ___GFP_DMA) * GFP_ZONES_SHIFT) \
355+ | (ZONE_MOVABLE << (___GFP_MOVABLE | ___GFP_HIGHMEM) * GFP_ZONES_SHIFT)\
356+ | (OPT_ZONE_DMA32 << (___GFP_MOVABLE | ___GFP_DMA32) * GFP_ZONES_SHIFT)\
352357 )
353358
354359 /*
@@ -373,8 +378,8 @@ static inline enum zone_type gfp_zone(gfp_t flags) · 18 unmodified lines
373378 enum zone_type z;
374379 int bit = (__force int) (flags & GFP_ZONEMASK);
375380
376− z = (GFP_ZONE_TABLE >> (bit * ZONES_SHIFT)) &
377− ((1 << ZONES_SHIFT) - 1);
381+ z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) &
382+ ((1 << GFP_ZONES_SHIFT) - 1);
378383 VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1);
379384 return z;
380385 }
include/linux/hrtimer.h+7−5
@@ -220,7 +220,7 @@ static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time · 219 unmodified lines
220220 timer->node.expires = ktime_add_safe(time, delta);
221221 }
222222
223−static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, unsigned long delta)
223+static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
224224 {
225225 timer->_softexpires = time;
226226 timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
@@ -378,7 +378,7 @@ static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { } · 151 unmodified lines
378378
379379 /* Basic timer operations: */
380380 extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
381− unsigned long range_ns, const enum hrtimer_mode mode);
381+ u64 range_ns, const enum hrtimer_mode mode);
382382
383383 /**
384384 * hrtimer_start - (re)start an hrtimer on the current CPU
@@ -399,7 +399,7 @@ extern int hrtimer_try_to_cancel(struct hrtimer *timer); · 14 unmodified lines
399399 static inline void hrtimer_start_expires(struct hrtimer *timer,
400400 enum hrtimer_mode mode)
401401 {
402− unsigned long delta;
402+ u64 delta;
403403 ktime_t soft, hard;
404404 soft = hrtimer_get_softexpires(timer);
405405 hard = hrtimer_get_expires(timer);
@@ -477,10 +477,12 @@ extern long hrtimer_nanosleep_restart(struct restart_block *restart_block); · 71 unmodified lines
477477 extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
478478 struct task_struct *tsk);
479479
480−extern int schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
480+extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
481481 const enum hrtimer_mode mode);
482482 extern int schedule_hrtimeout_range_clock(ktime_t *expires,
483− unsigned long delta, const enum hrtimer_mode mode, int clock);
483+ u64 delta,
484+ const enum hrtimer_mode mode,
485+ int clock);
484486 extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
485487
486488 /* Soft interrupt function to run the hrtimer queues: */
include/linux/huge_mm.h+13−9
@@ -41,7 +41,8 @@ int vmf_insert_pfn_pmd(struct vm_area_struct *, unsigned long addr, pmd_t *, · 40 unmodified lines
4141 enum transparent_hugepage_flag {
4242 TRANSPARENT_HUGEPAGE_FLAG,
4343 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
44− TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
44+ TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
45+ TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
4546 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
4647 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
4748 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
@@ -71,12 +72,6 @@ extern bool is_vma_temporary_stack(struct vm_area_struct *vma); · 23 unmodified lines
7172 ((__vma)->vm_flags & VM_HUGEPAGE))) && \
7273 !((__vma)->vm_flags & VM_NOHUGEPAGE) && \
7374 !is_vma_temporary_stack(__vma))
74−#define transparent_hugepage_defrag(__vma) \
75− ((transparent_hugepage_flags & \
76− (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) || \
77− (transparent_hugepage_flags & \
78− (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \
79− (__vma)->vm_flags & VM_HUGEPAGE))
8075 #define transparent_hugepage_use_zero_page() \
8176 (transparent_hugepage_flags & \
8277 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
@@ -101,16 +96,21 @@ static inline int split_huge_page(struct page *page) · 18 unmodified lines
10196 void deferred_split_huge_page(struct page *page);
10297
10398 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
104− unsigned long address);
99+ unsigned long address, bool freeze);
105100
106101 #define split_huge_pmd(__vma, __pmd, __address) \
107102 do { \
108103 pmd_t *____pmd = (__pmd); \
109104 if (pmd_trans_huge(*____pmd) \
110105 || pmd_devmap(*____pmd)) \
111− __split_huge_pmd(__vma, __pmd, __address); \
106+ __split_huge_pmd(__vma, __pmd, __address, \
107+ false); \
112108 } while (0)
113109
110+
111+void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
112+ bool freeze, struct page *page);
113+
114114 #if HPAGE_PMD_ORDER >= MAX_ORDER
115115 #error "hugepages can't be allocated by the buddy allocator"
116116 #endif
@@ -178,6 +178,10 @@ static inline int split_huge_page(struct page *page) · 61 unmodified lines
178178 static inline void deferred_split_huge_page(struct page *page) {}
179179 #define split_huge_pmd(__vma, __pmd, __address) \
180180 do { } while (0)
181+
182+static inline void split_huge_pmd_address(struct vm_area_struct *vma,
183+ unsigned long address, bool freeze, struct page *page) {}
184+
181185 static inline int hugepage_madvise(struct vm_area_struct *vma,
182186 unsigned long *vm_flags, int advice)
183187 {
include/linux/kernel.h+2−0
@@ -357,6 +357,7 @@ int __must_check kstrtou16(const char *s, unsigned int base, u16 *res); · 356 unmodified lines
357357 int __must_check kstrtos16(const char *s, unsigned int base, s16 *res);
358358 int __must_check kstrtou8(const char *s, unsigned int base, u8 *res);
359359 int __must_check kstrtos8(const char *s, unsigned int base, s8 *res);
360+int __must_check kstrtobool(const char *s, bool *res);
360361
361362 int __must_check kstrtoull_from_user(const char __user *s, size_t count, unsigned int base, unsigned long long *res);
362363 int __must_check kstrtoll_from_user(const char __user *s, size_t count, unsigned int base, long long *res);
@@ -368,6 +369,7 @@ int __must_check kstrtou16_from_user(const char __user *s, size_t count, unsigne · 5 unmodified lines
368369 int __must_check kstrtos16_from_user(const char __user *s, size_t count, unsigned int base, s16 *res);
369370 int __must_check kstrtou8_from_user(const char __user *s, size_t count, unsigned int base, u8 *res);
370371 int __must_check kstrtos8_from_user(const char __user *s, size_t count, unsigned int base, s8 *res);
372+int __must_check kstrtobool_from_user(const char __user *s, size_t count, bool *res);
371373
372374 static inline int __must_check kstrtou64_from_user(const char __user *s, size_t count, unsigned int base, u64 *res)
373375 {
include/linux/list_bl.h+2−2
@@ -48,7 +48,7 @@ static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) · 47 unmodified lines
4848
4949 #define hlist_bl_entry(ptr, type, member) container_of(ptr,type,member)
5050
51−static inline int hlist_bl_unhashed(const struct hlist_bl_node *h)
51+static inline bool hlist_bl_unhashed(const struct hlist_bl_node *h)
5252 {
5353 return !h->pprev;
5454 }
@@ -68,7 +68,7 @@ static inline void hlist_bl_set_first(struct hlist_bl_head *h, · 13 unmodified lines
6868 h->first = (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK);
6969 }
7070
71−static inline int hlist_bl_empty(const struct hlist_bl_head *h)
71+static inline bool hlist_bl_empty(const struct hlist_bl_head *h)
7272 {
7373 return !((unsigned long)READ_ONCE(h->first) & ~LIST_BL_LOCKMASK);
7474 }
include/linux/memcontrol.h+33−11
@@ -52,7 +52,10 @@ enum mem_cgroup_stat_index { · 51 unmodified lines
5252 MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
5353 MEM_CGROUP_STAT_NSTATS,
5454 /* default hierarchy stats */
55− MEMCG_SOCK = MEM_CGROUP_STAT_NSTATS,
55+ MEMCG_KERNEL_STACK = MEM_CGROUP_STAT_NSTATS,
56+ MEMCG_SLAB_RECLAIMABLE,
57+ MEMCG_SLAB_UNRECLAIMABLE,
58+ MEMCG_SOCK,
5659 MEMCG_NR_STAT,
5760 };
5861
@@ -400,6 +403,9 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); · 341 unmodified lines
400403 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
401404 int nr_pages);
402405
406+unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
407+ int nid, unsigned int lru_mask);
408+
403409 static inline
404410 unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
405411 {
@@ -658,6 +664,13 @@ mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, · 252 unmodified lines
658664 {
659665 }
660666
667+static inline unsigned long
668+mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
669+ int nid, unsigned int lru_mask)
670+{
671+ return 0;
672+}
673+
661674 static inline void
662675 mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
663676 {
@@ -792,11 +805,6 @@ static inline bool memcg_kmem_enabled(void) · 128 unmodified lines
792805 return static_branch_unlikely(&memcg_kmem_enabled_key);
793806 }
794807
795−static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
796−{
797− return memcg->kmem_state == KMEM_ONLINE;
798−}
799−
800808 /*
801809 * In general, we'll do everything in our power to not incur in any overhead
802810 * for non-memcg users for the kmem functions. Not even a function call, if we
@@ -883,6 +891,20 @@ static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep) · 80 unmodified lines
883891 if (memcg_kmem_enabled())
884892 __memcg_kmem_put_cache(cachep);
885893 }
894+
895+/**
896+ * memcg_kmem_update_page_stat - update kmem page state statistics
897+ * @page: the page
898+ * @idx: page state item to account
899+ * @val: number of pages (positive or negative)
900+ */
901+static inline void memcg_kmem_update_page_stat(struct page *page,
902+ enum mem_cgroup_stat_index idx, int val)
903+{
904+ if (memcg_kmem_enabled() && page->mem_cgroup)
905+ this_cpu_add(page->mem_cgroup->stat->count[idx], val);
906+}
907+
886908 #else
887909 #define for_each_memcg_cache_index(_idx) \
888910 for (; NULL; )
@@ -892,11 +914,6 @@ static inline bool memcg_kmem_enabled(void) · 3 unmodified lines
892914 return false;
893915 }
894916
895−static inline bool memcg_kmem_online(struct mem_cgroup *memcg)
896−{
897− return false;
898−}
899−
900917 static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
901918 {
902919 return 0;
@@ -928,6 +945,11 @@ memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) · 25 unmodified lines
928945 static inline void memcg_kmem_put_cache(struct kmem_cache *cachep)
929946 {
930947 }
948+
949+static inline void memcg_kmem_update_page_stat(struct page *page,
950+ enum mem_cgroup_stat_index idx, int val)
951+{
952+}
931953 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */
932954
933955 #endif /* _LINUX_MEMCONTROL_H */
include/linux/mm.h+38−30
@@ -22,6 +22,7 @@ · 21 unmodified lines
2222 #include <linux/resource.h>
2323 #include <linux/page_ext.h>
2424 #include <linux/err.h>
25+#include <linux/page_ref.h>
2526
2627 struct mempolicy;
2728 struct anon_vma;
@@ -82,6 +83,27 @@ extern int mmap_rnd_compat_bits __read_mostly; · 54 unmodified lines
8283 #define mm_forbids_zeropage(X) (0)
8384 #endif
8485
86+/*
87+ * Default maximum number of active map areas, this limits the number of vmas
88+ * per mm struct. Users can overwrite this number by sysctl but there is a
89+ * problem.
90+ *
91+ * When a program's coredump is generated as ELF format, a section is created
92+ * per a vma. In ELF, the number of sections is represented in unsigned short.
93+ * This means the number of sections should be smaller than 65535 at coredump.
94+ * Because the kernel adds some informative sections to a image of program at
95+ * generating coredump, we need some margin. The number of extra sections is
96+ * 1-3 now and depends on arch. We use "5" as safe margin, here.
97+ *
98+ * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
99+ * not a hard limit any more. Although some userspace tools can be surprised by
100+ * that.
101+ */
102+#define MAPCOUNT_ELF_CORE_MARGIN (5)
103+#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
104+
105+extern int sysctl_max_map_count;
106+
85107 extern unsigned long sysctl_user_reserve_kbytes;
86108 extern unsigned long sysctl_admin_reserve_kbytes;
87109
@@ -122,6 +144,7 @@ extern unsigned int kobjsize(const void *objp); · 34 unmodified lines
122144
123145 /*
124146 * vm_flags in vm_area_struct, see mm_types.h.
147+ * When changing, update also include/trace/events/mmflags.h
125148 */
126149 #define VM_NONE 0x00000000
127150
@@ -364,8 +387,8 @@ static inline int pmd_devmap(pmd_t pmd) · 236 unmodified lines
364387 */
365388 static inline int put_page_testzero(struct page *page)
366389 {
367− VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);
368− return atomic_dec_and_test(&page->_count);
390+ VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
391+ return page_ref_dec_and_test(page);
369392 }
370393
371394 /*
@@ -376,7 +399,7 @@ static inline int put_page_testzero(struct page *page) · 4 unmodified lines
376399 */
377400 static inline int get_page_unless_zero(struct page *page)
378401 {
379− return atomic_inc_not_zero(&page->_count);
402+ return page_ref_add_unless(page, 1, 0);
380403 }
381404
382405 extern int page_is_ram(unsigned long pfn);
@@ -464,11 +487,6 @@ static inline int total_mapcount(struct page *page) · 81 unmodified lines
464487 }
465488 #endif
466489
467−static inline int page_count(struct page *page)
468−{
469− return atomic_read(&compound_head(page)->_count);
470−}
471−
472490 static inline struct page *virt_to_head_page(const void *x)
473491 {
474492 struct page *page = virt_to_page(x);
@@ -476,15 +494,6 @@ static inline struct page *virt_to_head_page(const void *x) · 1 unmodified line
476494 return compound_head(page);
477495 }
478496
479−/*
480− * Setup the page count before being freed into the page allocator for
481− * the first time (boot or memory hotplug)
482− */
483−static inline void init_page_count(struct page *page)
484−{
485− atomic_set(&page->_count, 1);
486−}
487−
488497 void __put_page(struct page *page);
489498
490499 void put_pages_list(struct list_head *pages);
@@ -694,8 +703,8 @@ static inline void get_page(struct page *page) · 203 unmodified lines
694703 * Getting a normal page or the head of a compound page
695704 * requires to already have an elevated page->_count.
696705 */
697− VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);
698− atomic_inc(&page->_count);
706+ VM_BUG_ON_PAGE(page_ref_count(page) <= 0, page);
707+ page_ref_inc(page);
699708
700709 if (unlikely(is_zone_device_page(page)))
701710 get_zone_device_page(page);
@@ -1043,8 +1052,6 @@ static inline void clear_page_pfmemalloc(struct page *page) · 341 unmodified lines
10431052 * just gets major/minor fault counters bumped up.
10441053 */
10451054
1046−#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
1047−
10481055 #define VM_FAULT_OOM 0x0001
10491056 #define VM_FAULT_SIGBUS 0x0002
10501057 #define VM_FAULT_MAJOR 0x0004
@@ -1523,8 +1530,7 @@ static inline void mm_dec_nr_pmds(struct mm_struct *mm) · 472 unmodified lines
15231530 }
15241531 #endif
15251532
1526−int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
1527− pmd_t *pmd, unsigned long address);
1533+int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
15281534 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
15291535
15301536 /*
@@ -1650,15 +1656,15 @@ static inline void pgtable_page_dtor(struct page *page) · 119 unmodified lines
16501656 pte_unmap(pte); \
16511657 } while (0)
16521658
1653−#define pte_alloc_map(mm, vma, pmd, address) \
1654− ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \
1655− pmd, address))? \
1656− NULL: pte_offset_map(pmd, address))
1659+#define pte_alloc(mm, pmd, address) \
1660+ (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd, address))
1661+
1662+#define pte_alloc_map(mm, pmd, address) \
1663+ (pte_alloc(mm, pmd, address) ? NULL : pte_offset_map(pmd, address))
16571664
16581665 #define pte_alloc_map_lock(mm, pmd, address, ptlp) \
1659− ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \
1660− pmd, address))? \
1661− NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
1666+ (pte_alloc(mm, pmd, address) ? \
1667+ NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
16621668
16631669 #define pte_alloc_kernel(pmd, address) \
16641670 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
@@ -1853,6 +1859,7 @@ extern int __meminit init_per_zone_wmark_min(void); · 188 unmodified lines
18531859 extern void mem_init(void);
18541860 extern void __init mmap_init(void);
18551861 extern void show_mem(unsigned int flags);
1862+extern long si_mem_available(void);
18561863 extern void si_meminfo(struct sysinfo * val);
18571864 extern void si_meminfo_node(struct sysinfo *val, int nid);
18581865
@@ -1867,6 +1874,7 @@ extern void zone_pcp_reset(struct zone *zone); · 8 unmodified lines
18671874
18681875 /* page_alloc.c */
18691876 extern int min_free_kbytes;
1877+extern int watermark_scale_factor;
18701878
18711879 /* nommu.c */
18721880 extern atomic_long_t mmap_pages_allocated;
include/linux/mmzone.h+8−0
@@ -668,6 +668,12 @@ typedef struct pglist_data { · 667 unmodified lines
668668 mem_hotplug_begin/end() */
669669 int kswapd_max_order;
670670 enum zone_type classzone_idx;
671+#ifdef CONFIG_COMPACTION
672+ int kcompactd_max_order;
673+ enum zone_type kcompactd_classzone_idx;
674+ wait_queue_head_t kcompactd_wait;
675+ struct task_struct *kcompactd;
676+#endif
671677 #ifdef CONFIG_NUMA_BALANCING
672678 /* Lock serializing the migrate rate limiting window */
673679 spinlock_t numabalancing_migrate_lock;
@@ -835,6 +841,8 @@ static inline int is_highmem(struct zone *zone) · 161 unmodified lines
835841 struct ctl_table;
836842 int min_free_kbytes_sysctl_handler(struct ctl_table *, int,
837843 void __user *, size_t *, loff_t *);
844+int watermark_scale_factor_sysctl_handler(struct ctl_table *, int,
845+ void __user *, size_t *, loff_t *);
838846 extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1];
839847 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int,
840848 void __user *, size_t *, loff_t *);
include/linux/page-flags-layout.h+2−0
@@ -17,6 +17,8 @@ · 16 unmodified lines
1717 #define ZONES_SHIFT 1
1818 #elif MAX_NR_ZONES <= 4
1919 #define ZONES_SHIFT 2
20+#elif MAX_NR_ZONES <= 8
21+#define ZONES_SHIFT 3
2022 #else
2123 #error ZONES_SHIFT -- too many zones configured adjust calculation
2224 #endif
include/linux/page-flags.h+17−15
@@ -144,12 +144,12 @@ static inline struct page *compound_head(struct page *page) · 143 unmodified lines
144144 return page;
145145 }
146146
147−static inline int PageTail(struct page *page)
147+static __always_inline int PageTail(struct page *page)
148148 {
149149 return READ_ONCE(page->compound_head) & 1;
150150 }
151151
152−static inline int PageCompound(struct page *page)
152+static __always_inline int PageCompound(struct page *page)
153153 {
154154 return test_bit(PG_head, &page->flags) || PageTail(page);
155155 }
@@ -184,31 +184,31 @@ static inline int PageCompound(struct page *page) · 28 unmodified lines
184184 * Macros to create function definitions for page flags
185185 */
186186 #define TESTPAGEFLAG(uname, lname, policy) \
187−static inline int Page##uname(struct page *page) \
187+static __always_inline int Page##uname(struct page *page) \
188188 { return test_bit(PG_##lname, &policy(page, 0)->flags); }
189189
190190 #define SETPAGEFLAG(uname, lname, policy) \
191−static inline void SetPage##uname(struct page *page) \
191+static __always_inline void SetPage##uname(struct page *page) \
192192 { set_bit(PG_##lname, &policy(page, 1)->flags); }
193193
194194 #define CLEARPAGEFLAG(uname, lname, policy) \
195−static inline void ClearPage##uname(struct page *page) \
195+static __always_inline void ClearPage##uname(struct page *page) \
196196 { clear_bit(PG_##lname, &policy(page, 1)->flags); }
197197
198198 #define __SETPAGEFLAG(uname, lname, policy) \
199−static inline void __SetPage##uname(struct page *page) \
199+static __always_inline void __SetPage##uname(struct page *page) \
200200 { __set_bit(PG_##lname, &policy(page, 1)->flags); }
201201
202202 #define __CLEARPAGEFLAG(uname, lname, policy) \
203−static inline void __ClearPage##uname(struct page *page) \
203+static __always_inline void __ClearPage##uname(struct page *page) \
204204 { __clear_bit(PG_##lname, &policy(page, 1)->flags); }
205205
206206 #define TESTSETFLAG(uname, lname, policy) \
207−static inline int TestSetPage##uname(struct page *page) \
207+static __always_inline int TestSetPage##uname(struct page *page) \
208208 { return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
209209
210210 #define TESTCLEARFLAG(uname, lname, policy) \
211−static inline int TestClearPage##uname(struct page *page) \
211+static __always_inline int TestClearPage##uname(struct page *page) \
212212 { return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
213213
214214 #define PAGEFLAG(uname, lname, policy) \
@@ -371,7 +371,7 @@ PAGEFLAG(Idle, idle, PF_ANY) · 156 unmodified lines
371371 #define PAGE_MAPPING_KSM 2
372372 #define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
373373
374−static inline int PageAnon(struct page *page)
374+static __always_inline int PageAnon(struct page *page)
375375 {
376376 page = compound_head(page);
377377 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
@@ -384,7 +384,7 @@ static inline int PageAnon(struct page *page) · 6 unmodified lines
384384 * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
385385 * anon_vma, but to that page's node of the stable tree.
386386 */
387−static inline int PageKsm(struct page *page)
387+static __always_inline int PageKsm(struct page *page)
388388 {
389389 page = compound_head(page);
390390 return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
@@ -415,14 +415,14 @@ static inline int PageUptodate(struct page *page) · 24 unmodified lines
415415 return ret;
416416 }
417417
418−static inline void __SetPageUptodate(struct page *page)
418+static __always_inline void __SetPageUptodate(struct page *page)
419419 {
420420 VM_BUG_ON_PAGE(PageTail(page), page);
421421 smp_wmb();
422422 __set_bit(PG_uptodate, &page->flags);
423423 }
424424
425−static inline void SetPageUptodate(struct page *page)
425+static __always_inline void SetPageUptodate(struct page *page)
426426 {
427427 VM_BUG_ON_PAGE(PageTail(page), page);
428428 /*
@@ -456,12 +456,12 @@ static inline void set_page_writeback_keepwrite(struct page *page) · 27 unmodified lines
456456
457457 __PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
458458
459−static inline void set_compound_head(struct page *page, struct page *head)
459+static __always_inline void set_compound_head(struct page *page, struct page *head)
460460 {
461461 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
462462 }
463463
464−static inline void clear_compound_head(struct page *page)
464+static __always_inline void clear_compound_head(struct page *page)
465465 {
466466 WRITE_ONCE(page->compound_head, 0);
467467 }
@@ -593,6 +593,8 @@ static inline void __ClearPageBuddy(struct page *page) · 125 unmodified lines
593593 atomic_set(&page->_mapcount, -1);
594594 }
595595
596+extern bool is_free_buddy_page(struct page *page);
597+
596598 #define PAGE_BALLOON_MAPCOUNT_VALUE (-256)
597599
598600 static inline int PageBalloon(struct page *page)
include/linux/page_ref.h+173−0
@@ -0,0 +1,173 @@
1+#ifndef _LINUX_PAGE_REF_H
2+#define _LINUX_PAGE_REF_H
3+
4+#include <linux/atomic.h>
5+#include <linux/mm_types.h>
6+#include <linux/page-flags.h>
7+#include <linux/tracepoint-defs.h>
8+
9+extern struct tracepoint __tracepoint_page_ref_set;
10+extern struct tracepoint __tracepoint_page_ref_mod;
11+extern struct tracepoint __tracepoint_page_ref_mod_and_test;
12+extern struct tracepoint __tracepoint_page_ref_mod_and_return;
13+extern struct tracepoint __tracepoint_page_ref_mod_unless;
14+extern struct tracepoint __tracepoint_page_ref_freeze;
15+extern struct tracepoint __tracepoint_page_ref_unfreeze;
16+
17+#ifdef CONFIG_DEBUG_PAGE_REF
18+
19+/*
20+ * Ideally we would want to use the trace_<tracepoint>_enabled() helper
21+ * functions. But due to include header file issues, that is not
22+ * feasible. Instead we have to open code the static key functions.
23+ *
24+ * See trace_##name##_enabled(void) in include/linux/tracepoint.h
25+ */
26+#define page_ref_tracepoint_active(t) static_key_false(&(t).key)
27+
28+extern void __page_ref_set(struct page *page, int v);
29+extern void __page_ref_mod(struct page *page, int v);
30+extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
31+extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
32+extern void __page_ref_mod_unless(struct page *page, int v, int u);
33+extern void __page_ref_freeze(struct page *page, int v, int ret);
34+extern void __page_ref_unfreeze(struct page *page, int v);
35+
36+#else
37+
38+#define page_ref_tracepoint_active(t) false
39+
40+static inline void __page_ref_set(struct page *page, int v)
41+{
42+}
43+static inline void __page_ref_mod(struct page *page, int v)
44+{
45+}
46+static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)
47+{
48+}
49+static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)
50+{
51+}
52+static inline void __page_ref_mod_unless(struct page *page, int v, int u)
53+{
54+}
55+static inline void __page_ref_freeze(struct page *page, int v, int ret)
56+{
57+}
58+static inline void __page_ref_unfreeze(struct page *page, int v)
59+{
60+}
61+
62+#endif
63+
64+static inline int page_ref_count(struct page *page)
65+{
66+ return atomic_read(&page->_count);
67+}
68+
69+static inline int page_count(struct page *page)
70+{
71+ return atomic_read(&compound_head(page)->_count);
72+}
73+
74+static inline void set_page_count(struct page *page, int v)
75+{
76+ atomic_set(&page->_count, v);
77+ if (page_ref_tracepoint_active(__tracepoint_page_ref_set))
78+ __page_ref_set(page, v);
79+}
80+
81+/*
82+ * Setup the page count before being freed into the page allocator for
83+ * the first time (boot or memory hotplug)
84+ */
85+static inline void init_page_count(struct page *page)
86+{
87+ set_page_count(page, 1);
88+}
89+
90+static inline void page_ref_add(struct page *page, int nr)
91+{
92+ atomic_add(nr, &page->_count);
93+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
94+ __page_ref_mod(page, nr);
95+}
96+
97+static inline void page_ref_sub(struct page *page, int nr)
98+{
99+ atomic_sub(nr, &page->_count);
100+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
101+ __page_ref_mod(page, -nr);
102+}
103+
104+static inline void page_ref_inc(struct page *page)
105+{
106+ atomic_inc(&page->_count);
107+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
108+ __page_ref_mod(page, 1);
109+}
110+
111+static inline void page_ref_dec(struct page *page)
112+{
113+ atomic_dec(&page->_count);
114+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod))
115+ __page_ref_mod(page, -1);
116+}
117+
118+static inline int page_ref_sub_and_test(struct page *page, int nr)
119+{
120+ int ret = atomic_sub_and_test(nr, &page->_count);
121+
122+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
123+ __page_ref_mod_and_test(page, -nr, ret);
124+ return ret;
125+}
126+
127+static inline int page_ref_dec_and_test(struct page *page)
128+{
129+ int ret = atomic_dec_and_test(&page->_count);
130+
131+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_test))
132+ __page_ref_mod_and_test(page, -1, ret);
133+ return ret;
134+}
135+
136+static inline int page_ref_dec_return(struct page *page)
137+{
138+ int ret = atomic_dec_return(&page->_count);
139+
140+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_and_return))
141+ __page_ref_mod_and_return(page, -1, ret);
142+ return ret;
143+}
144+
145+static inline int page_ref_add_unless(struct page *page, int nr, int u)
146+{
147+ int ret = atomic_add_unless(&page->_count, nr, u);
148+
149+ if (page_ref_tracepoint_active(__tracepoint_page_ref_mod_unless))
150+ __page_ref_mod_unless(page, nr, ret);
151+ return ret;
152+}
153+
154+static inline int page_ref_freeze(struct page *page, int count)
155+{
156+ int ret = likely(atomic_cmpxchg(&page->_count, count, 0) == count);
157+
158+ if (page_ref_tracepoint_active(__tracepoint_page_ref_freeze))
159+ __page_ref_freeze(page, count, ret);
160+ return ret;
161+}
162+
163+static inline void page_ref_unfreeze(struct page *page, int count)
164+{
165+ VM_BUG_ON_PAGE(page_count(page) != 0, page);
166+ VM_BUG_ON(count == 0);
167+
168+ atomic_set(&page->_count, count);
169+ if (page_ref_tracepoint_active(__tracepoint_page_ref_unfreeze))
170+ __page_ref_unfreeze(page, count);
171+}
172+
173+#endif
include/linux/pagemap.h+3−16
@@ -165,7 +165,7 @@ static inline int page_cache_get_speculative(struct page *page) · 164 unmodified lines
165165 * SMP requires.
166166 */
167167 VM_BUG_ON_PAGE(page_count(page) == 0, page);
168− atomic_inc(&page->_count);
168+ page_ref_inc(page);
169169
170170 #else
171171 if (unlikely(!get_page_unless_zero(page))) {
@@ -194,10 +194,10 @@ static inline int page_cache_add_speculative(struct page *page, int count) · 22 unmodified lines
194194 VM_BUG_ON(!in_atomic());
195195 # endif
196196 VM_BUG_ON_PAGE(page_count(page) == 0, page);
197− atomic_add(count, &page->_count);
197+ page_ref_add(page, count);
198198
199199 #else
200− if (unlikely(!atomic_add_unless(&page->_count, count, 0)))
200+ if (unlikely(!page_ref_add_unless(page, count, 0)))
201201 return 0;
202202 #endif
203203 VM_BUG_ON_PAGE(PageCompound(page) && page != compound_head(page), page);
@@ -205,19 +205,6 @@ static inline int page_cache_add_speculative(struct page *page, int count) · 1 unmodified line
205205 return 1;
206206 }
207207
208−static inline int page_freeze_refs(struct page *page, int count)
209−{
210− return likely(atomic_cmpxchg(&page->_count, count, 0) == count);
211−}
212−
213−static inline void page_unfreeze_refs(struct page *page, int count)
214−{
215− VM_BUG_ON_PAGE(page_count(page) != 0, page);
216− VM_BUG_ON(count == 0);
217−
218− atomic_set(&page->_count, count);
219−}
220−
221208 #ifdef CONFIG_NUMA
222209 extern struct page *__page_cache_alloc(gfp_t gfp);
223210 #else
include/linux/poll.h+1−1
@@ -96,7 +96,7 @@ extern void poll_initwait(struct poll_wqueues *pwq); · 95 unmodified lines
9696 extern void poll_freewait(struct poll_wqueues *pwq);
9797 extern int poll_schedule_timeout(struct poll_wqueues *pwq, int state,
9898 ktime_t *expires, unsigned long slack);
99−extern long select_estimate_accuracy(struct timespec *tv);
99+extern u64 select_estimate_accuracy(struct timespec *tv);
100100
101101
102102 static inline int poll_schedule(struct poll_wqueues *pwq, int state)
include/linux/quicklist.h+1−1
@@ -5,7 +5,7 @@ · 4 unmodified lines
55 * as needed after allocation when they are freed. Per cpu lists of pages
66 * are kept that only contain node local pages.
77 *
8− * (C) 2007, SGI. Christoph Lameter <clameter@sgi.com>
8+ * (C) 2007, SGI. Christoph Lameter <cl@linux.com>
99 */
1010 #include <linux/kernel.h>
1111 #include <linux/gfp.h>
include/linux/radix-tree.h+26−2
@@ -21,6 +21,7 @@ · 20 unmodified lines
2121 #ifndef _LINUX_RADIX_TREE_H
2222 #define _LINUX_RADIX_TREE_H
2323
24+#include <linux/bitops.h>
2425 #include <linux/preempt.h>
2526 #include <linux/types.h>
2627 #include <linux/bug.h>
@@ -270,8 +271,15 @@ static inline void radix_tree_replace_slot(void **pslot, void *item) · 243 unmodified lines
270271 }
271272
272273 int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
273− struct radix_tree_node **nodep, void ***slotp);
274−int radix_tree_insert(struct radix_tree_root *, unsigned long, void *);
274+ unsigned order, struct radix_tree_node **nodep,
275+ void ***slotp);
276+int __radix_tree_insert(struct radix_tree_root *, unsigned long index,
277+ unsigned order, void *);
278+static inline int radix_tree_insert(struct radix_tree_root *root,
279+ unsigned long index, void *entry)
280+{
281+ return __radix_tree_insert(root, index, 0, entry);
282+}
275283 void *__radix_tree_lookup(struct radix_tree_root *root, unsigned long index,
276284 struct radix_tree_node **nodep, void ***slotp);
277285 void *radix_tree_lookup(struct radix_tree_root *, unsigned long);
@@ -394,6 +402,22 @@ void **radix_tree_iter_retry(struct radix_tree_iter *iter) · 116 unmodified lines
394402 return NULL;
395403 }
396404
405+/**
406+ * radix_tree_iter_next - resume iterating when the chunk may be invalid
407+ * @iter: iterator state
408+ *
409+ * If the iterator needs to release then reacquire a lock, the chunk may
410+ * have been invalidated by an insertion or deletion. Call this function
411+ * to continue the iteration from the next index.
412+ */
413+static inline __must_check
414+void **radix_tree_iter_next(struct radix_tree_iter *iter)
415+{
416+ iter->next_index = iter->index + 1;
417+ iter->tags = 0;
418+ return NULL;
419+}
420+
397421 /**
398422 * radix_tree_chunk_size - get current chunk size
399423 *
include/linux/rmap.h+6−0
@@ -86,6 +86,7 @@ enum ttu_flags { · 85 unmodified lines
8686 TTU_MIGRATION = 2, /* migration mode */
8787 TTU_MUNLOCK = 4, /* munlock mode */
8888 TTU_LZFREE = 8, /* lazy free mode */
89+ TTU_SPLIT_HUGE_PMD = 16, /* split huge PMD if any */
8990
9091 TTU_IGNORE_MLOCK = (1 << 8), /* ignore mlock */
9192 TTU_IGNORE_ACCESS = (1 << 9), /* don't age */
@@ -93,6 +94,8 @@ enum ttu_flags { · 1 unmodified line
9394 TTU_BATCH_FLUSH = (1 << 11), /* Batch TLB flushes where possible
9495 * and caller guarantees they will
9596 * do a final flush if necessary */
97+ TTU_RMAP_LOCKED = (1 << 12) /* do not grab rmap lock:
98+ * caller holds it */
9699 };
97100
98101 #ifdef CONFIG_MMU
@@ -240,6 +243,8 @@ int page_mkclean(struct page *); · 141 unmodified lines
240243 */
241244 int try_to_munlock(struct page *);
242245
246+void remove_migration_ptes(struct page *old, struct page *new, bool locked);
247+
243248 /*
244249 * Called by memory-failure.c to kill processes.
245250 */
@@ -266,6 +271,7 @@ struct rmap_walk_control { · 20 unmodified lines
266271 };
267272
268273 int rmap_walk(struct page *page, struct rmap_walk_control *rwc);
274+int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc);
269275
270276 #else /* !CONFIG_MMU */
271277
include/linux/sched.h+2−2
@@ -1791,8 +1791,8 @@ struct task_struct { · 1790 unmodified lines
17911791 * time slack values; these are used to round up poll() and
17921792 * select() etc timeout values. These are in nanoseconds.
17931793 */
1794− unsigned long timer_slack_ns;
1795− unsigned long default_timer_slack_ns;
1794+ u64 timer_slack_ns;
1795+ u64 default_timer_slack_ns;
17961796
17971797 #ifdef CONFIG_KASAN
17981798 unsigned int kasan_depth;
include/linux/sched/sysctl.h+0−21
@@ -14,27 +14,6 @@ extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, · 13 unmodified lines
1414 enum { sysctl_hung_task_timeout_secs = 0 };
1515 #endif
1616
17−/*
18− * Default maximum number of active map areas, this limits the number of vmas
19− * per mm struct. Users can overwrite this number by sysctl but there is a
20− * problem.
21− *
22− * When a program's coredump is generated as ELF format, a section is created
23− * per a vma. In ELF, the number of sections is represented in unsigned short.
24− * This means the number of sections should be smaller than 65535 at coredump.
25− * Because the kernel adds some informative sections to a image of program at
26− * generating coredump, we need some margin. The number of extra sections is
27− * 1-3 now and depends on arch. We use "5" as safe margin, here.
28− *
29− * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
30− * not a hard limit any more. Although some userspace tools can be surprised by
31− * that.
32− */
33−#define MAPCOUNT_ELF_CORE_MARGIN (5)
34−#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
35−
36−extern int sysctl_max_map_count;
37−
3817 extern unsigned int sysctl_sched_latency;
3918 extern unsigned int sysctl_sched_min_granularity;
4019 extern unsigned int sysctl_sched_wakeup_granularity;
include/linux/string.h+7−1
@@ -128,7 +128,13 @@ extern char **argv_split(gfp_t gfp, const char *str, int *argcp); · 127 unmodified lines
128128 extern void argv_free(char **argv);
129129
130130 extern bool sysfs_streq(const char *s1, const char *s2);
131−extern int strtobool(const char *s, bool *res);
131+extern int kstrtobool(const char *s, bool *res);
132+static inline int strtobool(const char *s, bool *res)
133+{
134+ return kstrtobool(s, res);
135+}
136+
137+int match_string(const char * const *array, size_t n, const char *string);
132138
133139 #ifdef CONFIG_BINARY_PRINTF
134140 int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args);
include/linux/tick.h+1−1
@@ -111,7 +111,7 @@ enum tick_dep_bits { · 110 unmodified lines
111111 #define TICK_DEP_MASK_CLOCK_UNSTABLE (1 << TICK_DEP_BIT_CLOCK_UNSTABLE)
112112
113113 #ifdef CONFIG_NO_HZ_COMMON
114−extern int tick_nohz_enabled;
114+extern bool tick_nohz_enabled;
115115 extern int tick_nohz_tick_stopped(void);
116116 extern void tick_nohz_idle_enter(void);
117117 extern void tick_nohz_idle_exit(void);
include/linux/unaligned/access_ok.h+12−12
@@ -4,62 +4,62 @@ · 3 unmodified lines
44 #include <linux/kernel.h>
55 #include <asm/byteorder.h>
66
7−static inline u16 get_unaligned_le16(const void *p)
7+static __always_inline u16 get_unaligned_le16(const void *p)
88 {
99 return le16_to_cpup((__le16 *)p);
1010 }
1111
12−static inline u32 get_unaligned_le32(const void *p)
12+static __always_inline u32 get_unaligned_le32(const void *p)
1313 {
1414 return le32_to_cpup((__le32 *)p);
1515 }
1616
17−static inline u64 get_unaligned_le64(const void *p)
17+static __always_inline u64 get_unaligned_le64(const void *p)
1818 {
1919 return le64_to_cpup((__le64 *)p);
2020 }
2121
22−static inline u16 get_unaligned_be16(const void *p)
22+static __always_inline u16 get_unaligned_be16(const void *p)
2323 {
2424 return be16_to_cpup((__be16 *)p);
2525 }
2626
27−static inline u32 get_unaligned_be32(const void *p)
27+static __always_inline u32 get_unaligned_be32(const void *p)
2828 {
2929 return be32_to_cpup((__be32 *)p);
3030 }
3131
32−static inline u64 get_unaligned_be64(const void *p)
32+static __always_inline u64 get_unaligned_be64(const void *p)
3333 {
3434 return be64_to_cpup((__be64 *)p);
3535 }
3636
37−static inline void put_unaligned_le16(u16 val, void *p)
37+static __always_inline void put_unaligned_le16(u16 val, void *p)
3838 {
3939 *((__le16 *)p) = cpu_to_le16(val);
4040 }
4141
42−static inline void put_unaligned_le32(u32 val, void *p)
42+static __always_inline void put_unaligned_le32(u32 val, void *p)
4343 {
4444 *((__le32 *)p) = cpu_to_le32(val);
4545 }
4646
47−static inline void put_unaligned_le64(u64 val, void *p)
47+static __always_inline void put_unaligned_le64(u64 val, void *p)
4848 {
4949 *((__le64 *)p) = cpu_to_le64(val);
5050 }
5151
52−static inline void put_unaligned_be16(u16 val, void *p)
52+static __always_inline void put_unaligned_be16(u16 val, void *p)
5353 {
5454 *((__be16 *)p) = cpu_to_be16(val);
5555 }
5656
57−static inline void put_unaligned_be32(u32 val, void *p)
57+static __always_inline void put_unaligned_be32(u32 val, void *p)
5858 {
5959 *((__be32 *)p) = cpu_to_be32(val);
6060 }
6161
62−static inline void put_unaligned_be64(u64 val, void *p)
62+static __always_inline void put_unaligned_be64(u64 val, void *p)
6363 {
6464 *((__be64 *)p) = cpu_to_be64(val);
6565 }
include/linux/vm_event_item.h+2−0
@@ -53,6 +53,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, · 52 unmodified lines
5353 COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED,
5454 COMPACTISOLATED,
5555 COMPACTSTALL, COMPACTFAIL, COMPACTSUCCESS,
56+ KCOMPACTD_WAKE,
5657 #endif
5758 #ifdef CONFIG_HUGETLB_PAGE
5859 HTLB_BUDDY_PGALLOC, HTLB_BUDDY_PGALLOC_FAIL,
@@ -71,6 +72,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, · 12 unmodified lines
7172 THP_COLLAPSE_ALLOC_FAILED,
7273 THP_SPLIT_PAGE,
7374 THP_SPLIT_PAGE_FAILED,
75+ THP_DEFERRED_SPLIT_PAGE,
7476 THP_SPLIT_PMD,
7577 THP_ZERO_PAGE_ALLOC,
7678 THP_ZERO_PAGE_ALLOC_FAILED,
include/trace/events/compaction.h+55−0
@@ -350,6 +350,61 @@ DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_defer_reset, · 349 unmodified lines
350350 );
351351 #endif
352352
353+TRACE_EVENT(mm_compaction_kcompactd_sleep,
354+
355+ TP_PROTO(int nid),
356+
357+ TP_ARGS(nid),
358+
359+ TP_STRUCT__entry(
360+ __field(int, nid)
361+ ),
362+
363+ TP_fast_assign(
364+ __entry->nid = nid;
365+ ),
366+
367+ TP_printk("nid=%d", __entry->nid)
368+);
369+
370+DECLARE_EVENT_CLASS(kcompactd_wake_template,
371+
372+ TP_PROTO(int nid, int order, enum zone_type classzone_idx),
373+
374+ TP_ARGS(nid, order, classzone_idx),
375+
376+ TP_STRUCT__entry(
377+ __field(int, nid)
378+ __field(int, order)
379+ __field(enum zone_type, classzone_idx)
380+ ),
381+
382+ TP_fast_assign(
383+ __entry->nid = nid;
384+ __entry->order = order;
385+ __entry->classzone_idx = classzone_idx;
386+ ),
387+
388+ TP_printk("nid=%d order=%d classzone_idx=%-8s",
389+ __entry->nid,
390+ __entry->order,
391+ __print_symbolic(__entry->classzone_idx, ZONE_TYPE))
392+);
393+
394+DEFINE_EVENT(kcompactd_wake_template, mm_compaction_wakeup_kcompactd,
395+
396+ TP_PROTO(int nid, int order, enum zone_type classzone_idx),
397+
398+ TP_ARGS(nid, order, classzone_idx)
399+);
400+
401+DEFINE_EVENT(kcompactd_wake_template, mm_compaction_kcompactd_wake,
402+
403+ TP_PROTO(int nid, int order, enum zone_type classzone_idx),
404+
405+ TP_ARGS(nid, order, classzone_idx)
406+);
407+
353408 #endif /* _TRACE_COMPACTION_H */
354409
355410 /* This part must be outside protection */
include/trace/events/mmflags.h+16−7
@@ -111,15 +111,21 @@ IF_HAVE_PG_IDLE(PG_idle, "idle" ) · 110 unmodified lines
111111 ) : "none"
112112
113113 #if defined(CONFIG_X86)
114−#define __VM_ARCH_SPECIFIC {VM_PAT, "pat" }
114+#define __VM_ARCH_SPECIFIC_1 {VM_PAT, "pat" }
115115 #elif defined(CONFIG_PPC)
116−#define __VM_ARCH_SPECIFIC {VM_SAO, "sao" }
116+#define __VM_ARCH_SPECIFIC_1 {VM_SAO, "sao" }
117117 #elif defined(CONFIG_PARISC) || defined(CONFIG_METAG) || defined(CONFIG_IA64)
118−#define __VM_ARCH_SPECIFIC {VM_GROWSUP, "growsup" }
118+#define __VM_ARCH_SPECIFIC_1 {VM_GROWSUP, "growsup" }
119119 #elif !defined(CONFIG_MMU)
120−#define __VM_ARCH_SPECIFIC {VM_MAPPED_COPY,"mappedcopy" }
120+#define __VM_ARCH_SPECIFIC_1 {VM_MAPPED_COPY,"mappedcopy" }
121121 #else
122−#define __VM_ARCH_SPECIFIC {VM_ARCH_1, "arch_1" }
122+#define __VM_ARCH_SPECIFIC_1 {VM_ARCH_1, "arch_1" }
123+#endif
124+
125+#if defined(CONFIG_X86)
126+#define __VM_ARCH_SPECIFIC_2 {VM_MPX, "mpx" }
127+#else
128+#define __VM_ARCH_SPECIFIC_2 {VM_ARCH_2, "arch_2" }
123129 #endif
124130
125131 #ifdef CONFIG_MEM_SOFT_DIRTY
@@ -138,19 +144,22 @@ IF_HAVE_PG_IDLE(PG_idle, "idle" ) · 12 unmodified lines
138144 {VM_MAYEXEC, "mayexec" }, \
139145 {VM_MAYSHARE, "mayshare" }, \
140146 {VM_GROWSDOWN, "growsdown" }, \
147+ {VM_UFFD_MISSING, "uffd_missing" }, \
141148 {VM_PFNMAP, "pfnmap" }, \
142149 {VM_DENYWRITE, "denywrite" }, \
143− {VM_LOCKONFAULT, "lockonfault" }, \
150+ {VM_UFFD_WP, "uffd_wp" }, \
144151 {VM_LOCKED, "locked" }, \
145152 {VM_IO, "io" }, \
146153 {VM_SEQ_READ, "seqread" }, \
147154 {VM_RAND_READ, "randread" }, \
148155 {VM_DONTCOPY, "dontcopy" }, \
149156 {VM_DONTEXPAND, "dontexpand" }, \
157+ {VM_LOCKONFAULT, "lockonfault" }, \
150158 {VM_ACCOUNT, "account" }, \
151159 {VM_NORESERVE, "noreserve" }, \
152160 {VM_HUGETLB, "hugetlb" }, \
153− __VM_ARCH_SPECIFIC , \
161+ __VM_ARCH_SPECIFIC_1 , \
162+ __VM_ARCH_SPECIFIC_2 , \
154163 {VM_DONTDUMP, "dontdump" }, \
155164 IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY, "softdirty" ) \
156165 {VM_MIXEDMAP, "mixedmap" }, \
include/trace/events/page_ref.h+134−0
@@ -0,0 +1,134 @@
1+#undef TRACE_SYSTEM
2+#define TRACE_SYSTEM page_ref
3+
4+#if !defined(_TRACE_PAGE_REF_H) || defined(TRACE_HEADER_MULTI_READ)
5+#define _TRACE_PAGE_REF_H
6+
7+#include <linux/types.h>
8+#include <linux/page_ref.h>
9+#include <linux/tracepoint.h>
10+#include <trace/events/mmflags.h>
11+
12+DECLARE_EVENT_CLASS(page_ref_mod_template,
13+
14+ TP_PROTO(struct page *page, int v),
15+
16+ TP_ARGS(page, v),
17+
18+ TP_STRUCT__entry(
19+ __field(unsigned long, pfn)
20+ __field(unsigned long, flags)
21+ __field(int, count)
22+ __field(int, mapcount)
23+ __field(void *, mapping)
24+ __field(int, mt)
25+ __field(int, val)
26+ ),
27+
28+ TP_fast_assign(
29+ __entry->pfn = page_to_pfn(page);
30+ __entry->flags = page->flags;
31+ __entry->count = page_ref_count(page);
32+ __entry->mapcount = page_mapcount(page);
33+ __entry->mapping = page->mapping;
34+ __entry->mt = get_pageblock_migratetype(page);
35+ __entry->val = v;
36+ ),
37+
38+ TP_printk("pfn=0x%lx flags=%s count=%d mapcount=%d mapping=%p mt=%d val=%d",
39+ __entry->pfn,
40+ show_page_flags(__entry->flags & ((1UL << NR_PAGEFLAGS) - 1)),
41+ __entry->count,
42+ __entry->mapcount, __entry->mapping, __entry->mt,
43+ __entry->val)
44+);
45+
46+DEFINE_EVENT(page_ref_mod_template, page_ref_set,
47+
48+ TP_PROTO(struct page *page, int v),
49+
50+ TP_ARGS(page, v)
51+);
52+
53+DEFINE_EVENT(page_ref_mod_template, page_ref_mod,
54+
55+ TP_PROTO(struct page *page, int v),
56+
57+ TP_ARGS(page, v)
58+);
59+
60+DECLARE_EVENT_CLASS(page_ref_mod_and_test_template,
61+
62+ TP_PROTO(struct page *page, int v, int ret),
63+
64+ TP_ARGS(page, v, ret),
65+
66+ TP_STRUCT__entry(
67+ __field(unsigned long, pfn)
68+ __field(unsigned long, flags)
69+ __field(int, count)
70+ __field(int, mapcount)
71+ __field(void *, mapping)
72+ __field(int, mt)
73+ __field(int, val)
74+ __field(int, ret)
75+ ),
76+
77+ TP_fast_assign(
78+ __entry->pfn = page_to_pfn(page);
79+ __entry->flags = page->flags;
80+ __entry->count = page_ref_count(page);
81+ __entry->mapcount = page_mapcount(page);
82+ __entry->mapping = page->mapping;
83+ __entry->mt = get_pageblock_migratetype(page);
84+ __entry->val = v;
85+ __entry->ret = ret;
86+ ),
87+
88+ TP_printk("pfn=0x%lx flags=%s count=%d mapcount=%d mapping=%p mt=%d val=%d ret=%d",
89+ __entry->pfn,
90+ show_page_flags(__entry->flags & ((1UL << NR_PAGEFLAGS) - 1)),
91+ __entry->count,
92+ __entry->mapcount, __entry->mapping, __entry->mt,
93+ __entry->val, __entry->ret)
94+);
95+
96+DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_mod_and_test,
97+
98+ TP_PROTO(struct page *page, int v, int ret),
99+
100+ TP_ARGS(page, v, ret)
101+);
102+
103+DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_mod_and_return,
104+
105+ TP_PROTO(struct page *page, int v, int ret),
106+
107+ TP_ARGS(page, v, ret)
108+);
109+
110+DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_mod_unless,
111+
112+ TP_PROTO(struct page *page, int v, int ret),
113+
114+ TP_ARGS(page, v, ret)
115+);
116+
117+DEFINE_EVENT(page_ref_mod_and_test_template, page_ref_freeze,
118+
119+ TP_PROTO(struct page *page, int v, int ret),
120+
121+ TP_ARGS(page, v, ret)
122+);
123+
124+DEFINE_EVENT(page_ref_mod_template, page_ref_unfreeze,
125+
126+ TP_PROTO(struct page *page, int v),
127+
128+ TP_ARGS(page, v)
129+);
130+
131+#endif /* _TRACE_PAGE_COUNT_H */
132+
133+/* This part must be outside protection */
134+#include <trace/define_trace.h>
include/uapi/linux/byteorder/big_endian.h+12−12
@@ -40,51 +40,51 @@ · 39 unmodified lines
4040 #define __cpu_to_be16(x) ((__force __be16)(__u16)(x))
4141 #define __be16_to_cpu(x) ((__force __u16)(__be16)(x))
4242
43−static inline __le64 __cpu_to_le64p(const __u64 *p)
43+static __always_inline __le64 __cpu_to_le64p(const __u64 *p)
4444 {
4545 return (__force __le64)__swab64p(p);
4646 }
47−static inline __u64 __le64_to_cpup(const __le64 *p)
47+static __always_inline __u64 __le64_to_cpup(const __le64 *p)
4848 {
4949 return __swab64p((__u64 *)p);
5050 }
51−static inline __le32 __cpu_to_le32p(const __u32 *p)
51+static __always_inline __le32 __cpu_to_le32p(const __u32 *p)
5252 {
5353 return (__force __le32)__swab32p(p);
5454 }
55−static inline __u32 __le32_to_cpup(const __le32 *p)
55+static __always_inline __u32 __le32_to_cpup(const __le32 *p)
5656 {
5757 return __swab32p((__u32 *)p);
5858 }
59−static inline __le16 __cpu_to_le16p(const __u16 *p)
59+static __always_inline __le16 __cpu_to_le16p(const __u16 *p)
6060 {
6161 return (__force __le16)__swab16p(p);
6262 }
63−static inline __u16 __le16_to_cpup(const __le16 *p)
63+static __always_inline __u16 __le16_to_cpup(const __le16 *p)
6464 {
6565 return __swab16p((__u16 *)p);
6666 }
67−static inline __be64 __cpu_to_be64p(const __u64 *p)
67+static __always_inline __be64 __cpu_to_be64p(const __u64 *p)
6868 {
6969 return (__force __be64)*p;
7070 }
71−static inline __u64 __be64_to_cpup(const __be64 *p)
71+static __always_inline __u64 __be64_to_cpup(const __be64 *p)
7272 {
7373 return (__force __u64)*p;
7474 }
75−static inline __be32 __cpu_to_be32p(const __u32 *p)
75+static __always_inline __be32 __cpu_to_be32p(const __u32 *p)
7676 {
7777 return (__force __be32)*p;
7878 }
79−static inline __u32 __be32_to_cpup(const __be32 *p)
79+static __always_inline __u32 __be32_to_cpup(const __be32 *p)
8080 {
8181 return (__force __u32)*p;
8282 }
83−static inline __be16 __cpu_to_be16p(const __u16 *p)
83+static __always_inline __be16 __cpu_to_be16p(const __u16 *p)
8484 {
8585 return (__force __be16)*p;
8686 }
87−static inline __u16 __be16_to_cpup(const __be16 *p)
87+static __always_inline __u16 __be16_to_cpup(const __be16 *p)
8888 {
8989 return (__force __u16)*p;
9090 }
include/uapi/linux/byteorder/little_endian.h+12−12
@@ -40,51 +40,51 @@ · 39 unmodified lines
4040 #define __cpu_to_be16(x) ((__force __be16)__swab16((x)))
4141 #define __be16_to_cpu(x) __swab16((__force __u16)(__be16)(x))
4242
43−static inline __le64 __cpu_to_le64p(const __u64 *p)
43+static __always_inline __le64 __cpu_to_le64p(const __u64 *p)
4444 {
4545 return (__force __le64)*p;
4646 }
47−static inline __u64 __le64_to_cpup(const __le64 *p)
47+static __always_inline __u64 __le64_to_cpup(const __le64 *p)
4848 {
4949 return (__force __u64)*p;
5050 }
51−static inline __le32 __cpu_to_le32p(const __u32 *p)
51+static __always_inline __le32 __cpu_to_le32p(const __u32 *p)
5252 {
5353 return (__force __le32)*p;
5454 }
55−static inline __u32 __le32_to_cpup(const __le32 *p)
55+static __always_inline __u32 __le32_to_cpup(const __le32 *p)
5656 {
5757 return (__force __u32)*p;
5858 }
59−static inline __le16 __cpu_to_le16p(const __u16 *p)
59+static __always_inline __le16 __cpu_to_le16p(const __u16 *p)
6060 {
6161 return (__force __le16)*p;
6262 }
63−static inline __u16 __le16_to_cpup(const __le16 *p)
63+static __always_inline __u16 __le16_to_cpup(const __le16 *p)
6464 {
6565 return (__force __u16)*p;
6666 }
67−static inline __be64 __cpu_to_be64p(const __u64 *p)
67+static __always_inline __be64 __cpu_to_be64p(const __u64 *p)
6868 {
6969 return (__force __be64)__swab64p(p);
7070 }
71−static inline __u64 __be64_to_cpup(const __be64 *p)
71+static __always_inline __u64 __be64_to_cpup(const __be64 *p)
7272 {
7373 return __swab64p((__u64 *)p);
7474 }
75−static inline __be32 __cpu_to_be32p(const __u32 *p)
75+static __always_inline __be32 __cpu_to_be32p(const __u32 *p)
7676 {
7777 return (__force __be32)__swab32p(p);
7878 }
79−static inline __u32 __be32_to_cpup(const __be32 *p)
79+static __always_inline __u32 __be32_to_cpup(const __be32 *p)
8080 {
8181 return __swab32p((__u32 *)p);
8282 }
83−static inline __be16 __cpu_to_be16p(const __u16 *p)
83+static __always_inline __be16 __cpu_to_be16p(const __u16 *p)
8484 {
8585 return (__force __be16)__swab16p(p);
8686 }
87−static inline __u16 __be16_to_cpup(const __be16 *p)
87+static __always_inline __u16 __be16_to_cpup(const __be16 *p)
8888 {
8989 return __swab16p((__u16 *)p);
9090 }
include/uapi/linux/elf-em.h+0−3
@@ -30,7 +30,6 @@ · 29 unmodified lines
3030 #define EM_X86_64 62 /* AMD x86-64 */
3131 #define EM_S390 22 /* IBM S/390 */
3232 #define EM_CRIS 76 /* Axis Communications 32-bit embedded processor */
33−#define EM_V850 87 /* NEC v850 */
3433 #define EM_M32R 88 /* Renesas M32R */
3534 #define EM_MN10300 89 /* Panasonic/MEI MN10300, AM33 */
3635 #define EM_OPENRISC 92 /* OpenRISC 32-bit embedded processor */
@@ -50,8 +49,6 @@ · 13 unmodified lines
5049 */
5150 #define EM_ALPHA 0x9026
5251
53−/* Bogus old v850 magic number, used by old tools. */
54−#define EM_CYGNUS_V850 0x9080
5552 /* Bogus old m32r magic number, used by old tools. */
5653 #define EM_CYGNUS_M32R 0x9041
5754 /* This is the old interim value for S/390 architecture */
include/uapi/linux/swab.h+5−5
@@ -151,7 +151,7 @@ static inline __attribute_const__ __u32 __fswahb32(__u32 val) · 150 unmodified lines
151151 * __swab16p - return a byteswapped 16-bit value from a pointer
152152 * @p: pointer to a naturally-aligned 16-bit value
153153 */
154−static inline __u16 __swab16p(const __u16 *p)
154+static __always_inline __u16 __swab16p(const __u16 *p)
155155 {
156156 #ifdef __arch_swab16p
157157 return __arch_swab16p(p);
@@ -164,7 +164,7 @@ static inline __u16 __swab16p(const __u16 *p) · 6 unmodified lines
164164 * __swab32p - return a byteswapped 32-bit value from a pointer
165165 * @p: pointer to a naturally-aligned 32-bit value
166166 */
167−static inline __u32 __swab32p(const __u32 *p)
167+static __always_inline __u32 __swab32p(const __u32 *p)
168168 {
169169 #ifdef __arch_swab32p
170170 return __arch_swab32p(p);
@@ -177,7 +177,7 @@ static inline __u32 __swab32p(const __u32 *p) · 6 unmodified lines
177177 * __swab64p - return a byteswapped 64-bit value from a pointer
178178 * @p: pointer to a naturally-aligned 64-bit value
179179 */
180−static inline __u64 __swab64p(const __u64 *p)
180+static __always_inline __u64 __swab64p(const __u64 *p)
181181 {
182182 #ifdef __arch_swab64p
183183 return __arch_swab64p(p);
@@ -232,7 +232,7 @@ static inline void __swab16s(__u16 *p) · 48 unmodified lines
232232 * __swab32s - byteswap a 32-bit value in-place
233233 * @p: pointer to a naturally-aligned 32-bit value
234234 */
235−static inline void __swab32s(__u32 *p)
235+static __always_inline void __swab32s(__u32 *p)
236236 {
237237 #ifdef __arch_swab32s
238238 __arch_swab32s(p);
@@ -245,7 +245,7 @@ static inline void __swab32s(__u32 *p) · 6 unmodified lines
245245 * __swab64s - byteswap a 64-bit value in-place
246246 * @p: pointer to a naturally-aligned 64-bit value
247247 */
248−static inline void __swab64s(__u64 *p)
248+static __always_inline void __swab64s(__u64 *p)
249249 {
250250 #ifdef __arch_swab64s
251251 __arch_swab64s(p);
include/uapi/linux/virtio_balloon.h+2−1
@@ -51,7 +51,8 @@ struct virtio_balloon_config { · 50 unmodified lines
5151 #define VIRTIO_BALLOON_S_MINFLT 3 /* Number of minor faults */
5252 #define VIRTIO_BALLOON_S_MEMFREE 4 /* Total amount of free memory */
5353 #define VIRTIO_BALLOON_S_MEMTOT 5 /* Total amount of memory */
54−#define VIRTIO_BALLOON_S_NR 6
54+#define VIRTIO_BALLOON_S_AVAIL 6 /* Available memory as in /proc */
55+#define VIRTIO_BALLOON_S_NR 7
5556
5657 /*
5758 * Memory statistics structure.
kernel/fork.c+9−1
@@ -164,12 +164,20 @@ static struct thread_info *alloc_thread_info_node(struct task_struct *tsk, · 163 unmodified lines
164164 struct page *page = alloc_kmem_pages_node(node, THREADINFO_GFP,
165165 THREAD_SIZE_ORDER);
166166
167+ if (page)
168+ memcg_kmem_update_page_stat(page, MEMCG_KERNEL_STACK,
169+ 1 << THREAD_SIZE_ORDER);
170+
167171 return page ? page_address(page) : NULL;
168172 }
169173
170174 static inline void free_thread_info(struct thread_info *ti)
171175 {
172− free_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER);
176+ struct page *page = virt_to_page(ti);
177+
178+ memcg_kmem_update_page_stat(page, MEMCG_KERNEL_STACK,
179+ -(1 << THREAD_SIZE_ORDER));
180+ __free_kmem_pages(page, THREAD_SIZE_ORDER);
173181 }
174182 # else
175183 static struct kmem_cache *thread_info_cache;
kernel/panic.c+26−15
@@ -24,6 +24,7 @@ · 23 unmodified lines
2424 #include <linux/init.h>
2525 #include <linux/nmi.h>
2626 #include <linux/console.h>
27+#include <linux/bug.h>
2728
2829 #define PANIC_TIMER_STEP 100
2930 #define PANIC_BLINK_SPD 18
@@ -449,20 +450,25 @@ void oops_exit(void) · 419 unmodified lines
449450 kmsg_dump(KMSG_DUMP_OOPS);
450451 }
451452
452−#ifdef WANT_WARN_ON_SLOWPATH
453−struct slowpath_args {
453+struct warn_args {
454454 const char *fmt;
455455 va_list args;
456456 };
457457
458−static void warn_slowpath_common(const char *file, int line, void *caller,
459− unsigned taint, struct slowpath_args *args)
458+void __warn(const char *file, int line, void *caller, unsigned taint,
459+ struct pt_regs *regs, struct warn_args *args)
460460 {
461461 disable_trace_on_warning();
462462
463463 pr_warn("------------[ cut here ]------------\n");
464− pr_warn("WARNING: CPU: %d PID: %d at %s:%d %pS()\n",
465− raw_smp_processor_id(), current->pid, file, line, caller);
464+
465+ if (file)
466+ pr_warn("WARNING: CPU: %d PID: %d at %s:%d %pS\n",
467+ raw_smp_processor_id(), current->pid, file, line,
468+ caller);
469+ else
470+ pr_warn("WARNING: CPU: %d PID: %d at %pS\n",
471+ raw_smp_processor_id(), current->pid, caller);
466472
467473 if (args)
468474 vprintk(args->fmt, args->args);
@@ -479,20 +485,27 @@ static void warn_slowpath_common(const char *file, int line, void *caller, · 10 unmodified lines
479485 }
480486
481487 print_modules();
482− dump_stack();
488+
489+ if (regs)
490+ show_regs(regs);
491+ else
492+ dump_stack();
493+
483494 print_oops_end_marker();
495+
484496 /* Just a warning, don't kill lockdep. */
485497 add_taint(taint, LOCKDEP_STILL_OK);
486498 }
487499
500+#ifdef WANT_WARN_ON_SLOWPATH
488501 void warn_slowpath_fmt(const char *file, int line, const char *fmt, ...)
489502 {
490− struct slowpath_args args;
503+ struct warn_args args;
491504
492505 args.fmt = fmt;
493506 va_start(args.args, fmt);
494− warn_slowpath_common(file, line, __builtin_return_address(0),
495− TAINT_WARN, &args);
507+ __warn(file, line, __builtin_return_address(0), TAINT_WARN, NULL,
508+ &args);
496509 va_end(args.args);
497510 }
498511 EXPORT_SYMBOL(warn_slowpath_fmt);
@@ -500,20 +513,18 @@ EXPORT_SYMBOL(warn_slowpath_fmt); · 1 unmodified line
500513 void warn_slowpath_fmt_taint(const char *file, int line,
501514 unsigned taint, const char *fmt, ...)
502515 {
503− struct slowpath_args args;
516+ struct warn_args args;
504517
505518 args.fmt = fmt;
506519 va_start(args.args, fmt);
507− warn_slowpath_common(file, line, __builtin_return_address(0),
508− taint, &args);
520+ __warn(file, line, __builtin_return_address(0), taint, NULL, &args);
509521 va_end(args.args);
510522 }
511523 EXPORT_SYMBOL(warn_slowpath_fmt_taint);
512524
513525 void warn_slowpath_null(const char *file, int line)
514526 {
515− warn_slowpath_common(file, line, __builtin_return_address(0),
516− TAINT_WARN, NULL);
527+ __warn(file, line, __builtin_return_address(0), TAINT_WARN, NULL, NULL);
517528 }
518529 EXPORT_SYMBOL(warn_slowpath_null);
519530 #endif
kernel/printk/printk.c+66−74
@@ -367,16 +367,20 @@ static int logbuf_has_space(u32 msg_size, bool empty) · 366 unmodified lines
367367
368368 static int log_make_free_space(u32 msg_size)
369369 {
370− while (log_first_seq < log_next_seq) {
371− if (logbuf_has_space(msg_size, false))
372− return 0;
370+ while (log_first_seq < log_next_seq &&
371+ !logbuf_has_space(msg_size, false)) {
373372 /* drop old messages until we have enough contiguous space */
374373 log_first_idx = log_next(log_first_idx);
375374 log_first_seq++;
376375 }
377376
377+ if (clear_seq < log_first_seq) {
378+ clear_seq = log_first_seq;
379+ clear_idx = log_first_idx;
380+ }
381+
378382 /* sequence numbers are equal, so the log buffer is empty */
379− if (logbuf_has_space(msg_size, true))
383+ if (logbuf_has_space(msg_size, log_first_seq == log_next_seq))
380384 return 0;
381385
382386 return -ENOMEM;
@@ -854,6 +858,7 @@ void log_buf_kexec_setup(void) · 471 unmodified lines
854858 VMCOREINFO_SYMBOL(log_buf);
855859 VMCOREINFO_SYMBOL(log_buf_len);
856860 VMCOREINFO_SYMBOL(log_first_idx);
861+ VMCOREINFO_SYMBOL(clear_idx);
857862 VMCOREINFO_SYMBOL(log_next_idx);
858863 /*
859864 * Export struct printk_log size and field offsets. User space tools can
@@ -1216,12 +1221,6 @@ static int syslog_print_all(char __user *buf, int size, bool clear) · 356 unmodified lines
12161221 u32 idx;
12171222 enum log_flags prev;
12181223
1219− if (clear_seq < log_first_seq) {
1220− /* messages are gone, move to first available one */
1221− clear_seq = log_first_seq;
1222− clear_idx = log_first_idx;
1223− }
1224−
12251224 /*
12261225 * Find first record that fits, including all following records,
12271226 * into the user-provided buffer for this dump.
@@ -1483,58 +1482,6 @@ static void zap_locks(void) · 255 unmodified lines
14831482 sema_init(&console_sem, 1);
14841483 }
14851484
1486−/*
1487− * Check if we have any console that is capable of printing while cpu is
1488− * booting or shutting down. Requires console_sem.
1489− */
1490−static int have_callable_console(void)
1491−{
1492− struct console *con;
1493−
1494− for_each_console(con)
1495− if (con->flags & CON_ANYTIME)
1496− return 1;
1497−
1498− return 0;
1499−}
1500−
1501−/*
1502− * Can we actually use the console at this time on this cpu?
1503− *
1504− * Console drivers may assume that per-cpu resources have been allocated. So
1505− * unless they're explicitly marked as being able to cope (CON_ANYTIME) don't
1506− * call them until this CPU is officially up.
1507− */
1508−static inline int can_use_console(unsigned int cpu)
1509−{
1510− return cpu_online(cpu) || have_callable_console();
1511−}
1512−
1513−/*
1514− * Try to get console ownership to actually show the kernel
1515− * messages from a 'printk'. Return true (and with the
1516− * console_lock held, and 'console_locked' set) if it
1517− * is successful, false otherwise.
1518− */
1519−static int console_trylock_for_printk(void)
1520−{
1521− unsigned int cpu = smp_processor_id();
1522−
1523− if (!console_trylock())
1524− return 0;
1525− /*
1526− * If we can't use the console, we need to release the console
1527− * semaphore by hand to avoid flushing the buffer. We need to hold the
1528− * console semaphore in order to do this test safely.
1529− */
1530− if (!can_use_console(cpu)) {
1531− console_locked = 0;
1532− up_console_sem();
1533− return 0;
1534− }
1535− return 1;
1536−}
1537−
15381485 int printk_delay_msec __read_mostly;
15391486
15401487 static inline void printk_delay(void)
@@ -1681,7 +1628,6 @@ asmlinkage int vprintk_emit(int facility, int level, · 140 unmodified lines
16811628 boot_delay_msec(level);
16821629 printk_delay();
16831630
1684− /* This stops the holder of console_sem just where we want him */
16851631 local_irq_save(flags);
16861632 this_cpu = smp_processor_id();
16871633
@@ -1705,6 +1651,7 @@ asmlinkage int vprintk_emit(int facility, int level, · 17 unmodified lines
17051651 }
17061652
17071653 lockdep_off();
1654+ /* This stops the holder of console_sem just where we want him */
17081655 raw_spin_lock(&logbuf_lock);
17091656 logbuf_cpu = this_cpu;
17101657
@@ -1809,21 +1756,13 @@ asmlinkage int vprintk_emit(int facility, int level, · 98 unmodified lines
18091756 /* If called from the scheduler, we can not call up(). */
18101757 if (!in_sched) {
18111758 lockdep_off();
1812− /*
1813− * Disable preemption to avoid being preempted while holding
1814− * console_sem which would prevent anyone from printing to
1815− * console
1816− */
1817− preempt_disable();
1818−
18191759 /*
18201760 * Try to acquire and then immediately release the console
18211761 * semaphore. The release will print out buffers and wake up
18221762 * /dev/kmsg and syslog() users.
18231763 */
1824− if (console_trylock_for_printk())
1764+ if (console_trylock())
18251765 console_unlock();
1826− preempt_enable();
18271766 lockdep_on();
18281767 }
18291768
@@ -2174,7 +2113,20 @@ int console_trylock(void) · 344 unmodified lines
21742113 return 0;
21752114 }
21762115 console_locked = 1;
2177− console_may_schedule = 0;
2116+ /*
2117+ * When PREEMPT_COUNT disabled we can't reliably detect if it's
2118+ * safe to schedule (e.g. calling printk while holding a spin_lock),
2119+ * because preempt_disable()/preempt_enable() are just barriers there
2120+ * and preempt_count() is always 0.
2121+ *
2122+ * RCU read sections have a separate preemption counter when
2123+ * PREEMPT_RCU enabled thus we must take extra care and check
2124+ * rcu_preempt_depth(), otherwise RCU read sections modify
2125+ * preempt_count().
2126+ */
2127+ console_may_schedule = !oops_in_progress &&
2128+ preemptible() &&
2129+ !rcu_preempt_depth();
21782130 return 1;
21792131 }
21802132 EXPORT_SYMBOL(console_trylock);
@@ -2184,6 +2136,34 @@ int is_console_locked(void) · 3 unmodified lines
21842136 return console_locked;
21852137 }
21862138
2139+/*
2140+ * Check if we have any console that is capable of printing while cpu is
2141+ * booting or shutting down. Requires console_sem.
2142+ */
2143+static int have_callable_console(void)
2144+{
2145+ struct console *con;
2146+
2147+ for_each_console(con)
2148+ if ((con->flags & CON_ENABLED) &&
2149+ (con->flags & CON_ANYTIME))
2150+ return 1;
2151+
2152+ return 0;
2153+}
2154+
2155+/*
2156+ * Can we actually use the console at this time on this cpu?
2157+ *
2158+ * Console drivers may assume that per-cpu resources have been allocated. So
2159+ * unless they're explicitly marked as being able to cope (CON_ANYTIME) don't
2160+ * call them until this CPU is officially up.
2161+ */
2162+static inline int can_use_console(void)
2163+{
2164+ return cpu_online(raw_smp_processor_id()) || have_callable_console();
2165+}
2166+
21872167 static void console_cont_flush(char *text, size_t size)
21882168 {
21892169 unsigned long flags;
@@ -2254,9 +2234,21 @@ void console_unlock(void) · 64 unmodified lines
22542234 do_cond_resched = console_may_schedule;
22552235 console_may_schedule = 0;
22562236
2237+again:
2238+ /*
2239+ * We released the console_sem lock, so we need to recheck if
2240+ * cpu is online and (if not) is there at least one CON_ANYTIME
2241+ * console.
2242+ */
2243+ if (!can_use_console()) {
2244+ console_locked = 0;
2245+ up_console_sem();
2246+ return;
2247+ }
2248+
22572249 /* flush buffered message fragment immediately to console */
22582250 console_cont_flush(text, sizeof(text));
2259−again:
2251+
22602252 for (;;) {
22612253 struct printk_log *msg;
22622254 size_t ext_len = 0;
kernel/sys.c+4−1
@@ -2169,7 +2169,10 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, · 2168 unmodified lines
21692169 error = perf_event_task_enable();
21702170 break;
21712171 case PR_GET_TIMERSLACK:
2172− error = current->timer_slack_ns;
2172+ if (current->timer_slack_ns > ULONG_MAX)
2173+ error = ULONG_MAX;
2174+ else
2175+ error = current->timer_slack_ns;
21732176 break;
21742177 case PR_SET_TIMERSLACK:
21752178 if (arg2 <= 0)
kernel/sysctl.c+10−0
@@ -126,6 +126,7 @@ static int __maybe_unused two = 2; · 125 unmodified lines
126126 static int __maybe_unused four = 4;
127127 static unsigned long one_ul = 1;
128128 static int one_hundred = 100;
129+static int one_thousand = 1000;
129130 #ifdef CONFIG_PRINTK
130131 static int ten_thousand = 10000;
131132 #endif
@@ -1403,6 +1404,15 @@ static struct ctl_table vm_table[] = { · 1271 unmodified lines
14031404 .proc_handler = min_free_kbytes_sysctl_handler,
14041405 .extra1 = &zero,
14051406 },
1407+ {
1408+ .procname = "watermark_scale_factor",
1409+ .data = &watermark_scale_factor,
1410+ .maxlen = sizeof(watermark_scale_factor),
1411+ .mode = 0644,
1412+ .proc_handler = watermark_scale_factor_sysctl_handler,
1413+ .extra1 = &one,
1414+ .extra2 = &one_thousand,
1415+ },
14061416 {
14071417 .procname = "percpu_pagelist_fraction",
14081418 .data = &percpu_pagelist_fraction,
kernel/time/hrtimer.c+6−12
@@ -515,7 +515,7 @@ static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) · 514 unmodified lines
515515 /*
516516 * High resolution timer enabled ?
517517 */
518−static int hrtimer_hres_enabled __read_mostly = 1;
518+static bool hrtimer_hres_enabled __read_mostly = true;
519519 unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
520520 EXPORT_SYMBOL_GPL(hrtimer_resolution);
521521
@@ -524,13 +524,7 @@ EXPORT_SYMBOL_GPL(hrtimer_resolution); · 2 unmodified lines
524524 */
525525 static int __init setup_hrtimer_hres(char *str)
526526 {
527− if (!strcmp(str, "off"))
528− hrtimer_hres_enabled = 0;
529− else if (!strcmp(str, "on"))
530− hrtimer_hres_enabled = 1;
531− else
532− return 0;
533− return 1;
527+ return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
534528 }
535529
536530 __setup("highres=", setup_hrtimer_hres);
@@ -979,7 +973,7 @@ static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim, · 442 unmodified lines
979973 * relative (HRTIMER_MODE_REL)
980974 */
981975 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
982− unsigned long delta_ns, const enum hrtimer_mode mode)
976+ u64 delta_ns, const enum hrtimer_mode mode)
983977 {
984978 struct hrtimer_clock_base *base, *new_base;
985979 unsigned long flags;
@@ -1548,7 +1542,7 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, · 562 unmodified lines
15481542 struct restart_block *restart;
15491543 struct hrtimer_sleeper t;
15501544 int ret = 0;
1551− unsigned long slack;
1545+ u64 slack;
15521546
15531547 slack = current->timer_slack_ns;
15541548 if (dl_task(current) || rt_task(current))
@@ -1724,7 +1718,7 @@ void __init hrtimers_init(void) · 169 unmodified lines
17241718 * @clock: timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME
17251719 */
17261720 int __sched
1727−schedule_hrtimeout_range_clock(ktime_t *expires, unsigned long delta,
1721+schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
17281722 const enum hrtimer_mode mode, int clock)
17291723 {
17301724 struct hrtimer_sleeper t;
@@ -1792,7 +1786,7 @@ schedule_hrtimeout_range_clock(ktime_t *expires, unsigned long delta, · 61 unmodified lines
17921786 *
17931787 * Returns 0 when the timer has expired otherwise -EINTR
17941788 */
1795−int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
1789+int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
17961790 const enum hrtimer_mode mode)
17971791 {
17981792 return schedule_hrtimeout_range_clock(expires, delta, mode,
kernel/time/tick-sched.c+2−8
@@ -486,20 +486,14 @@ void __init tick_nohz_init(void) · 485 unmodified lines
486486 /*
487487 * NO HZ enabled ?
488488 */
489−int tick_nohz_enabled __read_mostly = 1;
489+bool tick_nohz_enabled __read_mostly = true;
490490 unsigned long tick_nohz_active __read_mostly;
491491 /*
492492 * Enable / Disable tickless mode
493493 */
494494 static int __init setup_tick_nohz(char *str)
495495 {
496− if (!strcmp(str, "off"))
497− tick_nohz_enabled = 0;
498− else if (!strcmp(str, "on"))
499− tick_nohz_enabled = 1;
500− else
501− return 0;
502− return 1;
496+ return (kstrtobool(str, &tick_nohz_enabled) == 0);
503497 }
504498
505499 __setup("nohz=", setup_tick_nohz);
kernel/time/timer.c+2−2
@@ -1698,10 +1698,10 @@ EXPORT_SYMBOL(msleep_interruptible); · 1697 unmodified lines
16981698 static void __sched do_usleep_range(unsigned long min, unsigned long max)
16991699 {
17001700 ktime_t kmin;
1701− unsigned long delta;
1701+ u64 delta;
17021702
17031703 kmin = ktime_set(0, min * NSEC_PER_USEC);
1704− delta = (max - min) * NSEC_PER_USEC;
1704+ delta = (u64)(max - min) * NSEC_PER_USEC;
17051705 schedule_hrtimeout_range(&kmin, delta, HRTIMER_MODE_REL);
17061706 }
17071707
kernel/watchdog.c+8−1
@@ -923,6 +923,9 @@ static int proc_watchdog_common(int which, struct ctl_table *table, int write, · 922 unmodified lines
923923 * both lockup detectors are disabled if proc_watchdog_update()
924924 * returns an error.
925925 */
926+ if (old == new)
927+ goto out;
928+
926929 err = proc_watchdog_update();
927930 }
928931 out:
@@ -967,7 +970,7 @@ int proc_soft_watchdog(struct ctl_table *table, int write, · 38 unmodified lines
967970 int proc_watchdog_thresh(struct ctl_table *table, int write,
968971 void __user *buffer, size_t *lenp, loff_t *ppos)
969972 {
970− int err, old;
973+ int err, old, new;
971974
972975 get_online_cpus();
973976 mutex_lock(&watchdog_proc_mutex);
@@ -987,6 +990,10 @@ int proc_watchdog_thresh(struct ctl_table *table, int write, · 13 unmodified lines
987990 /*
988991 * Update the sample period. Restore on failure.
989992 */
993+ new = ACCESS_ONCE(watchdog_thresh);
994+ if (old == new)
995+ goto out;
996+
990997 set_sample_period();
991998 err = proc_watchdog_update();
992999 if (err) {
lib/bug.c+2−13
@@ -167,19 +167,8 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) · 166 unmodified lines
167167
168168 if (warning) {
169169 /* this is a WARN_ON rather than BUG/BUG_ON */
170− pr_warn("------------[ cut here ]------------\n");
171−
172− if (file)
173− pr_warn("WARNING: at %s:%u\n", file, line);
174− else
175− pr_warn("WARNING: at %p [verbose debug info unavailable]\n",
176− (void *)bugaddr);
177−
178− print_modules();
179− show_regs(regs);
180− print_oops_end_marker();
181− /* Just a warning, don't kill lockdep. */
182− add_taint(BUG_GET_TAINT(bug), LOCKDEP_STILL_OK);
170+ __warn(file, line, (void *)bugaddr, BUG_GET_TAINT(bug), regs,
171+ NULL);
183172 return BUG_TRAP_TYPE_WARN;
184173 }
185174
lib/kstrtox.c+64−0
@@ -321,6 +321,70 @@ int kstrtos8(const char *s, unsigned int base, s8 *res) · 320 unmodified lines
321321 }
322322 EXPORT_SYMBOL(kstrtos8);
323323
324+/**
325+ * kstrtobool - convert common user inputs into boolean values
326+ * @s: input string
327+ * @res: result
328+ *
329+ * This routine returns 0 iff the first character is one of 'Yy1Nn0', or
330+ * [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value
331+ * pointed to by res is updated upon finding a match.
332+ */
333+int kstrtobool(const char *s, bool *res)
334+{
335+ if (!s)
336+ return -EINVAL;
337+
338+ switch (s[0]) {
339+ case 'y':
340+ case 'Y':
341+ case '1':
342+ *res = true;
343+ return 0;
344+ case 'n':
345+ case 'N':
346+ case '0':
347+ *res = false;
348+ return 0;
349+ case 'o':
350+ case 'O':
351+ switch (s[1]) {
352+ case 'n':
353+ case 'N':
354+ *res = true;
355+ return 0;
356+ case 'f':
357+ case 'F':
358+ *res = false;
359+ return 0;
360+ default:
361+ break;
362+ }
363+ default:
364+ break;
365+ }
366+
367+ return -EINVAL;
368+}
369+EXPORT_SYMBOL(kstrtobool);
370+
371+/*
372+ * Since "base" would be a nonsense argument, this open-codes the
373+ * _from_user helper instead of using the helper macro below.
374+ */
375+int kstrtobool_from_user(const char __user *s, size_t count, bool *res)
376+{
377+ /* Longest string needed to differentiate, newline, terminator */
378+ char buf[4];
379+
380+ count = min(count, sizeof(buf) - 1);
381+ if (copy_from_user(buf, s, count))
382+ return -EFAULT;
383+ buf[count] = '\0';
384+ return kstrtobool(buf, res);
385+}
386+EXPORT_SYMBOL(kstrtobool_from_user);
387+
324388 #define kstrto_from_user(f, g, type) \
325389 int f(const char __user *s, size_t count, unsigned int base, type *res) \
326390 { \
lib/radix-tree.c+148−34
@@ -173,6 +173,41 @@ radix_tree_find_next_bit(const unsigned long *addr, · 172 unmodified lines
173173 return size;
174174 }
175175
176+#if 0
177+static void dump_node(void *slot, int height, int offset)
178+{
179+ struct radix_tree_node *node;
180+ int i;
181+
182+ if (!slot)
183+ return;
184+
185+ if (height == 0) {
186+ pr_debug("radix entry %p offset %d\n", slot, offset);
187+ return;
188+ }
189+
190+ node = indirect_to_ptr(slot);
191+ pr_debug("radix node: %p offset %d tags %lx %lx %lx path %x count %d parent %p\n",
192+ slot, offset, node->tags[0][0], node->tags[1][0],
193+ node->tags[2][0], node->path, node->count, node->parent);
194+
195+ for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
196+ dump_node(node->slots[i], height - 1, i);
197+}
198+
199+/* For debug */
200+static void radix_tree_dump(struct radix_tree_root *root)
201+{
202+ pr_debug("radix root: %p height %d rnode %p tags %x\n",
203+ root, root->height, root->rnode,
204+ root->gfp_mask >> __GFP_BITS_SHIFT);
205+ if (!radix_tree_is_indirect_ptr(root->rnode))
206+ return;
207+ dump_node(root->rnode, root->height, 0);
208+}
209+#endif
210+
176211 /*
177212 * This assumes that the caller has performed appropriate preallocation, and
178213 * that the caller has pinned this thread of control to the current CPU.
@@ -191,6 +226,15 @@ radix_tree_node_alloc(struct radix_tree_root *root) · 12 unmodified lines
191226 if (!gfpflags_allow_blocking(gfp_mask) && !in_interrupt()) {
192227 struct radix_tree_preload *rtp;
193228
229+ /*
230+ * Even if the caller has preloaded, try to allocate from the
231+ * cache first for the new node to get accounted.
232+ */
233+ ret = kmem_cache_alloc(radix_tree_node_cachep,
234+ gfp_mask | __GFP_ACCOUNT | __GFP_NOWARN);
235+ if (ret)
236+ goto out;
237+
194238 /*
195239 * Provided the caller has preloaded here, we will always
196240 * succeed in getting a node here (and never reach
@@ -208,10 +252,11 @@ radix_tree_node_alloc(struct radix_tree_root *root) · 11 unmodified lines
208252 * for debugging.
209253 */
210254 kmemleak_update_trace(ret);
255+ goto out;
211256 }
212− if (ret == NULL)
213− ret = kmem_cache_alloc(radix_tree_node_cachep, gfp_mask);
214−
257+ ret = kmem_cache_alloc(radix_tree_node_cachep,
258+ gfp_mask | __GFP_ACCOUNT);
259+out:
215260 BUG_ON(radix_tree_is_indirect_ptr(ret));
216261 return ret;
217262 }
@@ -323,7 +368,8 @@ static inline unsigned long radix_tree_maxindex(unsigned int height) · 105 unmodified lines
323368 /*
324369 * Extend a radix tree so it can store key @index.
325370 */
326−static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
371+static int radix_tree_extend(struct radix_tree_root *root,
372+ unsigned long index, unsigned order)
327373 {
328374 struct radix_tree_node *node;
329375 struct radix_tree_node *slot;
@@ -335,7 +381,7 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index) · 5 unmodified lines
335381 while (index > radix_tree_maxindex(height))
336382 height++;
337383
338− if (root->rnode == NULL) {
384+ if ((root->rnode == NULL) && (order == 0)) {
339385 root->height = height;
340386 goto out;
341387 }
@@ -358,9 +404,10 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index) · 16 unmodified lines
358404 node->count = 1;
359405 node->parent = NULL;
360406 slot = root->rnode;
361− if (newheight > 1) {
407+ if (radix_tree_is_indirect_ptr(slot) && newheight > 1) {
362408 slot = indirect_to_ptr(slot);
363409 slot->parent = node;
410+ slot = ptr_to_indirect(slot);
364411 }
365412 node->slots[0] = slot;
366413 node = ptr_to_indirect(node);
@@ -375,6 +422,7 @@ out: · 8 unmodified lines
375422 * __radix_tree_create - create a slot in a radix tree
376423 * @root: radix tree root
377424 * @index: index key
425+ * @order: index occupies 2^order aligned slots
378426 * @nodep: returns node
379427 * @slotp: returns slot
380428 *
@@ -388,26 +436,29 @@ out: · 7 unmodified lines
388436 * Returns -ENOMEM, or 0 for success.
389437 */
390438 int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
391− struct radix_tree_node **nodep, void ***slotp)
439+ unsigned order, struct radix_tree_node **nodep,
440+ void ***slotp)
392441 {
393442 struct radix_tree_node *node = NULL, *slot;
394443 unsigned int height, shift, offset;
395444 int error;
396445
446+ BUG_ON((0 < order) && (order < RADIX_TREE_MAP_SHIFT));
447+
397448 /* Make sure the tree is high enough. */
398449 if (index > radix_tree_maxindex(root->height)) {
399− error = radix_tree_extend(root, index);
450+ error = radix_tree_extend(root, index, order);
400451 if (error)
401452 return error;
402453 }
403454
404− slot = indirect_to_ptr(root->rnode);
455+ slot = root->rnode;
405456
406457 height = root->height;
407− shift = (height-1) * RADIX_TREE_MAP_SHIFT;
458+ shift = height * RADIX_TREE_MAP_SHIFT;
408459
409460 offset = 0; /* uninitialised var warning */
410− while (height > 0) {
461+ while (shift > order) {
411462 if (slot == NULL) {
412463 /* Have to add a child node. */
413464 if (!(slot = radix_tree_node_alloc(root)))
@@ -415,19 +466,38 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index, · 1 unmodified line
415466 slot->path = height;
416467 slot->parent = node;
417468 if (node) {
418− rcu_assign_pointer(node->slots[offset], slot);
469+ rcu_assign_pointer(node->slots[offset],
470+ ptr_to_indirect(slot));
419471 node->count++;
420472 slot->path |= offset << RADIX_TREE_HEIGHT_SHIFT;
421473 } else
422− rcu_assign_pointer(root->rnode, ptr_to_indirect(slot));
423− }
474+ rcu_assign_pointer(root->rnode,
475+ ptr_to_indirect(slot));
476+ } else if (!radix_tree_is_indirect_ptr(slot))
477+ break;
424478
425479 /* Go a level down */
480+ height--;
481+ shift -= RADIX_TREE_MAP_SHIFT;
426482 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
427− node = slot;
483+ node = indirect_to_ptr(slot);
428484 slot = node->slots[offset];
429− shift -= RADIX_TREE_MAP_SHIFT;
430− height--;
485+ }
486+
487+ /* Insert pointers to the canonical entry */
488+ if ((shift - order) > 0) {
489+ int i, n = 1 << (shift - order);
490+ offset = offset & ~(n - 1);
491+ slot = ptr_to_indirect(&node->slots[offset]);
492+ for (i = 0; i < n; i++) {
493+ if (node->slots[offset + i])
494+ return -EEXIST;
495+ }
496+
497+ for (i = 1; i < n; i++) {
498+ rcu_assign_pointer(node->slots[offset + i], slot);
499+ node->count++;
500+ }
431501 }
432502
433503 if (nodep)
@@ -438,15 +508,16 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index, · 4 unmodified lines
438508 }
439509
440510 /**
441− * radix_tree_insert - insert into a radix tree
511+ * __radix_tree_insert - insert into a radix tree
442512 * @root: radix tree root
443513 * @index: index key
514+ * @order: key covers the 2^order indices around index
444515 * @item: item to insert
445516 *
446517 * Insert an item into the radix tree at position @index.
447518 */
448−int radix_tree_insert(struct radix_tree_root *root,
449− unsigned long index, void *item)
519+int __radix_tree_insert(struct radix_tree_root *root, unsigned long index,
520+ unsigned order, void *item)
450521 {
451522 struct radix_tree_node *node;
452523 void **slot;
@@ -454,7 +525,7 @@ int radix_tree_insert(struct radix_tree_root *root, · 1 unmodified line
454525
455526 BUG_ON(radix_tree_is_indirect_ptr(item));
456527
457− error = __radix_tree_create(root, index, &node, &slot);
528+ error = __radix_tree_create(root, index, order, &node, &slot);
458529 if (error)
459530 return error;
460531 if (*slot != NULL)
@@ -472,7 +543,7 @@ int radix_tree_insert(struct radix_tree_root *root, · 11 unmodified lines
472543
473544 return 0;
474545 }
475−EXPORT_SYMBOL(radix_tree_insert);
546+EXPORT_SYMBOL(__radix_tree_insert);
476547
477548 /**
478549 * __radix_tree_lookup - lookup an item in a radix tree
@@ -523,6 +594,9 @@ void *__radix_tree_lookup(struct radix_tree_root *root, unsigned long index, · 44 unmodified lines
523594 node = rcu_dereference_raw(*slot);
524595 if (node == NULL)
525596 return NULL;
597+ if (!radix_tree_is_indirect_ptr(node))
598+ break;
599+ node = indirect_to_ptr(node);
526600
527601 shift -= RADIX_TREE_MAP_SHIFT;
528602 height--;
@@ -609,6 +683,9 @@ void *radix_tree_tag_set(struct radix_tree_root *root, · 80 unmodified lines
609683 tag_set(slot, tag, offset);
610684 slot = slot->slots[offset];
611685 BUG_ON(slot == NULL);
686+ if (!radix_tree_is_indirect_ptr(slot))
687+ break;
688+ slot = indirect_to_ptr(slot);
612689 shift -= RADIX_TREE_MAP_SHIFT;
613690 height--;
614691 }
@@ -648,11 +725,14 @@ void *radix_tree_tag_clear(struct radix_tree_root *root, · 33 unmodified lines
648725 goto out;
649726
650727 shift = height * RADIX_TREE_MAP_SHIFT;
651− slot = indirect_to_ptr(root->rnode);
728+ slot = root->rnode;
652729
653730 while (shift) {
654731 if (slot == NULL)
655732 goto out;
733+ if (!radix_tree_is_indirect_ptr(slot))
734+ break;
735+ slot = indirect_to_ptr(slot);
656736
657737 shift -= RADIX_TREE_MAP_SHIFT;
658738 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
@@ -728,6 +808,7 @@ int radix_tree_tag_get(struct radix_tree_root *root, · 69 unmodified lines
728808
729809 if (node == NULL)
730810 return 0;
811+ node = indirect_to_ptr(node);
731812
732813 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
733814 if (!tag_get(node, tag, offset))
@@ -735,6 +816,8 @@ int radix_tree_tag_get(struct radix_tree_root *root, · 1 unmodified line
735816 if (height == 1)
736817 return 1;
737818 node = rcu_dereference_raw(node->slots[offset]);
819+ if (!radix_tree_is_indirect_ptr(node))
820+ return 1;
738821 shift -= RADIX_TREE_MAP_SHIFT;
739822 height--;
740823 }
@@ -795,6 +878,7 @@ restart: · 54 unmodified lines
795878
796879 node = rnode;
797880 while (1) {
881+ struct radix_tree_node *slot;
798882 if ((flags & RADIX_TREE_ITER_TAGGED) ?
799883 !test_bit(offset, node->tags[tag]) :
800884 !node->slots[offset]) {
@@ -825,9 +909,12 @@ restart: · 24 unmodified lines
825909 if (!shift)
826910 break;
827911
828− node = rcu_dereference_raw(node->slots[offset]);
829− if (node == NULL)
912+ slot = rcu_dereference_raw(node->slots[offset]);
913+ if (slot == NULL)
830914 goto restart;
915+ if (!radix_tree_is_indirect_ptr(slot))
916+ break;
917+ node = indirect_to_ptr(slot);
831918 shift -= RADIX_TREE_MAP_SHIFT;
832919 offset = (index >> shift) & RADIX_TREE_MAP_MASK;
833920 }
@@ -925,15 +1012,20 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root, · 91 unmodified lines
9251012 if (!tag_get(slot, iftag, offset))
9261013 goto next;
9271014 if (shift) {
928− /* Go down one level */
929− shift -= RADIX_TREE_MAP_SHIFT;
9301015 node = slot;
9311016 slot = slot->slots[offset];
932− continue;
1017+ if (radix_tree_is_indirect_ptr(slot)) {
1018+ slot = indirect_to_ptr(slot);
1019+ shift -= RADIX_TREE_MAP_SHIFT;
1020+ continue;
1021+ } else {
1022+ slot = node;
1023+ node = node->parent;
1024+ }
9331025 }
9341026
9351027 /* tag the leaf */
936− tagged++;
1028+ tagged += 1 << shift;
9371029 tag_set(slot, settag, offset);
9381030
9391031 /* walk back up the path tagging interior nodes */
@@ -1181,10 +1273,20 @@ static unsigned long __locate(struct radix_tree_node *slot, void *item, · 241 unmodified lines
11811273 goto out;
11821274 }
11831275
1184− shift -= RADIX_TREE_MAP_SHIFT;
11851276 slot = rcu_dereference_raw(slot->slots[i]);
11861277 if (slot == NULL)
11871278 goto out;
1279+ if (!radix_tree_is_indirect_ptr(slot)) {
1280+ if (slot == item) {
1281+ *found_index = index + i;
1282+ index = 0;
1283+ } else {
1284+ index += shift;
1285+ }
1286+ goto out;
1287+ }
1288+ slot = indirect_to_ptr(slot);
1289+ shift -= RADIX_TREE_MAP_SHIFT;
11881290 }
11891291
11901292 /* Bottom level: check items */
@@ -1264,11 +1366,13 @@ static inline void radix_tree_shrink(struct radix_tree_root *root) · 73 unmodified lines
12641366
12651367 /*
12661368 * The candidate node has more than one child, or its child
1267− * is not at the leftmost slot, we cannot shrink.
1369+ * is not at the leftmost slot, or it is a multiorder entry,
1370+ * we cannot shrink.
12681371 */
12691372 if (to_free->count != 1)
12701373 break;
1271− if (!to_free->slots[0])
1374+ slot = to_free->slots[0];
1375+ if (!slot)
12721376 break;
12731377
12741378 /*
@@ -1278,8 +1382,11 @@ static inline void radix_tree_shrink(struct radix_tree_root *root) · 3 unmodified lines
12781382 * (to_free->slots[0]), it will be safe to dereference the new
12791383 * one (root->rnode) as far as dependent read barriers go.
12801384 */
1281− slot = to_free->slots[0];
12821385 if (root->height > 1) {
1386+ if (!radix_tree_is_indirect_ptr(slot))
1387+ break;
1388+
1389+ slot = indirect_to_ptr(slot);
12831390 slot->parent = NULL;
12841391 slot = ptr_to_indirect(slot);
12851392 }
@@ -1377,7 +1484,7 @@ void *radix_tree_delete_item(struct radix_tree_root *root, · 91 unmodified lines
13771484 unsigned long index, void *item)
13781485 {
13791486 struct radix_tree_node *node;
1380− unsigned int offset;
1487+ unsigned int offset, i;
13811488 void **slot;
13821489 void *entry;
13831490 int tag;
@@ -1406,6 +1513,13 @@ void *radix_tree_delete_item(struct radix_tree_root *root, · 22 unmodified lines
14061513 radix_tree_tag_clear(root, index, tag);
14071514 }
14081515
1516+ /* Delete any sibling slots pointing to this slot */
1517+ for (i = 1; offset + i < RADIX_TREE_MAP_SIZE; i++) {
1518+ if (node->slots[offset + i] != ptr_to_indirect(slot))
1519+ break;
1520+ node->slots[offset + i] = NULL;
1521+ node->count--;
1522+ }
14091523 node->slots[offset] = NULL;
14101524 node->count--;
14111525
lib/string.c+21−24
@@ -631,33 +631,30 @@ bool sysfs_streq(const char *s1, const char *s2) · 630 unmodified lines
631631 EXPORT_SYMBOL(sysfs_streq);
632632
633633 /**
634− * strtobool - convert common user inputs into boolean values
635− * @s: input string
636− * @res: result
634+ * match_string - matches given string in an array
635+ * @array: array of strings
636+ * @n: number of strings in the array or -1 for NULL terminated arrays
637+ * @string: string to match with
637638 *
638− * This routine returns 0 iff the first character is one of 'Yy1Nn0'.
639− * Otherwise it will return -EINVAL. Value pointed to by res is
640− * updated upon finding a match.
641− */
642−int strtobool(const char *s, bool *res)
643−{
644− switch (s[0]) {
645− case 'y':
646− case 'Y':
647− case '1':
648− *res = true;
649− break;
650− case 'n':
651− case 'N':
652− case '0':
653− *res = false;
654− break;
655− default:
656− return -EINVAL;
639+ * Return:
640+ * index of a @string in the @array if matches, or %-EINVAL otherwise.
641+ */
642+int match_string(const char * const *array, size_t n, const char *string)
643+{
644+ int index;
645+ const char *item;
646+
647+ for (index = 0; index < n; index++) {
648+ item = array[index];
649+ if (!item)
650+ break;
651+ if (!strcmp(item, string))
652+ return index;
657653 }
658− return 0;
654+
655+ return -EINVAL;
659656 }
660−EXPORT_SYMBOL(strtobool);
657+EXPORT_SYMBOL(match_string);
661658
662659 #ifndef __HAVE_ARCH_MEMSET
663660 /**
lib/vsprintf.c+58−1
@@ -2640,8 +2640,12 @@ int vsscanf(const char *buf, const char *fmt, va_list args) · 2639 unmodified lines
26402640 if (*fmt == '*') {
26412641 if (!*str)
26422642 break;
2643− while (!isspace(*fmt) && *fmt != '%' && *fmt)
2643+ while (!isspace(*fmt) && *fmt != '%' && *fmt) {
2644+ /* '%*[' not yet supported, invalid format */
2645+ if (*fmt == '[')
2646+ return num;
26442647 fmt++;
2648+ }
26452649 while (!isspace(*str) && *str)
26462650 str++;
26472651 continue;
@@ -2714,6 +2718,59 @@ int vsscanf(const char *buf, const char *fmt, va_list args) · 66 unmodified lines
27142718 num++;
27152719 }
27162720 continue;
2721+ /*
2722+ * Warning: This implementation of the '[' conversion specifier
2723+ * deviates from its glibc counterpart in the following ways:
2724+ * (1) It does NOT support ranges i.e. '-' is NOT a special
2725+ * character
2726+ * (2) It cannot match the closing bracket ']' itself
2727+ * (3) A field width is required
2728+ * (4) '%*[' (discard matching input) is currently not supported
2729+ *
2730+ * Example usage:
2731+ * ret = sscanf("00:0a:95","%2[^:]:%2[^:]:%2[^:]",
2732+ * buf1, buf2, buf3);
2733+ * if (ret < 3)
2734+ * // etc..
2735+ */
2736+ case '[':
2737+ {
2738+ char *s = (char *)va_arg(args, char *);
2739+ DECLARE_BITMAP(set, 256) = {0};
2740+ unsigned int len = 0;
2741+ bool negate = (*fmt == '^');
2742+
2743+ /* field width is required */
2744+ if (field_width == -1)
2745+ return num;
2746+
2747+ if (negate)
2748+ ++fmt;
2749+
2750+ for ( ; *fmt && *fmt != ']'; ++fmt, ++len)
2751+ set_bit((u8)*fmt, set);
2752+
2753+ /* no ']' or no character set found */
2754+ if (!*fmt || !len)
2755+ return num;
2756+ ++fmt;
2757+
2758+ if (negate) {
2759+ bitmap_complement(set, set, 256);
2760+ /* exclude null '\0' byte */
2761+ clear_bit(0, set);
2762+ }
2763+
2764+ /* match must be non-empty */
2765+ if (!test_bit((u8)*str, set))
2766+ return num;
2767+
2768+ while (test_bit((u8)*str, set) && field_width--)
2769+ *s++ = *str++;
2770+ *s = '\0';
2771+ ++num;
2772+ }
2773+ continue;
27172774 case 'o':
27182775 base = 8;
27192776 break;
mm/Kconfig+1−3
@@ -187,7 +187,6 @@ config MEMORY_HOTPLUG · 186 unmodified lines
187187 bool "Allow for memory hot-add"
188188 depends on SPARSEMEM || X86_64_ACPI_NUMA
189189 depends on ARCH_ENABLE_MEMORY_HOTPLUG
190− depends on (IA64 || X86 || PPC_BOOK3S_64 || SUPERH || S390)
191190
192191 config MEMORY_HOTPLUG_SPARSE
193192 def_bool y
@@ -652,10 +651,9 @@ config IDLE_PAGE_TRACKING · 458 unmodified lines
652651
653652 config ZONE_DEVICE
654653 bool "Device memory (pmem, etc...) hotplug support" if EXPERT
655− default !ZONE_DMA
656− depends on !ZONE_DMA
657654 depends on MEMORY_HOTPLUG
658655 depends on MEMORY_HOTREMOVE
656+ depends on SPARSEMEM_VMEMMAP
659657 depends on X86_64 #arch_add_memory() comprehends device memory
660658
661659 help
mm/Kconfig.debug+13−0
@@ -79,3 +79,16 @@ config PAGE_POISONING_ZERO · 78 unmodified lines
7979 Enabling page poisoning with this option will disable hibernation
8080
8181 If unsure, say N
82+ bool
83+
84+config DEBUG_PAGE_REF
85+ bool "Enable tracepoint to track down page reference manipulation"
86+ depends on DEBUG_KERNEL
87+ depends on TRACEPOINTS
88+ ---help---
89+ This is a feature to add tracepoint for tracking down page reference
90+ manipulation. This tracking is useful to diagnose functional failure
91+ due to migration failures caused by page reference mismatches. Be
92+ careful when enabling this feature because it adds about 30 KB to the
93+ kernel code. However the runtime performance overhead is virtually
94+ nil until the tracepoints are actually enabled.
mm/Makefile+1−0
@@ -81,3 +81,4 @@ obj-$(CONFIG_CMA_DEBUGFS) += cma_debug.o · 80 unmodified lines
8181 obj-$(CONFIG_USERFAULTFD) += userfaultfd.o
8282 obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o
8383 obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o
84+obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o
mm/backing-dev.c+2−2
@@ -1026,8 +1026,8 @@ int pdflush_proc_obsolete(struct ctl_table *table, int write, · 1025 unmodified lines
10261026
10271027 if (copy_to_user(buffer, kbuf, sizeof(kbuf)))
10281028 return -EFAULT;
1029− printk_once(KERN_WARNING "%s exported in /proc is scheduled for removal\n",
1030− table->procname);
1029+ pr_warn_once("%s exported in /proc is scheduled for removal\n",
1030+ table->procname);
10311031
10321032 *lenp = 2;
10331033 *ppos += *lenp;
mm/bootmem.c+3−4
@@ -50,8 +50,7 @@ early_param("bootmem_debug", bootmem_debug_setup); · 49 unmodified lines
5050
5151 #define bdebug(fmt, args...) ({ \
5252 if (unlikely(bootmem_debug)) \
53− printk(KERN_INFO \
54− "bootmem::%s " fmt, \
53+ pr_info("bootmem::%s " fmt, \
5554 __func__, ## args); \
5655 })
5756
@@ -680,7 +679,7 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, · 622 unmodified lines
680679 /*
681680 * Whoops, we cannot satisfy the allocation request.
682681 */
683− printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
682+ pr_alert("bootmem alloc of %lu bytes failed!\n", size);
684683 panic("Out of memory");
685684 return NULL;
686685 }
@@ -755,7 +754,7 @@ void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, · 68 unmodified lines
755754 if (ptr)
756755 return ptr;
757756
758− printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
757+ pr_alert("bootmem alloc of %lu bytes failed!\n", size);
759758 panic("Out of memory");
760759 return NULL;
761760 }
mm/compaction.c+227−5
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 *
88 * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
99 */
10+#include <linux/cpu.h>
1011 #include <linux/swap.h>
1112 #include <linux/migrate.h>
1213 #include <linux/compaction.h>
@@ -17,6 +18,8 @@ · 4 unmodified lines
1718 #include <linux/balloon_compaction.h>
1819 #include <linux/page-isolation.h>
1920 #include <linux/kasan.h>
21+#include <linux/kthread.h>
22+#include <linux/freezer.h>
2023 #include "internal.h"
2124
2225 #ifdef CONFIG_COMPACTION
@@ -1188,11 +1191,11 @@ static int __compact_finished(struct zone *zone, struct compact_control *cc, · 1165 unmodified lines
11881191
11891192 /*
11901193 * Mark that the PG_migrate_skip information should be cleared
1191− * by kswapd when it goes to sleep. kswapd does not set the
1194+ * by kswapd when it goes to sleep. kcompactd does not set the
11921195 * flag itself as the decision to be clear should be directly
11931196 * based on an allocation request.
11941197 */
1195− if (!current_is_kswapd())
1198+ if (cc->direct_compaction)
11961199 zone->compact_blockskip_flush = true;
11971200
11981201 return COMPACT_COMPLETE;
@@ -1335,10 +1338,9 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) · 136 unmodified lines
13351338
13361339 /*
13371340 * Clear pageblock skip if there were failures recently and compaction
1338− * is about to be retried after being deferred. kswapd does not do
1339− * this reset as it'll reset the cached information when going to sleep.
1341+ * is about to be retried after being deferred.
13401342 */
1341− if (compaction_restarting(zone, cc->order) && !current_is_kswapd())
1343+ if (compaction_restarting(zone, cc->order))
13421344 __reset_isolation_suitable(zone);
13431345
13441346 /*
@@ -1474,6 +1476,7 @@ static unsigned long compact_zone_order(struct zone *zone, int order, · 129 unmodified lines
14741476 .mode = mode,
14751477 .alloc_flags = alloc_flags,
14761478 .classzone_idx = classzone_idx,
1479+ .direct_compaction = true,
14771480 };
14781481 INIT_LIST_HEAD(&cc.freepages);
14791482 INIT_LIST_HEAD(&cc.migratepages);
@@ -1736,4 +1739,223 @@ void compaction_unregister_node(struct node *node) · 256 unmodified lines
17361739 }
17371740 #endif /* CONFIG_SYSFS && CONFIG_NUMA */
17381741
1742+static inline bool kcompactd_work_requested(pg_data_t *pgdat)
1743+{
1744+ return pgdat->kcompactd_max_order > 0;
1745+}
1746+
1747+static bool kcompactd_node_suitable(pg_data_t *pgdat)
1748+{
1749+ int zoneid;
1750+ struct zone *zone;
1751+ enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx;
1752+
1753+ for (zoneid = 0; zoneid < classzone_idx; zoneid++) {
1754+ zone = &pgdat->node_zones[zoneid];
1755+
1756+ if (!populated_zone(zone))
1757+ continue;
1758+
1759+ if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
1760+ classzone_idx) == COMPACT_CONTINUE)
1761+ return true;
1762+ }
1763+
1764+ return false;
1765+}
1766+
1767+static void kcompactd_do_work(pg_data_t *pgdat)
1768+{
1769+ /*
1770+ * With no special task, compact all zones so that a page of requested
1771+ * order is allocatable.
1772+ */
1773+ int zoneid;
1774+ struct zone *zone;
1775+ struct compact_control cc = {
1776+ .order = pgdat->kcompactd_max_order,
1777+ .classzone_idx = pgdat->kcompactd_classzone_idx,
1778+ .mode = MIGRATE_SYNC_LIGHT,
1779+ .ignore_skip_hint = true,
1780+
1781+ };
1782+ bool success = false;
1783+
1784+ trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
1785+ cc.classzone_idx);
1786+ count_vm_event(KCOMPACTD_WAKE);
1787+
1788+ for (zoneid = 0; zoneid < cc.classzone_idx; zoneid++) {
1789+ int status;
1790+
1791+ zone = &pgdat->node_zones[zoneid];
1792+ if (!populated_zone(zone))
1793+ continue;
1794+
1795+ if (compaction_deferred(zone, cc.order))
1796+ continue;
1797+
1798+ if (compaction_suitable(zone, cc.order, 0, zoneid) !=
1799+ COMPACT_CONTINUE)
1800+ continue;
1801+
1802+ cc.nr_freepages = 0;
1803+ cc.nr_migratepages = 0;
1804+ cc.zone = zone;
1805+ INIT_LIST_HEAD(&cc.freepages);
1806+ INIT_LIST_HEAD(&cc.migratepages);
1807+
1808+ status = compact_zone(zone, &cc);
1809+
1810+ if (zone_watermark_ok(zone, cc.order, low_wmark_pages(zone),
1811+ cc.classzone_idx, 0)) {
1812+ success = true;
1813+ compaction_defer_reset(zone, cc.order, false);
1814+ } else if (status == COMPACT_COMPLETE) {
1815+ /*
1816+ * We use sync migration mode here, so we defer like
1817+ * sync direct compaction does.
1818+ */
1819+ defer_compaction(zone, cc.order);
1820+ }
1821+
1822+ VM_BUG_ON(!list_empty(&cc.freepages));
1823+ VM_BUG_ON(!list_empty(&cc.migratepages));
1824+ }
1825+
1826+ /*
1827+ * Regardless of success, we are done until woken up next. But remember
1828+ * the requested order/classzone_idx in case it was higher/tighter than
1829+ * our current ones
1830+ */
1831+ if (pgdat->kcompactd_max_order <= cc.order)
1832+ pgdat->kcompactd_max_order = 0;
1833+ if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
1834+ pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
1835+}
1836+
1837+void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
1838+{
1839+ if (!order)
1840+ return;
1841+
1842+ if (pgdat->kcompactd_max_order < order)
1843+ pgdat->kcompactd_max_order = order;
1844+
1845+ if (pgdat->kcompactd_classzone_idx > classzone_idx)
1846+ pgdat->kcompactd_classzone_idx = classzone_idx;
1847+
1848+ if (!waitqueue_active(&pgdat->kcompactd_wait))
1849+ return;
1850+
1851+ if (!kcompactd_node_suitable(pgdat))
1852+ return;
1853+
1854+ trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
1855+ classzone_idx);
1856+ wake_up_interruptible(&pgdat->kcompactd_wait);
1857+}
1858+
1859+/*
1860+ * The background compaction daemon, started as a kernel thread
1861+ * from the init process.
1862+ */
1863+static int kcompactd(void *p)
1864+{
1865+ pg_data_t *pgdat = (pg_data_t*)p;
1866+ struct task_struct *tsk = current;
1867+
1868+ const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1869+
1870+ if (!cpumask_empty(cpumask))
1871+ set_cpus_allowed_ptr(tsk, cpumask);
1872+
1873+ set_freezable();
1874+
1875+ pgdat->kcompactd_max_order = 0;
1876+ pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
1877+
1878+ while (!kthread_should_stop()) {
1879+ trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
1880+ wait_event_freezable(pgdat->kcompactd_wait,
1881+ kcompactd_work_requested(pgdat));
1882+
1883+ kcompactd_do_work(pgdat);
1884+ }
1885+
1886+ return 0;
1887+}
1888+
1889+/*
1890+ * This kcompactd start function will be called by init and node-hot-add.
1891+ * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
1892+ */
1893+int kcompactd_run(int nid)
1894+{
1895+ pg_data_t *pgdat = NODE_DATA(nid);
1896+ int ret = 0;
1897+
1898+ if (pgdat->kcompactd)
1899+ return 0;
1900+
1901+ pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
1902+ if (IS_ERR(pgdat->kcompactd)) {
1903+ pr_err("Failed to start kcompactd on node %d\n", nid);
1904+ ret = PTR_ERR(pgdat->kcompactd);
1905+ pgdat->kcompactd = NULL;
1906+ }
1907+ return ret;
1908+}
1909+
1910+/*
1911+ * Called by memory hotplug when all memory in a node is offlined. Caller must
1912+ * hold mem_hotplug_begin/end().
1913+ */
1914+void kcompactd_stop(int nid)
1915+{
1916+ struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
1917+
1918+ if (kcompactd) {
1919+ kthread_stop(kcompactd);
1920+ NODE_DATA(nid)->kcompactd = NULL;
1921+ }
1922+}
1923+
1924+/*
1925+ * It's optimal to keep kcompactd on the same CPUs as their memory, but
1926+ * not required for correctness. So if the last cpu in a node goes
1927+ * away, we get changed to run anywhere: as the first one comes back,
1928+ * restore their cpu bindings.
1929+ */
1930+static int cpu_callback(struct notifier_block *nfb, unsigned long action,
1931+ void *hcpu)
1932+{
1933+ int nid;
1934+
1935+ if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
1936+ for_each_node_state(nid, N_MEMORY) {
1937+ pg_data_t *pgdat = NODE_DATA(nid);
1938+ const struct cpumask *mask;
1939+
1940+ mask = cpumask_of_node(pgdat->node_id);
1941+
1942+ if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
1943+ /* One of our CPUs online: restore mask */
1944+ set_cpus_allowed_ptr(pgdat->kcompactd, mask);
1945+ }
1946+ }
1947+ return NOTIFY_OK;
1948+}
1949+
1950+static int __init kcompactd_init(void)
1951+{
1952+ int nid;
1953+
1954+ for_each_node_state(nid, N_MEMORY)
1955+ kcompactd_run(nid);
1956+ hotcpu_notifier(cpu_callback, 0);
1957+ return 0;
1958+}
1959+subsys_initcall(kcompactd_init)
1960+
17391961 #endif /* CONFIG_COMPACTION */
mm/debug.c+1−1
@@ -43,7 +43,7 @@ const struct trace_print_flags vmaflag_names[] = { · 42 unmodified lines
4343 void __dump_page(struct page *page, const char *reason)
4444 {
4545 pr_emerg("page:%p count:%d mapcount:%d mapping:%p index:%#lx",
46− page, atomic_read(&page->_count), page_mapcount(page),
46+ page, page_ref_count(page), page_mapcount(page),
4747 page->mapping, page->index);
4848 if (PageCompound(page))
4949 pr_cont(" compound_mapcount: %d", compound_mapcount(page));
mm/debug_page_ref.c+54−0
@@ -0,0 +1,54 @@
1+#include <linux/mm_types.h>
2+#include <linux/tracepoint.h>
3+
4+#define CREATE_TRACE_POINTS
5+#include <trace/events/page_ref.h>
6+
7+void __page_ref_set(struct page *page, int v)
8+{
9+ trace_page_ref_set(page, v);
10+}
11+EXPORT_SYMBOL(__page_ref_set);
12+EXPORT_TRACEPOINT_SYMBOL(page_ref_set);
13+
14+void __page_ref_mod(struct page *page, int v)
15+{
16+ trace_page_ref_mod(page, v);
17+}
18+EXPORT_SYMBOL(__page_ref_mod);
19+EXPORT_TRACEPOINT_SYMBOL(page_ref_mod);
20+
21+void __page_ref_mod_and_test(struct page *page, int v, int ret)
22+{
23+ trace_page_ref_mod_and_test(page, v, ret);
24+}
25+EXPORT_SYMBOL(__page_ref_mod_and_test);
26+EXPORT_TRACEPOINT_SYMBOL(page_ref_mod_and_test);
27+
28+void __page_ref_mod_and_return(struct page *page, int v, int ret)
29+{
30+ trace_page_ref_mod_and_return(page, v, ret);
31+}
32+EXPORT_SYMBOL(__page_ref_mod_and_return);
33+EXPORT_TRACEPOINT_SYMBOL(page_ref_mod_and_return);
34+
35+void __page_ref_mod_unless(struct page *page, int v, int u)
36+{
37+ trace_page_ref_mod_unless(page, v, u);
38+}
39+EXPORT_SYMBOL(__page_ref_mod_unless);
40+EXPORT_TRACEPOINT_SYMBOL(page_ref_mod_unless);
41+
42+void __page_ref_freeze(struct page *page, int v, int ret)
43+{
44+ trace_page_ref_freeze(page, v, ret);
45+}
46+EXPORT_SYMBOL(__page_ref_freeze);
47+EXPORT_TRACEPOINT_SYMBOL(page_ref_freeze);
48+
49+void __page_ref_unfreeze(struct page *page, int v)
50+{
51+ trace_page_ref_unfreeze(page, v);
52+}
53+EXPORT_SYMBOL(__page_ref_unfreeze);
54+EXPORT_TRACEPOINT_SYMBOL(page_ref_unfreeze);
mm/dmapool.c+7−11
@@ -294,8 +294,7 @@ void dma_pool_destroy(struct dma_pool *pool) · 293 unmodified lines
294294 "dma_pool_destroy %s, %p busy\n",
295295 pool->name, page->vaddr);
296296 else
297− printk(KERN_ERR
298− "dma_pool_destroy %s, %p busy\n",
297+ pr_err("dma_pool_destroy %s, %p busy\n",
299298 pool->name, page->vaddr);
300299 /* leak the still-in-use consistent memory */
301300 list_del(&page->page_list);
@@ -424,7 +423,7 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma) · 122 unmodified lines
424423 "dma_pool_free %s, %p/%lx (bad dma)\n",
425424 pool->name, vaddr, (unsigned long)dma);
426425 else
427− printk(KERN_ERR "dma_pool_free %s, %p/%lx (bad dma)\n",
426+ pr_err("dma_pool_free %s, %p/%lx (bad dma)\n",
428427 pool->name, vaddr, (unsigned long)dma);
429428 return;
430429 }
@@ -438,8 +437,7 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma) · 7 unmodified lines
438437 "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
439438 pool->name, vaddr, (unsigned long long)dma);
440439 else
441− printk(KERN_ERR
442− "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
440+ pr_err("dma_pool_free %s, %p (bad vaddr)/%Lx\n",
443441 pool->name, vaddr, (unsigned long long)dma);
444442 return;
445443 }
@@ -452,13 +450,11 @@ void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma) · 6 unmodified lines
452450 }
453451 spin_unlock_irqrestore(&pool->lock, flags);
454452 if (pool->dev)
455− dev_err(pool->dev, "dma_pool_free %s, dma %Lx "
456− "already free\n", pool->name,
457− (unsigned long long)dma);
453+ dev_err(pool->dev, "dma_pool_free %s, dma %Lx already free\n",
454+ pool->name, (unsigned long long)dma);
458455 else
459− printk(KERN_ERR "dma_pool_free %s, dma %Lx "
460− "already free\n", pool->name,
461− (unsigned long long)dma);
456+ pr_err("dma_pool_free %s, dma %Lx already free\n",
457+ pool->name, (unsigned long long)dma);
462458 return;
463459 }
464460 }
mm/filemap.c+18−37
@@ -586,7 +586,7 @@ static int page_cache_tree_insert(struct address_space *mapping, · 585 unmodified lines
586586 void **slot;
587587 int error;
588588
589− error = __radix_tree_create(&mapping->page_tree, page->index,
589+ error = __radix_tree_create(&mapping->page_tree, page->index, 0,
590590 &node, &slot);
591591 if (error)
592592 return error;
@@ -1255,7 +1255,6 @@ unsigned find_get_entries(struct address_space *mapping, · 662 unmodified lines
12551255 return 0;
12561256
12571257 rcu_read_lock();
1258−restart:
12591258 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
12601259 struct page *page;
12611260 repeat:
@@ -1263,8 +1262,10 @@ repeat: · 1 unmodified line
12631262 if (unlikely(!page))
12641263 continue;
12651264 if (radix_tree_exception(page)) {
1266− if (radix_tree_deref_retry(page))
1267− goto restart;
1265+ if (radix_tree_deref_retry(page)) {
1266+ slot = radix_tree_iter_retry(&iter);
1267+ continue;
1268+ }
12681269 /*
12691270 * A shadow entry of a recently evicted page, a swap
12701271 * entry from shmem/tmpfs or a DAX entry. Return it
@@ -1317,7 +1318,6 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start, · 46 unmodified lines
13171318 return 0;
13181319
13191320 rcu_read_lock();
1320−restart:
13211321 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
13221322 struct page *page;
13231323 repeat:
@@ -1327,13 +1327,8 @@ repeat: · 3 unmodified lines
13271327
13281328 if (radix_tree_exception(page)) {
13291329 if (radix_tree_deref_retry(page)) {
1330− /*
1331− * Transient condition which can only trigger
1332− * when entry at index 0 moves out of or back
1333− * to root: none yet gotten, safe to restart.
1334− */
1335− WARN_ON(iter.index);
1336− goto restart;
1330+ slot = radix_tree_iter_retry(&iter);
1331+ continue;
13371332 }
13381333 /*
13391334 * A shadow entry of a recently evicted page,
@@ -1384,7 +1379,6 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, · 44 unmodified lines
13841379 return 0;
13851380
13861381 rcu_read_lock();
1387−restart:
13881382 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
13891383 struct page *page;
13901384 repeat:
@@ -1395,12 +1389,8 @@ repeat: · 4 unmodified lines
13951389
13961390 if (radix_tree_exception(page)) {
13971391 if (radix_tree_deref_retry(page)) {
1398− /*
1399− * Transient condition which can only trigger
1400− * when entry at index 0 moves out of or back
1401− * to root: none yet gotten, safe to restart.
1402− */
1403− goto restart;
1392+ slot = radix_tree_iter_retry(&iter);
1393+ continue;
14041394 }
14051395 /*
14061396 * A shadow entry of a recently evicted page,
@@ -1460,7 +1450,6 @@ unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, · 53 unmodified lines
14601450 return 0;
14611451
14621452 rcu_read_lock();
1463−restart:
14641453 radix_tree_for_each_tagged(slot, &mapping->page_tree,
14651454 &iter, *index, tag) {
14661455 struct page *page;
@@ -1471,12 +1460,8 @@ repeat: · 4 unmodified lines
14711460
14721461 if (radix_tree_exception(page)) {
14731462 if (radix_tree_deref_retry(page)) {
1474− /*
1475− * Transient condition which can only trigger
1476− * when entry at index 0 moves out of or back
1477− * to root: none yet gotten, safe to restart.
1478− */
1479− goto restart;
1463+ slot = radix_tree_iter_retry(&iter);
1464+ continue;
14801465 }
14811466 /*
14821467 * A shadow entry of a recently evicted page.
@@ -1539,7 +1524,6 @@ unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start, · 56 unmodified lines
15391524 return 0;
15401525
15411526 rcu_read_lock();
1542−restart:
15431527 radix_tree_for_each_tagged(slot, &mapping->page_tree,
15441528 &iter, start, tag) {
15451529 struct page *page;
@@ -1549,12 +1533,8 @@ repeat: · 3 unmodified lines
15491533 continue;
15501534 if (radix_tree_exception(page)) {
15511535 if (radix_tree_deref_retry(page)) {
1552− /*
1553− * Transient condition which can only trigger
1554− * when entry at index 0 moves out of or back
1555− * to root: none yet gotten, safe to restart.
1556− */
1557− goto restart;
1536+ slot = radix_tree_iter_retry(&iter);
1537+ continue;
15581538 }
15591539
15601540 /*
@@ -2171,10 +2151,11 @@ repeat: · 610 unmodified lines
21712151 if (unlikely(!page))
21722152 goto next;
21732153 if (radix_tree_exception(page)) {
2174− if (radix_tree_deref_retry(page))
2175− break;
2176− else
2177− goto next;
2154+ if (radix_tree_deref_retry(page)) {
2155+ slot = radix_tree_iter_retry(&iter);
2156+ continue;
2157+ }
2158+ goto next;
21782159 }
21792160
21802161 if (!page_cache_get_speculative(page))
mm/huge_memory.c+109−228
@@ -78,7 +78,7 @@ unsigned long transparent_hugepage_flags __read_mostly = · 77 unmodified lines
7878 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
7979 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
8080 #endif
81− (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)|
81+ (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
8282 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
8383 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
8484
@@ -168,8 +168,7 @@ static void set_recommended_min_free_kbytes(void) · 83 unmodified lines
168168
169169 if (recommended_min > min_free_kbytes) {
170170 if (user_min_free_kbytes >= 0)
171− pr_info("raising min_free_kbytes from %d to %lu "
172− "to help transparent hugepage allocations\n",
171+ pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",
173172 min_free_kbytes, recommended_min);
174173
175174 min_free_kbytes = recommended_min;
@@ -270,37 +269,35 @@ static struct shrinker huge_zero_page_shrinker = { · 94 unmodified lines
270269
271270 #ifdef CONFIG_SYSFS
272271
273−static ssize_t double_flag_show(struct kobject *kobj,
274− struct kobj_attribute *attr, char *buf,
275− enum transparent_hugepage_flag enabled,
276− enum transparent_hugepage_flag req_madv)
277−{
278− if (test_bit(enabled, &transparent_hugepage_flags)) {
279− VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
280− return sprintf(buf, "[always] madvise never\n");
281− } else if (test_bit(req_madv, &transparent_hugepage_flags))
282− return sprintf(buf, "always [madvise] never\n");
283− else
284− return sprintf(buf, "always madvise [never]\n");
285−}
286−static ssize_t double_flag_store(struct kobject *kobj,
272+static ssize_t triple_flag_store(struct kobject *kobj,
287273 struct kobj_attribute *attr,
288274 const char *buf, size_t count,
289275 enum transparent_hugepage_flag enabled,
276+ enum transparent_hugepage_flag deferred,
290277 enum transparent_hugepage_flag req_madv)
291278 {
292− if (!memcmp("always", buf,
279+ if (!memcmp("defer", buf,
280+ min(sizeof("defer")-1, count))) {
281+ if (enabled == deferred)
282+ return -EINVAL;
283+ clear_bit(enabled, &transparent_hugepage_flags);
284+ clear_bit(req_madv, &transparent_hugepage_flags);
285+ set_bit(deferred, &transparent_hugepage_flags);
286+ } else if (!memcmp("always", buf,
293287 min(sizeof("always")-1, count))) {
294− set_bit(enabled, &transparent_hugepage_flags);
288+ clear_bit(deferred, &transparent_hugepage_flags);
295289 clear_bit(req_madv, &transparent_hugepage_flags);
290+ set_bit(enabled, &transparent_hugepage_flags);
296291 } else if (!memcmp("madvise", buf,
297292 min(sizeof("madvise")-1, count))) {
298293 clear_bit(enabled, &transparent_hugepage_flags);
294+ clear_bit(deferred, &transparent_hugepage_flags);
299295 set_bit(req_madv, &transparent_hugepage_flags);
300296 } else if (!memcmp("never", buf,
301297 min(sizeof("never")-1, count))) {
302298 clear_bit(enabled, &transparent_hugepage_flags);
303299 clear_bit(req_madv, &transparent_hugepage_flags);
300+ clear_bit(deferred, &transparent_hugepage_flags);
304301 } else
305302 return -EINVAL;
306303
@@ -310,17 +307,22 @@ static ssize_t double_flag_store(struct kobject *kobj, · 3 unmodified lines
310307 static ssize_t enabled_show(struct kobject *kobj,
311308 struct kobj_attribute *attr, char *buf)
312309 {
313− return double_flag_show(kobj, attr, buf,
314− TRANSPARENT_HUGEPAGE_FLAG,
315− TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
310+ if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
311+ return sprintf(buf, "[always] madvise never\n");
312+ else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
313+ return sprintf(buf, "always [madvise] never\n");
314+ else
315+ return sprintf(buf, "always madvise [never]\n");
316316 }
317+
317318 static ssize_t enabled_store(struct kobject *kobj,
318319 struct kobj_attribute *attr,
319320 const char *buf, size_t count)
320321 {
321322 ssize_t ret;
322323
323− ret = double_flag_store(kobj, attr, buf, count,
324+ ret = triple_flag_store(kobj, attr, buf, count,
325+ TRANSPARENT_HUGEPAGE_FLAG,
324326 TRANSPARENT_HUGEPAGE_FLAG,
325327 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
326328
@@ -378,16 +380,23 @@ static ssize_t single_flag_store(struct kobject *kobj, · 51 unmodified lines
378380 static ssize_t defrag_show(struct kobject *kobj,
379381 struct kobj_attribute *attr, char *buf)
380382 {
381− return double_flag_show(kobj, attr, buf,
382− TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
383− TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
383+ if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
384+ return sprintf(buf, "[always] defer madvise never\n");
385+ if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
386+ return sprintf(buf, "always [defer] madvise never\n");
387+ else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
388+ return sprintf(buf, "always defer [madvise] never\n");
389+ else
390+ return sprintf(buf, "always defer madvise [never]\n");
391+
384392 }
385393 static ssize_t defrag_store(struct kobject *kobj,
386394 struct kobj_attribute *attr,
387395 const char *buf, size_t count)
388396 {
389− return double_flag_store(kobj, attr, buf, count,
390− TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
397+ return triple_flag_store(kobj, attr, buf, count,
398+ TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
399+ TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
391400 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
392401 }
393402 static struct kobj_attribute defrag_attr =
@@ -843,9 +852,30 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, · 449 unmodified lines
843852 return 0;
844853 }
845854
846−static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp)
855+/*
856+ * If THP is set to always then directly reclaim/compact as necessary
857+ * If set to defer then do no reclaim and defer to khugepaged
858+ * If set to madvise and the VMA is flagged then directly reclaim/compact
859+ */
860+static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
861+{
862+ gfp_t reclaim_flags = 0;
863+
864+ if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags) &&
865+ (vma->vm_flags & VM_HUGEPAGE))
866+ reclaim_flags = __GFP_DIRECT_RECLAIM;
867+ else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
868+ reclaim_flags = __GFP_KSWAPD_RECLAIM;
869+ else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
870+ reclaim_flags = __GFP_DIRECT_RECLAIM;
871+
872+ return GFP_TRANSHUGE | reclaim_flags;
873+}
874+
875+/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
876+static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
847877 {
848− return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_RECLAIM)) | extra_gfp;
878+ return GFP_TRANSHUGE | (khugepaged_defrag() ? __GFP_DIRECT_RECLAIM : 0);
849879 }
850880
851881 /* Caller must hold page table lock. */
@@ -919,7 +949,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, · 67 unmodified lines
919949 }
920950 return ret;
921951 }
922− gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
952+ gfp = alloc_hugepage_direct_gfpmask(vma);
923953 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
924954 if (unlikely(!page)) {
925955 count_vm_event(THP_FAULT_FALLBACK);
@@ -1279,7 +1309,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, · 353 unmodified lines
12791309 alloc:
12801310 if (transparent_hugepage_enabled(vma) &&
12811311 !transparent_hugepage_debug_cow()) {
1282− huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
1312+ huge_gfp = alloc_hugepage_direct_gfpmask(vma);
12831313 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
12841314 } else
12851315 new_page = NULL;
@@ -2249,11 +2279,12 @@ static int khugepaged_find_target_node(void) · 963 unmodified lines
22492279 return 0;
22502280 }
22512281
2252−static inline struct page *alloc_hugepage(int defrag)
2282+static inline struct page *alloc_khugepaged_hugepage(void)
22532283 {
22542284 struct page *page;
22552285
2256− page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER);
2286+ page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
2287+ HPAGE_PMD_ORDER);
22572288 if (page)
22582289 prep_transhuge_page(page);
22592290 return page;
@@ -2264,7 +2295,7 @@ static struct page *khugepaged_alloc_hugepage(bool *wait) · 4 unmodified lines
22642295 struct page *hpage;
22652296
22662297 do {
2267− hpage = alloc_hugepage(khugepaged_defrag());
2298+ hpage = alloc_khugepaged_hugepage();
22682299 if (!hpage) {
22692300 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
22702301 if (!*wait)
@@ -2335,8 +2366,7 @@ static void collapse_huge_page(struct mm_struct *mm, · 64 unmodified lines
23352366 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
23362367
23372368 /* Only allocate from the target node */
2338− gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) |
2339− __GFP_THISNODE;
2369+ gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_OTHER_NODE | __GFP_THISNODE;
23402370
23412371 /* release the mmap_sem read lock. */
23422372 new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node);
@@ -2857,7 +2887,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 514 unmodified lines
28572887
28582888 page = pmd_page(*pmd);
28592889 VM_BUG_ON_PAGE(!page_count(page), page);
2860− atomic_add(HPAGE_PMD_NR - 1, &page->_count);
2890+ page_ref_add(page, HPAGE_PMD_NR - 1);
28612891 write = pmd_write(*pmd);
28622892 young = pmd_young(*pmd);
28632893 dirty = pmd_dirty(*pmd);
@@ -2947,44 +2977,33 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 83 unmodified lines
29472977 }
29482978
29492979 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2950− unsigned long address)
2980+ unsigned long address, bool freeze)
29512981 {
29522982 spinlock_t *ptl;
29532983 struct mm_struct *mm = vma->vm_mm;
2954− struct page *page = NULL;
29552984 unsigned long haddr = address & HPAGE_PMD_MASK;
29562985
29572986 mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
29582987 ptl = pmd_lock(mm, pmd);
29592988 if (pmd_trans_huge(*pmd)) {
2960− page = pmd_page(*pmd);
2989+ struct page *page = pmd_page(*pmd);
29612990 if (PageMlocked(page))
2962− get_page(page);
2963− else
2964− page = NULL;
2991+ clear_page_mlock(page);
29652992 } else if (!pmd_devmap(*pmd))
29662993 goto out;
2967− __split_huge_pmd_locked(vma, pmd, haddr, false);
2994+ __split_huge_pmd_locked(vma, pmd, haddr, freeze);
29682995 out:
29692996 spin_unlock(ptl);
29702997 mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
2971− if (page) {
2972− lock_page(page);
2973− munlock_vma_page(page);
2974− unlock_page(page);
2975− put_page(page);
2976− }
29772998 }
29782999
2979−static void split_huge_pmd_address(struct vm_area_struct *vma,
2980− unsigned long address)
3000+void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
3001+ bool freeze, struct page *page)
29813002 {
29823003 pgd_t *pgd;
29833004 pud_t *pud;
29843005 pmd_t *pmd;
29853006
2986− VM_BUG_ON(!(address & ~HPAGE_PMD_MASK));
2987−
29883007 pgd = pgd_offset(vma->vm_mm, address);
29893008 if (!pgd_present(*pgd))
29903009 return;
@@ -2996,11 +3015,20 @@ static void split_huge_pmd_address(struct vm_area_struct *vma, · 5 unmodified lines
29963015 pmd = pmd_offset(pud, address);
29973016 if (!pmd_present(*pmd) || (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)))
29983017 return;
3018+
3019+ /*
3020+ * If caller asks to setup a migration entries, we need a page to check
3021+ * pmd against. Otherwise we can end up replacing wrong page.
3022+ */
3023+ VM_BUG_ON(freeze && !page);
3024+ if (page && page != pmd_page(*pmd))
3025+ return;
3026+
29993027 /*
30003028 * Caller holds the mmap_sem write mode, so a huge pmd cannot
30013029 * materialize from under us.
30023030 */
3003− split_huge_pmd(vma, pmd, address);
3031+ __split_huge_pmd(vma, pmd, address, freeze);
30043032 }
30053033
30063034 void vma_adjust_trans_huge(struct vm_area_struct *vma,
@@ -3016,7 +3044,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 9 unmodified lines
30163044 if (start & ~HPAGE_PMD_MASK &&
30173045 (start & HPAGE_PMD_MASK) >= vma->vm_start &&
30183046 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
3019− split_huge_pmd_address(vma, start);
3047+ split_huge_pmd_address(vma, start, false, NULL);
30203048
30213049 /*
30223050 * If the new end address isn't hpage aligned and it could
@@ -3026,7 +3054,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 3 unmodified lines
30263054 if (end & ~HPAGE_PMD_MASK &&
30273055 (end & HPAGE_PMD_MASK) >= vma->vm_start &&
30283056 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
3029− split_huge_pmd_address(vma, end);
3057+ split_huge_pmd_address(vma, end, false, NULL);
30303058
30313059 /*
30323060 * If we're also updating the vma->vm_next->vm_start, if the new
@@ -3040,184 +3068,36 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, · 7 unmodified lines
30403068 if (nstart & ~HPAGE_PMD_MASK &&
30413069 (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
30423070 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
3043− split_huge_pmd_address(next, nstart);
3071+ split_huge_pmd_address(next, nstart, false, NULL);
30443072 }
30453073 }
30463074
3047−static void freeze_page_vma(struct vm_area_struct *vma, struct page *page,
3048− unsigned long address)
3075+static void freeze_page(struct page *page)
30493076 {
3050− unsigned long haddr = address & HPAGE_PMD_MASK;
3051− spinlock_t *ptl;
3052− pgd_t *pgd;
3053− pud_t *pud;
3054− pmd_t *pmd;
3055− pte_t *pte;
3056− int i, nr = HPAGE_PMD_NR;
3057−
3058− /* Skip pages which doesn't belong to the VMA */
3059− if (address < vma->vm_start) {
3060− int off = (vma->vm_start - address) >> PAGE_SHIFT;
3061− page += off;
3062− nr -= off;
3063− address = vma->vm_start;
3064− }
3065−
3066− pgd = pgd_offset(vma->vm_mm, address);
3067− if (!pgd_present(*pgd))
3068− return;
3069− pud = pud_offset(pgd, address);
3070− if (!pud_present(*pud))
3071− return;
3072− pmd = pmd_offset(pud, address);
3073− ptl = pmd_lock(vma->vm_mm, pmd);
3074− if (!pmd_present(*pmd)) {
3075− spin_unlock(ptl);
3076− return;
3077− }
3078− if (pmd_trans_huge(*pmd)) {
3079− if (page == pmd_page(*pmd))
3080− __split_huge_pmd_locked(vma, pmd, haddr, true);
3081− spin_unlock(ptl);
3082− return;
3083− }
3084− spin_unlock(ptl);
3085−
3086− pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
3087− for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
3088− pte_t entry, swp_pte;
3089− swp_entry_t swp_entry;
3090−
3091− /*
3092− * We've just crossed page table boundary: need to map next one.
3093− * It can happen if THP was mremaped to non PMD-aligned address.
3094− */
3095− if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3096− pte_unmap_unlock(pte - 1, ptl);
3097− pmd = mm_find_pmd(vma->vm_mm, address);
3098− if (!pmd)
3099− return;
3100− pte = pte_offset_map_lock(vma->vm_mm, pmd,
3101− address, &ptl);
3102− }
3103−
3104− if (!pte_present(*pte))
3105− continue;
3106− if (page_to_pfn(page) != pte_pfn(*pte))
3107− continue;
3108− flush_cache_page(vma, address, page_to_pfn(page));
3109− entry = ptep_clear_flush(vma, address, pte);
3110− if (pte_dirty(entry))
3111− SetPageDirty(page);
3112− swp_entry = make_migration_entry(page, pte_write(entry));
3113− swp_pte = swp_entry_to_pte(swp_entry);
3114− if (pte_soft_dirty(entry))
3115− swp_pte = pte_swp_mksoft_dirty(swp_pte);
3116− set_pte_at(vma->vm_mm, address, pte, swp_pte);
3117− page_remove_rmap(page, false);
3118− put_page(page);
3119− }
3120− pte_unmap_unlock(pte - 1, ptl);
3121−}
3122−
3123−static void freeze_page(struct anon_vma *anon_vma, struct page *page)
3124−{
3125− struct anon_vma_chain *avc;
3126− pgoff_t pgoff = page_to_pgoff(page);
3077+ enum ttu_flags ttu_flags = TTU_MIGRATION | TTU_IGNORE_MLOCK |
3078+ TTU_IGNORE_ACCESS | TTU_RMAP_LOCKED;
3079+ int i, ret;
31273080
31283081 VM_BUG_ON_PAGE(!PageHead(page), page);
31293082
3130− anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff,
3131− pgoff + HPAGE_PMD_NR - 1) {
3132− unsigned long address = __vma_address(page, avc->vma);
3133−
3134− mmu_notifier_invalidate_range_start(avc->vma->vm_mm,
3135− address, address + HPAGE_PMD_SIZE);
3136− freeze_page_vma(avc->vma, page, address);
3137− mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
3138− address, address + HPAGE_PMD_SIZE);
3139− }
3140−}
3141−
3142−static void unfreeze_page_vma(struct vm_area_struct *vma, struct page *page,
3143− unsigned long address)
3144−{
3145− spinlock_t *ptl;
3146− pmd_t *pmd;
3147− pte_t *pte, entry;
3148− swp_entry_t swp_entry;
3149− unsigned long haddr = address & HPAGE_PMD_MASK;
3150− int i, nr = HPAGE_PMD_NR;
3151−
3152− /* Skip pages which doesn't belong to the VMA */
3153− if (address < vma->vm_start) {
3154− int off = (vma->vm_start - address) >> PAGE_SHIFT;
3155− page += off;
3156− nr -= off;
3157− address = vma->vm_start;
3158− }
3159−
3160− pmd = mm_find_pmd(vma->vm_mm, address);
3161− if (!pmd)
3162− return;
3163−
3164− pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
3165− for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
3166− /*
3167− * We've just crossed page table boundary: need to map next one.
3168− * It can happen if THP was mremaped to non-PMD aligned address.
3169− */
3170− if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3171− pte_unmap_unlock(pte - 1, ptl);
3172− pmd = mm_find_pmd(vma->vm_mm, address);
3173− if (!pmd)
3174− return;
3175− pte = pte_offset_map_lock(vma->vm_mm, pmd,
3176− address, &ptl);
3177− }
3178−
3179− if (!is_swap_pte(*pte))
3180− continue;
3181−
3182− swp_entry = pte_to_swp_entry(*pte);
3183− if (!is_migration_entry(swp_entry))
3184− continue;
3185− if (migration_entry_to_page(swp_entry) != page)
3186− continue;
3187−
3188− get_page(page);
3189− page_add_anon_rmap(page, vma, address, false);
3190−
3191− entry = pte_mkold(mk_pte(page, vma->vm_page_prot));
3192− if (PageDirty(page))
3193− entry = pte_mkdirty(entry);
3194− if (is_write_migration_entry(swp_entry))
3195− entry = maybe_mkwrite(entry, vma);
3196−
3197− flush_dcache_page(page);
3198− set_pte_at(vma->vm_mm, address, pte, entry);
3083+ /* We only need TTU_SPLIT_HUGE_PMD once */
3084+ ret = try_to_unmap(page, ttu_flags | TTU_SPLIT_HUGE_PMD);
3085+ for (i = 1; !ret && i < HPAGE_PMD_NR; i++) {
3086+ /* Cut short if the page is unmapped */
3087+ if (page_count(page) == 1)
3088+ return;
31993089
3200− /* No need to invalidate - it was non-present before */
3201− update_mmu_cache(vma, address, pte);
3090+ ret = try_to_unmap(page + i, ttu_flags);
32023091 }
3203− pte_unmap_unlock(pte - 1, ptl);
3092+ VM_BUG_ON(ret);
32043093 }
32053094
3206−static void unfreeze_page(struct anon_vma *anon_vma, struct page *page)
3095+static void unfreeze_page(struct page *page)
32073096 {
3208− struct anon_vma_chain *avc;
3209− pgoff_t pgoff = page_to_pgoff(page);
3210−
3211− anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
3212− pgoff, pgoff + HPAGE_PMD_NR - 1) {
3213− unsigned long address = __vma_address(page, avc->vma);
3097+ int i;
32143098
3215− mmu_notifier_invalidate_range_start(avc->vma->vm_mm,
3216− address, address + HPAGE_PMD_SIZE);
3217− unfreeze_page_vma(avc->vma, page, address);
3218− mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
3219− address, address + HPAGE_PMD_SIZE);
3220− }
3099+ for (i = 0; i < HPAGE_PMD_NR; i++)
3100+ remove_migration_ptes(page + i, page + i, true);
32213101 }
32223102
32233103 static void __split_huge_page_tail(struct page *head, int tail,
@@ -3226,7 +3106,7 @@ static void __split_huge_page_tail(struct page *head, int tail, · 2 unmodified lines
32263106 struct page *page_tail = head + tail;
32273107
32283108 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
3229− VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail);
3109+ VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail);
32303110
32313111 /*
32323112 * tail_page->_count is zero and not changing from under us. But
@@ -3239,7 +3119,7 @@ static void __split_huge_page_tail(struct page *head, int tail, · 6 unmodified lines
32393119 * atomic_set() here would be safe on all archs (and not only on x86),
32403120 * it's safer to use atomic_inc().
32413121 */
3242− atomic_inc(&page_tail->_count);
3122+ page_ref_inc(page_tail);
32433123
32443124 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
32453125 page_tail->flags |= (head->flags &
@@ -3295,7 +3175,7 @@ static void __split_huge_page(struct page *page, struct list_head *list) · 49 unmodified lines
32953175 ClearPageCompound(head);
32963176 spin_unlock_irq(&zone->lru_lock);
32973177
3298− unfreeze_page(page_anon_vma(head), head);
3178+ unfreeze_page(head);
32993179
33003180 for (i = 0; i < HPAGE_PMD_NR; i++) {
33013181 struct page *subpage = head + i;
@@ -3391,7 +3271,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) · 89 unmodified lines
33913271 }
33923272
33933273 mlocked = PageMlocked(page);
3394− freeze_page(anon_vma, head);
3274+ freeze_page(head);
33953275 VM_BUG_ON_PAGE(compound_mapcount(head), head);
33963276
33973277 /* Make sure the page is not on per-CPU pagevec as it takes pin */
@@ -3420,7 +3300,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) · 22 unmodified lines
34203300 BUG();
34213301 } else {
34223302 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
3423− unfreeze_page(anon_vma, head);
3303+ unfreeze_page(head);
34243304 ret = -EBUSY;
34253305 }
34263306
@@ -3455,6 +3335,7 @@ void deferred_split_huge_page(struct page *page) · 28 unmodified lines
34553335
34563336 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
34573337 if (list_empty(page_deferred_list(page))) {
3338+ count_vm_event(THP_DEFERRED_SPLIT_PAGE);
34583339 list_add_tail(page_deferred_list(page), &pgdata->split_queue);
34593340 pgdata->split_queue_len++;
34603341 }
mm/hugetlb.c+2−3
@@ -2665,7 +2665,7 @@ void __init hugetlb_add_hstate(unsigned int order) · 2664 unmodified lines
26652665 unsigned long i;
26662666
26672667 if (size_to_hstate(PAGE_SIZE << order)) {
2668− pr_warning("hugepagesz= specified twice, ignoring\n");
2668+ pr_warn("hugepagesz= specified twice, ignoring\n");
26692669 return;
26702670 }
26712671 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
@@ -2701,8 +2701,7 @@ static int __init hugetlb_nrpages_setup(char *s) · 29 unmodified lines
27012701 mhp = &parsed_hstate->max_huge_pages;
27022702
27032703 if (mhp == last_mhp) {
2704− pr_warning("hugepages= specified twice without "
2705− "interleaving hugepagesz=, ignoring\n");
2704+ pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
27062705 return 1;
27072706 }
27082707
mm/internal.h+3−10
@@ -38,11 +38,6 @@ · 37 unmodified lines
3838 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
3939 unsigned long floor, unsigned long ceiling);
4040
41−static inline void set_page_count(struct page *page, int v)
42−{
43− atomic_set(&page->_count, v);
44−}
45−
4641 extern int __do_page_cache_readahead(struct address_space *mapping,
4742 struct file *filp, pgoff_t offset, unsigned long nr_to_read,
4843 unsigned long lookahead_size);
@@ -64,7 +59,7 @@ static inline unsigned long ra_submit(struct file_ra_state *ra, · 15 unmodified lines
6459 static inline void set_page_refcounted(struct page *page)
6560 {
6661 VM_BUG_ON_PAGE(PageTail(page), page);
67− VM_BUG_ON_PAGE(atomic_read(&page->_count), page);
62+ VM_BUG_ON_PAGE(page_ref_count(page), page);
6863 set_page_count(page, 1);
6964 }
7065
@@ -148,9 +143,6 @@ extern int __isolate_free_page(struct page *page, unsigned int order); · 77 unmodified lines
148143 extern void __free_pages_bootmem(struct page *page, unsigned long pfn,
149144 unsigned int order);
150145 extern void prep_compound_page(struct page *page, unsigned int order);
151−#ifdef CONFIG_MEMORY_FAILURE
152−extern bool is_free_buddy_page(struct page *page);
153−#endif
154146 extern int user_min_free_kbytes;
155147
156148 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
@@ -175,6 +167,7 @@ struct compact_control { · 18 unmodified lines
175167 unsigned long last_migrated_pfn;/* Not yet flushed page being freed */
176168 enum migrate_mode mode; /* Async or sync migration mode */
177169 bool ignore_skip_hint; /* Scan blocks even if marked skip */
170+ bool direct_compaction; /* False from kcompactd or /proc/... */
178171 int order; /* order a direct compactor needs */
179172 const gfp_t gfp_mask; /* gfp mask of a direct compactor */
180173 const int alloc_flags; /* alloc flags of a direct compactor */
@@ -393,7 +386,7 @@ extern int mminit_loglevel; · 212 unmodified lines
393386 do { \
394387 if (level < mminit_loglevel) { \
395388 if (level <= MMINIT_WARNING) \
396− printk(KERN_WARNING "mminit::" prefix " " fmt, ##arg); \
389+ pr_warn("mminit::" prefix " " fmt, ##arg); \
397390 else \
398391 printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
399392 } \
mm/kasan/report.c+2−4
@@ -214,8 +214,7 @@ static void kasan_report_error(struct kasan_access_info *info) · 213 unmodified lines
214214 */
215215 kasan_disable_current();
216216 spin_lock_irqsave(&report_lock, flags);
217− pr_err("================================="
218− "=================================\n");
217+ pr_err("==================================================================\n");
219218 if (info->access_addr <
220219 kasan_shadow_to_mem((void *)KASAN_SHADOW_START)) {
221220 if ((unsigned long)info->access_addr < PAGE_SIZE)
@@ -236,8 +235,7 @@ static void kasan_report_error(struct kasan_access_info *info) · 14 unmodified lines
236235 print_address_description(info);
237236 print_shadow_for_address(info->first_bad_addr);
238237 }
239− pr_err("================================="
240− "=================================\n");
238+ pr_err("==================================================================\n");
241239 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
242240 spin_unlock_irqrestore(&report_lock, flags);
243241 kasan_enable_current();
mm/kmemcheck.c+1−2
@@ -20,8 +20,7 @@ void kmemcheck_alloc_shadow(struct page *page, int order, gfp_t flags, int node) · 19 unmodified lines
2020 shadow = alloc_pages_node(node, flags | __GFP_NOTRACK, order);
2121 if (!shadow) {
2222 if (printk_ratelimit())
23− printk(KERN_ERR "kmemcheck: failed to allocate "
24− "shadow bitmap\n");
23+ pr_err("kmemcheck: failed to allocate shadow bitmap\n");
2524 return;
2625 }
2726
mm/kmemleak-test.c+1−1
@@ -49,7 +49,7 @@ static int __init kmemleak_test_init(void) · 48 unmodified lines
4949 struct test_node *elem;
5050 int i;
5151
52− printk(KERN_INFO "Kmemleak testing\n");
52+ pr_info("Kmemleak testing\n");
5353
5454 /* make some orphan objects */
5555 pr_info("kmalloc(32) = %p\n", kmalloc(32, GFP_KERNEL));
mm/kmemleak.c+15−17
@@ -276,7 +276,7 @@ static void kmemleak_disable(void); · 275 unmodified lines
276276 * Print a warning and dump the stack trace.
277277 */
278278 #define kmemleak_warn(x...) do { \
279− pr_warning(x); \
279+ pr_warn(x); \
280280 dump_stack(); \
281281 kmemleak_warning = 1; \
282282 } while (0)
@@ -543,7 +543,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size, · 260 unmodified lines
543543
544544 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
545545 if (!object) {
546− pr_warning("Cannot allocate a kmemleak_object structure\n");
546+ pr_warn("Cannot allocate a kmemleak_object structure\n");
547547 kmemleak_disable();
548548 return NULL;
549549 }
@@ -596,8 +596,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size, · 46 unmodified lines
596596 else if (parent->pointer + parent->size <= ptr)
597597 link = &parent->rb_node.rb_right;
598598 else {
599− kmemleak_stop("Cannot insert 0x%lx into the object "
600− "search tree (overlaps existing)\n",
599+ kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
601600 ptr);
602601 /*
603602 * No need for parent->lock here since "parent" cannot
@@ -670,8 +669,8 @@ static void delete_object_part(unsigned long ptr, size_t size) · 66 unmodified lines
670669 object = find_and_remove_object(ptr, 1);
671670 if (!object) {
672671 #ifdef DEBUG
673− kmemleak_warn("Partially freeing unknown object at 0x%08lx "
674− "(size %zu)\n", ptr, size);
672+ kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
673+ ptr, size);
675674 #endif
676675 return;
677676 }
@@ -717,8 +716,8 @@ static void paint_ptr(unsigned long ptr, int color) · 39 unmodified lines
717716
718717 object = find_and_get_object(ptr, 0);
719718 if (!object) {
720− kmemleak_warn("Trying to color unknown object "
721− "at 0x%08lx as %s\n", ptr,
719+ kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
720+ ptr,
722721 (color == KMEMLEAK_GREY) ? "Grey" :
723722 (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
724723 return;
@@ -764,7 +763,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) · 39 unmodified lines
764763
765764 area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
766765 if (!area) {
767− pr_warning("Cannot allocate a scan area\n");
766+ pr_warn("Cannot allocate a scan area\n");
768767 goto out;
769768 }
770769
@@ -1463,8 +1462,8 @@ static void kmemleak_scan(void) · 692 unmodified lines
14631462 if (new_leaks) {
14641463 kmemleak_found_leaks = true;
14651464
1466− pr_info("%d new suspected memory leaks (see "
1467− "/sys/kernel/debug/kmemleak)\n", new_leaks);
1465+ pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1466+ new_leaks);
14681467 }
14691468
14701469 }
@@ -1515,7 +1514,7 @@ static void start_scan_thread(void) · 44 unmodified lines
15151514 return;
15161515 scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
15171516 if (IS_ERR(scan_thread)) {
1518− pr_warning("Failed to create the scan thread\n");
1517+ pr_warn("Failed to create the scan thread\n");
15191518 scan_thread = NULL;
15201519 }
15211520 }
@@ -1795,8 +1794,7 @@ static void kmemleak_do_cleanup(struct work_struct *work) · 273 unmodified lines
17951794 if (!kmemleak_found_leaks)
17961795 __kmemleak_do_cleanup();
17971796 else
1798− pr_info("Kmemleak disabled without freeing internal data. "
1799− "Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\"\n");
1797+ pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
18001798 }
18011799
18021800 static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
@@ -1874,8 +1872,8 @@ void __init kmemleak_init(void) · 71 unmodified lines
18741872 scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
18751873
18761874 if (crt_early_log > ARRAY_SIZE(early_log))
1877− pr_warning("Early log buffer exceeded (%d), please increase "
1878− "DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n", crt_early_log);
1875+ pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
1876+ crt_early_log);
18791877
18801878 /* the kernel is still in UP mode, so disabling the IRQs is enough */
18811879 local_irq_save(flags);
@@ -1960,7 +1958,7 @@ static int __init kmemleak_late_init(void) · 78 unmodified lines
19601958 dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
19611959 &kmemleak_fops);
19621960 if (!dentry)
1963− pr_warning("Failed to create the debugfs kmemleak file\n");
1961+ pr_warn("Failed to create the debugfs kmemleak file\n");
19641962 mutex_lock(&scan_mutex);
19651963 start_scan_thread();
19661964 mutex_unlock(&scan_mutex);
mm/memblock.c+1−2
@@ -238,8 +238,7 @@ phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, · 237 unmodified lines
238238 * so we use WARN_ONCE() here to see the stack trace if
239239 * fail happens.
240240 */
241− WARN_ONCE(1, "memblock: bottom-up allocation failed, "
242− "memory hotunplug may be affected\n");
241+ WARN_ONCE(1, "memblock: bottom-up allocation failed, memory hotunplug may be affected\n");
243242 }
244243
245244 return __memblock_find_range_top_down(start, end, size, align, nid,
mm/memcontrol.c+108−83
@@ -638,9 +638,8 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, · 637 unmodified lines
638638 __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
639639 }
640640
641−static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
642− int nid,
643− unsigned int lru_mask)
641+unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
642+ int nid, unsigned int lru_mask)
644643 {
645644 unsigned long nr = 0;
646645 int zid;
@@ -1151,12 +1150,9 @@ static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) · 504 unmodified lines
11511150 */
11521151 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
11531152 {
1154− /* oom_info_lock ensures that parallel ooms do not interleave */
1155− static DEFINE_MUTEX(oom_info_lock);
11561153 struct mem_cgroup *iter;
11571154 unsigned int i;
11581155
1159− mutex_lock(&oom_info_lock);
11601156 rcu_read_lock();
11611157
11621158 if (p) {
@@ -1200,7 +1196,6 @@ void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) · 37 unmodified lines
12001196
12011197 pr_cont("\n");
12021198 }
1203− mutex_unlock(&oom_info_lock);
12041199 }
12051200
12061201 /*
@@ -1237,7 +1232,7 @@ static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg) · 30 unmodified lines
12371232 return limit;
12381233 }
12391234
1240−static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1235+static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
12411236 int order)
12421237 {
12431238 struct oom_control oc = {
@@ -1315,6 +1310,7 @@ static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, · 71 unmodified lines
13151310 }
13161311 unlock:
13171312 mutex_unlock(&oom_lock);
1313+ return chosen;
13181314 }
13191315
13201316 #if MAX_NUMNODES > 1
@@ -2325,9 +2321,6 @@ int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, · 1004 unmodified lines
23252321 struct page_counter *counter;
23262322 int ret;
23272323
2328− if (!memcg_kmem_online(memcg))
2329− return 0;
2330−
23312324 ret = try_charge(memcg, gfp, nr_pages);
23322325 if (ret)
23332326 return ret;
@@ -2346,10 +2339,11 @@ int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order, · 12 unmodified lines
23462339 int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
23472340 {
23482341 struct mem_cgroup *memcg;
2349− int ret;
2342+ int ret = 0;
23502343
23512344 memcg = get_mem_cgroup_from_mm(current->mm);
2352− ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);
2345+ if (!mem_cgroup_is_root(memcg))
2346+ ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);
23532347 css_put(&memcg->css);
23542348 return ret;
23552349 }
@@ -2719,39 +2713,48 @@ static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, · 363 unmodified lines
27192713 return retval;
27202714 }
27212715
2722−static unsigned long tree_stat(struct mem_cgroup *memcg,
2723− enum mem_cgroup_stat_index idx)
2716+static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
27242717 {
27252718 struct mem_cgroup *iter;
2726− unsigned long val = 0;
2719+ int i;
27272720
2728− for_each_mem_cgroup_tree(iter, memcg)
2729− val += mem_cgroup_read_stat(iter, idx);
2721+ memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
27302722
2731− return val;
2723+ for_each_mem_cgroup_tree(iter, memcg) {
2724+ for (i = 0; i < MEMCG_NR_STAT; i++)
2725+ stat[i] += mem_cgroup_read_stat(iter, i);
2726+ }
27322727 }
27332728
2734−static unsigned long tree_events(struct mem_cgroup *memcg,
2735− enum mem_cgroup_events_index idx)
2729+static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
27362730 {
27372731 struct mem_cgroup *iter;
2738− unsigned long val = 0;
2732+ int i;
27392733
2740− for_each_mem_cgroup_tree(iter, memcg)
2741− val += mem_cgroup_read_events(iter, idx);
2734+ memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
27422735
2743− return val;
2736+ for_each_mem_cgroup_tree(iter, memcg) {
2737+ for (i = 0; i < MEMCG_NR_EVENTS; i++)
2738+ events[i] += mem_cgroup_read_events(iter, i);
2739+ }
27442740 }
27452741
27462742 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
27472743 {
2748− unsigned long val;
2744+ unsigned long val = 0;
27492745
27502746 if (mem_cgroup_is_root(memcg)) {
2751− val = tree_stat(memcg, MEM_CGROUP_STAT_CACHE);
2752− val += tree_stat(memcg, MEM_CGROUP_STAT_RSS);
2753− if (swap)
2754− val += tree_stat(memcg, MEM_CGROUP_STAT_SWAP);
2747+ struct mem_cgroup *iter;
2748+
2749+ for_each_mem_cgroup_tree(iter, memcg) {
2750+ val += mem_cgroup_read_stat(iter,
2751+ MEM_CGROUP_STAT_CACHE);
2752+ val += mem_cgroup_read_stat(iter,
2753+ MEM_CGROUP_STAT_RSS);
2754+ if (swap)
2755+ val += mem_cgroup_read_stat(iter,
2756+ MEM_CGROUP_STAT_SWAP);
2757+ }
27552758 } else {
27562759 if (!swap)
27572760 val = page_counter_read(&memcg->memory);
@@ -2817,6 +2820,9 @@ static int memcg_online_kmem(struct mem_cgroup *memcg) · 59 unmodified lines
28172820 {
28182821 int memcg_id;
28192822
2823+ if (cgroup_memory_nokmem)
2824+ return 0;
2825+
28202826 BUG_ON(memcg->kmemcg_id >= 0);
28212827 BUG_ON(memcg->kmem_state);
28222828
@@ -2837,24 +2843,6 @@ static int memcg_online_kmem(struct mem_cgroup *memcg) · 14 unmodified lines
28372843 return 0;
28382844 }
28392845
2840−static int memcg_propagate_kmem(struct mem_cgroup *parent,
2841− struct mem_cgroup *memcg)
2842−{
2843− int ret = 0;
2844−
2845− mutex_lock(&memcg_limit_mutex);
2846− /*
2847− * If the parent cgroup is not kmem-online now, it cannot be
2848− * onlined after this point, because it has at least one child
2849− * already.
2850− */
2851− if (memcg_kmem_online(parent) ||
2852− (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nokmem))
2853− ret = memcg_online_kmem(memcg);
2854− mutex_unlock(&memcg_limit_mutex);
2855− return ret;
2856−}
2857−
28582846 static void memcg_offline_kmem(struct mem_cgroup *memcg)
28592847 {
28602848 struct cgroup_subsys_state *css;
@@ -2913,10 +2901,6 @@ static void memcg_free_kmem(struct mem_cgroup *memcg) · 52 unmodified lines
29132901 }
29142902 }
29152903 #else
2916−static int memcg_propagate_kmem(struct mem_cgroup *parent, struct mem_cgroup *memcg)
2917−{
2918− return 0;
2919−}
29202904 static int memcg_online_kmem(struct mem_cgroup *memcg)
29212905 {
29222906 return 0;
@@ -2932,22 +2916,10 @@ static void memcg_free_kmem(struct mem_cgroup *memcg) · 9 unmodified lines
29322916 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
29332917 unsigned long limit)
29342918 {
2935− int ret = 0;
2919+ int ret;
29362920
29372921 mutex_lock(&memcg_limit_mutex);
2938− /* Top-level cgroup doesn't propagate from root */
2939− if (!memcg_kmem_online(memcg)) {
2940− if (cgroup_is_populated(memcg->css.cgroup) ||
2941− (memcg->use_hierarchy && memcg_has_children(memcg)))
2942− ret = -EBUSY;
2943− if (ret)
2944− goto out;
2945− ret = memcg_online_kmem(memcg);
2946− if (ret)
2947− goto out;
2948− }
29492922 ret = page_counter_limit(&memcg->kmem, limit);
2950−out:
29512923 mutex_unlock(&memcg_limit_mutex);
29522924 return ret;
29532925 }
@@ -4198,7 +4170,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) · 1244 unmodified lines
41984170 return &memcg->css;
41994171 }
42004172
4201− error = memcg_propagate_kmem(parent, memcg);
4173+ error = memcg_online_kmem(memcg);
42024174 if (error)
42034175 goto fail;
42044176
@@ -4282,9 +4254,11 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) · 77 unmodified lines
42824254 {
42834255 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
42844256
4285− mem_cgroup_resize_limit(memcg, PAGE_COUNTER_MAX);
4286− mem_cgroup_resize_memsw_limit(memcg, PAGE_COUNTER_MAX);
4287− memcg_update_kmem_limit(memcg, PAGE_COUNTER_MAX);
4257+ page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4258+ page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4259+ page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4260+ page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4261+ page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
42884262 memcg->low = 0;
42894263 memcg->high = PAGE_COUNTER_MAX;
42904264 memcg->soft_limit = PAGE_COUNTER_MAX;
@@ -5015,6 +4989,7 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 724 unmodified lines
50154989 char *buf, size_t nbytes, loff_t off)
50164990 {
50174991 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
4992+ unsigned long nr_pages;
50184993 unsigned long high;
50194994 int err;
50204995
@@ -5025,6 +5000,11 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 4 unmodified lines
50255000
50265001 memcg->high = high;
50275002
5003+ nr_pages = page_counter_read(&memcg->memory);
5004+ if (nr_pages > high)
5005+ try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5006+ GFP_KERNEL, true);
5007+
50285008 memcg_wb_domain_size_changed(memcg);
50295009 return nbytes;
50305010 }
@@ -5046,6 +5026,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, · 15 unmodified lines
50465026 char *buf, size_t nbytes, loff_t off)
50475027 {
50485028 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5029+ unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5030+ bool drained = false;
50495031 unsigned long max;
50505032 int err;
50515033
@@ -5054,9 +5036,36 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, · 2 unmodified lines
50545036 if (err)
50555037 return err;
50565038
5057− err = mem_cgroup_resize_limit(memcg, max);
5058− if (err)
5059− return err;
5039+ xchg(&memcg->memory.limit, max);
5040+
5041+ for (;;) {
5042+ unsigned long nr_pages = page_counter_read(&memcg->memory);
5043+
5044+ if (nr_pages <= max)
5045+ break;
5046+
5047+ if (signal_pending(current)) {
5048+ err = -EINTR;
5049+ break;
5050+ }
5051+
5052+ if (!drained) {
5053+ drain_all_stock(memcg);
5054+ drained = true;
5055+ continue;
5056+ }
5057+
5058+ if (nr_reclaims) {
5059+ if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5060+ GFP_KERNEL, true))
5061+ nr_reclaims--;
5062+ continue;
5063+ }
5064+
5065+ mem_cgroup_events(memcg, MEMCG_OOM, 1);
5066+ if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5067+ break;
5068+ }
50605069
50615070 memcg_wb_domain_size_changed(memcg);
50625071 return nbytes;
@@ -5077,6 +5086,8 @@ static int memory_events_show(struct seq_file *m, void *v) · 14 unmodified lines
50775086 static int memory_stat_show(struct seq_file *m, void *v)
50785087 {
50795088 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5089+ unsigned long stat[MEMCG_NR_STAT];
5090+ unsigned long events[MEMCG_NR_EVENTS];
50805091 int i;
50815092
50825093 /*
@@ -5090,22 +5101,27 @@ static int memory_stat_show(struct seq_file *m, void *v) · 7 unmodified lines
50905101 * Current memory state:
50915102 */
50925103
5104+ tree_stat(memcg, stat);
5105+ tree_events(memcg, events);
5106+
50935107 seq_printf(m, "anon %llu\n",
5094− (u64)tree_stat(memcg, MEM_CGROUP_STAT_RSS) * PAGE_SIZE);
5108+ (u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE);
50955109 seq_printf(m, "file %llu\n",
5096− (u64)tree_stat(memcg, MEM_CGROUP_STAT_CACHE) * PAGE_SIZE);
5110+ (u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE);
5111+ seq_printf(m, "kernel_stack %llu\n",
5112+ (u64)stat[MEMCG_KERNEL_STACK] * PAGE_SIZE);
5113+ seq_printf(m, "slab %llu\n",
5114+ (u64)(stat[MEMCG_SLAB_RECLAIMABLE] +
5115+ stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
50975116 seq_printf(m, "sock %llu\n",
5098− (u64)tree_stat(memcg, MEMCG_SOCK) * PAGE_SIZE);
5117+ (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
50995118
51005119 seq_printf(m, "file_mapped %llu\n",
5101− (u64)tree_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED) *
5102− PAGE_SIZE);
5120+ (u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE);
51035121 seq_printf(m, "file_dirty %llu\n",
5104− (u64)tree_stat(memcg, MEM_CGROUP_STAT_DIRTY) *
5105− PAGE_SIZE);
5122+ (u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE);
51065123 seq_printf(m, "file_writeback %llu\n",
5107− (u64)tree_stat(memcg, MEM_CGROUP_STAT_WRITEBACK) *
5108− PAGE_SIZE);
5124+ (u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE);
51095125
51105126 for (i = 0; i < NR_LRU_LISTS; i++) {
51115127 struct mem_cgroup *mi;
@@ -5117,12 +5133,17 @@ static int memory_stat_show(struct seq_file *m, void *v) · 5 unmodified lines
51175133 mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
51185134 }
51195135
5136+ seq_printf(m, "slab_reclaimable %llu\n",
5137+ (u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE);
5138+ seq_printf(m, "slab_unreclaimable %llu\n",
5139+ (u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5140+
51205141 /* Accumulated memory events */
51215142
51225143 seq_printf(m, "pgfault %lu\n",
5123− tree_events(memcg, MEM_CGROUP_EVENTS_PGFAULT));
5144+ events[MEM_CGROUP_EVENTS_PGFAULT]);
51245145 seq_printf(m, "pgmajfault %lu\n",
5125− tree_events(memcg, MEM_CGROUP_EVENTS_PGMAJFAULT));
5146+ events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
51265147
51275148 return 0;
51285149 }
@@ -5395,6 +5416,10 @@ static void uncharge_list(struct list_head *page_list) · 266 unmodified lines
53955416 struct list_head *next;
53965417 struct page *page;
53975418
5419+ /*
5420+ * Note that the list can be a single page->lru; hence the
5421+ * do-while loop instead of a simple list_for_each_entry().
5422+ */
53985423 next = page_list->next;
53995424 do {
54005425 unsigned int nr_pages = 1;
mm/memory-failure.c+21−31
@@ -184,9 +184,8 @@ static int kill_proc(struct task_struct *t, unsigned long addr, int trapno, · 183 unmodified lines
184184 struct siginfo si;
185185 int ret;
186186
187− printk(KERN_ERR
188− "MCE %#lx: Killing %s:%d due to hardware memory corruption\n",
189− pfn, t->comm, t->pid);
187+ pr_err("MCE %#lx: Killing %s:%d due to hardware memory corruption\n",
188+ pfn, t->comm, t->pid);
190189 si.si_signo = SIGBUS;
191190 si.si_errno = 0;
192191 si.si_addr = (void *)addr;
@@ -209,8 +208,8 @@ static int kill_proc(struct task_struct *t, unsigned long addr, int trapno, · 16 unmodified lines
209208 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */
210209 }
211210 if (ret < 0)
212− printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
213− t->comm, t->pid, ret);
211+ pr_info("MCE: Error sending signal to %s:%d: %d\n",
212+ t->comm, t->pid, ret);
214213 return ret;
215214 }
216215
@@ -290,8 +289,7 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, · 73 unmodified lines
290289 } else {
291290 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
292291 if (!tk) {
293− printk(KERN_ERR
294− "MCE: Out of memory while machine check handling\n");
292+ pr_err("MCE: Out of memory while machine check handling\n");
295293 return;
296294 }
297295 }
@@ -336,9 +334,8 @@ static void kill_procs(struct list_head *to_kill, int forcekill, int trapno, · 38 unmodified lines
336334 * signal and then access the memory. Just kill it.
337335 */
338336 if (fail || tk->addr_valid == 0) {
339− printk(KERN_ERR
340− "MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
341− pfn, tk->tsk->comm, tk->tsk->pid);
337+ pr_err("MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
338+ pfn, tk->tsk->comm, tk->tsk->pid);
342339 force_sig(SIGKILL, tk->tsk);
343340 }
344341
@@ -350,9 +347,8 @@ static void kill_procs(struct list_head *to_kill, int forcekill, int trapno, · 5 unmodified lines
350347 */
351348 else if (kill_proc(tk->tsk, tk->addr, trapno,
352349 pfn, page, flags) < 0)
353− printk(KERN_ERR
354− "MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
355− pfn, tk->tsk->comm, tk->tsk->pid);
350+ pr_err("MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
351+ pfn, tk->tsk->comm, tk->tsk->pid);
356352 }
357353 put_task_struct(tk->tsk);
358354 kfree(tk);
@@ -563,7 +559,7 @@ static int me_kernel(struct page *p, unsigned long pfn) · 204 unmodified lines
563559 */
564560 static int me_unknown(struct page *p, unsigned long pfn)
565561 {
566− printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
562+ pr_err("MCE %#lx: Unknown page state\n", pfn);
567563 return MF_FAILED;
568564 }
569565
@@ -608,8 +604,8 @@ static int me_pagecache_clean(struct page *p, unsigned long pfn) · 38 unmodified lines
608604 if (mapping->a_ops->error_remove_page) {
609605 err = mapping->a_ops->error_remove_page(mapping, p);
610606 if (err != 0) {
611− printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
612− pfn, err);
607+ pr_info("MCE %#lx: Failed to punch page: %d\n",
608+ pfn, err);
613609 } else if (page_has_private(p) &&
614610 !try_to_release_page(p, GFP_NOIO)) {
615611 pr_info("MCE %#lx: failed to release buffers\n", pfn);
@@ -624,8 +620,7 @@ static int me_pagecache_clean(struct page *p, unsigned long pfn) · 8 unmodified lines
624620 if (invalidate_inode_page(p))
625621 ret = MF_RECOVERED;
626622 else
627− printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
628− pfn);
623+ pr_info("MCE %#lx: Failed to invalidate\n", pfn);
629624 }
630625 return ret;
631626 }
@@ -854,8 +849,7 @@ static int page_action(struct page_state *ps, struct page *p, · 222 unmodified lines
854849 if (ps->action == me_swapcache_dirty && result == MF_DELAYED)
855850 count--;
856851 if (count != 0) {
857− printk(KERN_ERR
858− "MCE %#lx: %s still referenced by %d users\n",
852+ pr_err("MCE %#lx: %s still referenced by %d users\n",
859853 pfn, action_page_types[ps->type], count);
860854 result = MF_FAILED;
861855 }
@@ -934,8 +928,7 @@ static int hwpoison_user_mappings(struct page *p, unsigned long pfn, · 72 unmodified lines
934928 }
935929
936930 if (PageSwapCache(p)) {
937− printk(KERN_ERR
938− "MCE %#lx: keeping poisoned page in swap cache\n", pfn);
931+ pr_err("MCE %#lx: keeping poisoned page in swap cache\n", pfn);
939932 ttu |= TTU_IGNORE_HWPOISON;
940933 }
941934
@@ -953,8 +946,7 @@ static int hwpoison_user_mappings(struct page *p, unsigned long pfn, · 11 unmodified lines
953946 } else {
954947 kill = 0;
955948 ttu |= TTU_IGNORE_HWPOISON;
956− printk(KERN_INFO
957− "MCE %#lx: corrupted page was clean: dropped without side effects\n",
949+ pr_info("MCE %#lx: corrupted page was clean: dropped without side effects\n",
958950 pfn);
959951 }
960952 }
@@ -972,8 +964,8 @@ static int hwpoison_user_mappings(struct page *p, unsigned long pfn, · 11 unmodified lines
972964
973965 ret = try_to_unmap(hpage, ttu);
974966 if (ret != SWAP_SUCCESS)
975− printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
976− pfn, page_mapcount(hpage));
967+ pr_err("MCE %#lx: failed to unmap page (mapcount=%d)\n",
968+ pfn, page_mapcount(hpage));
977969
978970 /*
979971 * Now that the dirty bit has been propagated to the
@@ -1040,16 +1032,14 @@ int memory_failure(unsigned long pfn, int trapno, int flags) · 60 unmodified lines
10401032 panic("Memory failure from trap %d on page %lx", trapno, pfn);
10411033
10421034 if (!pfn_valid(pfn)) {
1043− printk(KERN_ERR
1044− "MCE %#lx: memory outside kernel control\n",
1045− pfn);
1035+ pr_err("MCE %#lx: memory outside kernel control\n", pfn);
10461036 return -ENXIO;
10471037 }
10481038
10491039 p = pfn_to_page(pfn);
10501040 orig_head = hpage = compound_head(p);
10511041 if (TestSetPageHWPoison(p)) {
1052− printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
1042+ pr_err("MCE %#lx: already hardware poisoned\n", pfn);
10531043 return 0;
10541044 }
10551045
@@ -1180,7 +1170,7 @@ int memory_failure(unsigned long pfn, int trapno, int flags) · 124 unmodified lines
11801170 * unpoison always clear PG_hwpoison inside page lock
11811171 */
11821172 if (!PageHWPoison(p)) {
1183− printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
1173+ pr_err("MCE %#lx: just unpoisoned\n", pfn);
11841174 num_poisoned_pages_sub(nr_pages);
11851175 unlock_page(hpage);
11861176 put_hwpoison_page(hpage);
mm/memory.c+10−15
@@ -562,8 +562,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, · 561 unmodified lines
562562 }
563563 }
564564
565−int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
566− pmd_t *pmd, unsigned long address)
565+int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
567566 {
568567 spinlock_t *ptl;
569568 pgtable_t new = pte_alloc_one(mm, address);
@@ -661,9 +660,8 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, · 91 unmodified lines
661660 return;
662661 }
663662 if (nr_unshown) {
664− printk(KERN_ALERT
665− "BUG: Bad page map: %lu messages suppressed\n",
666− nr_unshown);
663+ pr_alert("BUG: Bad page map: %lu messages suppressed\n",
664+ nr_unshown);
667665 nr_unshown = 0;
668666 }
669667 nr_shown = 0;
@@ -674,15 +672,13 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, · 4 unmodified lines
674672 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
675673 index = linear_page_index(vma, addr);
676674
677− printk(KERN_ALERT
678− "BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
679− current->comm,
680− (long long)pte_val(pte), (long long)pmd_val(*pmd));
675+ pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
676+ current->comm,
677+ (long long)pte_val(pte), (long long)pmd_val(*pmd));
681678 if (page)
682679 dump_page(page, "bad pte");
683− printk(KERN_ALERT
684− "addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
685− (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
680+ pr_alert("addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
681+ (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
686682 /*
687683 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y
688684 */
@@ -3419,12 +3415,11 @@ static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, · 2730 unmodified lines
34193415 }
34203416
34213417 /*
3422− * Use __pte_alloc instead of pte_alloc_map, because we can't
3418+ * Use pte_alloc() instead of pte_alloc_map, because we can't
34233419 * run pte_offset_map on the pmd, if an huge pmd could
34243420 * materialize from under us from a different thread.
34253421 */
3426− if (unlikely(pmd_none(*pmd)) &&
3427− unlikely(__pte_alloc(mm, vma, pmd, address)))
3422+ if (unlikely(pte_alloc(mm, pmd, address)))
34283423 return VM_FAULT_OOM;
34293424 /*
34303425 * If a huge pmd materialized under us just retry later. Use
mm/memory_hotplug.c+29−25
@@ -33,6 +33,7 @@ · 32 unmodified lines
3333 #include <linux/hugetlb.h>
3434 #include <linux/memblock.h>
3535 #include <linux/bootmem.h>
36+#include <linux/compaction.h>
3637
3738 #include <asm/tlbflush.h>
3839
@@ -166,7 +167,7 @@ void get_page_bootmem(unsigned long info, struct page *page, · 127 unmodified lines
166167 page->lru.next = (struct list_head *) type;
167168 SetPagePrivate(page);
168169 set_page_private(page, info);
169− atomic_inc(&page->_count);
170+ page_ref_inc(page);
170171 }
171172
172173 void put_page_bootmem(struct page *page)
@@ -177,7 +178,7 @@ void put_page_bootmem(struct page *page) · 4 unmodified lines
177178 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
178179 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
179180
180− if (atomic_dec_return(&page->_count) == 1) {
181+ if (page_ref_dec_return(page) == 1) {
181182 ClearPagePrivate(page);
182183 set_page_private(page, 0);
183184 INIT_LIST_HEAD(&page->lru);
@@ -1054,14 +1055,13 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 870 unmodified lines
10541055 arg.nr_pages = nr_pages;
10551056 node_states_check_changes_online(nr_pages, zone, &arg);
10561057
1057− nid = pfn_to_nid(pfn);
1058+ nid = zone_to_nid(zone);
10581059
10591060 ret = memory_notify(MEM_GOING_ONLINE, &arg);
10601061 ret = notifier_to_errno(ret);
1061− if (ret) {
1062− memory_notify(MEM_CANCEL_ONLINE, &arg);
1063− return ret;
1064− }
1062+ if (ret)
1063+ goto failed_addition;
1064+
10651065 /*
10661066 * If this zone is not populated, then it is not in zonelist.
10671067 * This means the page allocator ignores this zone.
@@ -1079,12 +1079,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 11 unmodified lines
10791079 if (need_zonelists_rebuild)
10801080 zone_pcp_reset(zone);
10811081 mutex_unlock(&zonelists_mutex);
1082− printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n",
1083− (unsigned long long) pfn << PAGE_SHIFT,
1084− (((unsigned long long) pfn + nr_pages)
1085− << PAGE_SHIFT) - 1);
1086− memory_notify(MEM_CANCEL_ONLINE, &arg);
1087− return ret;
1082+ goto failed_addition;
10881083 }
10891084
10901085 zone->present_pages += onlined_pages;
@@ -1094,7 +1089,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 3 unmodified lines
10941089 pgdat_resize_unlock(zone->zone_pgdat, &flags);
10951090
10961091 if (onlined_pages) {
1097− node_states_set_node(zone_to_nid(zone), &arg);
1092+ node_states_set_node(nid, &arg);
10981093 if (need_zonelists_rebuild)
10991094 build_all_zonelists(NULL, NULL);
11001095 else
@@ -1105,8 +1100,10 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 4 unmodified lines
11051100
11061101 init_per_zone_wmark_min();
11071102
1108− if (onlined_pages)
1109− kswapd_run(zone_to_nid(zone));
1103+ if (onlined_pages) {
1104+ kswapd_run(nid);
1105+ kcompactd_run(nid);
1106+ }
11101107
11111108 vm_total_pages = nr_free_pagecache_pages();
11121109
@@ -1115,6 +1112,13 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ · 2 unmodified lines
11151112 if (onlined_pages)
11161113 memory_notify(MEM_ONLINE, &arg);
11171114 return 0;
1115+
1116+failed_addition:
1117+ pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
1118+ (unsigned long long) pfn << PAGE_SHIFT,
1119+ (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
1120+ memory_notify(MEM_CANCEL_ONLINE, &arg);
1121+ return ret;
11181122 }
11191123 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
11201124
@@ -1526,8 +1530,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · 405 unmodified lines
15261530
15271531 } else {
15281532 #ifdef CONFIG_DEBUG_VM
1529− printk(KERN_ALERT "removing pfn %lx from LRU failed\n",
1530− pfn);
1533+ pr_alert("removing pfn %lx from LRU failed\n", pfn);
15311534 dump_page(page, "failed to remove from LRU");
15321535 #endif
15331536 put_page(page);
@@ -1855,7 +1858,7 @@ repeat: · 321 unmodified lines
18551858 ret = -EBUSY;
18561859 goto failed_removal;
18571860 }
1858− printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages);
1861+ pr_info("Offlined Pages %ld\n", offlined_pages);
18591862 /* Ok, all of our target is isolated.
18601863 We cannot do rollback at this point. */
18611864 offline_isolated_pages(start_pfn, end_pfn);
@@ -1880,8 +1883,10 @@ repeat: · 18 unmodified lines
18801883 zone_pcp_update(zone);
18811884
18821885 node_states_clear_node(node, &arg);
1883− if (arg.status_change_nid >= 0)
1886+ if (arg.status_change_nid >= 0) {
18841887 kswapd_stop(node);
1888+ kcompactd_stop(node);
1889+ }
18851890
18861891 vm_total_pages = nr_free_pagecache_pages();
18871892 writeback_set_ratelimit();
@@ -1890,9 +1895,9 @@ repeat: · 2 unmodified lines
18901895 return 0;
18911896
18921897 failed_removal:
1893− printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n",
1894− (unsigned long long) start_pfn << PAGE_SHIFT,
1895− ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1898+ pr_debug("memory offlining [mem %#010llx-%#010llx] failed\n",
1899+ (unsigned long long) start_pfn << PAGE_SHIFT,
1900+ ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
18961901 memory_notify(MEM_CANCEL_OFFLINE, &arg);
18971902 /* pushback to free area */
18981903 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
@@ -1965,8 +1970,7 @@ static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) · 66 unmodified lines
19651970
19661971 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
19671972 endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
1968− pr_warn("removing memory fails, because memory "
1969− "[%pa-%pa] is onlined\n",
1973+ pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
19701974 &beginpa, &endpa);
19711975 }
19721976
mm/mempolicy.c+1−3
@@ -2559,9 +2559,7 @@ static void __init check_numabalancing_enable(void) · 2558 unmodified lines
25592559 set_numabalancing_state(numabalancing_override == 1);
25602560
25612561 if (num_online_nodes() > 1 && !numabalancing_override) {
2562− pr_info("%s automatic NUMA balancing. "
2563− "Configure with numa_balancing= or the "
2564− "kernel.numa_balancing sysctl",
2562+ pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
25652563 numabalancing_default ? "Enabling" : "Disabling");
25662564 set_numabalancing_state(numabalancing_default);
25672565 }
mm/mempool.c+16−4
@@ -310,25 +310,36 @@ EXPORT_SYMBOL(mempool_resize); · 309 unmodified lines
310310 * returns NULL. Note that due to preallocation, this function
311311 * *never* fails when called from process contexts. (it might
312312 * fail if called from an IRQ context.)
313− * Note: using __GFP_ZERO is not supported.
313+ * Note: neither __GFP_NOMEMALLOC nor __GFP_ZERO are supported.
314314 */
315−void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
315+void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
316316 {
317317 void *element;
318318 unsigned long flags;
319319 wait_queue_t wait;
320320 gfp_t gfp_temp;
321321
322+ /* If oom killed, memory reserves are essential to prevent livelock */
323+ VM_WARN_ON_ONCE(gfp_mask & __GFP_NOMEMALLOC);
324+ /* No element size to zero on allocation */
322325 VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
326+
323327 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
324328
325− gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
326329 gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */
327330 gfp_mask |= __GFP_NOWARN; /* failures are OK */
328331
329332 gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO);
330333
331334 repeat_alloc:
335+ if (likely(pool->curr_nr)) {
336+ /*
337+ * Don't allocate from emergency reserves if there are
338+ * elements available. This check is racy, but it will
339+ * be rechecked each loop.
340+ */
341+ gfp_temp |= __GFP_NOMEMALLOC;
342+ }
332343
333344 element = pool->alloc(gfp_temp, pool->pool_data);
334345 if (likely(element != NULL))
@@ -352,11 +363,12 @@ repeat_alloc: · 17 unmodified lines
352363 * We use gfp mask w/o direct reclaim or IO for the first round. If
353364 * alloc failed with that and @pool was empty, retry immediately.
354365 */
355− if (gfp_temp != gfp_mask) {
366+ if ((gfp_temp & ~__GFP_NOMEMALLOC) != gfp_mask) {
356367 spin_unlock_irqrestore(&pool->lock, flags);
357368 gfp_temp = gfp_mask;
358369 goto repeat_alloc;
359370 }
371+ gfp_temp = gfp_mask;
360372
361373 /* We must not sleep if !__GFP_DIRECT_RECLAIM */
362374 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
mm/migrate.c+19−14
@@ -172,7 +172,7 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, · 171 unmodified lines
172172 else
173173 page_add_file_rmap(new);
174174
175− if (vma->vm_flags & VM_LOCKED)
175+ if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
176176 mlock_vma_page(new);
177177
178178 /* No need to invalidate - it was non-present before */
@@ -187,14 +187,17 @@ out: · 8 unmodified lines
187187 * Get rid of all migration entries and replace them by
188188 * references to the indicated page.
189189 */
190−static void remove_migration_ptes(struct page *old, struct page *new)
190+void remove_migration_ptes(struct page *old, struct page *new, bool locked)
191191 {
192192 struct rmap_walk_control rwc = {
193193 .rmap_one = remove_migration_pte,
194194 .arg = old,
195195 };
196196
197− rmap_walk(new, &rwc);
197+ if (locked)
198+ rmap_walk_locked(new, &rwc);
199+ else
200+ rmap_walk(new, &rwc);
198201 }
199202
200203 /*
@@ -349,7 +352,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 148 unmodified lines
349352 return -EAGAIN;
350353 }
351354
352− if (!page_freeze_refs(page, expected_count)) {
355+ if (!page_ref_freeze(page, expected_count)) {
353356 spin_unlock_irq(&mapping->tree_lock);
354357 return -EAGAIN;
355358 }
@@ -363,7 +366,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 7 unmodified lines
363366 */
364367 if (mode == MIGRATE_ASYNC && head &&
365368 !buffer_migrate_lock_buffers(head, mode)) {
366− page_unfreeze_refs(page, expected_count);
369+ page_ref_unfreeze(page, expected_count);
367370 spin_unlock_irq(&mapping->tree_lock);
368371 return -EAGAIN;
369372 }
@@ -397,7 +400,7 @@ int migrate_page_move_mapping(struct address_space *mapping, · 27 unmodified lines
397400 * to one less reference.
398401 * We know this isn't the last reference.
399402 */
400− page_unfreeze_refs(page, expected_count - 1);
403+ page_ref_unfreeze(page, expected_count - 1);
401404
402405 spin_unlock(&mapping->tree_lock);
403406 /* Leave irq disabled to prevent preemption while updating stats */
@@ -451,7 +454,7 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, · 47 unmodified lines
451454 return -EAGAIN;
452455 }
453456
454− if (!page_freeze_refs(page, expected_count)) {
457+ if (!page_ref_freeze(page, expected_count)) {
455458 spin_unlock_irq(&mapping->tree_lock);
456459 return -EAGAIN;
457460 }
@@ -463,7 +466,7 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, · 5 unmodified lines
463466
464467 radix_tree_replace_slot(pslot, newpage);
465468
466− page_unfreeze_refs(page, expected_count - 1);
469+ page_ref_unfreeze(page, expected_count - 1);
467470
468471 spin_unlock_irq(&mapping->tree_lock);
469472
@@ -702,7 +705,7 @@ static int writeout(struct address_space *mapping, struct page *page) · 232 unmodified lines
702705 * At this point we know that the migration attempt cannot
703706 * be successful.
704707 */
705− remove_migration_ptes(page, page);
708+ remove_migration_ptes(page, page, false);
706709
707710 rc = mapping->a_ops->writepage(page, &wbc);
708711
@@ -900,7 +903,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, · 191 unmodified lines
900903
901904 if (page_was_mapped)
902905 remove_migration_ptes(page,
903− rc == MIGRATEPAGE_SUCCESS ? newpage : page);
906+ rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
904907
905908 out_unlock_both:
906909 unlock_page(newpage);
@@ -1070,7 +1073,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, · 163 unmodified lines
10701073
10711074 if (page_was_mapped)
10721075 remove_migration_ptes(hpage,
1073− rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage);
1076+ rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
10741077
10751078 unlock_page(new_hpage);
10761079
@@ -1773,7 +1776,10 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, · 696 unmodified lines
17731776 put_page(new_page);
17741777 goto out_fail;
17751778 }
1776−
1779+ /*
1780+ * We are not sure a pending tlb flush here is for a huge page
1781+ * mapping or not. Hence use the tlb range variant
1782+ */
17771783 if (mm_tlb_flush_pending(mm))
17781784 flush_tlb_range(vma, mmun_start, mmun_end);
17791785
@@ -1829,12 +1835,11 @@ fail_putback: · 49 unmodified lines
18291835 page_add_anon_rmap(new_page, vma, mmun_start, true);
18301836 pmdp_huge_clear_flush_notify(vma, mmun_start, pmd);
18311837 set_pmd_at(mm, mmun_start, pmd, entry);
1832− flush_tlb_range(vma, mmun_start, mmun_end);
18331838 update_mmu_cache_pmd(vma, address, &entry);
18341839
18351840 if (page_count(page) != 2) {
18361841 set_pmd_at(mm, mmun_start, pmd, orig_entry);
1837− flush_tlb_range(vma, mmun_start, mmun_end);
1842+ flush_pmd_tlb_range(vma, mmun_start, mmun_end);
18381843 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
18391844 update_mmu_cache_pmd(vma, address, &entry);
18401845 page_remove_rmap(new_page, true);
mm/mm_init.c+3−4
@@ -55,13 +55,12 @@ void __init mminit_verify_zonelist(void) · 54 unmodified lines
5555 /* Iterate the zonelist */
5656 for_each_zone_zonelist(zone, z, zonelist, zoneid) {
5757 #ifdef CONFIG_NUMA
58− printk(KERN_CONT "%d:%s ",
59− zone->node, zone->name);
58+ pr_cont("%d:%s ", zone->node, zone->name);
6059 #else
61− printk(KERN_CONT "0:%s ", zone->name);
60+ pr_cont("0:%s ", zone->name);
6261 #endif /* CONFIG_NUMA */
6362 }
64− printk(KERN_CONT "\n");
63+ pr_cont("\n");
6564 }
6665 }
6766 }
mm/mmap.c+3−130
@@ -37,7 +37,6 @@ · 36 unmodified lines
3737 #include <linux/khugepaged.h>
3838 #include <linux/uprobes.h>
3939 #include <linux/rbtree_augmented.h>
40−#include <linux/sched/sysctl.h>
4140 #include <linux/notifier.h>
4241 #include <linux/memory.h>
4342 #include <linux/printk.h>
@@ -123,130 +122,6 @@ void vma_set_page_prot(struct vm_area_struct *vma) · 79 unmodified lines
123122 }
124123 }
125124
126−
127−int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS; /* heuristic overcommit */
128−int sysctl_overcommit_ratio __read_mostly = 50; /* default is 50% */
129−unsigned long sysctl_overcommit_kbytes __read_mostly;
130−int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
131−unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
132−unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
133−/*
134− * Make sure vm_committed_as in one cacheline and not cacheline shared with
135− * other variables. It can be updated by several CPUs frequently.
136− */
137−struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
138−
139−/*
140− * The global memory commitment made in the system can be a metric
141− * that can be used to drive ballooning decisions when Linux is hosted
142− * as a guest. On Hyper-V, the host implements a policy engine for dynamically
143− * balancing memory across competing virtual machines that are hosted.
144− * Several metrics drive this policy engine including the guest reported
145− * memory commitment.
146− */
147−unsigned long vm_memory_committed(void)
148−{
149− return percpu_counter_read_positive(&vm_committed_as);
150−}
151−EXPORT_SYMBOL_GPL(vm_memory_committed);
152−
153−/*
154− * Check that a process has enough memory to allocate a new virtual
155− * mapping. 0 means there is enough memory for the allocation to
156− * succeed and -ENOMEM implies there is not.
157− *
158− * We currently support three overcommit policies, which are set via the
159− * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
160− *
161− * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
162− * Additional code 2002 Jul 20 by Robert Love.
163− *
164− * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
165− *
166− * Note this is a helper function intended to be used by LSMs which
167− * wish to use this logic.
168− */
169−int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
170−{
171− long free, allowed, reserve;
172−
173− VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) <
174− -(s64)vm_committed_as_batch * num_online_cpus(),
175− "memory commitment underflow");
176−
177− vm_acct_memory(pages);
178−
179− /*
180− * Sometimes we want to use more memory than we have
181− */
182− if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
183− return 0;
184−
185− if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
186− free = global_page_state(NR_FREE_PAGES);
187− free += global_page_state(NR_FILE_PAGES);
188−
189− /*
190− * shmem pages shouldn't be counted as free in this
191− * case, they can't be purged, only swapped out, and
192− * that won't affect the overall amount of available
193− * memory in the system.
194− */
195− free -= global_page_state(NR_SHMEM);
196−
197− free += get_nr_swap_pages();
198−
199− /*
200− * Any slabs which are created with the
201− * SLAB_RECLAIM_ACCOUNT flag claim to have contents
202− * which are reclaimable, under pressure. The dentry
203− * cache and most inode caches should fall into this
204− */
205− free += global_page_state(NR_SLAB_RECLAIMABLE);
206−
207− /*
208− * Leave reserved pages. The pages are not for anonymous pages.
209− */
210− if (free <= totalreserve_pages)
211− goto error;
212− else
213− free -= totalreserve_pages;
214−
215− /*
216− * Reserve some for root
217− */
218− if (!cap_sys_admin)
219− free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
220−
221− if (free > pages)
222− return 0;
223−
224− goto error;
225− }
226−
227− allowed = vm_commit_limit();
228− /*
229− * Reserve some for root
230− */
231− if (!cap_sys_admin)
232− allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
233−
234− /*
235− * Don't let a single process grow so big a user can't recover
236− */
237− if (mm) {
238− reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
239− allowed -= min_t(long, mm->total_vm / 32, reserve);
240− }
241−
242− if (percpu_counter_read_positive(&vm_committed_as) < allowed)
243− return 0;
244−error:
245− vm_unacct_memory(pages);
246−
247− return -ENOMEM;
248−}
249−
250125 /*
251126 * Requires inode->i_mapping->i_mmap_rwsem
252127 */
@@ -2642,9 +2517,8 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, · 2389 unmodified lines
26422517 unsigned long ret = -EINVAL;
26432518 struct file *file;
26442519
2645− pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. "
2646− "See Documentation/vm/remap_file_pages.txt.\n",
2647− current->comm, current->pid);
2520+ pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n",
2521+ current->comm, current->pid);
26482522
26492523 if (prot)
26502524 return ret;
@@ -3010,8 +2884,7 @@ bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages) · 359 unmodified lines
30102884 if (is_data_mapping(flags) &&
30112885 mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
30122886 if (ignore_rlimit_data)
3013− pr_warn_once("%s (%d): VmData %lu exceed data ulimit "
3014− "%lu. Will be forbidden soon.\n",
2887+ pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Will be forbidden soon.\n",
30152888 current->comm, current->pid,
30162889 (mm->data_vm + npages) << PAGE_SHIFT,
30172890 rlimit(RLIMIT_DATA));
mm/mmu_notifier.c+1−1
@@ -3,7 +3,7 @@ · 2 unmodified lines
33 *
44 * Copyright (C) 2008 Qumranet, Inc.
55 * Copyright (C) 2008 SGI
6− * Christoph Lameter <clameter@sgi.com>
6+ * Christoph Lameter <cl@linux.com>
77 *
88 * This work is licensed under the terms of the GNU GPL, version 2. See
99 * the COPYING file in the top-level directory.
mm/mremap.c+1−3
@@ -20,7 +20,6 @@ · 19 unmodified lines
2020 #include <linux/security.h>
2121 #include <linux/syscalls.h>
2222 #include <linux/mmu_notifier.h>
23−#include <linux/sched/sysctl.h>
2423 #include <linux/uaccess.h>
2524 #include <linux/mm-arch-hooks.h>
2625
@@ -214,8 +213,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma, · 187 unmodified lines
214213 continue;
215214 VM_BUG_ON(pmd_trans_huge(*old_pmd));
216215 }
217− if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma,
218− new_pmd, new_addr))
216+ if (pte_alloc(new_vma->vm_mm, new_pmd, new_addr))
219217 break;
220218 next = (new_addr + PMD_SIZE) & PMD_MASK;
221219 if (extent > next - new_addr)
mm/nobootmem.c+2−2
@@ -288,7 +288,7 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, · 287 unmodified lines
288288 /*
289289 * Whoops, we cannot satisfy the allocation request.
290290 */
291− printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
291+ pr_alert("bootmem alloc of %lu bytes failed!\n", size);
292292 panic("Out of memory");
293293 return NULL;
294294 }
@@ -360,7 +360,7 @@ static void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, · 65 unmodified lines
360360 if (ptr)
361361 return ptr;
362362
363− printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
363+ pr_alert("bootmem alloc of %lu bytes failed!\n", size);
364364 panic("Out of memory");
365365 return NULL;
366366 }
mm/nommu.c+0−117
@@ -33,7 +33,6 @@ · 32 unmodified lines
3333 #include <linux/security.h>
3434 #include <linux/syscalls.h>
3535 #include <linux/audit.h>
36−#include <linux/sched/sysctl.h>
3736 #include <linux/printk.h>
3837
3938 #include <asm/uaccess.h>
@@ -48,33 +47,11 @@ struct page *mem_map; · 8 unmodified lines
4847 unsigned long max_mapnr;
4948 EXPORT_SYMBOL(max_mapnr);
5049 unsigned long highest_memmap_pfn;
51−struct percpu_counter vm_committed_as;
52−int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
53−int sysctl_overcommit_ratio = 50; /* default is 50% */
54−unsigned long sysctl_overcommit_kbytes __read_mostly;
55−int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
5650 int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
57−unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
58−unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
5951 int heap_stack_gap = 0;
6052
6153 atomic_long_t mmap_pages_allocated;
6254
63−/*
64− * The global memory commitment made in the system can be a metric
65− * that can be used to drive ballooning decisions when Linux is hosted
66− * as a guest. On Hyper-V, the host implements a policy engine for dynamically
67− * balancing memory across competing virtual machines that are hosted.
68− * Several metrics drive this policy engine including the guest reported
69− * memory commitment.
70− */
71−unsigned long vm_memory_committed(void)
72−{
73− return percpu_counter_read_positive(&vm_committed_as);
74−}
75−
76−EXPORT_SYMBOL_GPL(vm_memory_committed);
77−
7855 EXPORT_SYMBOL(mem_map);
7956
8057 /* list of mapped, potentially shareable regions */
@@ -1829,100 +1806,6 @@ void unmap_mapping_range(struct address_space *mapping, · 1748 unmodified lines
18291806 }
18301807 EXPORT_SYMBOL(unmap_mapping_range);
18311808
1832−/*
1833− * Check that a process has enough memory to allocate a new virtual
1834− * mapping. 0 means there is enough memory for the allocation to
1835− * succeed and -ENOMEM implies there is not.
1836− *
1837− * We currently support three overcommit policies, which are set via the
1838− * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
1839− *
1840− * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
1841− * Additional code 2002 Jul 20 by Robert Love.
1842− *
1843− * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
1844− *
1845− * Note this is a helper function intended to be used by LSMs which
1846− * wish to use this logic.
1847− */
1848−int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
1849−{
1850− long free, allowed, reserve;
1851−
1852− vm_acct_memory(pages);
1853−
1854− /*
1855− * Sometimes we want to use more memory than we have
1856− */
1857− if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
1858− return 0;
1859−
1860− if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
1861− free = global_page_state(NR_FREE_PAGES);
1862− free += global_page_state(NR_FILE_PAGES);
1863−
1864− /*
1865− * shmem pages shouldn't be counted as free in this
1866− * case, they can't be purged, only swapped out, and
1867− * that won't affect the overall amount of available
1868− * memory in the system.
1869− */
1870− free -= global_page_state(NR_SHMEM);
1871−
1872− free += get_nr_swap_pages();
1873−
1874− /*
1875− * Any slabs which are created with the
1876− * SLAB_RECLAIM_ACCOUNT flag claim to have contents
1877− * which are reclaimable, under pressure. The dentry
1878− * cache and most inode caches should fall into this
1879− */
1880− free += global_page_state(NR_SLAB_RECLAIMABLE);
1881−
1882− /*
1883− * Leave reserved pages. The pages are not for anonymous pages.
1884− */
1885− if (free <= totalreserve_pages)
1886− goto error;
1887− else
1888− free -= totalreserve_pages;
1889−
1890− /*
1891− * Reserve some for root
1892− */
1893− if (!cap_sys_admin)
1894− free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
1895−
1896− if (free > pages)
1897− return 0;
1898−
1899− goto error;
1900− }
1901−
1902− allowed = vm_commit_limit();
1903− /*
1904− * Reserve some 3% for root
1905− */
1906− if (!cap_sys_admin)
1907− allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
1908−
1909− /*
1910− * Don't let a single process grow so big a user can't recover
1911− */
1912− if (mm) {
1913− reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
1914− allowed -= min_t(long, mm->total_vm / 32, reserve);
1915− }
1916−
1917− if (percpu_counter_read_positive(&vm_committed_as) < allowed)
1918− return 0;
1919−
1920−error:
1921− vm_unacct_memory(pages);
1922−
1923− return -ENOMEM;
1924−}
1925−
19261809 int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
19271810 {
19281811 BUG();
mm/oom_kill.c+4−12
@@ -287,9 +287,6 @@ enum oom_scan_t oom_scan_process_thread(struct oom_control *oc, · 286 unmodified lines
287287 if (oom_task_origin(task))
288288 return OOM_SCAN_SELECT;
289289
290− if (task_will_free_mem(task) && !is_sysrq_oom(oc))
291− return OOM_SCAN_ABORT;
292−
293290 return OOM_SCAN_OK;
294291 }
295292
@@ -386,8 +383,7 @@ static void dump_tasks(struct mem_cgroup *memcg, const nodemask_t *nodemask) · 90 unmodified lines
386383 static void dump_header(struct oom_control *oc, struct task_struct *p,
387384 struct mem_cgroup *memcg)
388385 {
389− pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), order=%d, "
390− "oom_score_adj=%hd\n",
386+ pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), order=%d, oom_score_adj=%hd\n",
391387 current->comm, oc->gfp_mask, &oc->gfp_mask, oc->order,
392388 current->signal->oom_score_adj);
393389
@@ -459,15 +455,11 @@ void exit_oom_victim(void) · 65 unmodified lines
459455 bool oom_killer_disable(void)
460456 {
461457 /*
462− * Make sure to not race with an ongoing OOM killer
463− * and that the current is not the victim.
458+ * Make sure to not race with an ongoing OOM killer. Check that the
459+ * current is not killed (possibly due to sharing the victim's memory).
464460 */
465− mutex_lock(&oom_lock);
466− if (test_thread_flag(TIF_MEMDIE)) {
467− mutex_unlock(&oom_lock);
461+ if (mutex_lock_killable(&oom_lock))
468462 return false;
469− }
470−
471463 oom_killer_disabled = true;
472464 mutex_unlock(&oom_lock);
473465
mm/page_alloc.c+124−68
@@ -249,6 +249,7 @@ compound_page_dtor * const compound_page_dtors[] = { · 248 unmodified lines
249249
250250 int min_free_kbytes = 1024;
251251 int user_min_free_kbytes = -1;
252+int watermark_scale_factor = 10;
252253
253254 static unsigned long __meminitdata nr_kernel_pages;
254255 static unsigned long __meminitdata nr_all_pages;
@@ -307,13 +308,20 @@ static inline bool update_defer_init(pg_data_t *pgdat, · 52 unmodified lines
307308 unsigned long pfn, unsigned long zone_end,
308309 unsigned long *nr_initialised)
309310 {
311+ unsigned long max_initialise;
312+
310313 /* Always populate low zones for address-contrained allocations */
311314 if (zone_end < pgdat_end_pfn(pgdat))
312315 return true;
316+ /*
317+ * Initialise at least 2G of a node but also take into account that
318+ * two large system hashes that can take up 1GB for 0.25TB/node.
319+ */
320+ max_initialise = max(2UL << (30 - PAGE_SHIFT),
321+ (pgdat->node_spanned_pages >> 8));
313322
314− /* Initialise at least 2G of the highest zone */
315323 (*nr_initialised)++;
316− if (*nr_initialised > (2UL << (30 - PAGE_SHIFT)) &&
324+ if ((*nr_initialised > max_initialise) &&
317325 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
318326 pgdat->first_deferred_pfn = pfn;
319327 return false;
@@ -498,6 +506,7 @@ void prep_compound_page(struct page *page, unsigned int order) · 178 unmodified lines
498506 unsigned int _debug_guardpage_minorder;
499507 bool _debug_pagealloc_enabled __read_mostly
500508 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
509+EXPORT_SYMBOL(_debug_pagealloc_enabled);
501510 bool _debug_guardpage_enabled __read_mostly;
502511
503512 static int __init early_debug_pagealloc(char *buf)
@@ -542,11 +551,11 @@ static int __init debug_guardpage_minorder_setup(char *buf) · 38 unmodified lines
542551 unsigned long res;
543552
544553 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
545− printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
554+ pr_err("Bad debug_guardpage_minorder value\n");
546555 return 0;
547556 }
548557 _debug_guardpage_minorder = res;
549− printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
558+ pr_info("Setting debug_guardpage_minorder to %lu\n", res);
550559 return 0;
551560 }
552561 __setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
@@ -764,7 +773,7 @@ static inline int free_pages_check(struct page *page) · 211 unmodified lines
764773 bad_reason = "nonzero mapcount";
765774 if (unlikely(page->mapping != NULL))
766775 bad_reason = "non-NULL mapping";
767− if (unlikely(atomic_read(&page->_count) != 0))
776+ if (unlikely(page_ref_count(page) != 0))
768777 bad_reason = "nonzero _count";
769778 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
770779 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
@@ -1460,7 +1469,7 @@ static inline int check_new_page(struct page *page) · 689 unmodified lines
14601469 bad_reason = "nonzero mapcount";
14611470 if (unlikely(page->mapping != NULL))
14621471 bad_reason = "non-NULL mapping";
1463− if (unlikely(atomic_read(&page->_count) != 0))
1472+ if (unlikely(page_ref_count(page) != 0))
14641473 bad_reason = "nonzero _count";
14651474 if (unlikely(page->flags & __PG_HWPOISON)) {
14661475 bad_reason = "HWPoisoned (hardware-corrupted)";
@@ -2348,19 +2357,11 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, · 881 unmodified lines
23482357 list_del(&page->lru);
23492358 pcp->count--;
23502359 } else {
2351− if (unlikely(gfp_flags & __GFP_NOFAIL)) {
2352− /*
2353− * __GFP_NOFAIL is not to be used in new code.
2354− *
2355− * All __GFP_NOFAIL callers should be fixed so that they
2356− * properly detect and handle allocation failures.
2357− *
2358− * We most definitely don't want callers attempting to
2359− * allocate greater than order-1 page units with
2360− * __GFP_NOFAIL.
2361− */
2362− WARN_ON_ONCE(order > 1);
2363− }
2360+ /*
2361+ * We most definitely don't want callers attempting to
2362+ * allocate greater than order-1 page units with __GFP_NOFAIL.
2363+ */
2364+ WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
23642365 spin_lock_irqsave(&zone->lock, flags);
23652366
23662367 page = NULL;
@@ -2857,8 +2858,12 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, · 490 unmodified lines
28572858 * XXX: Page reclaim didn't yield anything,
28582859 * and the OOM killer can't be invoked, but
28592860 * keep looping as per tradition.
2861+ *
2862+ * But do not keep looping if oom_killer_disable()
2863+ * was already called, for the system is trying to
2864+ * enter a quiescent state during suspend.
28602865 */
2861− *did_some_progress = 1;
2866+ *did_some_progress = !oom_killer_disabled;
28622867 goto out;
28632868 }
28642869 if (pm_suspended_storage())
@@ -3117,14 +3122,6 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, · 252 unmodified lines
31173122 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
31183123 gfp_mask &= ~__GFP_ATOMIC;
31193124
3120− /*
3121− * If this allocation cannot block and it is for a specific node, then
3122− * fail early. There's no need to wakeup kswapd or retry for a
3123− * speculative node-specific allocation.
3124− */
3125− if (IS_ENABLED(CONFIG_NUMA) && (gfp_mask & __GFP_THISNODE) && !can_direct_reclaim)
3126− goto nopage;
3127−
31283125 retry:
31293126 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
31303127 wake_all_kswapds(order, ac);
@@ -3481,7 +3478,7 @@ refill: · 350 unmodified lines
34813478 /* Even if we own the page, we do not use atomic_set().
34823479 * This would break get_page_unless_zero() users.
34833480 */
3484− atomic_add(size - 1, &page->_count);
3481+ page_ref_add(page, size - 1);
34853482
34863483 /* reset page count bias and offset to start of new frag */
34873484 nc->pfmemalloc = page_is_pfmemalloc(page);
@@ -3493,7 +3490,7 @@ refill: · 5 unmodified lines
34933490 if (unlikely(offset < 0)) {
34943491 page = virt_to_page(nc->va);
34953492
3496− if (!atomic_sub_and_test(nc->pagecnt_bias, &page->_count))
3493+ if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
34973494 goto refill;
34983495
34993496 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
@@ -3501,7 +3498,7 @@ refill: · 1 unmodified line
35013498 size = nc->size;
35023499 #endif
35033500 /* OK, page count is 0, we can safely set it */
3504− atomic_set(&page->_count, size);
3501+ set_page_count(page, size);
35053502
35063503 /* reset page count bias and offset to start of new frag */
35073504 nc->pagecnt_bias = size;
@@ -3712,6 +3709,49 @@ static inline void show_node(struct zone *zone) · 204 unmodified lines
37123709 printk("Node %d ", zone_to_nid(zone));
37133710 }
37143711
3712+long si_mem_available(void)
3713+{
3714+ long available;
3715+ unsigned long pagecache;
3716+ unsigned long wmark_low = 0;
3717+ unsigned long pages[NR_LRU_LISTS];
3718+ struct zone *zone;
3719+ int lru;
3720+
3721+ for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
3722+ pages[lru] = global_page_state(NR_LRU_BASE + lru);
3723+
3724+ for_each_zone(zone)
3725+ wmark_low += zone->watermark[WMARK_LOW];
3726+
3727+ /*
3728+ * Estimate the amount of memory available for userspace allocations,
3729+ * without causing swapping.
3730+ */
3731+ available = global_page_state(NR_FREE_PAGES) - totalreserve_pages;
3732+
3733+ /*
3734+ * Not all the page cache can be freed, otherwise the system will
3735+ * start swapping. Assume at least half of the page cache, or the
3736+ * low watermark worth of cache, needs to stay.
3737+ */
3738+ pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
3739+ pagecache -= min(pagecache / 2, wmark_low);
3740+ available += pagecache;
3741+
3742+ /*
3743+ * Part of the reclaimable slab consists of items that are in use,
3744+ * and cannot be freed. Cap this estimate at the low watermark.
3745+ */
3746+ available += global_page_state(NR_SLAB_RECLAIMABLE) -
3747+ min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
3748+
3749+ if (available < 0)
3750+ available = 0;
3751+ return available;
3752+}
3753+EXPORT_SYMBOL_GPL(si_mem_available);
3754+
37153755 void si_meminfo(struct sysinfo *val)
37163756 {
37173757 val->totalram = totalram_pages;
@@ -4044,9 +4084,7 @@ static int __parse_numa_zonelist_order(char *s) · 326 unmodified lines
40444084 } else if (*s == 'z' || *s == 'Z') {
40454085 user_zonelist_order = ZONELIST_ORDER_ZONE;
40464086 } else {
4047− printk(KERN_WARNING
4048− "Ignoring invalid numa_zonelist_order value: "
4049− "%s\n", s);
4087+ pr_warn("Ignoring invalid numa_zonelist_order value: %s\n", s);
40504088 return -EINVAL;
40514089 }
40524090 return 0;
@@ -4510,12 +4548,11 @@ void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) · 457 unmodified lines
45104548 else
45114549 page_group_by_mobility_disabled = 0;
45124550
4513− pr_info("Built %i zonelists in %s order, mobility grouping %s. "
4514− "Total pages: %ld\n",
4515− nr_online_nodes,
4516− zonelist_order_name[current_zonelist_order],
4517− page_group_by_mobility_disabled ? "off" : "on",
4518− vm_total_pages);
4551+ pr_info("Built %i zonelists in %s order, mobility grouping %s. Total pages: %ld\n",
4552+ nr_online_nodes,
4553+ zonelist_order_name[current_zonelist_order],
4554+ page_group_by_mobility_disabled ? "off" : "on",
4555+ vm_total_pages);
45194556 #ifdef CONFIG_NUMA
45204557 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
45214558 #endif
@@ -5404,6 +5441,9 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat) · 882 unmodified lines
54045441 #endif
54055442 init_waitqueue_head(&pgdat->kswapd_wait);
54065443 init_waitqueue_head(&pgdat->pfmemalloc_wait);
5444+#ifdef CONFIG_COMPACTION
5445+ init_waitqueue_head(&pgdat->kcompactd_wait);
5446+#endif
54075447 pgdat_page_ext_init(pgdat);
54085448
54095449 for (j = 0; j < MAX_NR_ZONES; j++) {
@@ -5428,8 +5468,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat) · 18 unmodified lines
54285468 " %s zone: %lu pages used for memmap\n",
54295469 zone_names[j], memmap_pages);
54305470 } else
5431− printk(KERN_WARNING
5432− " %s zone: %lu pages exceeds freesize %lu\n",
5471+ pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
54335472 zone_names[j], memmap_pages, freesize);
54345473 }
54355474
@@ -5637,8 +5676,7 @@ static unsigned long __init find_min_pfn_for_node(int nid) · 201 unmodified lines
56375676 min_pfn = min(min_pfn, start_pfn);
56385677
56395678 if (min_pfn == ULONG_MAX) {
5640− printk(KERN_WARNING
5641− "Could not find start_pfn for node %d\n", nid);
5679+ pr_warn("Could not find start_pfn for node %d\n", nid);
56425680 return 0;
56435681 }
56445682
@@ -6110,22 +6148,21 @@ void __init mem_init_print_info(const char *str) · 465 unmodified lines
61106148
61116149 #undef adj_init_size
61126150
6113− pr_info("Memory: %luK/%luK available "
6114− "(%luK kernel code, %luK rwdata, %luK rodata, "
6115− "%luK init, %luK bss, %luK reserved, %luK cma-reserved"
6151+ pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
61166152 #ifdef CONFIG_HIGHMEM
6117− ", %luK highmem"
6153+ ", %luK highmem"
61186154 #endif
6119− "%s%s)\n",
6120− nr_free_pages() << (PAGE_SHIFT-10), physpages << (PAGE_SHIFT-10),
6121− codesize >> 10, datasize >> 10, rosize >> 10,
6122− (init_data_size + init_code_size) >> 10, bss_size >> 10,
6123− (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT-10),
6124− totalcma_pages << (PAGE_SHIFT-10),
6155+ "%s%s)\n",
6156+ nr_free_pages() << (PAGE_SHIFT - 10),
6157+ physpages << (PAGE_SHIFT - 10),
6158+ codesize >> 10, datasize >> 10, rosize >> 10,
6159+ (init_data_size + init_code_size) >> 10, bss_size >> 10,
6160+ (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
6161+ totalcma_pages << (PAGE_SHIFT - 10),
61256162 #ifdef CONFIG_HIGHMEM
6126− totalhigh_pages << (PAGE_SHIFT-10),
6163+ totalhigh_pages << (PAGE_SHIFT - 10),
61276164 #endif
6128− str ? ", " : "", str ? str : "");
6165+ str ? ", " : "", str ? str : "");
61296166 }
61306167
61316168 /**
@@ -6300,8 +6337,17 @@ static void __setup_per_zone_wmarks(void) · 168 unmodified lines
63006337 zone->watermark[WMARK_MIN] = tmp;
63016338 }
63026339
6303− zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
6304− zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
6340+ /*
6341+ * Set the kswapd watermarks distance according to the
6342+ * scale factor in proportion to available memory, but
6343+ * ensure a minimum size on small systems.
6344+ */
6345+ tmp = max_t(u64, tmp >> 2,
6346+ mult_frac(zone->managed_pages,
6347+ watermark_scale_factor, 10000));
6348+
6349+ zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
6350+ zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
63056351
63066352 __mod_zone_page_state(zone, NR_ALLOC_BATCH,
63076353 high_wmark_pages(zone) - low_wmark_pages(zone) -
@@ -6442,6 +6488,21 @@ int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, · 134 unmodified lines
64426488 return 0;
64436489 }
64446490
6491+int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
6492+ void __user *buffer, size_t *length, loff_t *ppos)
6493+{
6494+ int rc;
6495+
6496+ rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6497+ if (rc)
6498+ return rc;
6499+
6500+ if (write)
6501+ setup_per_zone_wmarks();
6502+
6503+ return 0;
6504+}
6505+
64456506 #ifdef CONFIG_NUMA
64466507 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
64476508 void __user *buffer, size_t *length, loff_t *ppos)
@@ -6633,11 +6694,8 @@ void *__init alloc_large_system_hash(const char *tablename, · 185 unmodified lines
66336694 if (!table)
66346695 panic("Failed to allocate %s hash table\n", tablename);
66356696
6636− printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
6637− tablename,
6638− (1UL << log2qty),
6639− ilog2(size) - PAGE_SHIFT,
6640− size);
6697+ pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
6698+ tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
66416699
66426700 if (_hash_shift)
66436701 *_hash_shift = log2qty;
@@ -6788,7 +6846,7 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count, · 144 unmodified lines
67886846 * This check already skips compound tails of THP
67896847 * because their page->_count is zero at all time.
67906848 */
6791− if (!atomic_read(&page->_count)) {
6849+ if (!page_ref_count(page)) {
67926850 if (PageBuddy(page))
67936851 iter += (1 << page_order(page)) - 1;
67946852 continue;
@@ -7138,8 +7196,8 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 343 unmodified lines
71387196 BUG_ON(!PageBuddy(page));
71397197 order = page_order(page);
71407198 #ifdef CONFIG_DEBUG_VM
7141− printk(KERN_INFO "remove from free list %lx %d %lx\n",
7142− pfn, 1 << order, end_pfn);
7199+ pr_info("remove from free list %lx %d %lx\n",
7200+ pfn, 1 << order, end_pfn);
71437201 #endif
71447202 list_del(&page->lru);
71457203 rmv_page_order(page);
@@ -7152,7 +7210,6 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 6 unmodified lines
71527210 }
71537211 #endif
71547212
7155−#ifdef CONFIG_MEMORY_FAILURE
71567213 bool is_free_buddy_page(struct page *page)
71577214 {
71587215 struct zone *zone = page_zone(page);
@@ -7171,4 +7228,3 @@ bool is_free_buddy_page(struct page *page) · 12 unmodified lines
71717228
71727229 return order < MAX_ORDER;
71737230 }
7174−#endif
mm/page_io.c+11−11
@@ -56,10 +56,10 @@ void end_swap_bio_write(struct bio *bio) · 55 unmodified lines
5656 * Also clear PG_reclaim to avoid rotate_reclaimable_page()
5757 */
5858 set_page_dirty(page);
59− printk(KERN_ALERT "Write-error on swap-device (%u:%u:%Lu)\n",
60− imajor(bio->bi_bdev->bd_inode),
61− iminor(bio->bi_bdev->bd_inode),
62− (unsigned long long)bio->bi_iter.bi_sector);
59+ pr_alert("Write-error on swap-device (%u:%u:%llu)\n",
60+ imajor(bio->bi_bdev->bd_inode),
61+ iminor(bio->bi_bdev->bd_inode),
62+ (unsigned long long)bio->bi_iter.bi_sector);
6363 ClearPageReclaim(page);
6464 }
6565 end_page_writeback(page);
@@ -73,10 +73,10 @@ static void end_swap_bio_read(struct bio *bio) · 7 unmodified lines
7373 if (bio->bi_error) {
7474 SetPageError(page);
7575 ClearPageUptodate(page);
76− printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n",
77− imajor(bio->bi_bdev->bd_inode),
78− iminor(bio->bi_bdev->bd_inode),
79− (unsigned long long)bio->bi_iter.bi_sector);
76+ pr_alert("Read-error on swap-device (%u:%u:%llu)\n",
77+ imajor(bio->bi_bdev->bd_inode),
78+ iminor(bio->bi_bdev->bd_inode),
79+ (unsigned long long)bio->bi_iter.bi_sector);
8080 goto out;
8181 }
8282
@@ -216,7 +216,7 @@ reprobe: · 133 unmodified lines
216216 out:
217217 return ret;
218218 bad_bmap:
219− printk(KERN_ERR "swapon: swapfile has holes\n");
219+ pr_err("swapon: swapfile has holes\n");
220220 ret = -EINVAL;
221221 goto out;
222222 }
@@ -290,8 +290,8 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc, · 67 unmodified lines
290290 */
291291 set_page_dirty(page);
292292 ClearPageReclaim(page);
293− pr_err_ratelimited("Write error on dio swapfile (%Lu)\n",
294− page_file_offset(page));
293+ pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
294+ page_file_offset(page));
295295 }
296296 end_page_writeback(page);
297297 return ret;
mm/page_owner.c+2−3
@@ -198,9 +198,8 @@ void __dump_page_owner(struct page *page) · 197 unmodified lines
198198 return;
199199 }
200200
201− pr_alert("page allocated via order %u, migratetype %s, "
202− "gfp_mask %#x(%pGg)\n", page_ext->order,
203− migratetype_names[mt], gfp_mask, &gfp_mask);
201+ pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n",
202+ page_ext->order, migratetype_names[mt], gfp_mask, &gfp_mask);
204203 print_stack_trace(&trace, 0);
205204
206205 if (page_ext->last_migrate_reason != -1)
mm/percpu-km.c+3−3
@@ -95,7 +95,7 @@ static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai) · 94 unmodified lines
9595
9696 /* all units must be in a single group */
9797 if (ai->nr_groups != 1) {
98− printk(KERN_CRIT "percpu: can't handle more than one groups\n");
98+ pr_crit("can't handle more than one group\n");
9999 return -EINVAL;
100100 }
101101
@@ -103,8 +103,8 @@ static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai) · 1 unmodified line
103103 alloc_pages = roundup_pow_of_two(nr_pages);
104104
105105 if (alloc_pages > nr_pages)
106− printk(KERN_WARNING "percpu: wasting %zu pages per chunk\n",
107− alloc_pages - nr_pages);
106+ pr_warn("wasting %zu pages per chunk\n",
107+ alloc_pages - nr_pages);
108108
109109 return 0;
110110 }
mm/percpu.c+22−21
@@ -53,6 +53,8 @@ · 52 unmodified lines
5353 * setup the first chunk containing the kernel static percpu area
5454 */
5555
56+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
57+
5658 #include <linux/bitmap.h>
5759 #include <linux/bootmem.h>
5860 #include <linux/err.h>
@@ -888,8 +890,8 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved, · 829 unmodified lines
888890 size = ALIGN(size, 2);
889891
890892 if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
891− WARN(true, "illegal size (%zu) or align (%zu) for "
892− "percpu allocation\n", size, align);
893+ WARN(true, "illegal size (%zu) or align (%zu) for percpu allocation\n",
894+ size, align);
893895 return NULL;
894896 }
895897
@@ -1033,11 +1035,11 @@ fail_unlock: · 137 unmodified lines
10331035 spin_unlock_irqrestore(&pcpu_lock, flags);
10341036 fail:
10351037 if (!is_atomic && warn_limit) {
1036− pr_warning("PERCPU: allocation failed, size=%zu align=%zu atomic=%d, %s\n",
1037− size, align, is_atomic, err);
1038+ pr_warn("allocation failed, size=%zu align=%zu atomic=%d, %s\n",
1039+ size, align, is_atomic, err);
10381040 dump_stack();
10391041 if (!--warn_limit)
1040− pr_info("PERCPU: limit reached, disable warning\n");
1042+ pr_info("limit reached, disable warning\n");
10411043 }
10421044 if (is_atomic) {
10431045 /* see the flag handling in pcpu_blance_workfn() */
@@ -1449,20 +1451,20 @@ static void pcpu_dump_alloc_info(const char *lvl, · 405 unmodified lines
14491451 for (alloc_end += gi->nr_units / upa;
14501452 alloc < alloc_end; alloc++) {
14511453 if (!(alloc % apl)) {
1452− printk(KERN_CONT "\n");
1454+ pr_cont("\n");
14531455 printk("%spcpu-alloc: ", lvl);
14541456 }
1455− printk(KERN_CONT "[%0*d] ", group_width, group);
1457+ pr_cont("[%0*d] ", group_width, group);
14561458
14571459 for (unit_end += upa; unit < unit_end; unit++)
14581460 if (gi->cpu_map[unit] != NR_CPUS)
1459− printk(KERN_CONT "%0*d ", cpu_width,
1460− gi->cpu_map[unit]);
1461+ pr_cont("%0*d ",
1462+ cpu_width, gi->cpu_map[unit]);
14611463 else
1462− printk(KERN_CONT "%s ", empty_str);
1464+ pr_cont("%s ", empty_str);
14631465 }
14641466 }
1465− printk(KERN_CONT "\n");
1467+ pr_cont("\n");
14661468 }
14671469
14681470 /**
@@ -1538,8 +1540,8 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, · 69 unmodified lines
15381540
15391541 #define PCPU_SETUP_BUG_ON(cond) do { \
15401542 if (unlikely(cond)) { \
1541− pr_emerg("PERCPU: failed to initialize, %s", #cond); \
1542− pr_emerg("PERCPU: cpu_possible_mask=%*pb\n", \
1543+ pr_emerg("failed to initialize, %s\n", #cond); \
1544+ pr_emerg("cpu_possible_mask=%*pb\n", \
15431545 cpumask_pr_args(cpu_possible_mask)); \
15441546 pcpu_dump_alloc_info(KERN_EMERG, ai); \
15451547 BUG(); \
@@ -1723,7 +1725,7 @@ static int __init percpu_alloc_setup(char *str) · 177 unmodified lines
17231725 pcpu_chosen_fc = PCPU_FC_PAGE;
17241726 #endif
17251727 else
1726− pr_warning("PERCPU: unknown allocator %s specified\n", str);
1728+ pr_warn("unknown allocator %s specified\n", str);
17271729
17281730 return 0;
17291731 }
@@ -2016,9 +2018,8 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, · 286 unmodified lines
20162018
20172019 /* warn if maximum distance is further than 75% of vmalloc space */
20182020 if (max_distance > VMALLOC_TOTAL * 3 / 4) {
2019− pr_warning("PERCPU: max_distance=0x%zx too large for vmalloc "
2020− "space 0x%lx\n", max_distance,
2021− VMALLOC_TOTAL);
2021+ pr_warn("max_distance=0x%zx too large for vmalloc space 0x%lx\n",
2022+ max_distance, VMALLOC_TOTAL);
20222023 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
20232024 /* and fail if we have fallback */
20242025 rc = -EINVAL;
@@ -2026,7 +2027,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, · 1 unmodified line
20262027 #endif
20272028 }
20282029
2029− pr_info("PERCPU: Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n",
2030+ pr_info("Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n",
20302031 PFN_DOWN(size_sum), base, ai->static_size, ai->reserved_size,
20312032 ai->dyn_size, ai->unit_size);
20322033
@@ -2100,8 +2101,8 @@ int __init pcpu_page_first_chunk(size_t reserved_size, · 67 unmodified lines
21002101
21012102 ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE);
21022103 if (!ptr) {
2103− pr_warning("PERCPU: failed to allocate %s page "
2104− "for cpu%u\n", psize_str, cpu);
2104+ pr_warn("failed to allocate %s page for cpu%u\n",
2105+ psize_str, cpu);
21052106 goto enomem;
21062107 }
21072108 /* kmemleak tracks the percpu allocations separately */
@@ -2140,7 +2141,7 @@ int __init pcpu_page_first_chunk(size_t reserved_size, · 32 unmodified lines
21402141 }
21412142
21422143 /* we're ready, commit */
2143− pr_info("PERCPU: %d %s pages/cpu @%p s%zu r%zu d%zu\n",
2144+ pr_info("%d %s pages/cpu @%p s%zu r%zu d%zu\n",
21442145 unit_pages, psize_str, vm.addr, ai->static_size,
21452146 ai->reserved_size, ai->dyn_size);
21462147
mm/pgtable-generic.c+0−14
@@ -84,20 +84,6 @@ pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, · 83 unmodified lines
8484
8585 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
8686
87−#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
88−
89−/*
90− * ARCHes with special requirements for evicting THP backing TLB entries can
91− * implement this. Otherwise also, it can help optimize normal TLB flush in
92− * THP regime. stock flush_tlb_range() typically has optimization to nuke the
93− * entire TLB if flush span is greater than a threshold, which will
94− * likely be true for a single huge page. Thus a single thp flush will
95− * invalidate the entire TLB which is not desirable.
96− * e.g. see arch/arc: flush_pmd_tlb_range
97− */
98−#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
99−#endif
100−
10187 #ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
10288 int pmdp_set_access_flags(struct vm_area_struct *vma,
10389 unsigned long address, pmd_t *pmdp,
mm/quicklist.c+1−1
@@ -8,7 +8,7 @@ · 7 unmodified lines
88 * improved on it.
99 *
1010 * Copyright (C) 2007 SGI,
11− * Christoph Lameter <clameter@sgi.com>
11+ * Christoph Lameter <cl@linux.com>
1212 * Generalized, added support for multiple lists and
1313 * constructors / destructors.
1414 */
mm/rmap.c+54−17
@@ -1431,6 +1431,14 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · 1430 unmodified lines
14311431 if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
14321432 goto out;
14331433
1434+ if (flags & TTU_SPLIT_HUGE_PMD) {
1435+ split_huge_pmd_address(vma, address,
1436+ flags & TTU_MIGRATION, page);
1437+ /* check if we have anything to do after split */
1438+ if (page_mapcount(page) == 0)
1439+ goto out;
1440+ }
1441+
14341442 pte = page_check_address(page, mm, address, &ptl, 0);
14351443 if (!pte)
14361444 goto out;
@@ -1576,10 +1584,10 @@ static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) · 139 unmodified lines
15761584 return is_vma_temporary_stack(vma);
15771585 }
15781586
1579−static int page_not_mapped(struct page *page)
1587+static int page_mapcount_is_zero(struct page *page)
15801588 {
1581− return !page_mapped(page);
1582−};
1589+ return !page_mapcount(page);
1590+}
15831591
15841592 /**
15851593 * try_to_unmap - try to remove all page table mappings to a page
@@ -1606,12 +1614,10 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) · 20 unmodified lines
16061614 struct rmap_walk_control rwc = {
16071615 .rmap_one = try_to_unmap_one,
16081616 .arg = &rp,
1609− .done = page_not_mapped,
1617+ .done = page_mapcount_is_zero,
16101618 .anon_lock = page_lock_anon_vma_read,
16111619 };
16121620
1613− VM_BUG_ON_PAGE(!PageHuge(page) && PageTransHuge(page), page);
1614−
16151621 /*
16161622 * During exec, a temporary VMA is setup and later moved.
16171623 * The VMA is moved under the anon_vma lock but not the
@@ -1623,9 +1629,12 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) · 5 unmodified lines
16231629 if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page))
16241630 rwc.invalid_vma = invalid_migration_vma;
16251631
1626− ret = rmap_walk(page, &rwc);
1632+ if (flags & TTU_RMAP_LOCKED)
1633+ ret = rmap_walk_locked(page, &rwc);
1634+ else
1635+ ret = rmap_walk(page, &rwc);
16271636
1628− if (ret != SWAP_MLOCK && !page_mapped(page)) {
1637+ if (ret != SWAP_MLOCK && !page_mapcount(page)) {
16291638 ret = SWAP_SUCCESS;
16301639 if (rp.lazyfreed && !PageDirty(page))
16311640 ret = SWAP_LZFREE;
@@ -1633,6 +1642,11 @@ int try_to_unmap(struct page *page, enum ttu_flags flags) · 1 unmodified line
16331642 return ret;
16341643 }
16351644
1645+static int page_not_mapped(struct page *page)
1646+{
1647+ return !page_mapped(page);
1648+};
1649+
16361650 /**
16371651 * try_to_munlock - try to munlock a page
16381652 * @page: the page to be munlocked
@@ -1715,14 +1729,21 @@ static struct anon_vma *rmap_walk_anon_lock(struct page *page, · 76 unmodified lines
17151729 * vm_flags for that VMA. That should be OK, because that vma shouldn't be
17161730 * LOCKED.
17171731 */
1718−static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc)
1732+static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,
1733+ bool locked)
17191734 {
17201735 struct anon_vma *anon_vma;
17211736 pgoff_t pgoff;
17221737 struct anon_vma_chain *avc;
17231738 int ret = SWAP_AGAIN;
17241739
1725− anon_vma = rmap_walk_anon_lock(page, rwc);
1740+ if (locked) {
1741+ anon_vma = page_anon_vma(page);
1742+ /* anon_vma disappear under us? */
1743+ VM_BUG_ON_PAGE(!anon_vma, page);
1744+ } else {
1745+ anon_vma = rmap_walk_anon_lock(page, rwc);
1746+ }
17261747 if (!anon_vma)
17271748 return ret;
17281749
@@ -1742,7 +1763,9 @@ static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc) · 13 unmodified lines
17421763 if (rwc->done && rwc->done(page))
17431764 break;
17441765 }
1745− anon_vma_unlock_read(anon_vma);
1766+
1767+ if (!locked)
1768+ anon_vma_unlock_read(anon_vma);
17461769 return ret;
17471770 }
17481771
@@ -1759,9 +1782,10 @@ static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc) · 10 unmodified lines
17591782 * vm_flags for that VMA. That should be OK, because that vma shouldn't be
17601783 * LOCKED.
17611784 */
1762−static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc)
1785+static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,
1786+ bool locked)
17631787 {
1764− struct address_space *mapping = page->mapping;
1788+ struct address_space *mapping = page_mapping(page);
17651789 pgoff_t pgoff;
17661790 struct vm_area_struct *vma;
17671791 int ret = SWAP_AGAIN;
@@ -1778,7 +1802,8 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc) · 10 unmodified lines
17781802 return ret;
17791803
17801804 pgoff = page_to_pgoff(page);
1781− i_mmap_lock_read(mapping);
1805+ if (!locked)
1806+ i_mmap_lock_read(mapping);
17821807 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
17831808 unsigned long address = vma_address(page, vma);
17841809
@@ -1795,7 +1820,8 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc) · 10 unmodified lines
17951820 }
17961821
17971822 done:
1798− i_mmap_unlock_read(mapping);
1823+ if (!locked)
1824+ i_mmap_unlock_read(mapping);
17991825 return ret;
18001826 }
18011827
@@ -1804,9 +1830,20 @@ int rmap_walk(struct page *page, struct rmap_walk_control *rwc) · 2 unmodified lines
18041830 if (unlikely(PageKsm(page)))
18051831 return rmap_walk_ksm(page, rwc);
18061832 else if (PageAnon(page))
1807− return rmap_walk_anon(page, rwc);
1833+ return rmap_walk_anon(page, rwc, false);
1834+ else
1835+ return rmap_walk_file(page, rwc, false);
1836+}
1837+
1838+/* Like rmap_walk, but caller holds relevant rmap lock */
1839+int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)
1840+{
1841+ /* no ksm support for now */
1842+ VM_BUG_ON_PAGE(PageKsm(page), page);
1843+ if (PageAnon(page))
1844+ return rmap_walk_anon(page, rwc, true);
18081845 else
1809− return rmap_walk_file(page, rwc);
1846+ return rmap_walk_file(page, rwc, true);
18101847 }
18111848
18121849 #ifdef CONFIG_HUGETLB_PAGE
mm/shmem.c+21−28
@@ -376,28 +376,23 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, · 375 unmodified lines
376376
377377 rcu_read_lock();
378378
379−restart:
380379 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
381380 if (iter.index >= end)
382381 break;
383382
384383 page = radix_tree_deref_slot(slot);
385384
386− /*
387− * This should only be possible to happen at index 0, so we
388− * don't need to reset the counter, nor do we risk infinite
389− * restarts.
390− */
391− if (radix_tree_deref_retry(page))
392− goto restart;
385+ if (radix_tree_deref_retry(page)) {
386+ slot = radix_tree_iter_retry(&iter);
387+ continue;
388+ }
393389
394390 if (radix_tree_exceptional_entry(page))
395391 swapped++;
396392
397393 if (need_resched()) {
398394 cond_resched_rcu();
399− start = iter.index + 1;
400− goto restart;
395+ slot = radix_tree_iter_next(&iter);
401396 }
402397 }
403398
@@ -1947,12 +1942,13 @@ static void shmem_tag_pins(struct address_space *mapping) · 1543 unmodified lines
19471942 start = 0;
19481943 rcu_read_lock();
19491944
1950−restart:
19511945 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
19521946 page = radix_tree_deref_slot(slot);
19531947 if (!page || radix_tree_exception(page)) {
1954− if (radix_tree_deref_retry(page))
1955− goto restart;
1948+ if (radix_tree_deref_retry(page)) {
1949+ slot = radix_tree_iter_retry(&iter);
1950+ continue;
1951+ }
19561952 } else if (page_count(page) - page_mapcount(page) > 1) {
19571953 spin_lock_irq(&mapping->tree_lock);
19581954 radix_tree_tag_set(&mapping->page_tree, iter.index,
@@ -1962,8 +1958,7 @@ restart: · 3 unmodified lines
19621958
19631959 if (need_resched()) {
19641960 cond_resched_rcu();
1965− start = iter.index + 1;
1966− goto restart;
1961+ slot = radix_tree_iter_next(&iter);
19671962 }
19681963 }
19691964 rcu_read_unlock();
@@ -2000,14 +1995,15 @@ static int shmem_wait_for_pins(struct address_space *mapping) · 30 unmodified lines
20001995
20011996 start = 0;
20021997 rcu_read_lock();
2003−restart:
20041998 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter,
20051999 start, SHMEM_TAG_PINNED) {
20062000
20072001 page = radix_tree_deref_slot(slot);
20082002 if (radix_tree_exception(page)) {
2009− if (radix_tree_deref_retry(page))
2010− goto restart;
2003+ if (radix_tree_deref_retry(page)) {
2004+ slot = radix_tree_iter_retry(&iter);
2005+ continue;
2006+ }
20112007
20122008 page = NULL;
20132009 }
@@ -2032,8 +2028,7 @@ restart: · 18 unmodified lines
20322028 continue_resched:
20332029 if (need_resched()) {
20342030 cond_resched_rcu();
2035− start = iter.index + 1;
2036− goto restart;
2031+ slot = radix_tree_iter_next(&iter);
20372032 }
20382033 }
20392034 rcu_read_unlock();
@@ -2823,9 +2818,8 @@ static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, · 783 unmodified lines
28232818 if ((value = strchr(this_char,'=')) != NULL) {
28242819 *value++ = 0;
28252820 } else {
2826− printk(KERN_ERR
2827− "tmpfs: No value for mount option '%s'\n",
2828− this_char);
2821+ pr_err("tmpfs: No value for mount option '%s'\n",
2822+ this_char);
28292823 goto error;
28302824 }
28312825
@@ -2880,8 +2874,7 @@ static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, · 48 unmodified lines
28802874 if (mpol_parse_str(value, &mpol))
28812875 goto bad_val;
28822876 } else {
2883− printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2884− this_char);
2877+ pr_err("tmpfs: Bad mount option %s\n", this_char);
28852878 goto error;
28862879 }
28872880 }
@@ -2889,7 +2882,7 @@ static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, · 1 unmodified line
28892882 return 0;
28902883
28912884 bad_val:
2892− printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2885+ pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
28932886 value, this_char);
28942887 error:
28952888 mpol_put(mpol);
@@ -3286,14 +3279,14 @@ int __init shmem_init(void) · 390 unmodified lines
32863279
32873280 error = register_filesystem(&shmem_fs_type);
32883281 if (error) {
3289− printk(KERN_ERR "Could not register tmpfs\n");
3282+ pr_err("Could not register tmpfs\n");
32903283 goto out2;
32913284 }
32923285
32933286 shm_mnt = kern_mount(&shmem_fs_type);
32943287 if (IS_ERR(shm_mnt)) {
32953288 error = PTR_ERR(shm_mnt);
3296− printk(KERN_ERR "Could not kern_mount tmpfs\n");
3289+ pr_err("Could not kern_mount tmpfs\n");
32973290 goto out1;
32983291 }
32993292 return 0;
mm/slab.c+40−49
@@ -474,7 +474,7 @@ static unsigned int cache_estimate(unsigned long gfporder, size_t buffer_size, · 473 unmodified lines
474474 static void __slab_error(const char *function, struct kmem_cache *cachep,
475475 char *msg)
476476 {
477− printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
477+ pr_err("slab error in %s(): cache `%s': %s\n",
478478 function, cachep->name, msg);
479479 dump_stack();
480480 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
@@ -670,7 +670,7 @@ static inline void *____cache_alloc_node(struct kmem_cache *cachep, · 189 unmodified lines
670670
671671 static inline gfp_t gfp_exact_node(gfp_t flags)
672672 {
673− return flags;
673+ return flags & ~__GFP_NOFAIL;
674674 }
675675
676676 #else /* CONFIG_NUMA */
@@ -841,12 +841,12 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) · 164 unmodified lines
841841 }
842842
843843 /*
844− * Construct gfp mask to allocate from a specific node but do not direct reclaim
845− * or warn about failures. kswapd may still wake to reclaim in the background.
844+ * Construct gfp mask to allocate from a specific node but do not reclaim or
845+ * warn about failures.
846846 */
847847 static inline gfp_t gfp_exact_node(gfp_t flags)
848848 {
849− return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~__GFP_DIRECT_RECLAIM;
849+ return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
850850 }
851851 #endif
852852
@@ -1442,9 +1442,10 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, · 589 unmodified lines
14421442 */
14431443 static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
14441444 {
1445− const unsigned long nr_freed = (1 << cachep->gfporder);
1445+ int order = cachep->gfporder;
1446+ unsigned long nr_freed = (1 << order);
14461447
1447− kmemcheck_free_shadow(page, cachep->gfporder);
1448+ kmemcheck_free_shadow(page, order);
14481449
14491450 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
14501451 sub_zone_page_state(page_zone(page),
@@ -1461,7 +1462,8 @@ static void kmem_freepages(struct kmem_cache *cachep, struct page *page) · 10 unmodified lines
14611462
14621463 if (current->reclaim_state)
14631464 current->reclaim_state->reclaimed_slab += nr_freed;
1464− __free_kmem_pages(page, cachep->gfporder);
1465+ memcg_uncharge_slab(page, order, cachep);
1466+ __free_pages(page, order);
14651467 }
14661468
14671469 static void kmem_rcu_free(struct rcu_head *head)
@@ -1551,7 +1553,7 @@ static void dump_line(char *data, int offset, int limit) · 83 unmodified lines
15511553 unsigned char error = 0;
15521554 int bad_count = 0;
15531555
1554− printk(KERN_ERR "%03x: ", offset);
1556+ pr_err("%03x: ", offset);
15551557 for (i = 0; i < limit; i++) {
15561558 if (data[offset + i] != POISON_FREE) {
15571559 error = data[offset + i];
@@ -1564,13 +1566,11 @@ static void dump_line(char *data, int offset, int limit) · 6 unmodified lines
15641566 if (bad_count == 1) {
15651567 error ^= POISON_FREE;
15661568 if (!(error & (error - 1))) {
1567− printk(KERN_ERR "Single bit error detected. Probably "
1568− "bad RAM.\n");
1569+ pr_err("Single bit error detected. Probably bad RAM.\n");
15691570 #ifdef CONFIG_X86
1570− printk(KERN_ERR "Run memtest86+ or a similar memory "
1571− "test tool.\n");
1571+ pr_err("Run memtest86+ or a similar memory test tool.\n");
15721572 #else
1573− printk(KERN_ERR "Run a memory test tool.\n");
1573+ pr_err("Run a memory test tool.\n");
15741574 #endif
15751575 }
15761576 }
@@ -1585,13 +1585,13 @@ static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) · 8 unmodified lines
15851585 char *realobj;
15861586
15871587 if (cachep->flags & SLAB_RED_ZONE) {
1588− printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
1589− *dbg_redzone1(cachep, objp),
1590− *dbg_redzone2(cachep, objp));
1588+ pr_err("Redzone: 0x%llx/0x%llx\n",
1589+ *dbg_redzone1(cachep, objp),
1590+ *dbg_redzone2(cachep, objp));
15911591 }
15921592
15931593 if (cachep->flags & SLAB_STORE_USER) {
1594− printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
1594+ pr_err("Last user: [<%p>](%pSR)\n",
15951595 *dbg_userword(cachep, objp),
15961596 *dbg_userword(cachep, objp));
15971597 }
@@ -1627,9 +1627,9 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp) · 29 unmodified lines
16271627 /* Mismatch ! */
16281628 /* Print header */
16291629 if (lines == 0) {
1630− printk(KERN_ERR
1631− "Slab corruption (%s): %s start=%p, len=%d\n",
1632− print_tainted(), cachep->name, realobj, size);
1630+ pr_err("Slab corruption (%s): %s start=%p, len=%d\n",
1631+ print_tainted(), cachep->name,
1632+ realobj, size);
16331633 print_objinfo(cachep, objp, 0);
16341634 }
16351635 /* Hexdump the affected line */
@@ -1656,15 +1656,13 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp) · 20 unmodified lines
16561656 if (objnr) {
16571657 objp = index_to_obj(cachep, page, objnr - 1);
16581658 realobj = (char *)objp + obj_offset(cachep);
1659− printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
1660− realobj, size);
1659+ pr_err("Prev obj: start=%p, len=%d\n", realobj, size);
16611660 print_objinfo(cachep, objp, 2);
16621661 }
16631662 if (objnr + 1 < cachep->num) {
16641663 objp = index_to_obj(cachep, page, objnr + 1);
16651664 realobj = (char *)objp + obj_offset(cachep);
1666− printk(KERN_ERR "Next obj: start=%p, len=%d\n",
1667− realobj, size);
1665+ pr_err("Next obj: start=%p, len=%d\n", realobj, size);
16681666 print_objinfo(cachep, objp, 2);
16691667 }
16701668 }
@@ -1691,11 +1689,9 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, · 20 unmodified lines
16911689 }
16921690 if (cachep->flags & SLAB_RED_ZONE) {
16931691 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1694− slab_error(cachep, "start of a freed object "
1695− "was overwritten");
1692+ slab_error(cachep, "start of a freed object was overwritten");
16961693 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1697− slab_error(cachep, "end of a freed object "
1698− "was overwritten");
1694+ slab_error(cachep, "end of a freed object was overwritten");
16991695 }
17001696 }
17011697 }
@@ -2396,11 +2392,9 @@ static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page) · 694 unmodified lines
23962392
23972393 if (cachep->flags & SLAB_RED_ZONE) {
23982394 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2399− slab_error(cachep, "constructor overwrote the"
2400− " end of an object");
2395+ slab_error(cachep, "constructor overwrote the end of an object");
24012396 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2402− slab_error(cachep, "constructor overwrote the"
2403− " start of an object");
2397+ slab_error(cachep, "constructor overwrote the start of an object");
24042398 }
24052399 /* need to poison the objs? */
24062400 if (cachep->flags & SLAB_POISON) {
@@ -2467,8 +2461,8 @@ static void slab_put_obj(struct kmem_cache *cachep, · 60 unmodified lines
24672461 /* Verify double free bug */
24682462 for (i = page->active; i < cachep->num; i++) {
24692463 if (get_free_obj(page, i) == objnr) {
2470− printk(KERN_ERR "slab: double free detected in cache "
2471− "'%s', objp %p\n", cachep->name, objp);
2464+ pr_err("slab: double free detected in cache '%s', objp %p\n",
2465+ cachep->name, objp);
24722466 BUG();
24732467 }
24742468 }
@@ -2587,7 +2581,7 @@ failed: · 112 unmodified lines
25872581 static void kfree_debugcheck(const void *objp)
25882582 {
25892583 if (!virt_addr_valid(objp)) {
2590− printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
2584+ pr_err("kfree_debugcheck: out of range ptr %lxh\n",
25912585 (unsigned long)objp);
25922586 BUG();
25932587 }
@@ -2611,8 +2605,8 @@ static inline void verify_redzone_free(struct kmem_cache *cache, void *obj) · 17 unmodified lines
26112605 else
26122606 slab_error(cache, "memory outside object was overwritten");
26132607
2614− printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
2615− obj, redzone1, redzone2);
2608+ pr_err("%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
2609+ obj, redzone1, redzone2);
26162610 }
26172611
26182612 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
@@ -2899,12 +2893,10 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, · 280 unmodified lines
28992893 if (cachep->flags & SLAB_RED_ZONE) {
29002894 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
29012895 *dbg_redzone2(cachep, objp) != RED_INACTIVE) {
2902− slab_error(cachep, "double free, or memory outside"
2903− " object was overwritten");
2904− printk(KERN_ERR
2905− "%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
2906− objp, *dbg_redzone1(cachep, objp),
2907− *dbg_redzone2(cachep, objp));
2896+ slab_error(cachep, "double free, or memory outside object was overwritten");
2897+ pr_err("%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
2898+ objp, *dbg_redzone1(cachep, objp),
2899+ *dbg_redzone2(cachep, objp));
29082900 }
29092901 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
29102902 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
@@ -2915,7 +2907,7 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, · 4 unmodified lines
29152907 cachep->ctor(objp);
29162908 if (ARCH_SLAB_MINALIGN &&
29172909 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
2918− printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
2910+ pr_err("0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
29192911 objp, (int)ARCH_SLAB_MINALIGN);
29202912 }
29212913 return objp;
@@ -3842,7 +3834,7 @@ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) · 920 unmodified lines
38423834 skip_setup:
38433835 err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
38443836 if (err)
3845− printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
3837+ pr_err("enable_cpucache failed for %s, error %d\n",
38463838 cachep->name, -err);
38473839 return err;
38483840 }
@@ -3998,7 +3990,7 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) · 149 unmodified lines
39983990
39993991 name = cachep->name;
40003992 if (error)
4001− printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
3993+ pr_err("slab: cache %s error: %s\n", name, error);
40023994
40033995 sinfo->active_objs = active_objs;
40043996 sinfo->num_objs = num_objs;
@@ -4026,8 +4018,7 @@ void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep) · 21 unmodified lines
40264018 unsigned long node_frees = cachep->node_frees;
40274019 unsigned long overflows = cachep->node_overflow;
40284020
4029− seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
4030− "%4lu %4lu %4lu %4lu %4lu",
4021+ seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu %4lu",
40314022 allocs, high, grown,
40324023 reaped, errors, max_freeable, node_allocs,
40334024 node_frees, overflows);
mm/slab.h+28−2
@@ -246,12 +246,33 @@ static __always_inline int memcg_charge_slab(struct page *page, · 245 unmodified lines
246246 gfp_t gfp, int order,
247247 struct kmem_cache *s)
248248 {
249+ int ret;
250+
249251 if (!memcg_kmem_enabled())
250252 return 0;
251253 if (is_root_cache(s))
252254 return 0;
253− return __memcg_kmem_charge_memcg(page, gfp, order,
254− s->memcg_params.memcg);
255+
256+ ret = __memcg_kmem_charge_memcg(page, gfp, order,
257+ s->memcg_params.memcg);
258+ if (ret)
259+ return ret;
260+
261+ memcg_kmem_update_page_stat(page,
262+ (s->flags & SLAB_RECLAIM_ACCOUNT) ?
263+ MEMCG_SLAB_RECLAIMABLE : MEMCG_SLAB_UNRECLAIMABLE,
264+ 1 << order);
265+ return 0;
266+}
267+
268+static __always_inline void memcg_uncharge_slab(struct page *page, int order,
269+ struct kmem_cache *s)
270+{
271+ memcg_kmem_update_page_stat(page,
272+ (s->flags & SLAB_RECLAIM_ACCOUNT) ?
273+ MEMCG_SLAB_RECLAIMABLE : MEMCG_SLAB_UNRECLAIMABLE,
274+ -(1 << order));
275+ memcg_kmem_uncharge(page, order);
255276 }
256277
257278 extern void slab_init_memcg_params(struct kmem_cache *);
@@ -294,6 +315,11 @@ static inline int memcg_charge_slab(struct page *page, gfp_t gfp, int order, · 36 unmodified lines
294315 return 0;
295316 }
296317
318+static inline void memcg_uncharge_slab(struct page *page, int order,
319+ struct kmem_cache *s)
320+{
321+}
322+
297323 static inline void slab_init_memcg_params(struct kmem_cache *s)
298324 {
299325 }
mm/slab_common.c+6−8
@@ -442,7 +442,7 @@ out_unlock: · 441 unmodified lines
442442 panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n",
443443 name, err);
444444 else {
445− printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d",
445+ pr_warn("kmem_cache_create(%s) failed with error %d\n",
446446 name, err);
447447 dump_stack();
448448 }
@@ -510,7 +510,7 @@ void memcg_create_kmem_cache(struct mem_cgroup *memcg, · 61 unmodified lines
510510 * The memory cgroup could have been offlined while the cache
511511 * creation work was pending.
512512 */
513− if (!memcg_kmem_online(memcg))
513+ if (memcg->kmem_state != KMEM_ONLINE)
514514 goto out_unlock;
515515
516516 idx = memcg_cache_id(memcg);
@@ -726,8 +726,8 @@ void kmem_cache_destroy(struct kmem_cache *s) · 209 unmodified lines
726726 err = shutdown_cache(s, &release, &need_rcu_barrier);
727727
728728 if (err) {
729− pr_err("kmem_cache_destroy %s: "
730− "Slab cache still has objects\n", s->name);
729+ pr_err("kmem_cache_destroy %s: Slab cache still has objects\n",
730+ s->name);
731731 dump_stack();
732732 }
733733 out_unlock:
@@ -1047,13 +1047,11 @@ static void print_slabinfo_header(struct seq_file *m) · 313 unmodified lines
10471047 #else
10481048 seq_puts(m, "slabinfo - version: 2.1\n");
10491049 #endif
1050− seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
1051− "<objperslab> <pagesperslab>");
1050+ seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
10521051 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
10531052 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
10541053 #ifdef CONFIG_DEBUG_SLAB
1055− seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
1056− "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
1054+ seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
10571055 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
10581056 #endif
10591057 seq_putc(m, '\n');
mm/slub.c+15−16
@@ -254,11 +254,10 @@ static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) · 253 unmodified lines
254254 {
255255 void *p;
256256
257−#ifdef CONFIG_DEBUG_PAGEALLOC
257+ if (!debug_pagealloc_enabled())
258+ return get_freepointer(s, object);
259+
258260 probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
259−#else
260− p = get_freepointer(s, object);
261−#endif
262261 return p;
263262 }
264263
@@ -951,14 +950,14 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search) · 686 unmodified lines
951950 max_objects = MAX_OBJS_PER_PAGE;
952951
953952 if (page->objects != max_objects) {
954− slab_err(s, page, "Wrong number of objects. Found %d but "
955− "should be %d", page->objects, max_objects);
953+ slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
954+ page->objects, max_objects);
956955 page->objects = max_objects;
957956 slab_fix(s, "Number of objects adjusted.");
958957 }
959958 if (page->inuse != page->objects - nr) {
960− slab_err(s, page, "Wrong object count. Counter is %d but "
961− "counted were %d", page->inuse, page->objects - nr);
959+ slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
960+ page->inuse, page->objects - nr);
962961 page->inuse = page->objects - nr;
963962 slab_fix(s, "Object count adjusted.");
964963 }
@@ -1118,8 +1117,8 @@ static inline int free_consistency_checks(struct kmem_cache *s, · 153 unmodified lines
11181117
11191118 if (unlikely(s != page->slab_cache)) {
11201119 if (!PageSlab(page)) {
1121− slab_err(s, page, "Attempt to free object(0x%p) "
1122− "outside of slab", object);
1120+ slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
1121+ object);
11231122 } else if (!page->slab_cache) {
11241123 pr_err("SLUB <none>: no slab for object 0x%p.\n",
11251124 object);
@@ -1427,7 +1426,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) · 301 unmodified lines
14271426 */
14281427 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
14291428 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
1430− alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_DIRECT_RECLAIM;
1429+ alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
14311430
14321431 page = alloc_slab_page(s, alloc_gfp, node, oo);
14331432 if (unlikely(!page)) {
@@ -1540,7 +1539,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page) · 106 unmodified lines
15401539 page_mapcount_reset(page);
15411540 if (current->reclaim_state)
15421541 current->reclaim_state->reclaimed_slab += pages;
1543− __free_kmem_pages(page, order);
1542+ memcg_uncharge_slab(page, order, s);
1543+ __free_pages(page, order);
15441544 }
15451545
15461546 #define need_reserve_slab_rcu \
@@ -3439,10 +3439,9 @@ static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) · 1892 unmodified lines
34393439 free_kmem_cache_nodes(s);
34403440 error:
34413441 if (flags & SLAB_PANIC)
3442− panic("Cannot create slab %s size=%lu realsize=%u "
3443− "order=%u offset=%u flags=%lx\n",
3444− s->name, (unsigned long)s->size, s->size,
3445− oo_order(s->oo), s->offset, flags);
3442+ panic("Cannot create slab %s size=%lu realsize=%u order=%u offset=%u flags=%lx\n",
3443+ s->name, (unsigned long)s->size, s->size,
3444+ oo_order(s->oo), s->offset, flags);
34463445 return -EINVAL;
34473446 }
34483447
mm/sparse-vmemmap.c+4−4
@@ -166,8 +166,8 @@ void __meminit vmemmap_verify(pte_t *pte, int node, · 165 unmodified lines
166166 int actual_node = early_pfn_to_nid(pfn);
167167
168168 if (node_distance(actual_node, node) > LOCAL_DISTANCE)
169− printk(KERN_WARNING "[%lx-%lx] potential offnode "
170− "page_structs\n", start, end - 1);
169+ pr_warn("[%lx-%lx] potential offnode page_structs\n",
170+ start, end - 1);
171171 }
172172
173173 pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node)
@@ -292,8 +292,8 @@ void __init sparse_mem_maps_populate_node(struct page **map_map, · 118 unmodified lines
292292 if (map_map[pnum])
293293 continue;
294294 ms = __nr_to_section(pnum);
295− printk(KERN_ERR "%s: sparsemem memory map backing failed "
296− "some memory will not be available.\n", __func__);
295+ pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
296+ __func__);
297297 ms->section_mem_map = 0;
298298 }
299299
mm/sparse.c+9−12
@@ -313,9 +313,8 @@ static void __init check_usemap_section_nr(int nid, unsigned long *usemap) · 312 unmodified lines
313313
314314 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
315315 if (usemap_nid != nid) {
316− printk(KERN_INFO
317− "node %d must be removed before remove section %ld\n",
318− nid, usemap_snr);
316+ pr_info("node %d must be removed before remove section %ld\n",
317+ nid, usemap_snr);
319318 return;
320319 }
321320 /*
@@ -324,10 +323,8 @@ static void __init check_usemap_section_nr(int nid, unsigned long *usemap) · 2 unmodified lines
324323 * gather other removable sections for dynamic partitioning.
325324 * Just notify un-removable section's number here.
326325 */
327− printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr,
328− pgdat_snr, nid);
329− printk(KERN_CONT
330− " have a circular dependency on usemap and pgdat allocations\n");
326+ pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
327+ usemap_snr, pgdat_snr, nid);
331328 }
332329 #else
333330 static unsigned long * __init
@@ -355,7 +352,7 @@ static void __init sparse_early_usemaps_alloc_node(void *data, · 21 unmodified lines
355352 usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
356353 size * usemap_count);
357354 if (!usemap) {
358− printk(KERN_WARNING "%s: allocation failed\n", __func__);
355+ pr_warn("%s: allocation failed\n", __func__);
359356 return;
360357 }
361358
@@ -428,8 +425,8 @@ void __init sparse_mem_maps_populate_node(struct page **map_map, · 66 unmodified lines
428425 if (map_map[pnum])
429426 continue;
430427 ms = __nr_to_section(pnum);
431− printk(KERN_ERR "%s: sparsemem memory map backing failed "
432− "some memory will not be available.\n", __func__);
428+ pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
429+ __func__);
433430 ms->section_mem_map = 0;
434431 }
435432 }
@@ -456,8 +453,8 @@ static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) · 20 unmodified lines
456453 if (map)
457454 return map;
458455
459− printk(KERN_ERR "%s: sparsemem memory map backing failed "
460− "some memory will not be available.\n", __func__);
456+ pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
457+ __func__);
461458 ms->section_mem_map = 0;
462459 return NULL;
463460 }
mm/swap_cgroup.c+2−3
@@ -174,9 +174,8 @@ int swap_cgroup_swapon(int type, unsigned long max_pages) · 173 unmodified lines
174174
175175 return 0;
176176 nomem:
177− printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
178− printk(KERN_INFO
179− "swap_cgroup can be disabled by swapaccount=0 boot option\n");
177+ pr_info("couldn't allocate enough memory for swap_cgroup\n");
178+ pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
180179 return -ENOMEM;
181180 }
182181
mm/swapfile.c+1−2
@@ -2526,8 +2526,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) · 2525 unmodified lines
25262526 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
25272527 enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
25282528
2529− pr_info("Adding %uk swap on %s. "
2530− "Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2529+ pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
25312530 p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
25322531 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
25332532 (p->flags & SWP_SOLIDSTATE) ? "SS" : "",
mm/userfaultfd.c+1−2
@@ -230,8 +230,7 @@ retry: · 229 unmodified lines
230230 break;
231231 }
232232 if (unlikely(pmd_none(dst_pmdval)) &&
233− unlikely(__pte_alloc(dst_mm, dst_vma, dst_pmd,
234− dst_addr))) {
233+ unlikely(__pte_alloc(dst_mm, dst_pmd, dst_addr))) {
235234 err = -ENOMEM;
236235 break;
237236 }
mm/util.c+124−0
@@ -396,6 +396,13 @@ int __page_mapcount(struct page *page) · 395 unmodified lines
396396 }
397397 EXPORT_SYMBOL_GPL(__page_mapcount);
398398
399+int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;
400+int sysctl_overcommit_ratio __read_mostly = 50;
401+unsigned long sysctl_overcommit_kbytes __read_mostly;
402+int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
403+unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
404+unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
405+
399406 int overcommit_ratio_handler(struct ctl_table *table, int write,
400407 void __user *buffer, size_t *lenp,
401408 loff_t *ppos)
@@ -437,6 +444,123 @@ unsigned long vm_commit_limit(void) · 35 unmodified lines
437444 return allowed;
438445 }
439446
447+/*
448+ * Make sure vm_committed_as in one cacheline and not cacheline shared with
449+ * other variables. It can be updated by several CPUs frequently.
450+ */
451+struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
452+
453+/*
454+ * The global memory commitment made in the system can be a metric
455+ * that can be used to drive ballooning decisions when Linux is hosted
456+ * as a guest. On Hyper-V, the host implements a policy engine for dynamically
457+ * balancing memory across competing virtual machines that are hosted.
458+ * Several metrics drive this policy engine including the guest reported
459+ * memory commitment.
460+ */
461+unsigned long vm_memory_committed(void)
462+{
463+ return percpu_counter_read_positive(&vm_committed_as);
464+}
465+EXPORT_SYMBOL_GPL(vm_memory_committed);
466+
467+/*
468+ * Check that a process has enough memory to allocate a new virtual
469+ * mapping. 0 means there is enough memory for the allocation to
470+ * succeed and -ENOMEM implies there is not.
471+ *
472+ * We currently support three overcommit policies, which are set via the
473+ * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
474+ *
475+ * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
476+ * Additional code 2002 Jul 20 by Robert Love.
477+ *
478+ * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
479+ *
480+ * Note this is a helper function intended to be used by LSMs which
481+ * wish to use this logic.
482+ */
483+int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
484+{
485+ long free, allowed, reserve;
486+
487+ VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) <
488+ -(s64)vm_committed_as_batch * num_online_cpus(),
489+ "memory commitment underflow");
490+
491+ vm_acct_memory(pages);
492+
493+ /*
494+ * Sometimes we want to use more memory than we have
495+ */
496+ if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
497+ return 0;
498+
499+ if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
500+ free = global_page_state(NR_FREE_PAGES);
501+ free += global_page_state(NR_FILE_PAGES);
502+
503+ /*
504+ * shmem pages shouldn't be counted as free in this
505+ * case, they can't be purged, only swapped out, and
506+ * that won't affect the overall amount of available
507+ * memory in the system.
508+ */
509+ free -= global_page_state(NR_SHMEM);
510+
511+ free += get_nr_swap_pages();
512+
513+ /*
514+ * Any slabs which are created with the
515+ * SLAB_RECLAIM_ACCOUNT flag claim to have contents
516+ * which are reclaimable, under pressure. The dentry
517+ * cache and most inode caches should fall into this
518+ */
519+ free += global_page_state(NR_SLAB_RECLAIMABLE);
520+
521+ /*
522+ * Leave reserved pages. The pages are not for anonymous pages.
523+ */
524+ if (free <= totalreserve_pages)
525+ goto error;
526+ else
527+ free -= totalreserve_pages;
528+
529+ /*
530+ * Reserve some for root
531+ */
532+ if (!cap_sys_admin)
533+ free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
534+
535+ if (free > pages)
536+ return 0;
537+
538+ goto error;
539+ }
540+
541+ allowed = vm_commit_limit();
542+ /*
543+ * Reserve some for root
544+ */
545+ if (!cap_sys_admin)
546+ allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
547+
548+ /*
549+ * Don't let a single process grow so big a user can't recover
550+ */
551+ if (mm) {
552+ reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
553+ allowed -= min_t(long, mm->total_vm / 32, reserve);
554+ }
555+
556+ if (percpu_counter_read_positive(&vm_committed_as) < allowed)
557+ return 0;
558+error:
559+ vm_unacct_memory(pages);
560+
561+ return -ENOMEM;
562+}
563+
440564 /**
441565 * get_cmdline() - copy the cmdline value to a buffer.
442566 * @task: the task whose cmdline value to copy.
mm/vmalloc.c+15−16
@@ -469,8 +469,8 @@ overflow: · 468 unmodified lines
469469 goto retry;
470470 }
471471 if (printk_ratelimit())
472− pr_warn("vmap allocation for size %lu failed: "
473− "use vmalloc=<size> to increase size.\n", size);
472+ pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
473+ size);
474474 kfree(va);
475475 return ERR_PTR(-EBUSY);
476476 }
@@ -531,22 +531,21 @@ static void unmap_vmap_area(struct vmap_area *va) · 54 unmodified lines
531531 static void vmap_debug_free_range(unsigned long start, unsigned long end)
532532 {
533533 /*
534− * Unmap page tables and force a TLB flush immediately if
535− * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
536− * bugs similarly to those in linear kernel virtual address
537− * space after a page has been freed.
534+ * Unmap page tables and force a TLB flush immediately if pagealloc
535+ * debugging is enabled. This catches use after free bugs similarly to
536+ * those in linear kernel virtual address space after a page has been
537+ * freed.
538538 *
539− * All the lazy freeing logic is still retained, in order to
540− * minimise intrusiveness of this debugging feature.
539+ * All the lazy freeing logic is still retained, in order to minimise
540+ * intrusiveness of this debugging feature.
541541 *
542− * This is going to be *slow* (linear kernel virtual address
543− * debugging doesn't do a broadcast TLB flush so it is a lot
544− * faster).
542+ * This is going to be *slow* (linear kernel virtual address debugging
543+ * doesn't do a broadcast TLB flush so it is a lot faster).
545544 */
546−#ifdef CONFIG_DEBUG_PAGEALLOC
547− vunmap_page_range(start, end);
548− flush_tlb_kernel_range(start, end);
549−#endif
545+ if (debug_pagealloc_enabled()) {
546+ vunmap_page_range(start, end);
547+ flush_tlb_kernel_range(start, end);
548+ }
550549 }
551550
552551 /*
@@ -1086,7 +1085,7 @@ void vm_unmap_ram(const void *mem, unsigned int count) · 533 unmodified lines
10861085 BUG_ON(!addr);
10871086 BUG_ON(addr < VMALLOC_START);
10881087 BUG_ON(addr > VMALLOC_END);
1089− BUG_ON(!IS_ALIGNED(addr, PAGE_SIZE));
1088+ BUG_ON(!PAGE_ALIGNED(addr));
10901089
10911090 debug_check_no_locks_freed(mem, size);
10921091 vmap_debug_free_range(addr, addr+size);
mm/vmscan.c+61−108
@@ -382,9 +382,8 @@ static unsigned long do_shrink_slab(struct shrink_control *shrinkctl, · 381 unmodified lines
382382 *
383383 * @memcg specifies the memory cgroup to target. If it is not NULL,
384384 * only shrinkers with SHRINKER_MEMCG_AWARE set will be called to scan
385− * objects from the memory cgroup specified. Otherwise all shrinkers
386− * are called, and memcg aware shrinkers are supposed to scan the
387− * global list then.
385+ * objects from the memory cgroup specified. Otherwise, only unaware
386+ * shrinkers are called.
388387 *
389388 * @nr_scanned and @nr_eligible form a ratio that indicate how much of
390389 * the available objects should be scanned. Page reclaim for example
@@ -404,7 +403,7 @@ static unsigned long shrink_slab(gfp_t gfp_mask, int nid, · 13 unmodified lines
404403 struct shrinker *shrinker;
405404 unsigned long freed = 0;
406405
407− if (memcg && !memcg_kmem_online(memcg))
406+ if (memcg && (!memcg_kmem_enabled() || !mem_cgroup_online(memcg)))
408407 return 0;
409408
410409 if (nr_scanned == 0)
@@ -428,7 +427,13 @@ static unsigned long shrink_slab(gfp_t gfp_mask, int nid, · 17 unmodified lines
428427 .memcg = memcg,
429428 };
430429
431− if (memcg && !(shrinker->flags & SHRINKER_MEMCG_AWARE))
430+ /*
431+ * If kernel memory accounting is disabled, we ignore
432+ * SHRINKER_MEMCG_AWARE flag and call all shrinkers
433+ * passing NULL for memcg.
434+ */
435+ if (memcg_kmem_enabled() &&
436+ !!memcg != !!(shrinker->flags & SHRINKER_MEMCG_AWARE))
432437 continue;
433438
434439 if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
@@ -633,11 +638,11 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, · 198 unmodified lines
633638 * Note that if SetPageDirty is always performed via set_page_dirty,
634639 * and thus under tree_lock, then this ordering is not required.
635640 */
636− if (!page_freeze_refs(page, 2))
641+ if (!page_ref_freeze(page, 2))
637642 goto cannot_free;
638643 /* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
639644 if (unlikely(PageDirty(page))) {
640− page_unfreeze_refs(page, 2);
645+ page_ref_unfreeze(page, 2);
641646 goto cannot_free;
642647 }
643648
@@ -699,7 +704,7 @@ int remove_mapping(struct address_space *mapping, struct page *page) · 55 unmodified lines
699704 * drops the pagecache ref for us without requiring another
700705 * atomic operation.
701706 */
702− page_unfreeze_refs(page, 1);
707+ page_ref_unfreeze(page, 1);
703708 return 1;
704709 }
705710 return 0;
@@ -2968,18 +2973,23 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc) · 2262 unmodified lines
29682973 } while (memcg);
29692974 }
29702975
2971−static bool zone_balanced(struct zone *zone, int order,
2972− unsigned long balance_gap, int classzone_idx)
2976+static bool zone_balanced(struct zone *zone, int order, bool highorder,
2977+ unsigned long balance_gap, int classzone_idx)
29732978 {
2974− if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone) +
2975− balance_gap, classzone_idx))
2976− return false;
2979+ unsigned long mark = high_wmark_pages(zone) + balance_gap;
29772980
2978− if (IS_ENABLED(CONFIG_COMPACTION) && order && compaction_suitable(zone,
2979− order, 0, classzone_idx) == COMPACT_SKIPPED)
2980− return false;
2981+ /*
2982+ * When checking from pgdat_balanced(), kswapd should stop and sleep
2983+ * when it reaches the high order-0 watermark and let kcompactd take
2984+ * over. Other callers such as wakeup_kswapd() want to determine the
2985+ * true high-order watermark.
2986+ */
2987+ if (IS_ENABLED(CONFIG_COMPACTION) && !highorder) {
2988+ mark += (1UL << order);
2989+ order = 0;
2990+ }
29812991
2982− return true;
2992+ return zone_watermark_ok_safe(zone, order, mark, classzone_idx);
29832993 }
29842994
29852995 /*
@@ -3029,7 +3039,7 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx) · 43 unmodified lines
30293039 continue;
30303040 }
30313041
3032− if (zone_balanced(zone, order, 0, i))
3042+ if (zone_balanced(zone, order, false, 0, i))
30333043 balanced_pages += zone->managed_pages;
30343044 else if (!order)
30353045 return false;
@@ -3083,27 +3093,14 @@ static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, long remaining, · 47 unmodified lines
30833093 */
30843094 static bool kswapd_shrink_zone(struct zone *zone,
30853095 int classzone_idx,
3086− struct scan_control *sc,
3087− unsigned long *nr_attempted)
3096+ struct scan_control *sc)
30883097 {
3089− int testorder = sc->order;
30903098 unsigned long balance_gap;
30913099 bool lowmem_pressure;
30923100
30933101 /* Reclaim above the high watermark. */
30943102 sc->nr_to_reclaim = max(SWAP_CLUSTER_MAX, high_wmark_pages(zone));
30953103
3096− /*
3097− * Kswapd reclaims only single pages with compaction enabled. Trying
3098− * too hard to reclaim until contiguous free pages have become
3099− * available can hurt performance by evicting too much useful data
3100− * from memory. Do not reclaim more than needed for compaction.
3101− */
3102− if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
3103− compaction_suitable(zone, sc->order, 0, classzone_idx)
3104− != COMPACT_SKIPPED)
3105− testorder = 0;
3106−
31073104 /*
31083105 * We put equal pressure on every zone, unless one zone has way too
31093106 * many pages free already. The "too many pages" is defined as the
@@ -3118,15 +3115,12 @@ static bool kswapd_shrink_zone(struct zone *zone, · 8 unmodified lines
31183115 * reclaim is necessary
31193116 */
31203117 lowmem_pressure = (buffer_heads_over_limit && is_highmem(zone));
3121− if (!lowmem_pressure && zone_balanced(zone, testorder,
3118+ if (!lowmem_pressure && zone_balanced(zone, sc->order, false,
31223119 balance_gap, classzone_idx))
31233120 return true;
31243121
31253122 shrink_zone(zone, sc, zone_idx(zone) == classzone_idx);
31263123
3127− /* Account for the number of pages attempted to reclaim */
3128− *nr_attempted += sc->nr_to_reclaim;
3129−
31303124 clear_bit(ZONE_WRITEBACK, &zone->flags);
31313125
31323126 /*
@@ -3136,7 +3130,7 @@ static bool kswapd_shrink_zone(struct zone *zone, · 3 unmodified lines
31363130 * waits.
31373131 */
31383132 if (zone_reclaimable(zone) &&
3139− zone_balanced(zone, testorder, 0, classzone_idx)) {
3133+ zone_balanced(zone, sc->order, false, 0, classzone_idx)) {
31403134 clear_bit(ZONE_CONGESTED, &zone->flags);
31413135 clear_bit(ZONE_DIRTY, &zone->flags);
31423136 }
@@ -3148,7 +3142,7 @@ static bool kswapd_shrink_zone(struct zone *zone, · 5 unmodified lines
31483142 * For kswapd, balance_pgdat() will work across all this node's zones until
31493143 * they are all at high_wmark_pages(zone).
31503144 *
3151− * Returns the final order kswapd was reclaiming at
3145+ * Returns the highest zone idx kswapd was reclaiming at
31523146 *
31533147 * There is special handling here for zones which are full of pinned pages.
31543148 * This can happen if the pages are all mlocked, or if they are all used by
@@ -3165,8 +3159,7 @@ static bool kswapd_shrink_zone(struct zone *zone, · 10 unmodified lines
31653159 * interoperates with the page allocator fallback scheme to ensure that aging
31663160 * of pages is balanced across the zones.
31673161 */
3168−static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
3169− int *classzone_idx)
3162+static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
31703163 {
31713164 int i;
31723165 int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */
@@ -3183,9 +3176,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, · 10 unmodified lines
31833176 count_vm_event(PAGEOUTRUN);
31843177
31853178 do {
3186− unsigned long nr_attempted = 0;
31873179 bool raise_priority = true;
3188− bool pgdat_needs_compaction = (order > 0);
31893180
31903181 sc.nr_reclaimed = 0;
31913182
@@ -3220,7 +3211,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, · 28 unmodified lines
32203211 break;
32213212 }
32223213
3223− if (!zone_balanced(zone, order, 0, 0)) {
3214+ if (!zone_balanced(zone, order, false, 0, 0)) {
32243215 end_zone = i;
32253216 break;
32263217 } else {
@@ -3236,24 +3227,6 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, · 9 unmodified lines
32363227 if (i < 0)
32373228 goto out;
32383229
3239− for (i = 0; i <= end_zone; i++) {
3240− struct zone *zone = pgdat->node_zones + i;
3241−
3242− if (!populated_zone(zone))
3243− continue;
3244−
3245− /*
3246− * If any zone is currently balanced then kswapd will
3247− * not call compaction as it is expected that the
3248− * necessary pages are already available.
3249− */
3250− if (pgdat_needs_compaction &&
3251− zone_watermark_ok(zone, order,
3252− low_wmark_pages(zone),
3253− *classzone_idx, 0))
3254− pgdat_needs_compaction = false;
3255− }
3256−
32573230 /*
32583231 * If we're getting trouble reclaiming, start doing writepage
32593232 * even in laptop mode.
@@ -3297,8 +3270,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, · 37 unmodified lines
32973270 * that that high watermark would be met at 100%
32983271 * efficiency.
32993272 */
3300− if (kswapd_shrink_zone(zone, end_zone,
3301− &sc, &nr_attempted))
3273+ if (kswapd_shrink_zone(zone, end_zone, &sc))
33023274 raise_priority = false;
33033275 }
33043276
@@ -3311,28 +3283,10 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, · 6 unmodified lines
33113283 pfmemalloc_watermark_ok(pgdat))
33123284 wake_up_all(&pgdat->pfmemalloc_wait);
33133285
3314− /*
3315− * Fragmentation may mean that the system cannot be rebalanced
3316− * for high-order allocations in all zones. If twice the
3317− * allocation size has been reclaimed and the zones are still
3318− * not balanced then recheck the watermarks at order-0 to
3319− * prevent kswapd reclaiming excessively. Assume that a
3320− * process requested a high-order can direct reclaim/compact.
3321− */
3322− if (order && sc.nr_reclaimed >= 2UL << order)
3323− order = sc.order = 0;
3324−
33253286 /* Check if kswapd should be suspending */
33263287 if (try_to_freeze() || kthread_should_stop())
33273288 break;
33283289
3329− /*
3330− * Compact if necessary and kswapd is reclaiming at least the
3331− * high watermark number of pages as requsted
3332− */
3333− if (pgdat_needs_compaction && sc.nr_reclaimed > nr_attempted)
3334− compact_pgdat(pgdat, order);
3335−
33363290 /*
33373291 * Raise priority if scanning rate is too low or there was no
33383292 * progress in reclaiming pages
@@ -3340,20 +3294,18 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, · 1 unmodified line
33403294 if (raise_priority || !sc.nr_reclaimed)
33413295 sc.priority--;
33423296 } while (sc.priority >= 1 &&
3343− !pgdat_balanced(pgdat, order, *classzone_idx));
3297+ !pgdat_balanced(pgdat, order, classzone_idx));
33443298
33453299 out:
33463300 /*
3347− * Return the order we were reclaiming at so prepare_kswapd_sleep()
3348− * makes a decision on the order we were last reclaiming at. However,
3349− * if another caller entered the allocator slow path while kswapd
3350− * was awake, order will remain at the higher level
3301+ * Return the highest zone idx we were reclaiming at so
3302+ * prepare_kswapd_sleep() makes the same decisions as here.
33513303 */
3352− *classzone_idx = end_zone;
3353− return order;
3304+ return end_zone;
33543305 }
33553306
3356−static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx)
3307+static void kswapd_try_to_sleep(pg_data_t *pgdat, int order,
3308+ int classzone_idx, int balanced_classzone_idx)
33573309 {
33583310 long remaining = 0;
33593311 DEFINE_WAIT(wait);
@@ -3364,7 +3316,8 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx) · 4 unmodified lines
33643316 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
33653317
33663318 /* Try to sleep for a short interval */
3367− if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
3319+ if (prepare_kswapd_sleep(pgdat, order, remaining,
3320+ balanced_classzone_idx)) {
33683321 remaining = schedule_timeout(HZ/10);
33693322 finish_wait(&pgdat->kswapd_wait, &wait);
33703323 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
@@ -3374,7 +3327,8 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx) · 3 unmodified lines
33743327 * After a short sleep, check if it was a premature sleep. If not, then
33753328 * go fully to sleep until explicitly woken up.
33763329 */
3377− if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
3330+ if (prepare_kswapd_sleep(pgdat, order, remaining,
3331+ balanced_classzone_idx)) {
33783332 trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
33793333
33803334 /*
@@ -3395,6 +3349,12 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx) · 14 unmodified lines
33953349 */
33963350 reset_isolation_suitable(pgdat);
33973351
3352+ /*
3353+ * We have freed the memory, now we should compact it to make
3354+ * allocation of the requested order possible.
3355+ */
3356+ wakeup_kcompactd(pgdat, order, classzone_idx);
3357+
33983358 if (!kthread_should_stop())
33993359 schedule();
34003360
@@ -3424,7 +3384,6 @@ static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx) · 23 unmodified lines
34243384 static int kswapd(void *p)
34253385 {
34263386 unsigned long order, new_order;
3427− unsigned balanced_order;
34283387 int classzone_idx, new_classzone_idx;
34293388 int balanced_classzone_idx;
34303389 pg_data_t *pgdat = (pg_data_t*)p;
@@ -3457,24 +3416,19 @@ static int kswapd(void *p) · 26 unmodified lines
34573416 set_freezable();
34583417
34593418 order = new_order = 0;
3460− balanced_order = 0;
34613419 classzone_idx = new_classzone_idx = pgdat->nr_zones - 1;
34623420 balanced_classzone_idx = classzone_idx;
34633421 for ( ; ; ) {
34643422 bool ret;
34653423
34663424 /*
3467− * If the last balance_pgdat was unsuccessful it's unlikely a
3468− * new request of a similar or harder type will succeed soon
3469− * so consider going to sleep on the basis we reclaimed at
3425+ * While we were reclaiming, there might have been another
3426+ * wakeup, so check the values.
34703427 */
3471− if (balanced_classzone_idx >= new_classzone_idx &&
3472− balanced_order == new_order) {
3473− new_order = pgdat->kswapd_max_order;
3474− new_classzone_idx = pgdat->classzone_idx;
3475− pgdat->kswapd_max_order = 0;
3476− pgdat->classzone_idx = pgdat->nr_zones - 1;
3477− }
3428+ new_order = pgdat->kswapd_max_order;
3429+ new_classzone_idx = pgdat->classzone_idx;
3430+ pgdat->kswapd_max_order = 0;
3431+ pgdat->classzone_idx = pgdat->nr_zones - 1;
34783432
34793433 if (order < new_order || classzone_idx > new_classzone_idx) {
34803434 /*
@@ -3484,7 +3438,7 @@ static int kswapd(void *p) · 3 unmodified lines
34843438 order = new_order;
34853439 classzone_idx = new_classzone_idx;
34863440 } else {
3487− kswapd_try_to_sleep(pgdat, balanced_order,
3441+ kswapd_try_to_sleep(pgdat, order, classzone_idx,
34883442 balanced_classzone_idx);
34893443 order = pgdat->kswapd_max_order;
34903444 classzone_idx = pgdat->classzone_idx;
@@ -3504,9 +3458,8 @@ static int kswapd(void *p) · 13 unmodified lines
35043458 */
35053459 if (!ret) {
35063460 trace_mm_vmscan_kswapd_wake(pgdat->node_id, order);
3507− balanced_classzone_idx = classzone_idx;
3508− balanced_order = balance_pgdat(pgdat, order,
3509− &balanced_classzone_idx);
3461+ balanced_classzone_idx = balance_pgdat(pgdat, order,
3462+ classzone_idx);
35103463 }
35113464 }
35123465
@@ -3536,7 +3489,7 @@ void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx) · 23 unmodified lines
35363489 }
35373490 if (!waitqueue_active(&pgdat->kswapd_wait))
35383491 return;
3539− if (zone_balanced(zone, order, 0, 0))
3492+ if (zone_balanced(zone, order, true, 0, 0))
35403493 return;
35413494
35423495 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
mm/vmstat.c+2−0
@@ -826,6 +826,7 @@ const char * const vmstat_text[] = { · 825 unmodified lines
826826 "compact_stall",
827827 "compact_fail",
828828 "compact_success",
829+ "compact_daemon_wake",
829830 #endif
830831
831832 #ifdef CONFIG_HUGETLB_PAGE
@@ -847,6 +848,7 @@ const char * const vmstat_text[] = { · 15 unmodified lines
847848 "thp_collapse_alloc_failed",
848849 "thp_split_page",
849850 "thp_split_page_failed",
851+ "thp_deferred_split_page",
850852 "thp_split_pmd",
851853 "thp_zero_page_alloc",
852854 "thp_zero_page_alloc_failed",
mm/workingset.c+8−2
@@ -349,7 +349,13 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 348 unmodified lines
349349 shadow_nodes = list_lru_shrink_count(&workingset_shadow_nodes, sc);
350350 local_irq_enable();
351351
352− pages = node_present_pages(sc->nid);
352+ if (memcg_kmem_enabled())
353+ pages = mem_cgroup_node_nr_lru_pages(sc->memcg, sc->nid,
354+ LRU_ALL_FILE);
355+ else
356+ pages = node_page_state(sc->nid, NR_ACTIVE_FILE) +
357+ node_page_state(sc->nid, NR_INACTIVE_FILE);
358+
353359 /*
354360 * Active cache pages are limited to 50% of memory, and shadow
355361 * entries that represent a refault distance bigger than that
@@ -458,7 +464,7 @@ static struct shrinker workingset_shadow_shrinker = { · 102 unmodified lines
458464 .count_objects = count_shadow_nodes,
459465 .scan_objects = scan_shadow_nodes,
460466 .seeks = DEFAULT_SEEKS,
461− .flags = SHRINKER_NUMA_AWARE,
467+ .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE,
462468 };
463469
464470 /*
mm/zsmalloc.c+16−13
@@ -281,7 +281,6 @@ struct mapping_area { · 280 unmodified lines
281281 #endif
282282 char *vm_addr; /* address of kmap_atomic()'ed pages */
283283 enum zs_mapmode vm_mm; /* mapping mode */
284− bool huge;
285284 };
286285
287286 static int create_handle_cache(struct zs_pool *pool)
@@ -495,6 +494,8 @@ static void __exit zs_stat_exit(void) · 207 unmodified lines
495494 debugfs_remove_recursive(zs_stat_root);
496495 }
497496
497+static unsigned long zs_can_compact(struct size_class *class);
498+
498499 static int zs_stats_size_show(struct seq_file *s, void *v)
499500 {
500501 int i;
@@ -502,14 +503,15 @@ static int zs_stats_size_show(struct seq_file *s, void *v) · 1 unmodified line
502503 struct size_class *class;
503504 int objs_per_zspage;
504505 unsigned long class_almost_full, class_almost_empty;
505− unsigned long obj_allocated, obj_used, pages_used;
506+ unsigned long obj_allocated, obj_used, pages_used, freeable;
506507 unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
507508 unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
509+ unsigned long total_freeable = 0;
508510
509− seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s\n",
511+ seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
510512 "class", "size", "almost_full", "almost_empty",
511513 "obj_allocated", "obj_used", "pages_used",
512− "pages_per_zspage");
514+ "pages_per_zspage", "freeable");
513515
514516 for (i = 0; i < zs_size_classes; i++) {
515517 class = pool->size_class[i];
@@ -522,6 +524,7 @@ static int zs_stats_size_show(struct seq_file *s, void *v) · 6 unmodified lines
522524 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
523525 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
524526 obj_used = zs_stat_get(class, OBJ_USED);
527+ freeable = zs_can_compact(class);
525528 spin_unlock(&class->lock);
526529
527530 objs_per_zspage = get_maxobj_per_zspage(class->size,
@@ -529,23 +532,25 @@ static int zs_stats_size_show(struct seq_file *s, void *v) · 1 unmodified line
529532 pages_used = obj_allocated / objs_per_zspage *
530533 class->pages_per_zspage;
531534
532− seq_printf(s, " %5u %5u %11lu %12lu %13lu %10lu %10lu %16d\n",
535+ seq_printf(s, " %5u %5u %11lu %12lu %13lu"
536+ " %10lu %10lu %16d %8lu\n",
533537 i, class->size, class_almost_full, class_almost_empty,
534538 obj_allocated, obj_used, pages_used,
535− class->pages_per_zspage);
539+ class->pages_per_zspage, freeable);
536540
537541 total_class_almost_full += class_almost_full;
538542 total_class_almost_empty += class_almost_empty;
539543 total_objs += obj_allocated;
540544 total_used_objs += obj_used;
541545 total_pages += pages_used;
546+ total_freeable += freeable;
542547 }
543548
544549 seq_puts(s, "\n");
545− seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu\n",
550+ seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
546551 "Total", "", total_class_almost_full,
547552 total_class_almost_empty, total_objs,
548− total_used_objs, total_pages);
553+ total_used_objs, total_pages, "", total_freeable);
549554
550555 return 0;
551556 }
@@ -1127,11 +1132,9 @@ static void __zs_unmap_object(struct mapping_area *area, · 575 unmodified lines
11271132 goto out;
11281133
11291134 buf = area->vm_buf;
1130− if (!area->huge) {
1131− buf = buf + ZS_HANDLE_SIZE;
1132− size -= ZS_HANDLE_SIZE;
1133− off += ZS_HANDLE_SIZE;
1134− }
1135+ buf = buf + ZS_HANDLE_SIZE;
1136+ size -= ZS_HANDLE_SIZE;
1137+ off += ZS_HANDLE_SIZE;
11351138
11361139 sizes[0] = PAGE_SIZE - off;
11371140 sizes[1] = size - sizes[0];
net/core/sock.c+1−1
@@ -1903,7 +1903,7 @@ EXPORT_SYMBOL(sock_cmsg_send); · 1902 unmodified lines
19031903 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
19041904 {
19051905 if (pfrag->page) {
1906− if (atomic_read(&pfrag->page->_count) == 1) {
1906+ if (page_ref_count(pfrag->page) == 1) {
19071907 pfrag->offset = 0;
19081908 return true;
19091909 }
sound/drivers/pcsp/pcsp.c+5−4
@@ -14,6 +14,7 @@ · 13 unmodified lines
1414 #include <linux/input.h>
1515 #include <linux/delay.h>
1616 #include <linux/bitops.h>
17+#include <linux/mm.h>
1718 #include "pcsp_input.h"
1819 #include "pcsp.h"
1920
@@ -148,11 +149,11 @@ static int alsa_card_pcsp_init(struct device *dev) · 128 unmodified lines
148149 return err;
149150 }
150151
151−#ifdef CONFIG_DEBUG_PAGEALLOC
152152 /* Well, CONFIG_DEBUG_PAGEALLOC makes the sound horrible. Lets alert */
153− printk(KERN_WARNING "PCSP: CONFIG_DEBUG_PAGEALLOC is enabled, "
154− "which may make the sound noisy.\n");
155−#endif
153+ if (debug_pagealloc_enabled()) {
154+ printk(KERN_WARNING "PCSP: CONFIG_DEBUG_PAGEALLOC is enabled, "
155+ "which may make the sound noisy.\n");
156+ }
156157
157158 return 0;
158159 }
tools/testing/radix-tree/.gitignore+2−0
@@ -0,0 +1,2 @@
1+main
2+radix-tree.c
tools/testing/radix-tree/Makefile+19−0
@@ -0,0 +1,19 @@
1+
2+CFLAGS += -I. -g -Wall -D_LGPL_SOURCE
3+LDFLAGS += -lpthread -lurcu
4+TARGETS = main
5+OFILES = main.o radix-tree.o linux.o test.o tag_check.o find_next_bit.o \
6+ regression1.o regression2.o regression3.o
7+
8+targets: $(TARGETS)
9+
10+main: $(OFILES)
11+ $(CC) $(CFLAGS) $(LDFLAGS) $(OFILES) -o main
12+
13+clean:
14+ $(RM) -f $(TARGETS) *.o radix-tree.c
15+
16+$(OFILES): *.h */*.h
17+
18+radix-tree.c: ../../../lib/radix-tree.c
19+ sed -e 's/^static //' -e 's/__always_inline //' -e 's/inline //' < $< > $@
tools/testing/radix-tree/find_next_bit.c+57−0
@@ -0,0 +1,57 @@
1+/* find_next_bit.c: fallback find next bit implementation
2+ *
3+ * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
4+ * Written by David Howells (dhowells@redhat.com)
5+ *
6+ * This program is free software; you can redistribute it and/or
7+ * modify it under the terms of the GNU General Public License
8+ * as published by the Free Software Foundation; either version
9+ * 2 of the License, or (at your option) any later version.
10+ */
11+
12+#include <linux/types.h>
13+#include <linux/bitops.h>
14+
15+#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
16+
17+/*
18+ * Find the next set bit in a memory region.
19+ */
20+unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
21+ unsigned long offset)
22+{
23+ const unsigned long *p = addr + BITOP_WORD(offset);
24+ unsigned long result = offset & ~(BITS_PER_LONG-1);
25+ unsigned long tmp;
26+
27+ if (offset >= size)
28+ return size;
29+ size -= result;
30+ offset %= BITS_PER_LONG;
31+ if (offset) {
32+ tmp = *(p++);
33+ tmp &= (~0UL << offset);
34+ if (size < BITS_PER_LONG)
35+ goto found_first;
36+ if (tmp)
37+ goto found_middle;
38+ size -= BITS_PER_LONG;
39+ result += BITS_PER_LONG;
40+ }
41+ while (size & ~(BITS_PER_LONG-1)) {
42+ if ((tmp = *(p++)))
43+ goto found_middle;
44+ result += BITS_PER_LONG;
45+ size -= BITS_PER_LONG;
46+ }
47+ if (!size)
48+ return result;
49+ tmp = *p;
50+
51+found_first:
52+ tmp &= (~0UL >> (BITS_PER_LONG - size));
53+ if (tmp == 0UL) /* Are any bits set? */
54+ return result + size; /* Nope. */
55+found_middle:
56+ return result + __ffs(tmp);
57+}
tools/testing/radix-tree/linux.c+60−0
@@ -0,0 +1,60 @@
1+#include <stdlib.h>
2+#include <string.h>
3+#include <malloc.h>
4+#include <unistd.h>
5+#include <assert.h>
6+
7+#include <linux/mempool.h>
8+#include <linux/slab.h>
9+#include <urcu/uatomic.h>
10+
11+int nr_allocated;
12+
13+void *mempool_alloc(mempool_t *pool, int gfp_mask)
14+{
15+ return pool->alloc(gfp_mask, pool->data);
16+}
17+
18+void mempool_free(void *element, mempool_t *pool)
19+{
20+ pool->free(element, pool->data);
21+}
22+
23+mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
24+ mempool_free_t *free_fn, void *pool_data)
25+{
26+ mempool_t *ret = malloc(sizeof(*ret));
27+
28+ ret->alloc = alloc_fn;
29+ ret->free = free_fn;
30+ ret->data = pool_data;
31+ return ret;
32+}
33+
34+void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
35+{
36+ void *ret = malloc(cachep->size);
37+ if (cachep->ctor)
38+ cachep->ctor(ret);
39+ uatomic_inc(&nr_allocated);
40+ return ret;
41+}
42+
43+void kmem_cache_free(struct kmem_cache *cachep, void *objp)
44+{
45+ assert(objp);
46+ uatomic_dec(&nr_allocated);
47+ memset(objp, 0, cachep->size);
48+ free(objp);
49+}
50+
51+struct kmem_cache *
52+kmem_cache_create(const char *name, size_t size, size_t offset,
53+ unsigned long flags, void (*ctor)(void *))
54+{
55+ struct kmem_cache *ret = malloc(sizeof(*ret));
56+
57+ ret->size = size;
58+ ret->ctor = ctor;
59+ return ret;
60+}
tools/testing/radix-tree/linux/bitops.h+150−0
@@ -0,0 +1,150 @@
1+#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
2+#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
3+
4+#include <linux/types.h>
5+
6+#define BITOP_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
7+#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
8+
9+/**
10+ * __set_bit - Set a bit in memory
11+ * @nr: the bit to set
12+ * @addr: the address to start counting from
13+ *
14+ * Unlike set_bit(), this function is non-atomic and may be reordered.
15+ * If it's called on the same region of memory simultaneously, the effect
16+ * may be that only one operation succeeds.
17+ */
18+static inline void __set_bit(int nr, volatile unsigned long *addr)
19+{
20+ unsigned long mask = BITOP_MASK(nr);
21+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
22+
23+ *p |= mask;
24+}
25+
26+static inline void __clear_bit(int nr, volatile unsigned long *addr)
27+{
28+ unsigned long mask = BITOP_MASK(nr);
29+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
30+
31+ *p &= ~mask;
32+}
33+
34+/**
35+ * __change_bit - Toggle a bit in memory
36+ * @nr: the bit to change
37+ * @addr: the address to start counting from
38+ *
39+ * Unlike change_bit(), this function is non-atomic and may be reordered.
40+ * If it's called on the same region of memory simultaneously, the effect
41+ * may be that only one operation succeeds.
42+ */
43+static inline void __change_bit(int nr, volatile unsigned long *addr)
44+{
45+ unsigned long mask = BITOP_MASK(nr);
46+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
47+
48+ *p ^= mask;
49+}
50+
51+/**
52+ * __test_and_set_bit - Set a bit and return its old value
53+ * @nr: Bit to set
54+ * @addr: Address to count from
55+ *
56+ * This operation is non-atomic and can be reordered.
57+ * If two examples of this operation race, one can appear to succeed
58+ * but actually fail. You must protect multiple accesses with a lock.
59+ */
60+static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
61+{
62+ unsigned long mask = BITOP_MASK(nr);
63+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
64+ unsigned long old = *p;
65+
66+ *p = old | mask;
67+ return (old & mask) != 0;
68+}
69+
70+/**
71+ * __test_and_clear_bit - Clear a bit and return its old value
72+ * @nr: Bit to clear
73+ * @addr: Address to count from
74+ *
75+ * This operation is non-atomic and can be reordered.
76+ * If two examples of this operation race, one can appear to succeed
77+ * but actually fail. You must protect multiple accesses with a lock.
78+ */
79+static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
80+{
81+ unsigned long mask = BITOP_MASK(nr);
82+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
83+ unsigned long old = *p;
84+
85+ *p = old & ~mask;
86+ return (old & mask) != 0;
87+}
88+
89+/* WARNING: non atomic and it can be reordered! */
90+static inline int __test_and_change_bit(int nr,
91+ volatile unsigned long *addr)
92+{
93+ unsigned long mask = BITOP_MASK(nr);
94+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
95+ unsigned long old = *p;
96+
97+ *p = old ^ mask;
98+ return (old & mask) != 0;
99+}
100+
101+/**
102+ * test_bit - Determine whether a bit is set
103+ * @nr: bit number to test
104+ * @addr: Address to start counting from
105+ */
106+static inline int test_bit(int nr, const volatile unsigned long *addr)
107+{
108+ return 1UL & (addr[BITOP_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
109+}
110+
111+/**
112+ * __ffs - find first bit in word.
113+ * @word: The word to search
114+ *
115+ * Undefined if no bit exists, so code should check against 0 first.
116+ */
117+static inline unsigned long __ffs(unsigned long word)
118+{
119+ int num = 0;
120+
121+ if ((word & 0xffffffff) == 0) {
122+ num += 32;
123+ word >>= 32;
124+ }
125+ if ((word & 0xffff) == 0) {
126+ num += 16;
127+ word >>= 16;
128+ }
129+ if ((word & 0xff) == 0) {
130+ num += 8;
131+ word >>= 8;
132+ }
133+ if ((word & 0xf) == 0) {
134+ num += 4;
135+ word >>= 4;
136+ }
137+ if ((word & 0x3) == 0) {
138+ num += 2;
139+ word >>= 2;
140+ }
141+ if ((word & 0x1) == 0)
142+ num += 1;
143+ return num;
144+}
145+
146+unsigned long find_next_bit(const unsigned long *addr,
147+ unsigned long size,
148+ unsigned long offset);
149+
150+#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */
tools/testing/radix-tree/linux/bitops/__ffs.h+43−0
@@ -0,0 +1,43 @@
1+#ifndef _ASM_GENERIC_BITOPS___FFS_H_
2+#define _ASM_GENERIC_BITOPS___FFS_H_
3+
4+#include <asm/types.h>
5+
6+/**
7+ * __ffs - find first bit in word.
8+ * @word: The word to search
9+ *
10+ * Undefined if no bit exists, so code should check against 0 first.
11+ */
12+static inline unsigned long __ffs(unsigned long word)
13+{
14+ int num = 0;
15+
16+#if BITS_PER_LONG == 64
17+ if ((word & 0xffffffff) == 0) {
18+ num += 32;
19+ word >>= 32;
20+ }
21+#endif
22+ if ((word & 0xffff) == 0) {
23+ num += 16;
24+ word >>= 16;
25+ }
26+ if ((word & 0xff) == 0) {
27+ num += 8;
28+ word >>= 8;
29+ }
30+ if ((word & 0xf) == 0) {
31+ num += 4;
32+ word >>= 4;
33+ }
34+ if ((word & 0x3) == 0) {
35+ num += 2;
36+ word >>= 2;
37+ }
38+ if ((word & 0x1) == 0)
39+ num += 1;
40+ return num;
41+}
42+
43+#endif /* _ASM_GENERIC_BITOPS___FFS_H_ */
tools/testing/radix-tree/linux/bitops/ffs.h+41−0
@@ -0,0 +1,41 @@
1+#ifndef _ASM_GENERIC_BITOPS_FFS_H_
2+#define _ASM_GENERIC_BITOPS_FFS_H_
3+
4+/**
5+ * ffs - find first bit set
6+ * @x: the word to search
7+ *
8+ * This is defined the same way as
9+ * the libc and compiler builtin ffs routines, therefore
10+ * differs in spirit from the above ffz (man ffs).
11+ */
12+static inline int ffs(int x)
13+{
14+ int r = 1;
15+
16+ if (!x)
17+ return 0;
18+ if (!(x & 0xffff)) {
19+ x >>= 16;
20+ r += 16;
21+ }
22+ if (!(x & 0xff)) {
23+ x >>= 8;
24+ r += 8;
25+ }
26+ if (!(x & 0xf)) {
27+ x >>= 4;
28+ r += 4;
29+ }
30+ if (!(x & 3)) {
31+ x >>= 2;
32+ r += 2;
33+ }
34+ if (!(x & 1)) {
35+ x >>= 1;
36+ r += 1;
37+ }
38+ return r;
39+}
40+
41+#endif /* _ASM_GENERIC_BITOPS_FFS_H_ */
tools/testing/radix-tree/linux/bitops/ffz.h+12−0
@@ -0,0 +1,12 @@
1+#ifndef _ASM_GENERIC_BITOPS_FFZ_H_
2+#define _ASM_GENERIC_BITOPS_FFZ_H_
3+
4+/*
5+ * ffz - find first zero in word.
6+ * @word: The word to search
7+ *
8+ * Undefined if no zero exists, so code should check against ~0UL first.
9+ */
10+#define ffz(x) __ffs(~(x))
11+
12+#endif /* _ASM_GENERIC_BITOPS_FFZ_H_ */
tools/testing/radix-tree/linux/bitops/find.h+13−0
@@ -0,0 +1,13 @@
1+#ifndef _ASM_GENERIC_BITOPS_FIND_H_
2+#define _ASM_GENERIC_BITOPS_FIND_H_
3+
4+extern unsigned long find_next_bit(const unsigned long *addr, unsigned long
5+ size, unsigned long offset);
6+
7+extern unsigned long find_next_zero_bit(const unsigned long *addr, unsigned
8+ long size, unsigned long offset);
9+
10+#define find_first_bit(addr, size) find_next_bit((addr), (size), 0)
11+#define find_first_zero_bit(addr, size) find_next_zero_bit((addr), (size), 0)
12+
13+#endif /*_ASM_GENERIC_BITOPS_FIND_H_ */
tools/testing/radix-tree/linux/bitops/fls.h+41−0
@@ -0,0 +1,41 @@
1+#ifndef _ASM_GENERIC_BITOPS_FLS_H_
2+#define _ASM_GENERIC_BITOPS_FLS_H_
3+
4+/**
5+ * fls - find last (most-significant) bit set
6+ * @x: the word to search
7+ *
8+ * This is defined the same way as ffs.
9+ * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
10+ */
11+
12+static inline int fls(int x)
13+{
14+ int r = 32;
15+
16+ if (!x)
17+ return 0;
18+ if (!(x & 0xffff0000u)) {
19+ x <<= 16;
20+ r -= 16;
21+ }
22+ if (!(x & 0xff000000u)) {
23+ x <<= 8;
24+ r -= 8;
25+ }
26+ if (!(x & 0xf0000000u)) {
27+ x <<= 4;
28+ r -= 4;
29+ }
30+ if (!(x & 0xc0000000u)) {
31+ x <<= 2;
32+ r -= 2;
33+ }
34+ if (!(x & 0x80000000u)) {
35+ x <<= 1;
36+ r -= 1;
37+ }
38+ return r;
39+}
40+
41+#endif /* _ASM_GENERIC_BITOPS_FLS_H_ */
tools/testing/radix-tree/linux/bitops/fls64.h+14−0
@@ -0,0 +1,14 @@
1+#ifndef _ASM_GENERIC_BITOPS_FLS64_H_
2+#define _ASM_GENERIC_BITOPS_FLS64_H_
3+
4+#include <asm/types.h>
5+
6+static inline int fls64(__u64 x)
7+{
8+ __u32 h = x >> 32;
9+ if (h)
10+ return fls(h) + 32;
11+ return fls(x);
12+}
13+
14+#endif /* _ASM_GENERIC_BITOPS_FLS64_H_ */
tools/testing/radix-tree/linux/bitops/hweight.h+11−0
@@ -0,0 +1,11 @@
1+#ifndef _ASM_GENERIC_BITOPS_HWEIGHT_H_
2+#define _ASM_GENERIC_BITOPS_HWEIGHT_H_
3+
4+#include <asm/types.h>
5+
6+extern unsigned int hweight32(unsigned int w);
7+extern unsigned int hweight16(unsigned int w);
8+extern unsigned int hweight8(unsigned int w);
9+extern unsigned long hweight64(__u64 w);
10+
11+#endif /* _ASM_GENERIC_BITOPS_HWEIGHT_H_ */
tools/testing/radix-tree/linux/bitops/le.h+53−0
@@ -0,0 +1,53 @@
1+#ifndef _ASM_GENERIC_BITOPS_LE_H_
2+#define _ASM_GENERIC_BITOPS_LE_H_
3+
4+#include <asm/types.h>
5+#include <asm/byteorder.h>
6+
7+#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
8+#define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7)
9+
10+#if defined(__LITTLE_ENDIAN)
11+
12+#define generic_test_le_bit(nr, addr) test_bit(nr, addr)
13+#define generic___set_le_bit(nr, addr) __set_bit(nr, addr)
14+#define generic___clear_le_bit(nr, addr) __clear_bit(nr, addr)
15+
16+#define generic_test_and_set_le_bit(nr, addr) test_and_set_bit(nr, addr)
17+#define generic_test_and_clear_le_bit(nr, addr) test_and_clear_bit(nr, addr)
18+
19+#define generic___test_and_set_le_bit(nr, addr) __test_and_set_bit(nr, addr)
20+#define generic___test_and_clear_le_bit(nr, addr) __test_and_clear_bit(nr, addr)
21+
22+#define generic_find_next_zero_le_bit(addr, size, offset) find_next_zero_bit(addr, size, offset)
23+
24+#elif defined(__BIG_ENDIAN)
25+
26+#define generic_test_le_bit(nr, addr) \
27+ test_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
28+#define generic___set_le_bit(nr, addr) \
29+ __set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
30+#define generic___clear_le_bit(nr, addr) \
31+ __clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
32+
33+#define generic_test_and_set_le_bit(nr, addr) \
34+ test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
35+#define generic_test_and_clear_le_bit(nr, addr) \
36+ test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
37+
38+#define generic___test_and_set_le_bit(nr, addr) \
39+ __test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
40+#define generic___test_and_clear_le_bit(nr, addr) \
41+ __test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr))
42+
43+extern unsigned long generic_find_next_zero_le_bit(const unsigned long *addr,
44+ unsigned long size, unsigned long offset);
45+
46+#else
47+#error "Please fix <asm/byteorder.h>"
48+#endif
49+
50+#define generic_find_first_zero_le_bit(addr, size) \
51+ generic_find_next_zero_le_bit((addr), (size), 0)
52+
53+#endif /* _ASM_GENERIC_BITOPS_LE_H_ */
tools/testing/radix-tree/linux/bitops/non-atomic.h+111−0
@@ -0,0 +1,111 @@
1+#ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
2+#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_
3+
4+#include <asm/types.h>
5+
6+#define BITOP_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
7+#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
8+
9+/**
10+ * __set_bit - Set a bit in memory
11+ * @nr: the bit to set
12+ * @addr: the address to start counting from
13+ *
14+ * Unlike set_bit(), this function is non-atomic and may be reordered.
15+ * If it's called on the same region of memory simultaneously, the effect
16+ * may be that only one operation succeeds.
17+ */
18+static inline void __set_bit(int nr, volatile unsigned long *addr)
19+{
20+ unsigned long mask = BITOP_MASK(nr);
21+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
22+
23+ *p |= mask;
24+}
25+
26+static inline void __clear_bit(int nr, volatile unsigned long *addr)
27+{
28+ unsigned long mask = BITOP_MASK(nr);
29+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
30+
31+ *p &= ~mask;
32+}
33+
34+/**
35+ * __change_bit - Toggle a bit in memory
36+ * @nr: the bit to change
37+ * @addr: the address to start counting from
38+ *
39+ * Unlike change_bit(), this function is non-atomic and may be reordered.
40+ * If it's called on the same region of memory simultaneously, the effect
41+ * may be that only one operation succeeds.
42+ */
43+static inline void __change_bit(int nr, volatile unsigned long *addr)
44+{
45+ unsigned long mask = BITOP_MASK(nr);
46+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
47+
48+ *p ^= mask;
49+}
50+
51+/**
52+ * __test_and_set_bit - Set a bit and return its old value
53+ * @nr: Bit to set
54+ * @addr: Address to count from
55+ *
56+ * This operation is non-atomic and can be reordered.
57+ * If two examples of this operation race, one can appear to succeed
58+ * but actually fail. You must protect multiple accesses with a lock.
59+ */
60+static inline int __test_and_set_bit(int nr, volatile unsigned long *addr)
61+{
62+ unsigned long mask = BITOP_MASK(nr);
63+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
64+ unsigned long old = *p;
65+
66+ *p = old | mask;
67+ return (old & mask) != 0;
68+}
69+
70+/**
71+ * __test_and_clear_bit - Clear a bit and return its old value
72+ * @nr: Bit to clear
73+ * @addr: Address to count from
74+ *
75+ * This operation is non-atomic and can be reordered.
76+ * If two examples of this operation race, one can appear to succeed
77+ * but actually fail. You must protect multiple accesses with a lock.
78+ */
79+static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr)
80+{
81+ unsigned long mask = BITOP_MASK(nr);
82+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
83+ unsigned long old = *p;
84+
85+ *p = old & ~mask;
86+ return (old & mask) != 0;
87+}
88+
89+/* WARNING: non atomic and it can be reordered! */
90+static inline int __test_and_change_bit(int nr,
91+ volatile unsigned long *addr)
92+{
93+ unsigned long mask = BITOP_MASK(nr);
94+ unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr);
95+ unsigned long old = *p;
96+
97+ *p = old ^ mask;
98+ return (old & mask) != 0;
99+}
100+
101+/**
102+ * test_bit - Determine whether a bit is set
103+ * @nr: bit number to test
104+ * @addr: Address to start counting from
105+ */
106+static inline int test_bit(int nr, const volatile unsigned long *addr)
107+{
108+ return 1UL & (addr[BITOP_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
109+}
110+
111+#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */
tools/testing/radix-tree/linux/bug.h+1−0
@@ -0,0 +1 @@
1+#define WARN_ON_ONCE(x) assert(x)
tools/testing/radix-tree/linux/cpu.h+34−0
@@ -0,0 +1,34 @@
1+
2+#define hotcpu_notifier(a, b)
3+
4+#define CPU_ONLINE 0x0002 /* CPU (unsigned)v is up */
5+#define CPU_UP_PREPARE 0x0003 /* CPU (unsigned)v coming up */
6+#define CPU_UP_CANCELED 0x0004 /* CPU (unsigned)v NOT coming up */
7+#define CPU_DOWN_PREPARE 0x0005 /* CPU (unsigned)v going down */
8+#define CPU_DOWN_FAILED 0x0006 /* CPU (unsigned)v NOT going down */
9+#define CPU_DEAD 0x0007 /* CPU (unsigned)v dead */
10+#define CPU_DYING 0x0008 /* CPU (unsigned)v not running any task,
11+ * not handling interrupts, soon dead.
12+ * Called on the dying cpu, interrupts
13+ * are already disabled. Must not
14+ * sleep, must not fail */
15+#define CPU_POST_DEAD 0x0009 /* CPU (unsigned)v dead, cpu_hotplug
16+ * lock is dropped */
17+#define CPU_STARTING 0x000A /* CPU (unsigned)v soon running.
18+ * Called on the new cpu, just before
19+ * enabling interrupts. Must not sleep,
20+ * must not fail */
21+#define CPU_DYING_IDLE 0x000B /* CPU (unsigned)v dying, reached
22+ * idle loop. */
23+#define CPU_BROKEN 0x000C /* CPU (unsigned)v did not die properly,
24+ * perhaps due to preemption. */
25+#define CPU_TASKS_FROZEN 0x0010
26+
27+#define CPU_ONLINE_FROZEN (CPU_ONLINE | CPU_TASKS_FROZEN)
28+#define CPU_UP_PREPARE_FROZEN (CPU_UP_PREPARE | CPU_TASKS_FROZEN)
29+#define CPU_UP_CANCELED_FROZEN (CPU_UP_CANCELED | CPU_TASKS_FROZEN)
30+#define CPU_DOWN_PREPARE_FROZEN (CPU_DOWN_PREPARE | CPU_TASKS_FROZEN)
31+#define CPU_DOWN_FAILED_FROZEN (CPU_DOWN_FAILED | CPU_TASKS_FROZEN)
32+#define CPU_DEAD_FROZEN (CPU_DEAD | CPU_TASKS_FROZEN)
33+#define CPU_DYING_FROZEN (CPU_DYING | CPU_TASKS_FROZEN)
34+#define CPU_STARTING_FROZEN (CPU_STARTING | CPU_TASKS_FROZEN)
tools/testing/radix-tree/linux/export.h+2−0
@@ -0,0 +1,2 @@
1+
2+#define EXPORT_SYMBOL(sym)
tools/testing/radix-tree/linux/gfp.h+10−0
@@ -0,0 +1,10 @@
1+#ifndef _GFP_H
2+#define _GFP_H
3+
4+#define __GFP_BITS_SHIFT 22
5+#define __GFP_BITS_MASK ((gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
6+#define __GFP_WAIT 1
7+#define __GFP_ACCOUNT 0
8+#define __GFP_NOWARN 0
9+
10+#endif
tools/testing/radix-tree/linux/kernel.h+35−0
@@ -0,0 +1,35 @@
1+#ifndef _KERNEL_H
2+#define _KERNEL_H
3+
4+#include <assert.h>
5+#include <string.h>
6+#include <stdio.h>
7+#include <stddef.h>
8+#include <limits.h>
9+
10+#ifndef NULL
11+#define NULL 0
12+#endif
13+
14+#define BUG_ON(expr) assert(!(expr))
15+#define __init
16+#define __must_check
17+#define panic(expr)
18+#define printk printf
19+#define __force
20+#define likely(c) (c)
21+#define unlikely(c) (c)
22+#define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
23+
24+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
25+
26+#define container_of(ptr, type, member) ({ \
27+ const typeof( ((type *)0)->member ) *__mptr = (ptr); \
28+ (type *)( (char *)__mptr - offsetof(type, member) );})
29+#define min(a, b) ((a) < (b) ? (a) : (b))
30+
31+static inline int in_interrupt(void)
32+{
33+ return 0;
34+}
35+#endif /* _KERNEL_H */
tools/testing/radix-tree/linux/kmemleak.h+1−0
@@ -0,0 +1 @@
1+static inline void kmemleak_update_trace(const void *ptr) { }
tools/testing/radix-tree/linux/mempool.h+16−0
@@ -0,0 +1,16 @@
1+
2+#include <linux/slab.h>
3+
4+typedef void *(mempool_alloc_t)(int gfp_mask, void *pool_data);
5+typedef void (mempool_free_t)(void *element, void *pool_data);
6+
7+typedef struct {
8+ mempool_alloc_t *alloc;
9+ mempool_free_t *free;
10+ void *data;
11+} mempool_t;
12+
13+void *mempool_alloc(mempool_t *pool, int gfp_mask);
14+void mempool_free(void *element, mempool_t *pool);
15+mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
16+ mempool_free_t *free_fn, void *pool_data);
tools/testing/radix-tree/linux/notifier.h+8−0
@@ -0,0 +1,8 @@
1+#ifndef _NOTIFIER_H
2+#define _NOTIFIER_H
3+
4+struct notifier_block;
5+
6+#define NOTIFY_OK 0x0001 /* Suits me */
7+
8+#endif
tools/testing/radix-tree/linux/percpu.h+7−0
@@ -0,0 +1,7 @@
1+
2+#define DEFINE_PER_CPU(type, val) type val
3+
4+#define __get_cpu_var(var) var
5+#define this_cpu_ptr(var) var
6+#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
7+#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
tools/testing/radix-tree/linux/preempt.h+4−0
@@ -0,0 +1,4 @@
1+/* */
2+
3+#define preempt_disable() do { } while (0)
4+#define preempt_enable() do { } while (0)
tools/testing/radix-tree/linux/radix-tree.h+1−0
@@ -0,0 +1 @@
1+#include "../../../../include/linux/radix-tree.h"
tools/testing/radix-tree/linux/rcupdate.h+9−0
@@ -0,0 +1,9 @@
1+#ifndef _RCUPDATE_H
2+#define _RCUPDATE_H
3+
4+#include <urcu.h>
5+
6+#define rcu_dereference_raw(p) rcu_dereference(p)
7+#define rcu_dereference_protected(p, cond) rcu_dereference(p)
8+
9+#endif
tools/testing/radix-tree/linux/slab.h+28−0
@@ -0,0 +1,28 @@
1+#ifndef SLAB_H
2+#define SLAB_H
3+
4+#include <linux/types.h>
5+
6+#define GFP_KERNEL 1
7+#define SLAB_HWCACHE_ALIGN 1
8+#define SLAB_PANIC 2
9+#define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */
10+
11+static inline int gfpflags_allow_blocking(gfp_t mask)
12+{
13+ return 1;
14+}
15+
16+struct kmem_cache {
17+ int size;
18+ void (*ctor)(void *);
19+};
20+
21+void *kmem_cache_alloc(struct kmem_cache *cachep, int flags);
22+void kmem_cache_free(struct kmem_cache *cachep, void *objp);
23+
24+struct kmem_cache *
25+kmem_cache_create(const char *name, size_t size, size_t offset,
26+ unsigned long flags, void (*ctor)(void *));
27+
28+#endif /* SLAB_H */
tools/testing/radix-tree/linux/types.h+28−0
@@ -0,0 +1,28 @@
1+#ifndef _TYPES_H
2+#define _TYPES_H
3+
4+#define __rcu
5+#define __read_mostly
6+
7+#define BITS_PER_LONG (sizeof(long) * 8)
8+
9+struct list_head {
10+ struct list_head *next, *prev;
11+};
12+
13+static inline void INIT_LIST_HEAD(struct list_head *list)
14+{
15+ list->next = list;
16+ list->prev = list;
17+}
18+
19+typedef struct {
20+ unsigned int x;
21+} spinlock_t;
22+
23+#define uninitialized_var(x) x = x
24+
25+typedef unsigned gfp_t;
26+#include <linux/gfp.h>
27+
28+#endif
tools/testing/radix-tree/main.c+272−0
@@ -0,0 +1,272 @@
1+#include <stdio.h>
2+#include <stdlib.h>
3+#include <unistd.h>
4+#include <time.h>
5+#include <assert.h>
6+
7+#include <linux/slab.h>
8+#include <linux/radix-tree.h>
9+
10+#include "test.h"
11+#include "regression.h"
12+
13+void __gang_check(unsigned long middle, long down, long up, int chunk, int hop)
14+{
15+ long idx;
16+ RADIX_TREE(tree, GFP_KERNEL);
17+
18+ middle = 1 << 30;
19+
20+ for (idx = -down; idx < up; idx++)
21+ item_insert(&tree, middle + idx);
22+
23+ item_check_absent(&tree, middle - down - 1);
24+ for (idx = -down; idx < up; idx++)
25+ item_check_present(&tree, middle + idx);
26+ item_check_absent(&tree, middle + up);
27+
28+ item_gang_check_present(&tree, middle - down,
29+ up + down, chunk, hop);
30+ item_full_scan(&tree, middle - down, down + up, chunk);
31+ item_kill_tree(&tree);
32+}
33+
34+void gang_check(void)
35+{
36+ __gang_check(1 << 30, 128, 128, 35, 2);
37+ __gang_check(1 << 31, 128, 128, 32, 32);
38+ __gang_check(1 << 31, 128, 128, 32, 100);
39+ __gang_check(1 << 31, 128, 128, 17, 7);
40+ __gang_check(0xffff0000, 0, 65536, 17, 7);
41+ __gang_check(0xfffffffe, 1, 1, 17, 7);
42+}
43+
44+void __big_gang_check(void)
45+{
46+ unsigned long start;
47+ int wrapped = 0;
48+
49+ start = 0;
50+ do {
51+ unsigned long old_start;
52+
53+// printf("0x%08lx\n", start);
54+ __gang_check(start, rand() % 113 + 1, rand() % 71,
55+ rand() % 157, rand() % 91 + 1);
56+ old_start = start;
57+ start += rand() % 1000000;
58+ start %= 1ULL << 33;
59+ if (start < old_start)
60+ wrapped = 1;
61+ } while (!wrapped);
62+}
63+
64+void big_gang_check(void)
65+{
66+ int i;
67+
68+ for (i = 0; i < 1000; i++) {
69+ __big_gang_check();
70+ srand(time(0));
71+ printf("%d ", i);
72+ fflush(stdout);
73+ }
74+}
75+
76+void add_and_check(void)
77+{
78+ RADIX_TREE(tree, GFP_KERNEL);
79+
80+ item_insert(&tree, 44);
81+ item_check_present(&tree, 44);
82+ item_check_absent(&tree, 43);
83+ item_kill_tree(&tree);
84+}
85+
86+void dynamic_height_check(void)
87+{
88+ int i;
89+ RADIX_TREE(tree, GFP_KERNEL);
90+ tree_verify_min_height(&tree, 0);
91+
92+ item_insert(&tree, 42);
93+ tree_verify_min_height(&tree, 42);
94+
95+ item_insert(&tree, 1000000);
96+ tree_verify_min_height(&tree, 1000000);
97+
98+ assert(item_delete(&tree, 1000000));
99+ tree_verify_min_height(&tree, 42);
100+
101+ assert(item_delete(&tree, 42));
102+ tree_verify_min_height(&tree, 0);
103+
104+ for (i = 0; i < 1000; i++) {
105+ item_insert(&tree, i);
106+ tree_verify_min_height(&tree, i);
107+ }
108+
109+ i--;
110+ for (;;) {
111+ assert(item_delete(&tree, i));
112+ if (i == 0) {
113+ tree_verify_min_height(&tree, 0);
114+ break;
115+ }
116+ i--;
117+ tree_verify_min_height(&tree, i);
118+ }
119+
120+ item_kill_tree(&tree);
121+}
122+
123+void check_copied_tags(struct radix_tree_root *tree, unsigned long start, unsigned long end, unsigned long *idx, int count, int fromtag, int totag)
124+{
125+ int i;
126+
127+ for (i = 0; i < count; i++) {
128+/* if (i % 1000 == 0)
129+ putchar('.'); */
130+ if (idx[i] < start || idx[i] > end) {
131+ if (item_tag_get(tree, idx[i], totag)) {
132+ printf("%lu-%lu: %lu, tags %d-%d\n", start, end, idx[i], item_tag_get(tree, idx[i], fromtag), item_tag_get(tree, idx[i], totag));
133+ }
134+ assert(!item_tag_get(tree, idx[i], totag));
135+ continue;
136+ }
137+ if (item_tag_get(tree, idx[i], fromtag) ^
138+ item_tag_get(tree, idx[i], totag)) {
139+ printf("%lu-%lu: %lu, tags %d-%d\n", start, end, idx[i], item_tag_get(tree, idx[i], fromtag), item_tag_get(tree, idx[i], totag));
140+ }
141+ assert(!(item_tag_get(tree, idx[i], fromtag) ^
142+ item_tag_get(tree, idx[i], totag)));
143+ }
144+}
145+
146+#define ITEMS 50000
147+
148+void copy_tag_check(void)
149+{
150+ RADIX_TREE(tree, GFP_KERNEL);
151+ unsigned long idx[ITEMS];
152+ unsigned long start, end, count = 0, tagged, cur, tmp;
153+ int i;
154+
155+// printf("generating radix tree indices...\n");
156+ start = rand();
157+ end = rand();
158+ if (start > end && (rand() % 10)) {
159+ cur = start;
160+ start = end;
161+ end = cur;
162+ }
163+ /* Specifically create items around the start and the end of the range
164+ * with high probability to check for off by one errors */
165+ cur = rand();
166+ if (cur & 1) {
167+ item_insert(&tree, start);
168+ if (cur & 2) {
169+ if (start <= end)
170+ count++;
171+ item_tag_set(&tree, start, 0);
172+ }
173+ }
174+ if (cur & 4) {
175+ item_insert(&tree, start-1);
176+ if (cur & 8)
177+ item_tag_set(&tree, start-1, 0);
178+ }
179+ if (cur & 16) {
180+ item_insert(&tree, end);
181+ if (cur & 32) {
182+ if (start <= end)
183+ count++;
184+ item_tag_set(&tree, end, 0);
185+ }
186+ }
187+ if (cur & 64) {
188+ item_insert(&tree, end+1);
189+ if (cur & 128)
190+ item_tag_set(&tree, end+1, 0);
191+ }
192+
193+ for (i = 0; i < ITEMS; i++) {
194+ do {
195+ idx[i] = rand();
196+ } while (item_lookup(&tree, idx[i]));
197+
198+ item_insert(&tree, idx[i]);
199+ if (rand() & 1) {
200+ item_tag_set(&tree, idx[i], 0);
201+ if (idx[i] >= start && idx[i] <= end)
202+ count++;
203+ }
204+/* if (i % 1000 == 0)
205+ putchar('.'); */
206+ }
207+
208+// printf("\ncopying tags...\n");
209+ cur = start;
210+ tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, ITEMS, 0, 1);
211+
212+// printf("checking copied tags\n");
213+ assert(tagged == count);
214+ check_copied_tags(&tree, start, end, idx, ITEMS, 0, 1);
215+
216+ /* Copy tags in several rounds */
217+// printf("\ncopying tags...\n");
218+ cur = start;
219+ do {
220+ tmp = rand() % (count/10+2);
221+ tagged = radix_tree_range_tag_if_tagged(&tree, &cur, end, tmp, 0, 2);
222+ } while (tmp == tagged);
223+
224+// printf("%lu %lu %lu\n", tagged, tmp, count);
225+// printf("checking copied tags\n");
226+ check_copied_tags(&tree, start, end, idx, ITEMS, 0, 2);
227+ assert(tagged < tmp);
228+ verify_tag_consistency(&tree, 0);
229+ verify_tag_consistency(&tree, 1);
230+ verify_tag_consistency(&tree, 2);
231+// printf("\n");
232+ item_kill_tree(&tree);
233+}
234+
235+static void single_thread_tests(void)
236+{
237+ int i;
238+
239+ tag_check();
240+ printf("after tag_check: %d allocated\n", nr_allocated);
241+ gang_check();
242+ printf("after gang_check: %d allocated\n", nr_allocated);
243+ add_and_check();
244+ printf("after add_and_check: %d allocated\n", nr_allocated);
245+ dynamic_height_check();
246+ printf("after dynamic_height_check: %d allocated\n", nr_allocated);
247+ big_gang_check();
248+ printf("after big_gang_check: %d allocated\n", nr_allocated);
249+ for (i = 0; i < 2000; i++) {
250+ copy_tag_check();
251+ printf("%d ", i);
252+ fflush(stdout);
253+ }
254+ printf("after copy_tag_check: %d allocated\n", nr_allocated);
255+}
256+
257+int main(void)
258+{
259+ rcu_register_thread();
260+ radix_tree_init();
261+
262+ regression1_test();
263+ regression2_test();
264+ regression3_test();
265+ single_thread_tests();
266+
267+ sleep(1);
268+ printf("after sleep(1): %d allocated\n", nr_allocated);
269+ rcu_unregister_thread();
270+
271+ exit(0);
272+}
tools/testing/radix-tree/rcupdate.c+86−0
@@ -0,0 +1,86 @@
1+#include <linux/rcupdate.h>
2+#include <pthread.h>
3+#include <stdio.h>
4+#include <assert.h>
5+
6+static pthread_mutex_t rculock = PTHREAD_MUTEX_INITIALIZER;
7+static struct rcu_head *rcuhead_global = NULL;
8+static __thread int nr_rcuhead = 0;
9+static __thread struct rcu_head *rcuhead = NULL;
10+static __thread struct rcu_head *rcutail = NULL;
11+
12+static pthread_cond_t rcu_worker_cond = PTHREAD_COND_INITIALIZER;
13+
14+/* switch to urcu implementation when it is merged. */
15+void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *head))
16+{
17+ head->func = func;
18+ head->next = rcuhead;
19+ rcuhead = head;
20+ if (!rcutail)
21+ rcutail = head;
22+ nr_rcuhead++;
23+ if (nr_rcuhead >= 1000) {
24+ int signal = 0;
25+
26+ pthread_mutex_lock(&rculock);
27+ if (!rcuhead_global)
28+ signal = 1;
29+ rcutail->next = rcuhead_global;
30+ rcuhead_global = head;
31+ pthread_mutex_unlock(&rculock);
32+
33+ nr_rcuhead = 0;
34+ rcuhead = NULL;
35+ rcutail = NULL;
36+
37+ if (signal) {
38+ pthread_cond_signal(&rcu_worker_cond);
39+ }
40+ }
41+}
42+
43+static void *rcu_worker(void *arg)
44+{
45+ struct rcu_head *r;
46+
47+ rcupdate_thread_init();
48+
49+ while (1) {
50+ pthread_mutex_lock(&rculock);
51+ while (!rcuhead_global) {
52+ pthread_cond_wait(&rcu_worker_cond, &rculock);
53+ }
54+ r = rcuhead_global;
55+ rcuhead_global = NULL;
56+
57+ pthread_mutex_unlock(&rculock);
58+
59+ synchronize_rcu();
60+
61+ while (r) {
62+ struct rcu_head *tmp = r->next;
63+ r->func(r);
64+ r = tmp;
65+ }
66+ }
67+
68+ rcupdate_thread_exit();
69+
70+ return NULL;
71+}
72+
73+static pthread_t worker_thread;
74+void rcupdate_init(void)
75+{
76+ pthread_create(&worker_thread, NULL, rcu_worker, NULL);
77+}
78+
79+void rcupdate_thread_init(void)
80+{
81+ rcu_register_thread();
82+}
83+void rcupdate_thread_exit(void)
84+{
85+ rcu_unregister_thread();
86+}
tools/testing/radix-tree/regression.h+8−0
@@ -0,0 +1,8 @@
1+#ifndef __REGRESSION_H__
2+#define __REGRESSION_H__
3+
4+void regression1_test(void);
5+void regression2_test(void);
6+void regression3_test(void);
7+
8+#endif
tools/testing/radix-tree/regression1.c+220−0
@@ -0,0 +1,220 @@
1+/*
2+ * Regression1
3+ * Description:
4+ * Salman Qazi describes the following radix-tree bug:
5+ *
6+ * In the following case, we get can get a deadlock:
7+ *
8+ * 0. The radix tree contains two items, one has the index 0.
9+ * 1. The reader (in this case find_get_pages) takes the rcu_read_lock.
10+ * 2. The reader acquires slot(s) for item(s) including the index 0 item.
11+ * 3. The non-zero index item is deleted, and as a consequence the other item
12+ * is moved to the root of the tree. The place where it used to be is queued
13+ * for deletion after the readers finish.
14+ * 3b. The zero item is deleted, removing it from the direct slot, it remains in
15+ * the rcu-delayed indirect node.
16+ * 4. The reader looks at the index 0 slot, and finds that the page has 0 ref
17+ * count
18+ * 5. The reader looks at it again, hoping that the item will either be freed
19+ * or the ref count will increase. This never happens, as the slot it is
20+ * looking at will never be updated. Also, this slot can never be reclaimed
21+ * because the reader is holding rcu_read_lock and is in an infinite loop.
22+ *
23+ * The fix is to re-use the same "indirect" pointer case that requires a slot
24+ * lookup retry into a general "retry the lookup" bit.
25+ *
26+ * Running:
27+ * This test should run to completion in a few seconds. The above bug would
28+ * cause it to hang indefinitely.
29+ *
30+ * Upstream commit:
31+ * Not yet
32+ */
33+#include <linux/kernel.h>
34+#include <linux/gfp.h>
35+#include <linux/slab.h>
36+#include <linux/radix-tree.h>
37+#include <linux/rcupdate.h>
38+#include <stdlib.h>
39+#include <pthread.h>
40+#include <stdio.h>
41+#include <assert.h>
42+
43+#include "regression.h"
44+
45+static RADIX_TREE(mt_tree, GFP_KERNEL);
46+static pthread_mutex_t mt_lock;
47+
48+struct page {
49+ pthread_mutex_t lock;
50+ struct rcu_head rcu;
51+ int count;
52+ unsigned long index;
53+};
54+
55+static struct page *page_alloc(void)
56+{
57+ struct page *p;
58+ p = malloc(sizeof(struct page));
59+ p->count = 1;
60+ p->index = 1;
61+ pthread_mutex_init(&p->lock, NULL);
62+
63+ return p;
64+}
65+
66+static void page_rcu_free(struct rcu_head *rcu)
67+{
68+ struct page *p = container_of(rcu, struct page, rcu);
69+ assert(!p->count);
70+ pthread_mutex_destroy(&p->lock);
71+ free(p);
72+}
73+
74+static void page_free(struct page *p)
75+{
76+ call_rcu(&p->rcu, page_rcu_free);
77+}
78+
79+static unsigned find_get_pages(unsigned long start,
80+ unsigned int nr_pages, struct page **pages)
81+{
82+ unsigned int i;
83+ unsigned int ret;
84+ unsigned int nr_found;
85+
86+ rcu_read_lock();
87+restart:
88+ nr_found = radix_tree_gang_lookup_slot(&mt_tree,
89+ (void ***)pages, NULL, start, nr_pages);
90+ ret = 0;
91+ for (i = 0; i < nr_found; i++) {
92+ struct page *page;
93+repeat:
94+ page = radix_tree_deref_slot((void **)pages[i]);
95+ if (unlikely(!page))
96+ continue;
97+
98+ if (radix_tree_exception(page)) {
99+ if (radix_tree_deref_retry(page)) {
100+ /*
101+ * Transient condition which can only trigger
102+ * when entry at index 0 moves out of or back
103+ * to root: none yet gotten, safe to restart.
104+ */
105+ assert((start | i) == 0);
106+ goto restart;
107+ }
108+ /*
109+ * No exceptional entries are inserted in this test.
110+ */
111+ assert(0);
112+ }
113+
114+ pthread_mutex_lock(&page->lock);
115+ if (!page->count) {
116+ pthread_mutex_unlock(&page->lock);
117+ goto repeat;
118+ }
119+ /* don't actually update page refcount */
120+ pthread_mutex_unlock(&page->lock);
121+
122+ /* Has the page moved? */
123+ if (unlikely(page != *((void **)pages[i]))) {
124+ goto repeat;
125+ }
126+
127+ pages[ret] = page;
128+ ret++;
129+ }
130+ rcu_read_unlock();
131+ return ret;
132+}
133+
134+static pthread_barrier_t worker_barrier;
135+
136+static void *regression1_fn(void *arg)
137+{
138+ rcu_register_thread();
139+
140+ if (pthread_barrier_wait(&worker_barrier) ==
141+ PTHREAD_BARRIER_SERIAL_THREAD) {
142+ int j;
143+
144+ for (j = 0; j < 1000000; j++) {
145+ struct page *p;
146+
147+ p = page_alloc();
148+ pthread_mutex_lock(&mt_lock);
149+ radix_tree_insert(&mt_tree, 0, p);
150+ pthread_mutex_unlock(&mt_lock);
151+
152+ p = page_alloc();
153+ pthread_mutex_lock(&mt_lock);
154+ radix_tree_insert(&mt_tree, 1, p);
155+ pthread_mutex_unlock(&mt_lock);
156+
157+ pthread_mutex_lock(&mt_lock);
158+ p = radix_tree_delete(&mt_tree, 1);
159+ pthread_mutex_lock(&p->lock);
160+ p->count--;
161+ pthread_mutex_unlock(&p->lock);
162+ pthread_mutex_unlock(&mt_lock);
163+ page_free(p);
164+
165+ pthread_mutex_lock(&mt_lock);
166+ p = radix_tree_delete(&mt_tree, 0);
167+ pthread_mutex_lock(&p->lock);
168+ p->count--;
169+ pthread_mutex_unlock(&p->lock);
170+ pthread_mutex_unlock(&mt_lock);
171+ page_free(p);
172+ }
173+ } else {
174+ int j;
175+
176+ for (j = 0; j < 100000000; j++) {
177+ struct page *pages[10];
178+
179+ find_get_pages(0, 10, pages);
180+ }
181+ }
182+
183+ rcu_unregister_thread();
184+
185+ return NULL;
186+}
187+
188+static pthread_t *threads;
189+void regression1_test(void)
190+{
191+ int nr_threads;
192+ int i;
193+ long arg;
194+
195+ /* Regression #1 */
196+ printf("running regression test 1, should finish in under a minute\n");
197+ nr_threads = 2;
198+ pthread_barrier_init(&worker_barrier, NULL, nr_threads);
199+
200+ threads = malloc(nr_threads * sizeof(pthread_t *));
201+
202+ for (i = 0; i < nr_threads; i++) {
203+ arg = i;
204+ if (pthread_create(&threads[i], NULL, regression1_fn, (void *)arg)) {
205+ perror("pthread_create");
206+ exit(1);
207+ }
208+ }
209+
210+ for (i = 0; i < nr_threads; i++) {
211+ if (pthread_join(threads[i], NULL)) {
212+ perror("pthread_join");
213+ exit(1);
214+ }
215+ }
216+
217+ free(threads);
218+
219+ printf("regression test 1, done\n");
220+}
tools/testing/radix-tree/regression2.c+126−0
@@ -0,0 +1,126 @@
1+/*
2+ * Regression2
3+ * Description:
4+ * Toshiyuki Okajima describes the following radix-tree bug:
5+ *
6+ * In the following case, we can get a hangup on
7+ * radix_radix_tree_gang_lookup_tag_slot.
8+ *
9+ * 0. The radix tree contains RADIX_TREE_MAP_SIZE items. And the tag of
10+ * a certain item has PAGECACHE_TAG_DIRTY.
11+ * 1. radix_tree_range_tag_if_tagged(, start, end, , PAGECACHE_TAG_DIRTY,
12+ * PAGECACHE_TAG_TOWRITE) is called to add PAGECACHE_TAG_TOWRITE tag
13+ * for the tag which has PAGECACHE_TAG_DIRTY. However, there is no tag with
14+ * PAGECACHE_TAG_DIRTY within the range from start to end. As the result,
15+ * There is no tag with PAGECACHE_TAG_TOWRITE but the root tag has
16+ * PAGECACHE_TAG_TOWRITE.
17+ * 2. An item is added into the radix tree and then the level of it is
18+ * extended into 2 from 1. At that time, the new radix tree node succeeds
19+ * the tag status of the root tag. Therefore the tag of the new radix tree
20+ * node has PAGECACHE_TAG_TOWRITE but there is not slot with
21+ * PAGECACHE_TAG_TOWRITE tag in the child node of the new radix tree node.
22+ * 3. The tag of a certain item is cleared with PAGECACHE_TAG_DIRTY.
23+ * 4. All items within the index range from 0 to RADIX_TREE_MAP_SIZE - 1 are
24+ * released. (Only the item which index is RADIX_TREE_MAP_SIZE exist in the
25+ * radix tree.) As the result, the slot of the radix tree node is NULL but
26+ * the tag which corresponds to the slot has PAGECACHE_TAG_TOWRITE.
27+ * 5. radix_tree_gang_lookup_tag_slot(PAGECACHE_TAG_TOWRITE) calls
28+ * __lookup_tag. __lookup_tag returns with 0. And __lookup_tag doesn't
29+ * change the index that is the input and output parameter. Because the 1st
30+ * slot of the radix tree node is NULL, but the tag which corresponds to
31+ * the slot has PAGECACHE_TAG_TOWRITE.
32+ * Therefore radix_tree_gang_lookup_tag_slot tries to get some items by
33+ * calling __lookup_tag, but it cannot get any items forever.
34+ *
35+ * The fix is to change that radix_tree_tag_if_tagged doesn't tag the root tag
36+ * if it doesn't set any tags within the specified range.
37+ *
38+ * Running:
39+ * This test should run to completion immediately. The above bug would cause it
40+ * to hang indefinitely.
41+ *
42+ * Upstream commit:
43+ * Not yet
44+ */
45+#include <linux/kernel.h>
46+#include <linux/gfp.h>
47+#include <linux/slab.h>
48+#include <linux/radix-tree.h>
49+#include <stdlib.h>
50+#include <stdio.h>
51+
52+#include "regression.h"
53+
54+#ifdef __KERNEL__
55+#define RADIX_TREE_MAP_SHIFT (CONFIG_BASE_SMALL ? 4 : 6)
56+#else
57+#define RADIX_TREE_MAP_SHIFT 3 /* For more stressful testing */
58+#endif
59+
60+#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
61+#define PAGECACHE_TAG_DIRTY 0
62+#define PAGECACHE_TAG_WRITEBACK 1
63+#define PAGECACHE_TAG_TOWRITE 2
64+
65+static RADIX_TREE(mt_tree, GFP_KERNEL);
66+unsigned long page_count = 0;
67+
68+struct page {
69+ unsigned long index;
70+};
71+
72+static struct page *page_alloc(void)
73+{
74+ struct page *p;
75+ p = malloc(sizeof(struct page));
76+ p->index = page_count++;
77+
78+ return p;
79+}
80+
81+void regression2_test(void)
82+{
83+ int i;
84+ struct page *p;
85+ int max_slots = RADIX_TREE_MAP_SIZE;
86+ unsigned long int start, end;
87+ struct page *pages[1];
88+
89+ printf("running regression test 2 (should take milliseconds)\n");
90+ /* 0. */
91+ for (i = 0; i <= max_slots - 1; i++) {
92+ p = page_alloc();
93+ radix_tree_insert(&mt_tree, i, p);
94+ }
95+ radix_tree_tag_set(&mt_tree, max_slots - 1, PAGECACHE_TAG_DIRTY);
96+
97+ /* 1. */
98+ start = 0;
99+ end = max_slots - 2;
100+ radix_tree_range_tag_if_tagged(&mt_tree, &start, end, 1,
101+ PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
102+
103+ /* 2. */
104+ p = page_alloc();
105+ radix_tree_insert(&mt_tree, max_slots, p);
106+
107+ /* 3. */
108+ radix_tree_tag_clear(&mt_tree, max_slots - 1, PAGECACHE_TAG_DIRTY);
109+
110+ /* 4. */
111+ for (i = max_slots - 1; i >= 0; i--)
112+ radix_tree_delete(&mt_tree, i);
113+
114+ /* 5. */
115+ // NOTE: start should not be 0 because radix_tree_gang_lookup_tag_slot
116+ // can return.
117+ start = 1;
118+ end = max_slots - 2;
119+ radix_tree_gang_lookup_tag_slot(&mt_tree, (void ***)pages, start, end,
120+ PAGECACHE_TAG_TOWRITE);
121+
122+ /* We remove all the remained nodes */
123+ radix_tree_delete(&mt_tree, max_slots);
124+
125+ printf("regression test 2, done\n");
126+}
tools/testing/radix-tree/regression3.c+117−0
@@ -0,0 +1,117 @@
1+/*
2+ * Regression3
3+ * Description:
4+ * Helper radix_tree_iter_retry resets next_index to the current index.
5+ * In following radix_tree_next_slot current chunk size becomes zero.
6+ * This isn't checked and it tries to dereference null pointer in slot.
7+ *
8+ * Helper radix_tree_iter_next reset slot to NULL and next_index to index + 1,
9+ * for tagger iteraction it also must reset cached tags in iterator to abort
10+ * next radix_tree_next_slot and go to slow-path into radix_tree_next_chunk.
11+ *
12+ * Running:
13+ * This test should run to completion immediately. The above bug would
14+ * cause it to segfault.
15+ *
16+ * Upstream commit:
17+ * Not yet
18+ */
19+#include <linux/kernel.h>
20+#include <linux/gfp.h>
21+#include <linux/slab.h>
22+#include <linux/radix-tree.h>
23+#include <stdlib.h>
24+#include <stdio.h>
25+
26+#include "regression.h"
27+
28+void regression3_test(void)
29+{
30+ RADIX_TREE(root, GFP_KERNEL);
31+ void *ptr0 = (void *)4ul;
32+ void *ptr = (void *)8ul;
33+ struct radix_tree_iter iter;
34+ void **slot;
35+ bool first;
36+
37+ printf("running regression test 3 (should take milliseconds)\n");
38+
39+ radix_tree_insert(&root, 0, ptr0);
40+ radix_tree_tag_set(&root, 0, 0);
41+
42+ first = true;
43+ radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) {
44+ printf("tagged %ld %p\n", iter.index, *slot);
45+ if (first) {
46+ radix_tree_insert(&root, 1, ptr);
47+ radix_tree_tag_set(&root, 1, 0);
48+ first = false;
49+ }
50+ if (radix_tree_deref_retry(*slot)) {
51+ printf("retry at %ld\n", iter.index);
52+ slot = radix_tree_iter_retry(&iter);
53+ continue;
54+ }
55+ }
56+ radix_tree_delete(&root, 1);
57+
58+ first = true;
59+ radix_tree_for_each_slot(slot, &root, &iter, 0) {
60+ printf("slot %ld %p\n", iter.index, *slot);
61+ if (first) {
62+ radix_tree_insert(&root, 1, ptr);
63+ first = false;
64+ }
65+ if (radix_tree_deref_retry(*slot)) {
66+ printk("retry at %ld\n", iter.index);
67+ slot = radix_tree_iter_retry(&iter);
68+ continue;
69+ }
70+ }
71+ radix_tree_delete(&root, 1);
72+
73+ first = true;
74+ radix_tree_for_each_contig(slot, &root, &iter, 0) {
75+ printk("contig %ld %p\n", iter.index, *slot);
76+ if (first) {
77+ radix_tree_insert(&root, 1, ptr);
78+ first = false;
79+ }
80+ if (radix_tree_deref_retry(*slot)) {
81+ printk("retry at %ld\n", iter.index);
82+ slot = radix_tree_iter_retry(&iter);
83+ continue;
84+ }
85+ }
86+
87+ radix_tree_for_each_slot(slot, &root, &iter, 0) {
88+ printf("slot %ld %p\n", iter.index, *slot);
89+ if (!iter.index) {
90+ printf("next at %ld\n", iter.index);
91+ slot = radix_tree_iter_next(&iter);
92+ }
93+ }
94+
95+ radix_tree_for_each_contig(slot, &root, &iter, 0) {
96+ printf("contig %ld %p\n", iter.index, *slot);
97+ if (!iter.index) {
98+ printf("next at %ld\n", iter.index);
99+ slot = radix_tree_iter_next(&iter);
100+ }
101+ }
102+
103+ radix_tree_tag_set(&root, 0, 0);
104+ radix_tree_tag_set(&root, 1, 0);
105+ radix_tree_for_each_tagged(slot, &root, &iter, 0, 0) {
106+ printf("tagged %ld %p\n", iter.index, *slot);
107+ if (!iter.index) {
108+ printf("next at %ld\n", iter.index);
109+ slot = radix_tree_iter_next(&iter);
110+ }
111+ }
112+
113+ radix_tree_delete(&root, 0);
114+ radix_tree_delete(&root, 1);
115+
116+ printf("regression test 3 passed\n");
117+}
tools/testing/radix-tree/tag_check.c+332−0
@@ -0,0 +1,332 @@
1+#include <stdlib.h>
2+#include <assert.h>
3+#include <stdio.h>
4+#include <string.h>
5+
6+#include <linux/slab.h>
7+#include <linux/radix-tree.h>
8+
9+#include "test.h"
10+
11+
12+static void
13+__simple_checks(struct radix_tree_root *tree, unsigned long index, int tag)
14+{
15+ int ret;
16+
17+ item_check_absent(tree, index);
18+ assert(item_tag_get(tree, index, tag) == 0);
19+
20+ item_insert(tree, index);
21+ assert(item_tag_get(tree, index, tag) == 0);
22+ item_tag_set(tree, index, tag);
23+ ret = item_tag_get(tree, index, tag);
24+ assert(ret != 0);
25+ ret = item_delete(tree, index);
26+ assert(ret != 0);
27+ item_insert(tree, index);
28+ ret = item_tag_get(tree, index, tag);
29+ assert(ret == 0);
30+ ret = item_delete(tree, index);
31+ assert(ret != 0);
32+ ret = item_delete(tree, index);
33+ assert(ret == 0);
34+}
35+
36+void simple_checks(void)
37+{
38+ unsigned long index;
39+ RADIX_TREE(tree, GFP_KERNEL);
40+
41+ for (index = 0; index < 10000; index++) {
42+ __simple_checks(&tree, index, 0);
43+ __simple_checks(&tree, index, 1);
44+ }
45+ verify_tag_consistency(&tree, 0);
46+ verify_tag_consistency(&tree, 1);
47+ printf("before item_kill_tree: %d allocated\n", nr_allocated);
48+ item_kill_tree(&tree);
49+ printf("after item_kill_tree: %d allocated\n", nr_allocated);
50+}
51+
52+/*
53+ * Check that tags propagate correctly when extending a tree.
54+ */
55+static void extend_checks(void)
56+{
57+ RADIX_TREE(tree, GFP_KERNEL);
58+
59+ item_insert(&tree, 43);
60+ assert(item_tag_get(&tree, 43, 0) == 0);
61+ item_tag_set(&tree, 43, 0);
62+ assert(item_tag_get(&tree, 43, 0) == 1);
63+ item_insert(&tree, 1000000);
64+ assert(item_tag_get(&tree, 43, 0) == 1);
65+
66+ item_insert(&tree, 0);
67+ item_tag_set(&tree, 0, 0);
68+ item_delete(&tree, 1000000);
69+ assert(item_tag_get(&tree, 43, 0) != 0);
70+ item_delete(&tree, 43);
71+ assert(item_tag_get(&tree, 43, 0) == 0); /* crash */
72+ assert(item_tag_get(&tree, 0, 0) == 1);
73+
74+ verify_tag_consistency(&tree, 0);
75+
76+ item_kill_tree(&tree);
77+}
78+
79+/*
80+ * Check that tags propagate correctly when contracting a tree.
81+ */
82+static void contract_checks(void)
83+{
84+ struct item *item;
85+ int tmp;
86+ RADIX_TREE(tree, GFP_KERNEL);
87+
88+ tmp = 1<<RADIX_TREE_MAP_SHIFT;
89+ item_insert(&tree, tmp);
90+ item_insert(&tree, tmp+1);
91+ item_tag_set(&tree, tmp, 0);
92+ item_tag_set(&tree, tmp, 1);
93+ item_tag_set(&tree, tmp+1, 0);
94+ item_delete(&tree, tmp+1);
95+ item_tag_clear(&tree, tmp, 1);
96+
97+ assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 0) == 1);
98+ assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 1) == 0);
99+
100+ assert(item_tag_get(&tree, tmp, 0) == 1);
101+ assert(item_tag_get(&tree, tmp, 1) == 0);
102+
103+ verify_tag_consistency(&tree, 0);
104+ item_kill_tree(&tree);
105+}
106+
107+/*
108+ * Stupid tag thrasher
109+ *
110+ * Create a large linear array corresponding to the tree. Each element in
111+ * the array is coherent with each node in the tree
112+ */
113+
114+enum {
115+ NODE_ABSENT = 0,
116+ NODE_PRESENT = 1,
117+ NODE_TAGGED = 2,
118+};
119+
120+#define THRASH_SIZE 1000 * 1000
121+#define N 127
122+#define BATCH 33
123+
124+static void gang_check(struct radix_tree_root *tree,
125+ char *thrash_state, int tag)
126+{
127+ struct item *items[BATCH];
128+ int nr_found;
129+ unsigned long index = 0;
130+ unsigned long last_index = 0;
131+
132+ while ((nr_found = radix_tree_gang_lookup_tag(tree, (void **)items,
133+ index, BATCH, tag))) {
134+ int i;
135+
136+ for (i = 0; i < nr_found; i++) {
137+ struct item *item = items[i];
138+
139+ while (last_index < item->index) {
140+ assert(thrash_state[last_index] != NODE_TAGGED);
141+ last_index++;
142+ }
143+ assert(thrash_state[last_index] == NODE_TAGGED);
144+ last_index++;
145+ }
146+ index = items[nr_found - 1]->index + 1;
147+ }
148+}
149+
150+static void do_thrash(struct radix_tree_root *tree, char *thrash_state, int tag)
151+{
152+ int insert_chunk;
153+ int delete_chunk;
154+ int tag_chunk;
155+ int untag_chunk;
156+ int total_tagged = 0;
157+ int total_present = 0;
158+
159+ for (insert_chunk = 1; insert_chunk < THRASH_SIZE; insert_chunk *= N)
160+ for (delete_chunk = 1; delete_chunk < THRASH_SIZE; delete_chunk *= N)
161+ for (tag_chunk = 1; tag_chunk < THRASH_SIZE; tag_chunk *= N)
162+ for (untag_chunk = 1; untag_chunk < THRASH_SIZE; untag_chunk *= N) {
163+ int i;
164+ unsigned long index;
165+ int nr_inserted = 0;
166+ int nr_deleted = 0;
167+ int nr_tagged = 0;
168+ int nr_untagged = 0;
169+ int actual_total_tagged;
170+ int actual_total_present;
171+
172+ for (i = 0; i < insert_chunk; i++) {
173+ index = rand() % THRASH_SIZE;
174+ if (thrash_state[index] != NODE_ABSENT)
175+ continue;
176+ item_check_absent(tree, index);
177+ item_insert(tree, index);
178+ assert(thrash_state[index] != NODE_PRESENT);
179+ thrash_state[index] = NODE_PRESENT;
180+ nr_inserted++;
181+ total_present++;
182+ }
183+
184+ for (i = 0; i < delete_chunk; i++) {
185+ index = rand() % THRASH_SIZE;
186+ if (thrash_state[index] == NODE_ABSENT)
187+ continue;
188+ item_check_present(tree, index);
189+ if (item_tag_get(tree, index, tag)) {
190+ assert(thrash_state[index] == NODE_TAGGED);
191+ total_tagged--;
192+ } else {
193+ assert(thrash_state[index] == NODE_PRESENT);
194+ }
195+ item_delete(tree, index);
196+ assert(thrash_state[index] != NODE_ABSENT);
197+ thrash_state[index] = NODE_ABSENT;
198+ nr_deleted++;
199+ total_present--;
200+ }
201+
202+ for (i = 0; i < tag_chunk; i++) {
203+ index = rand() % THRASH_SIZE;
204+ if (thrash_state[index] != NODE_PRESENT) {
205+ if (item_lookup(tree, index))
206+ assert(item_tag_get(tree, index, tag));
207+ continue;
208+ }
209+ item_tag_set(tree, index, tag);
210+ item_tag_set(tree, index, tag);
211+ assert(thrash_state[index] != NODE_TAGGED);
212+ thrash_state[index] = NODE_TAGGED;
213+ nr_tagged++;
214+ total_tagged++;
215+ }
216+
217+ for (i = 0; i < untag_chunk; i++) {
218+ index = rand() % THRASH_SIZE;
219+ if (thrash_state[index] != NODE_TAGGED)
220+ continue;
221+ item_check_present(tree, index);
222+ assert(item_tag_get(tree, index, tag));
223+ item_tag_clear(tree, index, tag);
224+ item_tag_clear(tree, index, tag);
225+ assert(thrash_state[index] != NODE_PRESENT);
226+ thrash_state[index] = NODE_PRESENT;
227+ nr_untagged++;
228+ total_tagged--;
229+ }
230+
231+ actual_total_tagged = 0;
232+ actual_total_present = 0;
233+ for (index = 0; index < THRASH_SIZE; index++) {
234+ switch (thrash_state[index]) {
235+ case NODE_ABSENT:
236+ item_check_absent(tree, index);
237+ break;
238+ case NODE_PRESENT:
239+ item_check_present(tree, index);
240+ assert(!item_tag_get(tree, index, tag));
241+ actual_total_present++;
242+ break;
243+ case NODE_TAGGED:
244+ item_check_present(tree, index);
245+ assert(item_tag_get(tree, index, tag));
246+ actual_total_present++;
247+ actual_total_tagged++;
248+ break;
249+ }
250+ }
251+
252+ gang_check(tree, thrash_state, tag);
253+
254+ printf("%d(%d) %d(%d) %d(%d) %d(%d) / "
255+ "%d(%d) present, %d(%d) tagged\n",
256+ insert_chunk, nr_inserted,
257+ delete_chunk, nr_deleted,
258+ tag_chunk, nr_tagged,
259+ untag_chunk, nr_untagged,
260+ total_present, actual_total_present,
261+ total_tagged, actual_total_tagged);
262+ }
263+}
264+
265+static void thrash_tags(void)
266+{
267+ RADIX_TREE(tree, GFP_KERNEL);
268+ char *thrash_state;
269+
270+ thrash_state = malloc(THRASH_SIZE);
271+ memset(thrash_state, 0, THRASH_SIZE);
272+
273+ do_thrash(&tree, thrash_state, 0);
274+
275+ verify_tag_consistency(&tree, 0);
276+ item_kill_tree(&tree);
277+ free(thrash_state);
278+}
279+
280+static void leak_check(void)
281+{
282+ RADIX_TREE(tree, GFP_KERNEL);
283+
284+ item_insert(&tree, 1000000);
285+ item_delete(&tree, 1000000);
286+ item_kill_tree(&tree);
287+}
288+
289+static void __leak_check(void)
290+{
291+ RADIX_TREE(tree, GFP_KERNEL);
292+
293+ printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
294+ item_insert(&tree, 1000000);
295+ printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
296+ item_delete(&tree, 1000000);
297+ printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
298+ item_kill_tree(&tree);
299+ printf("%d: nr_allocated=%d\n", __LINE__, nr_allocated);
300+}
301+
302+static void single_check(void)
303+{
304+ struct item *items[BATCH];
305+ RADIX_TREE(tree, GFP_KERNEL);
306+ int ret;
307+
308+ item_insert(&tree, 0);
309+ item_tag_set(&tree, 0, 0);
310+ ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 0);
311+ assert(ret == 1);
312+ ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 1, BATCH, 0);
313+ assert(ret == 0);
314+ verify_tag_consistency(&tree, 0);
315+ verify_tag_consistency(&tree, 1);
316+ item_kill_tree(&tree);
317+}
318+
319+void tag_check(void)
320+{
321+ single_check();
322+ extend_checks();
323+ contract_checks();
324+ printf("after extend_checks: %d allocated\n", nr_allocated);
325+ __leak_check();
326+ leak_check();
327+ printf("after leak_check: %d allocated\n", nr_allocated);
328+ simple_checks();
329+ printf("after simple_checks: %d allocated\n", nr_allocated);
330+ thrash_tags();
331+ printf("after thrash_tags: %d allocated\n", nr_allocated);
332+}
tools/testing/radix-tree/test.c+219−0
@@ -0,0 +1,219 @@
1+#include <stdlib.h>
2+#include <assert.h>
3+#include <stdio.h>
4+#include <linux/types.h>
5+#include <linux/kernel.h>
6+#include <linux/bitops.h>
7+
8+#include "test.h"
9+
10+struct item *
11+item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
12+{
13+ return radix_tree_tag_set(root, index, tag);
14+}
15+
16+struct item *
17+item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
18+{
19+ return radix_tree_tag_clear(root, index, tag);
20+}
21+
22+int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
23+{
24+ return radix_tree_tag_get(root, index, tag);
25+}
26+
27+int __item_insert(struct radix_tree_root *root, struct item *item)
28+{
29+ return radix_tree_insert(root, item->index, item);
30+}
31+
32+int item_insert(struct radix_tree_root *root, unsigned long index)
33+{
34+ return __item_insert(root, item_create(index));
35+}
36+
37+int item_delete(struct radix_tree_root *root, unsigned long index)
38+{
39+ struct item *item = radix_tree_delete(root, index);
40+
41+ if (item) {
42+ assert(item->index == index);
43+ free(item);
44+ return 1;
45+ }
46+ return 0;
47+}
48+
49+struct item *item_create(unsigned long index)
50+{
51+ struct item *ret = malloc(sizeof(*ret));
52+
53+ ret->index = index;
54+ return ret;
55+}
56+
57+void item_check_present(struct radix_tree_root *root, unsigned long index)
58+{
59+ struct item *item;
60+
61+ item = radix_tree_lookup(root, index);
62+ assert(item != 0);
63+ assert(item->index == index);
64+}
65+
66+struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
67+{
68+ return radix_tree_lookup(root, index);
69+}
70+
71+void item_check_absent(struct radix_tree_root *root, unsigned long index)
72+{
73+ struct item *item;
74+
75+ item = radix_tree_lookup(root, index);
76+ assert(item == 0);
77+}
78+
79+/*
80+ * Scan only the passed (start, start+nr] for present items
81+ */
82+void item_gang_check_present(struct radix_tree_root *root,
83+ unsigned long start, unsigned long nr,
84+ int chunk, int hop)
85+{
86+ struct item *items[chunk];
87+ unsigned long into;
88+
89+ for (into = 0; into < nr; ) {
90+ int nfound;
91+ int nr_to_find = chunk;
92+ int i;
93+
94+ if (nr_to_find > (nr - into))
95+ nr_to_find = nr - into;
96+
97+ nfound = radix_tree_gang_lookup(root, (void **)items,
98+ start + into, nr_to_find);
99+ assert(nfound == nr_to_find);
100+ for (i = 0; i < nfound; i++)
101+ assert(items[i]->index == start + into + i);
102+ into += hop;
103+ }
104+}
105+
106+/*
107+ * Scan the entire tree, only expecting present items (start, start+nr]
108+ */
109+void item_full_scan(struct radix_tree_root *root, unsigned long start,
110+ unsigned long nr, int chunk)
111+{
112+ struct item *items[chunk];
113+ unsigned long into = 0;
114+ unsigned long this_index = start;
115+ int nfound;
116+ int i;
117+
118+// printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
119+
120+ while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
121+ chunk))) {
122+// printf("At 0x%08lx, nfound=%d\n", into, nfound);
123+ for (i = 0; i < nfound; i++) {
124+ assert(items[i]->index == this_index);
125+ this_index++;
126+ }
127+// printf("Found 0x%08lx->0x%08lx\n",
128+// items[0]->index, items[nfound-1]->index);
129+ into = this_index;
130+ }
131+ if (chunk)
132+ assert(this_index == start + nr);
133+ nfound = radix_tree_gang_lookup(root, (void **)items,
134+ this_index, chunk);
135+ assert(nfound == 0);
136+}
137+
138+static int verify_node(struct radix_tree_node *slot, unsigned int tag,
139+ unsigned int height, int tagged)
140+{
141+ int anyset = 0;
142+ int i;
143+ int j;
144+
145+ slot = indirect_to_ptr(slot);
146+
147+ /* Verify consistency at this level */
148+ for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
149+ if (slot->tags[tag][i]) {
150+ anyset = 1;
151+ break;
152+ }
153+ }
154+ if (tagged != anyset) {
155+ printf("tag: %u, height %u, tagged: %d, anyset: %d\n", tag, height, tagged, anyset);
156+ for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
157+ printf("tag %d: ", j);
158+ for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
159+ printf("%016lx ", slot->tags[j][i]);
160+ printf("\n");
161+ }
162+ return 1;
163+ }
164+ assert(tagged == anyset);
165+
166+ /* Go for next level */
167+ if (height > 1) {
168+ for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
169+ if (slot->slots[i])
170+ if (verify_node(slot->slots[i], tag, height - 1,
171+ !!test_bit(i, slot->tags[tag]))) {
172+ printf("Failure at off %d\n", i);
173+ for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
174+ printf("tag %d: ", j);
175+ for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
176+ printf("%016lx ", slot->tags[j][i]);
177+ printf("\n");
178+ }
179+ return 1;
180+ }
181+ }
182+ return 0;
183+}
184+
185+void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
186+{
187+ if (!root->height)
188+ return;
189+ verify_node(root->rnode, tag, root->height, !!root_tag_get(root, tag));
190+}
191+
192+void item_kill_tree(struct radix_tree_root *root)
193+{
194+ struct item *items[32];
195+ int nfound;
196+
197+ while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
198+ int i;
199+
200+ for (i = 0; i < nfound; i++) {
201+ void *ret;
202+
203+ ret = radix_tree_delete(root, items[i]->index);
204+ assert(ret == items[i]);
205+ free(items[i]);
206+ }
207+ }
208+ assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
209+ assert(root->rnode == NULL);
210+}
211+
212+void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
213+{
214+ assert(radix_tree_maxindex(root->height) >= maxindex);
215+ if (root->height > 1)
216+ assert(radix_tree_maxindex(root->height-1) < maxindex);
217+ else if (root->height == 1)
218+ assert(radix_tree_maxindex(root->height-1) <= maxindex);
219+}
tools/testing/radix-tree/test.h+40−0
@@ -0,0 +1,40 @@
1+#include <linux/gfp.h>
2+#include <linux/types.h>
3+#include <linux/radix-tree.h>
4+#include <linux/rcupdate.h>
5+
6+struct item {
7+ unsigned long index;
8+};
9+
10+struct item *item_create(unsigned long index);
11+int __item_insert(struct radix_tree_root *root, struct item *item);
12+int item_insert(struct radix_tree_root *root, unsigned long index);
13+int item_delete(struct radix_tree_root *root, unsigned long index);
14+struct item *item_lookup(struct radix_tree_root *root, unsigned long index);
15+
16+void item_check_present(struct radix_tree_root *root, unsigned long index);
17+void item_check_absent(struct radix_tree_root *root, unsigned long index);
18+void item_gang_check_present(struct radix_tree_root *root,
19+ unsigned long start, unsigned long nr,
20+ int chunk, int hop);
21+void item_full_scan(struct radix_tree_root *root, unsigned long start,
22+ unsigned long nr, int chunk);
23+void item_kill_tree(struct radix_tree_root *root);
24+
25+void tag_check(void);
26+
27+struct item *
28+item_tag_set(struct radix_tree_root *root, unsigned long index, int tag);
29+struct item *
30+item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag);
31+int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag);
32+void tree_verify_min_height(struct radix_tree_root *root, int maxindex);
33+void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag);
34+
35+extern int nr_allocated;
36+
37+/* Normally private parts of lib/radix-tree.c */
38+void *indirect_to_ptr(void *ptr);
39+int root_tag_get(struct radix_tree_root *root, unsigned int tag);
40+unsigned long radix_tree_maxindex(unsigned int height);
tools/vm/page-types.c+119−14
@@ -61,6 +61,8 @@ · 60 unmodified lines
6161 #define PM_PFRAME_BITS 55
6262 #define PM_PFRAME_MASK ((1LL << PM_PFRAME_BITS) - 1)
6363 #define PM_PFRAME(x) ((x) & PM_PFRAME_MASK)
64+#define MAX_SWAPFILES_SHIFT 5
65+#define PM_SWAP_OFFSET(x) (((x) & PM_PFRAME_MASK) >> MAX_SWAPFILES_SHIFT)
6466 #define PM_SOFT_DIRTY (1ULL << 55)
6567 #define PM_MMAP_EXCLUSIVE (1ULL << 56)
6668 #define PM_FILE (1ULL << 61)
@@ -73,6 +75,7 @@ · 6 unmodified lines
7375
7476 #define KPF_BYTES 8
7577 #define PROC_KPAGEFLAGS "/proc/kpageflags"
78+#define PROC_KPAGECGROUP "/proc/kpagecgroup"
7679
7780 /* [32-] kernel hacking assistances */
7881 #define KPF_RESERVED 32
@@ -92,7 +95,8 @@ · 13 unmodified lines
9295 #define KPF_SLOB_FREE 49
9396 #define KPF_SLUB_FROZEN 50
9497 #define KPF_SLUB_DEBUG 51
95−#define KPF_FILE 62
98+#define KPF_FILE 61
99+#define KPF_SWAP 62
96100 #define KPF_MMAP_EXCLUSIVE 63
97101
98102 #define KPF_ALL_BITS ((uint64_t)~0ULL)
@@ -146,6 +150,7 @@ static const char * const page_flag_names[] = { · 47 unmodified lines
146150 [KPF_SLUB_DEBUG] = "E:slub_debug",
147151
148152 [KPF_FILE] = "F:file",
153+ [KPF_SWAP] = "w:swap",
149154 [KPF_MMAP_EXCLUSIVE] = "1:mmap_exclusive",
150155 };
151156
@@ -164,7 +169,9 @@ static int opt_raw; /* for kernel developers */ · 12 unmodified lines
164169 static int opt_list; /* list pages (in ranges) */
165170 static int opt_no_summary; /* don't show summary */
166171 static pid_t opt_pid; /* process to walk */
167−const char * opt_file;
172+const char * opt_file; /* file or directory path */
173+static uint64_t opt_cgroup; /* cgroup inode */
174+static int opt_list_cgroup;/* list page cgroup */
168175
169176 #define MAX_ADDR_RANGES 1024
170177 static int nr_addr_ranges;
@@ -185,6 +192,7 @@ static int page_size; · 14 unmodified lines
185192
186193 static int pagemap_fd;
187194 static int kpageflags_fd;
195+static int kpagecgroup_fd = -1;
188196
189197 static int opt_hwpoison;
190198 static int opt_unpoison;
@@ -278,6 +286,16 @@ static unsigned long kpageflags_read(uint64_t *buf, · 87 unmodified lines
278286 return do_u64_read(kpageflags_fd, PROC_KPAGEFLAGS, buf, index, pages);
279287 }
280288
289+static unsigned long kpagecgroup_read(uint64_t *buf,
290+ unsigned long index,
291+ unsigned long pages)
292+{
293+ if (kpagecgroup_fd < 0)
294+ return pages;
295+
296+ return do_u64_read(kpagecgroup_fd, PROC_KPAGEFLAGS, buf, index, pages);
297+}
298+
281299 static unsigned long pagemap_read(uint64_t *buf,
282300 unsigned long index,
283301 unsigned long pages)
@@ -297,6 +315,10 @@ static unsigned long pagemap_pfn(uint64_t val) · 13 unmodified lines
297315 return pfn;
298316 }
299317
318+static unsigned long pagemap_swap_offset(uint64_t val)
319+{
320+ return val & PM_SWAP ? PM_SWAP_OFFSET(val) : 0;
321+}
300322
301323 /*
302324 * page flag names
@@ -346,14 +368,15 @@ static char *page_flag_longname(uint64_t flags) · 43 unmodified lines
346368 */
347369
348370 static void show_page_range(unsigned long voffset, unsigned long offset,
349− unsigned long size, uint64_t flags)
371+ unsigned long size, uint64_t flags, uint64_t cgroup)
350372 {
351373 static uint64_t flags0;
374+ static uint64_t cgroup0;
352375 static unsigned long voff;
353376 static unsigned long index;
354377 static unsigned long count;
355378
356− if (flags == flags0 && offset == index + count &&
379+ if (flags == flags0 && cgroup == cgroup0 && offset == index + count &&
357380 size && voffset == voff + count) {
358381 count += size;
359382 return;
@@ -364,11 +387,14 @@ static void show_page_range(unsigned long voffset, unsigned long offset, · 4 unmodified lines
364387 printf("%lx\t", voff);
365388 if (opt_file)
366389 printf("%lu\t", voff);
390+ if (opt_list_cgroup)
391+ printf("@%llu\t", (unsigned long long)cgroup0);
367392 printf("%lx\t%lx\t%s\n",
368393 index, count, page_flag_name(flags0));
369394 }
370395
371396 flags0 = flags;
397+ cgroup0= cgroup;
372398 index = offset;
373399 voff = voffset;
374400 count = size;
@@ -376,16 +402,18 @@ static void show_page_range(unsigned long voffset, unsigned long offset, · 1 unmodified line
376402
377403 static void flush_page_range(void)
378404 {
379− show_page_range(0, 0, 0, 0);
405+ show_page_range(0, 0, 0, 0, 0);
380406 }
381407
382−static void show_page(unsigned long voffset,
383− unsigned long offset, uint64_t flags)
408+static void show_page(unsigned long voffset, unsigned long offset,
409+ uint64_t flags, uint64_t cgroup)
384410 {
385411 if (opt_pid)
386412 printf("%lx\t", voffset);
387413 if (opt_file)
388414 printf("%lu\t", voffset);
415+ if (opt_list_cgroup)
416+ printf("@%llu\t", (unsigned long long)cgroup);
389417 printf("%lx\t%s\n", offset, page_flag_name(flags));
390418 }
391419
@@ -452,6 +480,8 @@ static uint64_t expand_overloaded_flags(uint64_t flags, uint64_t pme) · 60 unmodified lines
452480 flags |= BIT(SOFTDIRTY);
453481 if (pme & PM_FILE)
454482 flags |= BIT(FILE);
483+ if (pme & PM_SWAP)
484+ flags |= BIT(SWAP);
455485 if (pme & PM_MMAP_EXCLUSIVE)
456486 flags |= BIT(MMAP_EXCLUSIVE);
457487
@@ -566,23 +596,26 @@ static size_t hash_slot(uint64_t flags) · 108 unmodified lines
566596 exit(EXIT_FAILURE);
567597 }
568598
569−static void add_page(unsigned long voffset,
570− unsigned long offset, uint64_t flags, uint64_t pme)
599+static void add_page(unsigned long voffset, unsigned long offset,
600+ uint64_t flags, uint64_t cgroup, uint64_t pme)
571601 {
572602 flags = kpageflags_flags(flags, pme);
573603
574604 if (!bit_mask_ok(flags))
575605 return;
576606
607+ if (opt_cgroup && cgroup != (uint64_t)opt_cgroup)
608+ return;
609+
577610 if (opt_hwpoison)
578611 hwpoison_page(offset);
579612 if (opt_unpoison)
580613 unpoison_page(offset);
581614
582615 if (opt_list == 1)
583− show_page_range(voffset, offset, 1, flags);
616+ show_page_range(voffset, offset, 1, flags, cgroup);
584617 else if (opt_list == 2)
585− show_page(voffset, offset, flags);
618+ show_page(voffset, offset, flags, cgroup);
586619
587620 nr_pages[hash_slot(flags)]++;
588621 total_pages++;
@@ -595,24 +628,57 @@ static void walk_pfn(unsigned long voffset, · 6 unmodified lines
595628 uint64_t pme)
596629 {
597630 uint64_t buf[KPAGEFLAGS_BATCH];
631+ uint64_t cgi[KPAGEFLAGS_BATCH];
598632 unsigned long batch;
599633 unsigned long pages;
600634 unsigned long i;
601635
636+ /*
637+ * kpagecgroup_read() reads only if kpagecgroup were opened, but
638+ * /proc/kpagecgroup might even not exist, so it's better to fill
639+ * them with zeros here.
640+ */
641+ if (count == 1)
642+ cgi[0] = 0;
643+ else
644+ memset(cgi, 0, sizeof cgi);
645+
602646 while (count) {
603647 batch = min_t(unsigned long, count, KPAGEFLAGS_BATCH);
604648 pages = kpageflags_read(buf, index, batch);
605649 if (pages == 0)
606650 break;
607651
652+ if (kpagecgroup_read(cgi, index, pages) != pages)
653+ fatal("kpagecgroup returned fewer pages than expected");
654+
608655 for (i = 0; i < pages; i++)
609− add_page(voffset + i, index + i, buf[i], pme);
656+ add_page(voffset + i, index + i, buf[i], cgi[i], pme);
610657
611658 index += pages;
612659 count -= pages;
613660 }
614661 }
615662
663+static void walk_swap(unsigned long voffset, uint64_t pme)
664+{
665+ uint64_t flags = kpageflags_flags(0, pme);
666+
667+ if (!bit_mask_ok(flags))
668+ return;
669+
670+ if (opt_cgroup)
671+ return;
672+
673+ if (opt_list == 1)
674+ show_page_range(voffset, pagemap_swap_offset(pme), 1, flags, 0);
675+ else if (opt_list == 2)
676+ show_page(voffset, pagemap_swap_offset(pme), flags, 0);
677+
678+ nr_pages[hash_slot(flags)]++;
679+ total_pages++;
680+}
681+
616682 #define PAGEMAP_BATCH (64 << 10)
617683 static void walk_vma(unsigned long index, unsigned long count)
618684 {
@@ -632,6 +698,8 @@ static void walk_vma(unsigned long index, unsigned long count) · 13 unmodified lines
632698 pfn = pagemap_pfn(buf[i]);
633699 if (pfn)
634700 walk_pfn(index + i, pfn, 1, buf[i]);
701+ if (buf[i] & PM_SWAP)
702+ walk_swap(index + i, buf[i]);
635703 }
636704
637705 index += pages;
@@ -713,10 +781,12 @@ static void usage(void) · 75 unmodified lines
713781 " -d|--describe flags Describe flags\n"
714782 " -a|--addr addr-spec Walk a range of pages\n"
715783 " -b|--bits bits-spec Walk pages with specified bits\n"
784+" -c|--cgroup path|@inode Walk pages within memory cgroup\n"
716785 " -p|--pid pid Walk process address space\n"
717786 " -f|--file filename Walk file address space\n"
718787 " -l|--list Show page details in ranges\n"
719788 " -L|--list-each Show page details one by one\n"
789+" -C|--list-cgroup Show cgroup inode for pages\n"
720790 " -N|--no-summary Don't show summary info\n"
721791 " -X|--hwpoison hwpoison pages\n"
722792 " -x|--unpoison unpoison pages\n"
@@ -851,6 +921,7 @@ static void walk_file(const char *name, const struct stat *st) · 128 unmodified lines
851921 {
852922 uint8_t vec[PAGEMAP_BATCH];
853923 uint64_t buf[PAGEMAP_BATCH], flags;
924+ uint64_t cgroup = 0;
854925 unsigned long nr_pages, pfn, i;
855926 off_t off, end = st->st_size;
856927 int fd;
@@ -908,12 +979,15 @@ got_sigbus: · 51 unmodified lines
908979 continue;
909980 if (!kpageflags_read(&flags, pfn, 1))
910981 continue;
982+ if (!kpagecgroup_read(&cgroup, pfn, 1))
983+ fatal("kpagecgroup_read failed");
911984 if (first && opt_list) {
912985 first = 0;
913986 flush_page_range();
914987 show_file(name, st);
915988 }
916− add_page(off / page_size + i, pfn, flags, buf[i]);
989+ add_page(off / page_size + i, pfn,
990+ flags, cgroup, buf[i]);
917991 }
918992 }
919993
@@ -965,6 +1039,24 @@ static void parse_file(const char *name) · 45 unmodified lines
9651039 opt_file = name;
9661040 }
9671041
1042+static void parse_cgroup(const char *path)
1043+{
1044+ if (path[0] == '@') {
1045+ opt_cgroup = parse_number(path + 1);
1046+ return;
1047+ }
1048+
1049+ struct stat st;
1050+
1051+ if (stat(path, &st))
1052+ fatal("stat failed: %s: %m\n", path);
1053+
1054+ if (!S_ISDIR(st.st_mode))
1055+ fatal("cgroup supposed to be a directory: %s\n", path);
1056+
1057+ opt_cgroup = st.st_ino;
1058+}
1059+
9681060 static void parse_addr_range(const char *optarg)
9691061 {
9701062 unsigned long offset;
@@ -1088,9 +1180,11 @@ static const struct option opts[] = { · 117 unmodified lines
10881180 { "file" , 1, NULL, 'f' },
10891181 { "addr" , 1, NULL, 'a' },
10901182 { "bits" , 1, NULL, 'b' },
1183+ { "cgroup" , 1, NULL, 'c' },
10911184 { "describe" , 1, NULL, 'd' },
10921185 { "list" , 0, NULL, 'l' },
10931186 { "list-each" , 0, NULL, 'L' },
1187+ { "list-cgroup", 0, NULL, 'C' },
10941188 { "no-summary", 0, NULL, 'N' },
10951189 { "hwpoison" , 0, NULL, 'X' },
10961190 { "unpoison" , 0, NULL, 'x' },
@@ -1105,7 +1199,7 @@ int main(int argc, char *argv[]) · 8 unmodified lines
11051199 page_size = getpagesize();
11061200
11071201 while ((c = getopt_long(argc, argv,
1108− "rp:f:a:b:d:lLNXxh", opts, NULL)) != -1) {
1202+ "rp:f:a:b:d:c:ClLNXxh", opts, NULL)) != -1) {
11091203 switch (c) {
11101204 case 'r':
11111205 opt_raw = 1;
@@ -1122,6 +1216,12 @@ int main(int argc, char *argv[]) · 10 unmodified lines
11221216 case 'b':
11231217 parse_bits_mask(optarg);
11241218 break;
1219+ case 'c':
1220+ parse_cgroup(optarg);
1221+ break;
1222+ case 'C':
1223+ opt_list_cgroup = 1;
1224+ break;
11251225 case 'd':
11261226 describe_flags(optarg);
11271227 exit(0);
@@ -1151,10 +1251,15 @@ int main(int argc, char *argv[]) · 23 unmodified lines
11511251 }
11521252 }
11531253
1254+ if (opt_cgroup || opt_list_cgroup)
1255+ kpagecgroup_fd = checked_open(PROC_KPAGECGROUP, O_RDONLY);
1256+
11541257 if (opt_list && opt_pid)
11551258 printf("voffset\t");
11561259 if (opt_list && opt_file)
11571260 printf("foffset\t");
1261+ if (opt_list && opt_list_cgroup)
1262+ printf("cgroup\t");
11581263 if (opt_list == 1)
11591264 printf("offset\tlen\tflags\n");
11601265 if (opt_list == 2)