lael-labs/linux

Documentation/admin-guide/blockdev/zram.rst+33−1

@@ -109,14 +109,41 @@ path to the `dict` along with other parameters:: · 108 unmodified lines
109	#pass path to pre-trained zstd dictionary	109	#pass path to pre-trained zstd dictionary
110	echo "algo=zstd dict=/etc/dictionary" > /sys/block/zram0/algorithm_params	110	echo "algo=zstd dict=/etc/dictionary" > /sys/block/zram0/algorithm_params
111		111
		112	#pass path to pre-trained zstd dictionary and compression level
		113	echo "algo=zstd level=8 dict=/etc/dictionary" > \
		114	/sys/block/zram0/algorithm_params
		115
112	#same, but using algorithm priority	116	#same, but using algorithm priority
		117	echo "algo=zstd priority=1" > /sys/block/zram0/recomp_algorithm
113	echo "priority=1 dict=/etc/dictionary" > \	118	echo "priority=1 dict=/etc/dictionary" > \
114	/sys/block/zram0/algorithm_params	119	/sys/block/zram0/algorithm_params
115		120
116	#pass path to pre-trained zstd dictionary and compression level	121	Each write to `algorithm_params` replaces the entire set of parameters of
		122	the corresponding algorithm, parameters that are not listed in the write
		123	are reset to their default values. Configure all of the parameters of an
		124	algorithm in one write::
		125
		126	#WRONG: the second write resets level back to its default value
		127	echo "algo=zstd level=8" > /sys/block/zram0/algorithm_params
		128	echo "algo=zstd dict=/etc/dictionary" > /sys/block/zram0/algorithm_params
		129
		130	#RIGHT
117	echo "algo=zstd level=8 dict=/etc/dictionary" > \	131	echo "algo=zstd level=8 dict=/etc/dictionary" > \
118	/sys/block/zram0/algorithm_params	132	/sys/block/zram0/algorithm_params
119		133
		134	Select the compression algorithm before configuring its parameters. The
		135	parameters of one algorithm are not necessarily valid for another one, so
		136	changing the algorithm of a particular priority resets that priority's
		137	parameters::
		138
		139	#WRONG: comp_algorithm write resets the previously configured level
		140	echo "level=8" > /sys/block/zram0/algorithm_params
		141	echo zstd > /sys/block/zram0/comp_algorithm
		142
		143	#RIGHT
		144	echo zstd > /sys/block/zram0/comp_algorithm
		145	echo "algo=zstd level=8" > /sys/block/zram0/algorithm_params
		146
120	Parameters are algorithm specific: not all algorithms support pre-trained	147	Parameters are algorithm specific: not all algorithms support pre-trained
121	dictionaries, not all algorithms support `level`. Furthermore, for certain	148	dictionaries, not all algorithms support `level`. Furthermore, for certain
122	algorithms `level` controls the compression level (the higher the value the	149	algorithms `level` controls the compression level (the higher the value the
@@ -124,6 +151,11 @@ better the compression ratio, it even can take negatives values for some · 1 unmodified line
124	algorithms), for other algorithms `level` is acceleration level (the higher	151	algorithms), for other algorithms `level` is acceleration level (the higher
125	the value the lower the compression ratio).	152	the value the lower the compression ratio).
126		153
		154	Parameters are handed over to the compression algorithm when the device is
		155	initialised, hence invalid parameters (or parameters that the selected
		156	algorithm does not support) are reported by the `disksize` write, and not
		157	by the `algorithm_params` write that has configured them.
		158
127	Set Disksize	159	Set Disksize
128	============	160	============
129		161

Documentation/admin-guide/kernel-parameters.txt+14−2

 	hugetlb_cma=	[HW,CMA,EARLY] The size of a CMA area used for allocation
 			of gigantic hugepages. Or using node format, the size
 			of a CMA area per node can be specified.
-			Format: nn[KMGTPE] or (node format)
+			The size can be an absolute value (e.g., 2G) or a
-				<node>:nn[KMGTPE][,<node>:nn[KMGTPE]]
+			percentage of the total memory or node memory (e.g., 20%).
+			Percentage-derived sizes are rounded down to a multiple of
+			the architecture's gigantic hugepage size and may become
+			zero.
+			Format: nn[KMGTPE] or nn% or (node format)
+				<node>:nn[KMGTPE][,<node>:nn[KMGTPE]] or
+				<node>:nn%[,<node>:nn%]
 			The size must be a multiple of the gigantic page size.
 			When using node format, this applies to each per-node size.
 			Adjust the minimal page reporting order. The page
 			reporting is disabled when it exceeds MAX_PAGE_ORDER.
+	page_reporting.page_reporting_delay_ms=
+			[KNL] Free page reporting delay in milliseconds
+			Format: <unsigned integer>
+			Adjust the delay in milliseconds between free page
+			reporting intervals. Default is 2000 (2 seconds).
 	panic=		[KNL] Kernel behaviour on panic: delay <timeout>
 			timeout > 0: seconds before rebooting
 			timeout = 0: wait forever

Documentation/admin-guide/mm/transhuge.rst+4−2

 (either of the per-size anon control or the top-level control are set
 to "always" or "madvise"), and it'll be automatically shutdown when
 all THP sizes are disabled (when both the per-size anon control and the
-top-level control are "never")
+top-level control are "never").
 process THP controls
 --------------------
 hugepage of various sizes (PMD-sized or mTHP). Each may happen independently,
 or together, depending on the type of memory and the failures that occur.
 As such, this value should be interpreted roughly as a sign of progress,
-and counters in /proc/vmstat consulted for more accurate accounting)::
+and counters in /proc/vmstat consulted for more accurate accounting.
+Per-order mTHP collapse statistics are also available under
+/sys/kernel/mm/transparent_hugepage/hugepages-<size>kB/stats/)::
 	/sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed

Documentation/core-api/maple_tree.rst+20−4

 entry in a cache-efficient manner.  The tree can also be put into an RCU-safe
 mode of operation which allows reading and writing concurrently.  Writers must
 synchronize on a lock, which can be the default spinlock, or the user can set
-the lock to an external lock of a different type.
+the lock to an external lock of a different type.  Note that external locks may
+interfere with allocations in a low memory situation.
 The Maple Tree maintains a small memory footprint and was designed to use
 modern processor cache efficiently.  The majority of the users will be able to
 code segment when allocating cannot be done.  Allocations of nodes are
 relatively small at around 256 bytes.
+Since the maple tree uses internal nodes that are allocated and has rules on
+data density, erasing an entry may cause allocations to occur.  That is,
+erasing an entry may consume memory.  Users must take care to ensure that they
+do not violate the larger system constraints on when and how memory is
+allocated.  Most situations are fine to allocate, but the pre-allocation
+support is provided as a mechanism to avoid trickier situations.  There is also
+the possibility of using special entries and clean up the tree later, in
+extreme circumstances.
 .. _maple-tree-normal-api:
 Normal API
 but takes a range.  mtree_load() is used to retrieve the entry stored at a
 given index.  You can use mtree_erase() to erase an entire range by only
 knowing one value within that range, or mtree_store() call with an entry of
-NULL may be used to partially erase a range or many ranges at once.
+NULL may be used to partially erase a range or many ranges at once.  Note that
+mtree_erase() may use GFP_KERNEL | __GFP_NOFAIL for allocations and cannot
+fail.  mtree_erase() can sleep, so it must not be called from an atomic
+context.
 If you want to only store a new entry to a range (or index) if that range is
 currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which
 last of the maple state to the desired range to erase.  This will erase
 the first range that is found in that range, set the maple state index
 and last as the range that was erased and return the entry that existed
-at that location.
+at that location.  Note that mas_erase() may allocate with the GFP_KERNEL
+__GFP_NOFAIL and cannot fail, but may sleep.  If this is not okay, consider
+using mas_store_gfp() and pass it a ``NULL``,
+after setting up the correct range by walking to the entry.
 You can walk each entry within a range by using mas_for_each().  If you want
 to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as
 The maple tree uses a spinlock by default, but external locks can be used for
 tree updates as well.  To use an external lock, the tree must be initialized
-with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the
+with the ``MT_FLAGS_LOCK_EXTERN`` flag, this is usually done with the
 MTREE_INIT_EXT() #define, which takes an external lock as an argument.
 Functions and structures

Documentation/dev-tools/kmemleak.rst+10−0

@@ -198,6 +198,16 @@ systems, because of pointers temporarily stored in CPU registers or · 197 unmodified lines
198	stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing	198	stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing
199	the minimum age of an object to be reported as a memory leak.	199	the minimum age of an object to be reported as a memory leak.
200		200
		201	The ``min_unref_scans`` module parameter requires an object to be seen
		202	unreferenced in that many consecutive scans before it is reported. It
		203	defaults to 2 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the
		204	periodic scan thread confirms a leak on its own, and to 1 otherwise. A
		205	value of 1 preserves the historical behaviour; higher values filter the
		206	transient false positives described above, at the cost of delaying genuine
		207	reports by up to that many scans. It can be set at boot with
		208	``kmemleak.min_unref_scans=<n>`` or at run-time via
		209	``/sys/module/kmemleak/parameters/min_unref_scans``.
		210
201	Limitations and Drawbacks	211	Limitations and Drawbacks
202	-------------------------	212	-------------------------
203		213

Documentation/filesystems/fscrypt.rst+4−0

@@ -1237,6 +1237,10 @@ astute users may notice some differences in behavior: · 1236 unmodified lines
1237		1237
1238	- DAX (Direct Access) is not supported on encrypted files.	1238	- DAX (Direct Access) is not supported on encrypted files.
1239		1239
		1240	- Encrypted files cannot be used directly as swap files. To swap to
		1241	an encrypted file, set up a loopback device on top of it.
		1242	Alternatively, encrypted swap can use a dm-crypt device.
		1243
1240	- The maximum length of an encrypted symlink is 2 bytes shorter than	1244	- The maximum length of an encrypted symlink is 2 bytes shorter than
1241	the maximum length of an unencrypted symlink. For example, on an	1245	the maximum length of an unencrypted symlink. For example, on an
1242	EXT4 filesystem with a 4K block size, unencrypted symlinks can be up	1246	EXT4 filesystem with a 4K block size, unencrypted symlinks can be up

Documentation/filesystems/locking.rst+2−6

 	int (*error_remove_folio)(struct address_space *, struct folio *);
 	int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span)
 	int (*swap_deactivate)(struct file *);
-	int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
 locking rules:
 	All except dirty_folio and free_folio may block
 error_remove_folio:	yes
 swap_activate:		no
 swap_deactivate:	no
-swap_rw:		yes, unlocks
 ======================	======================== =========	===============
 ->write_begin(), ->write_end() and ->read_folio() may be called from
 writes can be performed with minimal memory allocation.  It should call
 add_swap_extent(), or the helper iomap_swapfile_activate(), and return
 the number of extents added.  If IO should be submitted through
-->swap_rw(), it should set SWP_FS_OPS, otherwise IO will be submitted
+the file system it should call swap_fs_activate, otherwise IO will be
-directly to the block device ``sis->bdev``.
+submitted directly to the block device ``sis->bdev``.
 ->swap_deactivate() will be called in the sys_swapoff()
 path after ->swap_activate() returned success.
-->swap_rw will be called for swap IO if SWP_FS_OPS was set by ->swap_activate().
 file_lock_operations
 ====================

Documentation/filesystems/vfs.rst+2−6

 		int (*error_remove_folio)(struct mapping *mapping, struct folio *);
 		int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span)
 		int (*swap_deactivate)(struct file *);
-		int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
 	};
 ``read_folio``
 	can be performed with minimal memory allocation.  It should call
 	add_swap_extent(), or the helper iomap_swapfile_activate(), and
 	return the number of extents added.  If IO should be submitted
-	through ->swap_rw(), it should set SWP_FS_OPS, otherwise IO will
+	through the file system it should call swap_fs_activate, otherwise IO
-	be submitted directly to the block device ``sis->bdev``.
+	will be submitted directly to the block device ``sis->bdev``.
 ``swap_deactivate``
 	Called during swapoff on files where swap_activate was
 	successful.
-``swap_rw``
-	Called to read or write swap pages when SWP_FS_OPS is set.
 The File Object
 ===============

Documentation/mm/allocation-profiling.rst+16−0

@@ -43,9 +43,25 @@ sysctl: · 42 unmodified lines
43	warnings produced by allocations made while profiling is disabled and freed	43	warnings produced by allocations made while profiling is disabled and freed
44	when it's enabled.	44	when it's enabled.
45		45
		46	/proc/sys/vm/mem_profiling_compressed
		47
		48	1: Page alloc tag compression is enabled.
		49
		50	0: Page alloc tag compression is disabled.
		51
		52	This reflects a static boot-time configuration of how page allocation tags are
		53	stored (in page flags when compression is enabled and in page_ext when disabled).
		54	Toggling ``mem_profiling`` at runtime does not change the state of
		55	``mem_profiling_compressed``.
		56
46	Runtime info:	57	Runtime info:
47	/proc/allocinfo	58	/proc/allocinfo
48		59
		60	Profiling data can be retrieved either by reading `/proc/allocinfo` directly as
		61	text or programmatically via `ioctl()` calls defined in `<uapi/linux/alloc_tag.h>`.
		62	The ioctl interface supports structured binary data extraction as well as filtering
		63	by module name, function, file, line number, accuracy, or allocation size limits.
		64
49	Example output::	65	Example output::
50		66
51	root@moria-kvm:~# sort -g /proc/allocinfo\|tail\|numfmt --to=iec	67	root@moria-kvm:~# sort -g /proc/allocinfo\|tail\|numfmt --to=iec

Documentation/mm/ksm.rst+2−2

@@ -24,13 +24,13 @@ tree. · 23 unmodified lines
24		24
25	If a KSM page is shared between less than ``max_page_sharing`` VMAs,	25	If a KSM page is shared between less than ``max_page_sharing`` VMAs,
26	the node of the stable tree that represents such KSM page points to a	26	the node of the stable tree that represents such KSM page points to a
27	list of struct ksm_rmap_item and the ``page->mapping`` of the	27	list of struct ksm_rmap_item and the ``folio->mapping`` of the
28	KSM page points to the stable tree node.	28	KSM page points to the stable tree node.
29		29
30	When the sharing passes this threshold, KSM adds a second dimension to	30	When the sharing passes this threshold, KSM adds a second dimension to
31	the stable tree. The tree node becomes a "chain" that links one or	31	the stable tree. The tree node becomes a "chain" that links one or
32	more "dups". Each "dup" keeps reverse mapping information for a KSM	32	more "dups". Each "dup" keeps reverse mapping information for a KSM
33	page with ``page->mapping`` pointing to that "dup".	33	page with ``folio->mapping`` pointing to that "dup".
34		34
35	Every "chain" and all "dups" linked into a "chain" enforce the	35	Every "chain" and all "dups" linked into a "chain" enforce the
36	invariant that they represent the same write protected memory content,	36	invariant that they represent the same write protected memory content,

Documentation/mm/page_migration.rst+3−3

@@ -110,13 +110,13 @@ Steps: · 109 unmodified lines
110	6. The refcount of the page is examined and we back out if references remain.	110	6. The refcount of the page is examined and we back out if references remain.
111	Otherwise, we know that we are the only one referencing this page.	111	Otherwise, we know that we are the only one referencing this page.
112		112
113	7. The radix tree is checked and if it does not contain the pointer to this	113	7. The page cache tree is checked and if it does not contain the pointer to this
114	page then we back out because someone else modified the radix tree.	114	page then we back out because someone else modified the page cache tree.
115		115
116	8. The new page is prepped with some settings from the old page so that	116	8. The new page is prepped with some settings from the old page so that
117	accesses to the new page will discover a page with the correct settings.	117	accesses to the new page will discover a page with the correct settings.
118		118
119	9. The radix tree is changed to point to the new page.	119	9. The page cache tree is changed to point to the new page.
120		120
121	10. The reference count of the old page is dropped because the address space	121	10. The reference count of the old page is dropped because the address space
122	reference is gone. A reference to the new page is established because	122	reference is gone. A reference to the new page is established because

Documentation/translations/zh_CN/mm/ksm.rst+2−2

@@ -31,10 +31,10 @@ KSM维护着稳定树中的KSM页的逆映射信息。 · 30 unmodified lines
31		31
32	当KSM页面的共享数小于 ``max_page_sharing`` 的虚拟内存区域(VMAs)时，则代表了	32	当KSM页面的共享数小于 ``max_page_sharing`` 的虚拟内存区域(VMAs)时，则代表了
33	KSM页的稳定树其中的节点指向了一个ksm_rmap_item结构体类型的列表。同时，这个KSM页	33	KSM页的稳定树其中的节点指向了一个ksm_rmap_item结构体类型的列表。同时，这个KSM页
34	的 ``page->mapping`` 指向了该稳定树节点。	34	的 ``folio->mapping`` 指向了该稳定树节点。
35		35
36	如果共享数超过了阈值，KSM将给稳定树添加第二个维度。稳定树就变成链接一个或多	36	如果共享数超过了阈值，KSM将给稳定树添加第二个维度。稳定树就变成链接一个或多
37	个稳定树"副本"的"链"。每个副本都保留KSM页的逆映射信息，其中 ``page->mapping``	37	个稳定树"副本"的"链"。每个副本都保留KSM页的逆映射信息，其中 ``folio->mapping``
38	指向该"副本"。	38	指向该"副本"。
39		39
40	每个链以及链接到该链中的所有"副本"强制不变的是，它们代表了相同的写保护内存	40	每个链以及链接到该链中的所有"副本"强制不变的是，它们代表了相同的写保护内存

Documentation/userspace-api/ioctl/ioctl-number.rst+2−0

                                                                        <mailto:luzmaximilian@gmail.com>
 0xA5  20-2F  linux/surface_aggregator/dtx.h                            Microsoft Surface DTX driver
                                                                        <mailto:luzmaximilian@gmail.com>
+0xA6  00-0F  uapi/linux/alloc_tag.h                                    Memory allocation profiling
+                                                                       <mailto:surenb@google.com>
 0xAA  00-3F  linux/uapi/linux/userfaultfd.h
 0xAB  00-1F  linux/nbd.h
 0xAC  00-1F  linux/raw.h

MAINTAINERS+6−0

 F:	Documentation/core-api/boot-time-mm.rst
 F:	include/linux/kho/abi/memblock.h
 F:	include/linux/memblock.h
+F:	mm/arch_numa.c
 F:	mm/memblock.c
 F:	mm/memtest.c
 F:	mm/mm_init.c
 F:	Documentation/mm/allocation-profiling.rst
 F:	include/linux/alloc_tag.h
 F:	include/linux/pgalloc_tag.h
+F:	include/uapi/linux/alloc_tag.h
 F:	mm/alloc_tag.c
+F:	tools/testing/selftests/alloc_tag/
 MEMORY MANAGEMENT - BALLOON
 M:	Andrew Morton <akpm@linux-foundation.org>
 F:	Documentation/ABI/testing/sysfs-kernel-mm-numa
 F:	Documentation/admin-guide/mm/
 F:	Documentation/mm/
+F:	drivers/char/mem.c
 F:	include/linux/cma.h
 F:	include/linux/dmapool.h
 F:	include/linux/ioremap.h
 F:	Documentation/ABI/testing/sysfs-kernel-mm-swap
 F:	Documentation/mm/swap-table.rst
 F:	include/linux/swap.h
+F:	include/linux/swap_ops.h
 F:	include/linux/swapfile.h
 F:	include/linux/swapops.h
 F:	mm/page_io.c
 S:	Maintained
 W:	http://www.linux-mm.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
+F:	drivers/char/mem.c
 F:	include/trace/events/mmap.h
 F:	fs/proc/task_mmu.c
 F:	fs/proc/task_nommu.c

arch/s390/mm/gmap_helpers.c+1−1

@@ -203,7 +203,7 @@ static int find_zeropage_pte_entry(pte_t *pte, unsigned long addr, · 202 unmodified lines
203	* currently only works in COW mappings, which is also where	203	* currently only works in COW mappings, which is also where
204	* mm_forbids_zeropage() is checked.	204	* mm_forbids_zeropage() is checked.
205	*/	205	*/
206	if (!is_cow_mapping(walk->vma->vm_flags))	206	if (!vma_is_cow_mapping(walk->vma))
207	return -EFAULT;	207	return -EFAULT;
208		208
209	*found_addr = addr;	209	*found_addr = addr;

drivers/base/Kconfig+0−7

@@ -230,13 +230,6 @@ config GENERIC_ARCH_TOPOLOGY · 229 unmodified lines
230	appropriate scaling, sysfs interface for reading capacity values at	230	appropriate scaling, sysfs interface for reading capacity values at
231	runtime.	231	runtime.
232		232
233	config GENERIC_ARCH_NUMA
234	bool
235	select NUMA_MEMBLKS
236	help
237	Enable support for generic NUMA implementation. Currently, RISC-V
238	and ARM64 use it.
239
240	config FW_DEVLINK_SYNC_STATE_TIMEOUT	233	config FW_DEVLINK_SYNC_STATE_TIMEOUT
241	bool "sync_state() behavior defaults to timeout instead of strict"	234	bool "sync_state() behavior defaults to timeout instead of strict"
242	help	235	help

drivers/base/Makefile+0−1

@@ -25,7 +25,6 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o · 24 unmodified lines
25	obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o	25	obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o
26	obj-$(CONFIG_GENERIC_MSI_IRQ) += platform-msi.o	26	obj-$(CONFIG_GENERIC_MSI_IRQ) += platform-msi.o
27	obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o	27	obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o
28	obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
29	obj-$(CONFIG_ACPI) += physical_location.o	28	obj-$(CONFIG_ACPI) += physical_location.o
30		29
31	obj-y += test/	30	obj-y += test/

drivers/block/zram/backend_842.c+10−0

 // SPDX-License-Identifier: GPL-2.0-or-later
+#define pr_fmt(fmt) "842: " fmt
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/sw842.h>
 static int setup_params_842(struct zcomp_params *params)
 {
+	if (params->dict_sz) {
+		pr_err("dictionary is not supported\n");
+		return -EOPNOTSUPP;
+	}
+	if (params->level != ZCOMP_PARAM_NOT_SET) {
+		pr_err("compression level is not supported\n");
+		return -EOPNOTSUPP;
+	}
 	return 0;
 }

drivers/block/zram/backend_deflate.c+23−2

@@ -1,5 +1,7 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later	1	// SPDX-License-Identifier: GPL-2.0-or-later
2		2
		3	#define pr_fmt(fmt) "deflate: " fmt
		4
3	#include <linux/kernel.h>	5	#include <linux/kernel.h>
4	#include <linux/slab.h>	6	#include <linux/slab.h>
5	#include <linux/vmalloc.h>	7	#include <linux/vmalloc.h>
@@ -22,10 +24,29 @@ static void deflate_release_params(struct zcomp_params *params) · 16 unmodified lines
22		24
23	static int deflate_setup_params(struct zcomp_params *params)	25	static int deflate_setup_params(struct zcomp_params *params)
24	{	26	{
25	if (params->level == ZCOMP_PARAM_NOT_SET)	27	if (params->dict_sz) {
		28	pr_err("dictionary is not supported\n");
		29	return -EOPNOTSUPP;
		30	}
		31
		32	if (params->level == ZCOMP_PARAM_NOT_SET) {
26	params->level = Z_DEFAULT_COMPRESSION;	33	params->level = Z_DEFAULT_COMPRESSION;
27	if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET)	34	} else if (params->level < Z_DEFAULT_COMPRESSION \|\|
		35	params->level > Z_BEST_COMPRESSION) {
		36	pr_err("invalid compression level %d\n", params->level);
		37	return -EINVAL;
		38	}
		39
		40	if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) {
28	params->deflate.winbits = DEFLATE_DEF_WINBITS;	41	params->deflate.winbits = DEFLATE_DEF_WINBITS;
		42	} else {
		43	s32 wb = params->deflate.winbits;
		44
		45	if ((wb < -15 \|\| wb > -9) && (wb < 9 \|\| wb > 15)) {
		46	pr_err("invalid winbits %d\n", wb);
		47	return -EINVAL;
		48	}
		49	}
29		50
30	return 0;	51	return 0;
31	}	52	}

drivers/block/zram/backend_lz4.c+9−1

+// SPDX-License-Identifier: GPL-2.0-or-later
+#define pr_fmt(fmt) "lz4: " fmt
 #include <linux/kernel.h>
 #include <linux/lz4.h>
 #include <linux/slab.h>
 	LZ4_stream_t *dict_stream;
 	int ret;
-	if (params->level == ZCOMP_PARAM_NOT_SET)
+	if (params->level == ZCOMP_PARAM_NOT_SET) {
 		params->level = LZ4_ACCELERATION_DEFAULT;
+	} else if (params->level < LZ4_ACCELERATION_DEFAULT) {
+		pr_err("invalid compression level %d\n", params->level);
+		return -EINVAL;
+	}
 	if (!params->dict || !params->dict_sz)
 		return 0;

drivers/block/zram/backend_lz4hc.c+15−1

@@ -1,3 +1,7 @@
		1	// SPDX-License-Identifier: GPL-2.0-or-later
		2
		3	#define pr_fmt(fmt) "lz4hc: " fmt
		4
1	#include <linux/kernel.h>	5	#include <linux/kernel.h>
2	#include <linux/lz4.h>	6	#include <linux/lz4.h>
3	#include <linux/slab.h>	7	#include <linux/slab.h>
@@ -18,8 +22,18 @@ static void lz4hc_release_params(struct zcomp_params *params) · 14 unmodified lines
18		22
19	static int lz4hc_setup_params(struct zcomp_params *params)	23	static int lz4hc_setup_params(struct zcomp_params *params)
20	{	24	{
21	if (params->level == ZCOMP_PARAM_NOT_SET)	25	if (params->level == ZCOMP_PARAM_NOT_SET) {
22	params->level = LZ4HC_DEFAULT_CLEVEL;	26	params->level = LZ4HC_DEFAULT_CLEVEL;
		27	} else if (params->level < 1 \|\| params->level > LZ4HC_MAX_CLEVEL) {
		28	/*
		29	* Use < 1 rather than < LZ4HC_MIN_CLEVEL here because
		30	* LZ4HC_compress_generic() only clamps levels below 1
		31	* (levels 1 and 2 are valid). LZ4HC_MIN_CLEVEL (3) is
		32	* advisory and not enforced by the library.
		33	*/
		34	pr_err("invalid compression level %d\n", params->level);
		35	return -EINVAL;
		36	}
23		37
24	return 0;	38	return 0;
25	}	39	}

drivers/block/zram/backend_lzo.c+10−0

 // SPDX-License-Identifier: GPL-2.0-or-later
+#define pr_fmt(fmt) "lzo: " fmt
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/lzo.h>
 static int lzo_setup_params(struct zcomp_params *params)
 {
+	if (params->dict_sz) {
+		pr_err("dictionary is not supported\n");
+		return -EOPNOTSUPP;
+	}
+	if (params->level != ZCOMP_PARAM_NOT_SET) {
+		pr_err("compression level is not supported\n");
+		return -EOPNOTSUPP;
+	}
 	return 0;
 }

drivers/block/zram/backend_lzorle.c+10−0

 // SPDX-License-Identifier: GPL-2.0-or-later
+#define pr_fmt(fmt) "lzo-rle: " fmt
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/lzo.h>
 static int lzorle_setup_params(struct zcomp_params *params)
 {
+	if (params->dict_sz) {
+		pr_err("dictionary is not supported\n");
+		return -EOPNOTSUPP;
+	}
+	if (params->level != ZCOMP_PARAM_NOT_SET) {
+		pr_err("compression level is not supported\n");
+		return -EOPNOTSUPP;
+	}
 	return 0;
 }

drivers/block/zram/backend_zstd.c+8−3

 // SPDX-License-Identifier: GPL-2.0-or-later
+#define pr_fmt(fmt) "zstd: " fmt
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 		return -ENOMEM;
 	params->drv_data = zp;
-	if (params->level == ZCOMP_PARAM_NOT_SET)
+	if (params->level == ZCOMP_PARAM_NOT_SET) {
 		params->level = zstd_default_clevel();
+	} else if (params->level < zstd_min_clevel() ||
+		   params->level > zstd_max_clevel()) {
+		pr_err("invalid compression level %d\n", params->level);
+		goto error;
+	}
 	zp->cprm = zstd_get_params(params->level, PAGE_SIZE);
 	return 0;
 error:
-	zstd_release_params(params);
 	return -EINVAL;
 }
 	return 0;
 error:
-	zstd_release_params(params);
 	zstd_destroy(ctx);
 	return -EINVAL;
 }

drivers/block/zram/zram_drv.c+109−79

 static const struct block_device_operations zram_devops;
-static void slot_free(struct zram *zram, u32 index);
+static void slot_free(struct zram *zram, unsigned long index);
 /*
  * entry locking rules:
  * 4) Use TRY lock variant when in atomic context
  *    - must check return value and handle locking failers
  */
-static __must_check bool slot_trylock(struct zram *zram, u32 index)
+static __must_check bool slot_trylock(struct zram *zram, unsigned long index)
 {
 	unsigned long *lock = &zram->table[index].__lock;
-	if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK, lock)) {
+	if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK_BIT, lock)) {
 		mutex_acquire(&zram->table_lock_map, 0, 1, _RET_IP_);
 		lock_acquired(&zram->table_lock_map, _RET_IP_);
 		return true;
 	return false;
 }
-static void slot_lock(struct zram *zram, u32 index)
+static void slot_lock(struct zram *zram, unsigned long index)
 {
 	unsigned long *lock = &zram->table[index].__lock;
 	mutex_acquire(&zram->table_lock_map, 0, 0, _RET_IP_);
-	wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK, TASK_UNINTERRUPTIBLE);
+	wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK_BIT, TASK_UNINTERRUPTIBLE);
 	lock_acquired(&zram->table_lock_map, _RET_IP_);
 }
-static void slot_unlock(struct zram *zram, u32 index)
+static void slot_unlock(struct zram *zram, unsigned long index)
 {
 	unsigned long *lock = &zram->table[index].__lock;
 	mutex_release(&zram->table_lock_map, _RET_IP_);
-	clear_and_wake_up_bit(ZRAM_ENTRY_LOCK, lock);
+	clear_and_wake_up_bit(ZRAM_ENTRY_LOCK_BIT, lock);
 }
 static inline bool init_done(struct zram *zram)
 	return (struct zram *)dev_to_disk(dev)->private_data;
 }
-static unsigned long get_slot_handle(struct zram *zram, u32 index)
+static unsigned long get_slot_handle(struct zram *zram, unsigned long index)
 {
 	return zram->table[index].handle;
 }
-static void set_slot_handle(struct zram *zram, u32 index, unsigned long handle)
+static void set_slot_handle(struct zram *zram, unsigned long index,
+			    unsigned long handle)
 {
 	zram->table[index].handle = handle;
 }
-static bool test_slot_flag(struct zram *zram, u32 index,
+static bool test_slot_flag(struct zram *zram, unsigned long index,
 			   enum zram_pageflags flag)
 {
 	return zram->table[index].attr.flags & BIT(flag);
 }
-static void set_slot_flag(struct zram *zram, u32 index,
+static void set_slot_flag(struct zram *zram, unsigned long index,
 			  enum zram_pageflags flag)
 {
 	zram->table[index].attr.flags |= BIT(flag);
 }
-static void clear_slot_flag(struct zram *zram, u32 index,
+static void clear_slot_flag(struct zram *zram, unsigned long index,
 			    enum zram_pageflags flag)
 {
 	zram->table[index].attr.flags &= ~BIT(flag);
 }
-static size_t get_slot_size(struct zram *zram, u32 index)
+static size_t get_slot_size(struct zram *zram, unsigned long index)
 {
 	return zram->table[index].attr.flags & (BIT(ZRAM_FLAG_SHIFT) - 1);
 }
-static void set_slot_size(struct zram *zram, u32 index, size_t size)
+static void set_slot_size(struct zram *zram, unsigned long index, size_t size)
 {
 	unsigned long flags = zram->table[index].attr.flags >> ZRAM_FLAG_SHIFT;
 	zram->table[index].attr.flags = (flags << ZRAM_FLAG_SHIFT) | size;
 }
-static inline bool slot_allocated(struct zram *zram, u32 index)
+static inline bool slot_allocated(struct zram *zram, unsigned long index)
 {
 	return get_slot_size(zram, index) ||
 		test_slot_flag(zram, index, ZRAM_SAME) ||
 		test_slot_flag(zram, index, ZRAM_WB);
 }
-static inline void set_slot_comp_priority(struct zram *zram, u32 index,
+static inline void set_slot_comp_priority(struct zram *zram,
-					  u32 prio)
+					  unsigned long index, u32 prio)
 {
 	prio &= ZRAM_COMP_PRIORITY_MASK;
 	/*
 	zram->table[index].attr.flags |= (prio << ZRAM_COMP_PRIORITY_BIT1);
 }
-static inline u32 get_slot_comp_priority(struct zram *zram, u32 index)
+static inline u32 get_slot_comp_priority(struct zram *zram, unsigned long index)
 {
 	u32 prio = zram->table[index].attr.flags >> ZRAM_COMP_PRIORITY_BIT1;
 	return prio & ZRAM_COMP_PRIORITY_MASK;
 }
-static void mark_slot_accessed(struct zram *zram, u32 index)
+static void mark_slot_accessed(struct zram *zram, unsigned long index)
 {
 	clear_slot_flag(zram, index, ZRAM_IDLE);
 	clear_slot_flag(zram, index, ZRAM_PP_SLOT);
 }
 static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,
-			  u32 index)
+			  unsigned long index)
 {
 	struct zram_pp_slot *pps;
 	u32 bid;
 {
 	int is_idle = 1;
 	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
-	int index;
+	unsigned long index;
 	for (index = 0; index < nr_pages; index++) {
 		/*
 #define INVALID_BDEV_BLOCK		(~0UL)
 static int read_from_zspool_raw(struct zram *zram, struct page *page,
-				u32 index);
+				unsigned long index);
-static int read_from_zspool(struct zram *zram, struct page *page, u32 index);
+static int read_from_zspool(struct zram *zram, struct page *page,
+			    unsigned long index);
 struct zram_wb_ctl {
 	/* idle list is accessed only by the writeback task, no concurency */
 		/* error status (sync read) */
 		int error;
 	};
-	u32 index;
+	unsigned long index;
 };
 #define FOUR_K(x) ((x) * (1 << (PAGE_SHIFT - 12)))
 static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req)
 {
-	u32 index = req->pps->index;
+	unsigned long index = req->pps->index;
 	int err;
 	err = blk_status_to_errno(req->bio.bi_status);
 	struct zram_wb_req *req = NULL;
 	struct zram_pp_slot *pps;
 	int ret = 0, err = 0;
-	u32 index = 0;
+	unsigned long index = 0;
 	while ((pps = select_pp_slot(ctl))) {
 		if (zram->wb_limit_enable && !zram->bd_wb_limit) {
 				     unsigned long lo, unsigned long hi,
 				     struct zram_pp_ctl *ctl)
 {
-	u32 index = lo;
+	unsigned long index = lo;
 	while (index < hi) {
 		bool ok = true;
 			       const char *buf, size_t len)
 {
 	struct zram *zram = dev_to_zram(dev);
-	u64 nr_pages = zram->disksize >> PAGE_SHIFT;
+	unsigned long nr_pages;
-	unsigned long lo = 0, hi = nr_pages;
+	unsigned long lo = 0, hi;
 	struct zram_pp_ctl *pp_ctl = NULL;
 	struct zram_wb_ctl *wb_ctl = NULL;
 	char *args, *param, *val;
 	if (!zram->backing_dev)
 		return -ENODEV;
+	nr_pages = zram->disksize >> PAGE_SHIFT;
+	hi = nr_pages;
 	pp_ctl = init_pp_ctl();
 	if (!pp_ctl)
 		return -ENOMEM;
 	return ret;
 }
-static int decompress_bdev_page(struct zram *zram, struct page *page, u32 index)
+static int decompress_bdev_page(struct zram *zram, struct page *page,
+				unsigned long index)
 {
 	struct zcomp_strm *zstrm;
 	unsigned int size;
 	struct zram_rb_req *req = container_of(w, struct zram_rb_req, work);
 	struct page *page = bio_first_page_all(req->bio);
 	struct zram *zram = req->zram;
-	u32 index = req->index;
+	unsigned long index = req->index;
 	int ret;
 	ret = decompress_bdev_page(zram, page, index);
 }
 static int read_from_bdev_async(struct zram *zram, struct page *page,
-				u32 index, unsigned long blk_idx,
+				unsigned long index, unsigned long blk_idx,
 				struct bio *parent)
 {
 	struct zram_rb_req *req;
  * chained IO with parent IO in same context, it's a deadlock. To avoid that,
  * use a worker thread context.
  */
-static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index,
+static int read_from_bdev_sync(struct zram *zram, struct page *page,
-			       unsigned long blk_idx)
+			       unsigned long index, unsigned long blk_idx)
 {
 	struct zram_rb_req req;
 	return decompress_bdev_page(zram, page, index);
 }
-static int read_from_bdev(struct zram *zram, struct page *page, u32 index,
+static int read_from_bdev(struct zram *zram, struct page *page,
-			  unsigned long blk_idx, struct bio *parent)
+			  unsigned long index, unsigned long blk_idx,
+			  struct bio *parent)
 {
 	atomic64_inc(&zram->stats.bd_reads);
 	if (!parent) {
 }
 #else
 static inline void reset_bdev(struct zram *zram) {};
-static int read_from_bdev(struct zram *zram, struct page *page, u32 index,
+static int read_from_bdev(struct zram *zram, struct page *page,
-			  unsigned long blk_idx, struct bio *parent)
+			  unsigned long index, unsigned long blk_idx,
+			  struct bio *parent)
 {
 	return -EIO;
 }
 				size_t count, loff_t *ppos)
 {
 	char *kbuf;
-	ssize_t index, written = 0;
+	unsigned long index;
+	ssize_t written = 0;
 	struct zram *zram = file->private_data;
-	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
+	unsigned long nr_pages;
 	kbuf = kvmalloc(count, GFP_KERNEL);
 	if (!kbuf)
 		return -EINVAL;
 	}
+	nr_pages = zram->disksize >> PAGE_SHIFT;
 	for (index = *ppos; index < nr_pages; index++) {
 		int copied;
 			goto next;
 		copied = snprintf(kbuf + written, count,
-			"%12zd %12u.%06d %c%c%c%c%c%c\n",
+			"%12lu %12u.%06d %c%c%c%c%c%c\n",
 			index, zram->table[index].attr.ac_time, 0,
 			test_slot_flag(zram, index, ZRAM_SAME) ? 's' : '.',
 			test_slot_flag(zram, index, ZRAM_WB) ? 'w' : '.',
 	zram->comp_algs[prio] = alg;
 }
+static void comp_params_reset(struct zram *zram, u32 prio)
+{
+	struct zcomp_params *params = &zram->params[prio];
+	vfree(params->dict);
+	params->level = ZCOMP_PARAM_NOT_SET;
+	params->deflate.winbits = ZCOMP_PARAM_NOT_SET;
+	params->dict_sz = 0;
+	params->dict = NULL;
+}
 static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf)
 {
 	const char *alg;
 	}
 	comp_algorithm_set(zram, prio, alg);
+	comp_params_reset(zram, prio);
 	return 0;
 }
-static void comp_params_reset(struct zram *zram, u32 prio)
-{
-	struct zcomp_params *params = &zram->params[prio];
-	vfree(params->dict);
-	params->level = ZCOMP_PARAM_NOT_SET;
-	params->deflate.winbits = ZCOMP_PARAM_NOT_SET;
-	params->dict_sz = 0;
-	params->dict = NULL;
-}
 static int comp_params_store(struct zram *zram, u32 prio, s32 level,
 			     const char *dict_path,
 			     struct deflate_params *deflate_params)
 						INT_MAX,
 						NULL,
 						READING_POLICY);
-		if (sz < 0)
+		if (sz < 0) {
+			pr_err("failed to load dictionary %s (err=%zd)\n",
+			       dict_path, sz);
+			return sz;
+		}
+		if (sz == 0) {
+			pr_err("failed to load dictionary %s (empty file)\n",
+			       dict_path);
 			return -EINVAL;
+		}
 	}
 	zram->params[prio].dict_sz = sz;
 static void zram_meta_free(struct zram *zram, u64 disksize)
 {
-	size_t num_pages = disksize >> PAGE_SHIFT;
+	unsigned long num_pages = disksize >> PAGE_SHIFT;
-	size_t index;
+	unsigned long index;
 	if (!zram->table)
 		return;
 static bool zram_meta_alloc(struct zram *zram, u64 disksize)
 {
-	size_t num_pages;
+	unsigned long num_pages;
 	num_pages = disksize >> PAGE_SHIFT;
 	zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table)));
 	return true;
 }
-static void slot_free(struct zram *zram, u32 index)
+static void slot_free(struct zram *zram, unsigned long index)
 {
 	unsigned long handle;
 }
 static int read_same_filled_page(struct zram *zram, struct page *page,
-				 u32 index)
+				 unsigned long index)
 {
 	void *mem;
 }
 static int read_incompressible_page(struct zram *zram, struct page *page,
-				    u32 index)
+				    unsigned long index)
 {
 	unsigned long handle;
 	void *src, *dst;
 	return 0;
 }
-static int read_compressed_page(struct zram *zram, struct page *page, u32 index)
+static int read_compressed_page(struct zram *zram, struct page *page,
+				unsigned long index)
 {
 	struct zcomp_strm *zstrm;
 	unsigned long handle;
 }
 #if defined CONFIG_ZRAM_WRITEBACK
-static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index)
+static int read_from_zspool_raw(struct zram *zram, struct page *page,
+				unsigned long index)
 {
 	struct zcomp_strm *zstrm;
 	unsigned long handle;
  * Reads (decompresses if needed) a page from zspool (zsmalloc).
  * Corresponding ZRAM slot should be locked.
  */
-static int read_from_zspool(struct zram *zram, struct page *page, u32 index)
+static int read_from_zspool(struct zram *zram, struct page *page,
+			    unsigned long index)
 {
 	if (test_slot_flag(zram, index, ZRAM_SAME) ||
 	    !get_slot_handle(zram, index))
 		return read_incompressible_page(zram, page, index);
 }
-static int zram_read_page(struct zram *zram, struct page *page, u32 index,
+static int zram_read_page(struct zram *zram, struct page *page,
-			  struct bio *parent)
+			  unsigned long index, struct bio *parent)
 {
 	int ret;
 	/* Should NEVER happen. Return bio error if it does. */
 	if (WARN_ON(ret < 0))
-		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
+		pr_err("Decompression failed! err=%d, page=%lu\n", ret, index);
 	return ret;
 }
  * always expects a full page for the output.
  */
 static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec,
-				  u32 index, int offset)
+				  unsigned long index, int offset)
 {
 	struct page *page = alloc_page(GFP_NOIO);
 	int ret;
 }
 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
-			  u32 index, int offset, struct bio *bio)
+			  unsigned long index, int offset, struct bio *bio)
 {
 	if (is_partial_io(bvec))
 		return zram_bvec_read_partial(zram, bvec, index, offset);
 }
 static int write_same_filled_page(struct zram *zram, unsigned long fill,
-				  u32 index)
+				  unsigned long index)
 {
 	slot_lock(zram, index);
 	slot_free(zram, index);
 }
 static int write_incompressible_page(struct zram *zram, struct page *page,
-				     u32 index)
+				     unsigned long index)
 {
 	unsigned long handle;
 	void *src;
 	return 0;
 }
-static int zram_write_page(struct zram *zram, struct page *page, u32 index)
+static int zram_write_page(struct zram *zram, struct page *page,
+			   unsigned long index)
 {
 	int ret = 0;
 	unsigned long handle;
  * This is a partial IO. Read the full page before writing the changes.
  */
 static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec,
-				   u32 index, int offset)
+				   unsigned long index, int offset)
 {
 	struct page *page = alloc_page(GFP_NOIO);
 	int ret;
 }
 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
-			   u32 index, int offset)
+			   unsigned long index, int offset)
 {
 	if (is_partial_io(bvec))
 		return zram_bvec_write_partial(zram, bvec, index, offset);
  *
  * Corresponding ZRAM slot should be locked.
  */
-static int recompress_slot(struct zram *zram, u32 index, struct page *page,
+static int recompress_slot(struct zram *zram, unsigned long index,
-			   u64 *num_recomp_pages, u32 threshold, u32 prio)
+			   struct page *page, u64 *num_recomp_pages,
+			   u32 threshold, u32 prio)
 {
 	struct zcomp_strm *zstrm = NULL;
 	unsigned long handle_old;
 static void zram_bio_discard(struct zram *zram, struct bio *bio)
 {
 	size_t n = bio->bi_iter.bi_size;
-	u32 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
+	unsigned long index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
 	u32 offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) <<
 			SECTOR_SHIFT;
 	struct bvec_iter iter = bio->bi_iter;
 	do {
-		u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
+		unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
 		u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) <<
 				SECTOR_SHIFT;
 		struct bio_vec bv = bio_iter_iovec(bio, iter);
 	struct bvec_iter iter = bio->bi_iter;
 	do {
-		u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
+		unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
 		u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) <<
 				SECTOR_SHIFT;
 		struct bio_vec bv = bio_iter_iovec(bio, iter);
 		zram->comp_algs[prio] = NULL;
 	zram_comp_params_reset(zram);
+	comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);
 }
 static void zram_reset_device(struct zram *zram)
 	zram_destroy_comps(zram);
 	memset(&zram->stats, 0, sizeof(zram->stats));
 	reset_bdev(zram);
-	comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);
 }
 static ssize_t disksize_store(struct device *dev, struct device_attribute *attr,
 			      const char *buf, size_t len)
 {
+	unsigned long num_pages;
 	u64 disksize;
 	struct zcomp *comp;
 	struct zram *zram = dev_to_zram(dev);
 	}
 	disksize = PAGE_ALIGN(disksize);
+	num_pages = disksize >> PAGE_SHIFT;
+	/* Slots are addressed by an unsigned long index */
+	if (!num_pages || ((u64)num_pages << PAGE_SHIFT) != disksize)
+		return -EINVAL;
 	if (!zram_meta_alloc(zram, disksize))
 		return -ENOMEM;

drivers/block/zram/zram_drv.h+14−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	#ifndef _ZRAM_DRV_H_	15	#ifndef _ZRAM_DRV_H_
16	#define _ZRAM_DRV_H_	16	#define _ZRAM_DRV_H_
17		17
		18	#include <asm/byteorder.h>
18	#include <linux/rwsem.h>	19	#include <linux/rwsem.h>
19	#include <linux/zsmalloc.h>	20	#include <linux/zsmalloc.h>
20		21
@@ -57,6 +58,19 @@ enum zram_pageflags { · 36 unmodified lines
57	__NR_ZRAM_PAGEFLAGS,	58	__NR_ZRAM_PAGEFLAGS,
58	};	59	};
59		60
		61	/*
		62	* The slot lock is a bit-wait lock on the whole __lock word, while
		63	* flags and ac_time alias that word as two u32s. The lock bit must
		64	* land in the slot that ZRAM_ENTRY_LOCK reserves in attr.flags; on
		65	* 64-bit big-endian the flags word maps to the upper half of __lock,
		66	* so the bit position has to be shifted up.
		67	*/
		68	#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN)
		69	#define ZRAM_ENTRY_LOCK_BIT (ZRAM_ENTRY_LOCK + 32)
		70	#else
		71	#define ZRAM_ENTRY_LOCK_BIT ZRAM_ENTRY_LOCK
		72	#endif
		73
60	/*	74	/*
61	* Allocated for each disk page. We use bit-lock (ZRAM_ENTRY_LOCK bit	75	* Allocated for each disk page. We use bit-lock (ZRAM_ENTRY_LOCK bit
62	* of flags) to save memory. There can be plenty of entries and standard	76	* of flags) to save memory. There can be plenty of entries and standard

drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c+2−2

@@ -377,9 +377,9 @@ static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_str · 376 unmodified lines
377	/* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings	377	/* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
378	* for debugger access to invisible VRAM. Should have used MAP_SHARED	378	* for debugger access to invisible VRAM. Should have used MAP_SHARED
379	* instead. Clearing VM_MAYWRITE prevents the mapping from ever	379	* instead. Clearing VM_MAYWRITE prevents the mapping from ever
380	* becoming writable and makes is_cow_mapping(vm_flags) false.	380	* becoming writable and makes vma_is_cow_mapping(vma) false.
381	*/	381	*/
382	if (is_cow_mapping(vma->vm_flags) &&	382	if (vma_is_cow_mapping(vma) &&
383	!(vma->vm_flags & VM_ACCESS_FLAGS))	383	!(vma->vm_flags & VM_ACCESS_FLAGS))
384	vm_flags_clear(vma, VM_MAYWRITE);	384	vm_flags_clear(vma, VM_MAYWRITE);
385		385

drivers/gpu/drm/drm_gem_shmem_helper.c+1−1

@@ -771,7 +771,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct · 770 unmodified lines
771	return ret;	771	return ret;
772	}	772	}
773		773
774	if (is_cow_mapping(vma->vm_flags))	774	if (vma_is_cow_mapping(vma))
775	return -EINVAL;	775	return -EINVAL;
776		776
777	dma_resv_lock(shmem->base.resv, NULL);	777	dma_resv_lock(shmem->base.resv, NULL);

drivers/gpu/drm/i915/gem/i915_gem_shmem.c+1−1

@@ -325,7 +325,7 @@ void __shmem_writeback(size_t size, struct address_space *mapping) · 324 unmodified lines
325	if (folio_mapped(folio))	325	if (folio_mapped(folio))
326	folio_redirty_for_writepage(&wbc, folio);	326	folio_redirty_for_writepage(&wbc, folio);
327	else	327	else
328	error = shmem_writeout(folio, NULL, NULL);	328	error = shmem_write_folio(folio);
329	}	329	}
330	}	330	}
331		331

drivers/gpu/drm/panthor/panthor_gem.c+1−1

@@ -761,7 +761,7 @@ static int panthor_gem_mmap(struct drm_gem_object obj, struct vm_area_struct v · 760 unmodified lines
761	return ret;	761	return ret;
762	}	762	}
763		763
764	if (is_cow_mapping(vma->vm_flags))	764	if (vma_is_cow_mapping(vma))
765	return -EINVAL;	765	return -EINVAL;
766		766
767	if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) {	767	if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) {

drivers/gpu/drm/ttm/ttm_backup.c+1−1

@@ -160,7 +160,7 @@ ttm_backup_backup_folio(struct file backup, struct folio folio, · 159 unmodified lines
160	if (writeback && !folio_mapped(to_folio) &&	160	if (writeback && !folio_mapped(to_folio) &&
161	folio_clear_dirty_for_io(to_folio)) {	161	folio_clear_dirty_for_io(to_folio)) {
162	folio_set_reclaim(to_folio);	162	folio_set_reclaim(to_folio);
163	ret = shmem_writeout(to_folio, NULL, NULL);	163	ret = shmem_write_folio(to_folio);
164	if (!folio_test_writeback(to_folio))	164	if (!folio_test_writeback(to_folio))
165	folio_clear_reclaim(to_folio);	165	folio_clear_reclaim(to_folio);
166	if (ret == AOP_WRITEPAGE_ACTIVATE)	166	if (ret == AOP_WRITEPAGE_ACTIVATE)

drivers/gpu/drm/ttm/ttm_bo_vm.c+1−1

@@ -489,7 +489,7 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { · 488 unmodified lines
489	int ttm_bo_mmap_obj(struct vm_area_struct vma, struct ttm_buffer_object bo)	489	int ttm_bo_mmap_obj(struct vm_area_struct vma, struct ttm_buffer_object bo)
490	{	490	{
491	/* Enforce no COW since would have really strange behavior with it. */	491	/* Enforce no COW since would have really strange behavior with it. */
492	if (is_cow_mapping(vma->vm_flags))	492	if (vma_is_cow_mapping(vma))
493	return -EINVAL;	493	return -EINVAL;
494		494
495	drm_gem_object_get(&bo->base);	495	drm_gem_object_get(&bo->base);

drivers/gpu/drm/xe/xe_device.c+1−1

@@ -331,7 +331,7 @@ static int xe_pci_barrier_mmap(struct file *filp, · 330 unmodified lines
331	if (vma->vm_end - vma->vm_start > SZ_4K)	331	if (vma->vm_end - vma->vm_start > SZ_4K)
332	return -EINVAL;	332	return -EINVAL;
333		333
334	if (is_cow_mapping(vma->vm_flags))	334	if (vma_is_cow_mapping(vma))
335	return -EINVAL;	335	return -EINVAL;
336		336
337	if (vma->vm_flags & (VM_READ \| VM_EXEC))	337	if (vma->vm_flags & (VM_READ \| VM_EXEC))

fs/nfs/direct.c+0−20

@@ -145,26 +145,6 @@ static void nfs_direct_file_adjust_size_locked(struct inode *inode, · 144 unmodified lines
145	}	145	}
146	}	146	}
147		147
148	/**
149	* nfs_swap_rw - NFS address space operation for swap I/O
150	* @iocb: target I/O control block
151	* @iter: I/O buffer
152	*
153	* Perform IO to the swap-file. This is much like direct IO.
154	*/
155	int nfs_swap_rw(struct kiocb iocb, struct iov_iter iter)
156	{
157	ssize_t ret;
158
159	if (iov_iter_rw(iter) == READ)
160	ret = nfs_file_direct_read(iocb, iter, true);
161	else
162	ret = nfs_file_direct_write(iocb, iter, true);
163	if (ret < 0)
164	return ret;
165	return 0;
166	}
167
168	static void nfs_direct_release_pages(struct page **pages, unsigned int npages)	148	static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
169	{	149	{
170	unsigned int i;	150	unsigned int i;

fs/nfs/file.c+38−6

@@ -29,9 +29,8 @@ · 28 unmodified lines
29	#include <linux/pagemap.h>	29	#include <linux/pagemap.h>
30	#include <linux/gfp.h>	30	#include <linux/gfp.h>
31	#include <linux/rmap.h>	31	#include <linux/rmap.h>
32	#include <linux/swap.h>
33	#include <linux/compaction.h>	32	#include <linux/compaction.h>
34		33	#include <linux/swap_ops.h>
35	#include <linux/uaccess.h>	34	#include <linux/uaccess.h>
36	#include <linux/filelock.h>	35	#include <linux/filelock.h>
37		36
@@ -575,6 +574,38 @@ static int nfs_launder_folio(struct folio *folio) · 537 unmodified lines
575	return ret;	574	return ret;
576	}	575	}
577		576
		577	#ifdef CONFIG_SWAP
		578	static void nfs_swap_submit_write(struct swap_io_ctx *ctx)
		579	{
		580	struct swap_iocb *sio = ctx->sio;
		581	struct iov_iter iter;
		582	int ret;
		583
		584	swap_fs_prepare_rw(ctx, WRITE, &iter);
		585	ret = nfs_file_direct_write(&sio->iocb, &iter, true);
		586	if (ret != -EIOCBQUEUED)
		587	sio->iocb.ki_complete(&sio->iocb, ret);
		588	}
		589
		590	static void nfs_swap_submit_read(struct swap_io_ctx *ctx)
		591	{
		592	struct swap_iocb *sio = ctx->sio;
		593	struct iov_iter iter;
		594	int ret;
		595
		596	swap_fs_prepare_rw(ctx, READ, &iter);
		597	ret = nfs_file_direct_read(&sio->iocb, &iter, true);
		598	if (ret != -EIOCBQUEUED)
		599	sio->iocb.ki_complete(&sio->iocb, ret);
		600	}
		601
		602	static const struct swap_ops nfs_swap_ops = {
		603	.flags = SWAP_OPS_F_REQUIRE_NOFS,
		604	.submit_write = nfs_swap_submit_write,
		605	.submit_read = nfs_swap_submit_read,
		606	.can_merge = swap_fs_can_merge,
		607	};
		608
578	static int nfs_swap_activate(struct swap_info_struct sis, struct file file,	609	static int nfs_swap_activate(struct swap_info_struct sis, struct file file,
579	sector_t *span)	610	sector_t *span)
580	{	611	{
@@ -597,7 +628,7 @@ static int nfs_swap_activate(struct swap_info_struct sis, struct file file, · 16 unmodified lines
597	ret = rpc_clnt_swap_activate(clnt);	628	ret = rpc_clnt_swap_activate(clnt);
598	if (ret)	629	if (ret)
599	return ret;	630	return ret;
600	ret = add_swap_extent(sis, 0, sis->max, 0);	631	ret = swap_fs_activate(sis, &nfs_swap_ops);
601	if (ret < 0) {	632	if (ret < 0) {
602	rpc_clnt_swap_deactivate(clnt);	633	rpc_clnt_swap_deactivate(clnt);
603	return ret;	634	return ret;
@@ -607,8 +638,6 @@ static int nfs_swap_activate(struct swap_info_struct sis, struct file file, · 3 unmodified lines
607		638
608	if (cl->rpc_ops->enable_swap)	639	if (cl->rpc_ops->enable_swap)
609	cl->rpc_ops->enable_swap(inode);	640	cl->rpc_ops->enable_swap(inode);
610
611	sis->flags \|= SWP_FS_OPS;
612	return ret;	641	return ret;
613	}	642	}
614		643
@@ -622,6 +651,10 @@ static void nfs_swap_deactivate(struct file *file) · 7 unmodified lines
622	if (cl->rpc_ops->disable_swap)	651	if (cl->rpc_ops->disable_swap)
623	cl->rpc_ops->disable_swap(file_inode(file));	652	cl->rpc_ops->disable_swap(file_inode(file));
624	}	653	}
		654	#else
		655	#define nfs_swap_activate NULL
		656	#define nfs_swap_deactivate NULL
		657	#endif /* CONFIG_SWAP */
625		658
626	const struct address_space_operations nfs_file_aops = {	659	const struct address_space_operations nfs_file_aops = {
627	.read_folio = nfs_read_folio,	660	.read_folio = nfs_read_folio,
@@ -638,7 +671,6 @@ const struct address_space_operations nfs_file_aops = { · 10 unmodified lines
638	.error_remove_folio = generic_error_remove_folio,	671	.error_remove_folio = generic_error_remove_folio,
639	.swap_activate = nfs_swap_activate,	672	.swap_activate = nfs_swap_activate,
640	.swap_deactivate = nfs_swap_deactivate,	673	.swap_deactivate = nfs_swap_deactivate,
641	.swap_rw = nfs_swap_rw,
642	};	674	};
643		675
644	/*	676	/*

fs/proc/task_mmu.c+1−1

@@ -1693,7 +1693,7 @@ static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, · 1692 unmodified lines
1693		1693
1694	if (!pte_write(pte))	1694	if (!pte_write(pte))
1695	return false;	1695	return false;
1696	if (!is_cow_mapping(vma->vm_flags))	1696	if (!vma_is_cow_mapping(vma))
1697	return false;	1697	return false;
1698	if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm)))	1698	if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm)))
1699	return false;	1699	return false;

fs/ramfs/file-nommu.c+3−0

@@ -69,6 +69,9 @@ int ramfs_nommu_expand_for_mapping(struct inode *inode, size_t newsize) · 68 unmodified lines
69	gfp_t gfp = mapping_gfp_mask(inode->i_mapping);	69	gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
70		70
71	/* make various checks */	71	/* make various checks */
		72	if (!newsize)
		73	return 0;
		74
72	order = get_order(newsize);	75	order = get_order(newsize);
73	if (unlikely(order > MAX_PAGE_ORDER))	76	if (unlikely(order > MAX_PAGE_ORDER))
74	return -EFBIG;	77	return -EFBIG;

fs/smb/client/file.c+39−26

@@ -20,7 +20,7 @@ · 19 unmodified lines
20	#include <linux/delay.h>	20	#include <linux/delay.h>
21	#include <linux/mount.h>	21	#include <linux/mount.h>
22	#include <linux/slab.h>	22	#include <linux/slab.h>
23	#include <linux/swap.h>	23	#include <linux/swap_ops.h>
24	#include <linux/mm.h>	24	#include <linux/mm.h>
25	#include <asm/div64.h>	25	#include <asm/div64.h>
26	#include "cifsfs.h"	26	#include "cifsfs.h"
@@ -3406,6 +3406,38 @@ out: · 3379 unmodified lines
3406	cifs_done_oplock_break(cinode);	3406	cifs_done_oplock_break(cinode);
3407	}	3407	}
3408		3408
		3409	#ifdef CONFIG_SWAP
		3410	static void cifs_swap_submit_write(struct swap_io_ctx *ctx)
		3411	{
		3412	struct swap_iocb *sio = ctx->sio;
		3413	struct iov_iter iter;
		3414	int ret;
		3415
		3416	swap_fs_prepare_rw(ctx, WRITE, &iter);
		3417	ret = netfs_unbuffered_write_iter_locked(&sio->iocb, &iter, NULL);
		3418	if (ret != -EIOCBQUEUED)
		3419	sio->iocb.ki_complete(&sio->iocb, ret);
		3420	}
		3421
		3422	static void cifs_swap_submit_read(struct swap_io_ctx *ctx)
		3423	{
		3424	struct swap_iocb *sio = ctx->sio;
		3425	struct iov_iter iter;
		3426	int ret;
		3427
		3428	swap_fs_prepare_rw(ctx, READ, &iter);
		3429	ret = netfs_unbuffered_read_iter_locked(&sio->iocb, &iter);
		3430	if (ret != -EIOCBQUEUED)
		3431	sio->iocb.ki_complete(&sio->iocb, ret);
		3432	}
		3433
		3434	static const struct swap_ops cifs_swap_ops = {
		3435	.flags = SWAP_OPS_F_REQUIRE_NOFS,
		3436	.submit_write = cifs_swap_submit_write,
		3437	.submit_read = cifs_swap_submit_read,
		3438	.can_merge = swap_fs_can_merge,
		3439	};
		3440
3409	static int cifs_swap_activate(struct swap_info_struct *sis,	3441	static int cifs_swap_activate(struct swap_info_struct *sis,
3410	struct file swap_file, sector_t span)	3442	struct file swap_file, sector_t span)
3411	{	3443	{
@@ -3416,7 +3448,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis, · 4 unmodified lines
3416		3448
3417	cifs_dbg(FYI, "swap activate\n");	3449	cifs_dbg(FYI, "swap activate\n");
3418		3450
3419	if (!swap_file->f_mapping->a_ops->swap_rw)	3451	if (swap_file->f_mapping->a_ops != &cifs_addr_ops)
3420	/* Cannot support swap */	3452	/* Cannot support swap */
3421	return -EINVAL;	3453	return -EINVAL;
3422		3454
@@ -3447,9 +3479,7 @@ static int cifs_swap_activate(struct swap_info_struct *sis, · 24 unmodified lines
3447	* but we could add call to grab a byte range lock to prevent others	3479	* but we could add call to grab a byte range lock to prevent others
3448	* from reading or writing the file	3480	* from reading or writing the file
3449	*/	3481	*/
3450		3482	return swap_fs_activate(sis, &cifs_swap_ops);
3451	sis->flags \|= SWP_FS_OPS;
3452	return add_swap_extent(sis, 0, sis->max, 0);
3453	}	3483	}
3454		3484
3455	static void cifs_swap_deactivate(struct file *file)	3485	static void cifs_swap_deactivate(struct file *file)
@@ -3465,26 +3495,10 @@ static void cifs_swap_deactivate(struct file *file) · 9 unmodified lines
3465		3495
3466	/* do we need to unpin (or unlock) the file */	3496	/* do we need to unpin (or unlock) the file */
3467	}	3497	}
3468		3498	#else
3469	/**	3499	#define cifs_swap_activate NULL
3470	* cifs_swap_rw - SMB3 address space operation for swap I/O	3500	#define cifs_swap_deactivate NULL
3471	* @iocb: target I/O control block	3501	#endif /* CONFIG_SWAP */
3472	* @iter: I/O buffer
3473	*
3474	* Perform IO to the swap-file. This is much like direct IO.
3475	*/
3476	static int cifs_swap_rw(struct kiocb iocb, struct iov_iter iter)
3477	{
3478	ssize_t ret;
3479
3480	if (iov_iter_rw(iter) == READ)
3481	ret = netfs_unbuffered_read_iter_locked(iocb, iter);
3482	else
3483	ret = netfs_unbuffered_write_iter_locked(iocb, iter, NULL);
3484	if (ret < 0)
3485	return ret;
3486	return 0;
3487	}
3488		3502
3489	const struct address_space_operations cifs_addr_ops = {	3503	const struct address_space_operations cifs_addr_ops = {
3490	.read_folio = netfs_read_folio,	3504	.read_folio = netfs_read_folio,
@@ -3501,7 +3515,6 @@ const struct address_space_operations cifs_addr_ops = { · 10 unmodified lines
3501	*/	3515	*/
3502	.swap_activate = cifs_swap_activate,	3516	.swap_activate = cifs_swap_activate,
3503	.swap_deactivate = cifs_swap_deactivate,	3517	.swap_deactivate = cifs_swap_deactivate,
3504	.swap_rw = cifs_swap_rw,
3505	};	3518	};
3506		3519
3507	/*	3520	/*

include/linux/codetag.h+2−0

@@ -77,6 +77,8 @@ struct codetag_iterator { · 76 unmodified lines
77	void codetag_lock_module_list(struct codetag_type *cttype);	77	void codetag_lock_module_list(struct codetag_type *cttype);
78	bool codetag_trylock_module_list(struct codetag_type *cttype);	78	bool codetag_trylock_module_list(struct codetag_type *cttype);
79	void codetag_unlock_module_list(struct codetag_type *cttype);	79	void codetag_unlock_module_list(struct codetag_type *cttype);
		80	unsigned long codetag_get_content_id(struct codetag_type *cttype);
		81	unsigned int codetag_get_count(struct codetag_type *cttype);
80	struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype);	82	struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype);
81	struct codetag codetag_next_ct(struct codetag_iterator iter);	83	struct codetag codetag_next_ct(struct codetag_iterator iter);
82		84

include/linux/execmem.h+1−1

@@ -89,7 +89,7 @@ static inline int execmem_restore_rox(void *ptr, size_t size) { return 0; } · 88 unmodified lines
89	* @end: address space end (inclusive)	89	* @end: address space end (inclusive)
90	* @fallback_start: start of the secondary address space range for fallback	90	* @fallback_start: start of the secondary address space range for fallback
91	* allocations on architectures that require it	91	* allocations on architectures that require it
92	* @fallback_end: start of the secondary address space (inclusive)	92	* @fallback_end: end of the secondary address space (inclusive)
93	* @pgprot: permissions for memory in this address space	93	* @pgprot: permissions for memory in this address space
94	* @alignment: alignment required for text allocations	94	* @alignment: alignment required for text allocations
95	* @flags: options for memory allocations for this range	95	* @flags: options for memory allocations for this range

include/linux/fs.h+0−1

@@ -438,7 +438,6 @@ struct address_space_operations { · 437 unmodified lines
438	int (swap_activate)(struct swap_info_struct sis, struct file *file,	438	int (swap_activate)(struct swap_info_struct sis, struct file *file,
439	sector_t *span);	439	sector_t *span);
440	void (swap_deactivate)(struct file file);	440	void (swap_deactivate)(struct file file);
441	int (swap_rw)(struct kiocb iocb, struct iov_iter *iter);
442	};	441	};
443		442
444	extern const struct address_space_operations empty_aops;	443	extern const struct address_space_operations empty_aops;

include/linux/lockdep.h+3−0

@@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock); · 272 unmodified lines
273	extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);	273	extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);
274	extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);	274	extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);
275		275
		276	extern u32 lock_sequence(struct lockdep_map *lock);
		277	#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map)
		278
276	#define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0)	279	#define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0)
277		280
278	#define lockdep_assert(cond) \	281	#define lockdep_assert(cond) \

include/linux/lockdep_types.h+2−1

@@ -253,7 +253,8 @@ struct held_lock { · 252 unmodified lines
253	unsigned int hardirqs_off:1;	253	unsigned int hardirqs_off:1;
254	unsigned int sync:1;	254	unsigned int sync:1;
255	unsigned int references:11; /* 32 bits */	255	unsigned int references:11; /* 32 bits */
256	unsigned int pin_count;	256	unsigned int pin_count:24;
		257	unsigned int seq_count:8;
257	};	258	};
258		259
259	#else /* !CONFIG_LOCKDEP */	260	#else /* !CONFIG_LOCKDEP */

include/linux/maple_tree.h+9−7

 #include <linux/kernel.h>
 #include <linux/rcupdate.h>
 #include <linux/spinlock.h>
-/* #define CONFIG_MAPLE_RCU_DISABLED */
 /*
  * Allocated nodes are mutable until they have been inserted into the tree,
 #define MAPLE_NODE_SLOTS	31	/* 256 bytes including ->parent */
 #define MAPLE_RANGE64_SLOTS	16	/* 256 bytes */
 #define MAPLE_ARANGE64_SLOTS	10	/* 240 bytes */
-#define MAPLE_ALLOC_SLOTS	(MAPLE_NODE_SLOTS - 1)
 #else
 /* 32bit sizes */
 #define MAPLE_NODE_SLOTS	63	/* 256 bytes including ->parent */
 #define MAPLE_RANGE64_SLOTS	32	/* 256 bytes */
 #define MAPLE_ARANGE64_SLOTS	21	/* 240 bytes */
-#define MAPLE_ALLOC_SLOTS	(MAPLE_NODE_SLOTS - 2)
 #endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */
 #define MAPLE_NODE_MASK		255UL
 	unsigned char mas_flags;
 	unsigned char end;		/* The end of the node */
 	enum store_type store_type;	/* The type of store needed for this operation */
+#ifdef CONFIG_LOCKDEP
+	u32 ld_seq;
+#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
+	unsigned long rcu_gp;
+#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
+#endif /* CONFIG_LOCKDEP */
 };
 struct ma_wr_state {
 		unsigned long *next, gfp_t gfp);
 bool mas_nomem(struct ma_state *mas, gfp_t gfp);
+bool mas_nomem_nofail(struct ma_state *mas, unsigned long index,
+		      unsigned long last);
 void mas_pause(struct ma_state *mas);
 void maple_tree_init(void);
 void mas_destroy(struct ma_state *mas);
 void *mas_prev(struct ma_state *mas, unsigned long min);
-void *mas_prev_range(struct ma_state *mas, unsigned long max);
+void *mas_prev_range(struct ma_state *mas, unsigned long min);
 void *mas_next(struct ma_state *mas, unsigned long max);
 void *mas_next_range(struct ma_state *mas, unsigned long max);
 static inline bool mt_in_rcu(struct maple_tree *mt)
 {
-#ifdef CONFIG_MAPLE_RCU_DISABLED
-	return false;
-#endif
 	return mt->ma_flags & MT_FLAGS_USE_RCU;
 }

include/linux/memcontrol.h+17−4

 	 */
 	bool oom_group;
-	int swappiness;
 	/* memory.events and memory.events.local */
 	struct cgroup_file events_file;
 	struct cgroup_file events_local_file;
 #endif
 	int kmemcg_id;
-	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
 #ifdef CONFIG_CGROUP_WRITEBACK
 	struct list_head cgwb_list;
+#endif
+	/* Keep the hot per-CPU stats pointer away from memory event counters. */
+	struct memcg_vmstats_percpu __percpu *vmstats_percpu
+		____cacheline_aligned_in_smp;
+#ifdef CONFIG_CGROUP_WRITEBACK
 	struct wb_domain cgwb_domain;
 	struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
 #endif
 	/* List of events which userspace want to receive */
 	struct list_head event_list;
 	spinlock_t event_list_lock;
+	int swappiness;
 #endif /* CONFIG_MEMCG_V1 */
 	struct mem_cgroup_per_node *nodeinfo[];
 bool memcg_stat_item_valid(int idx);
 bool memcg_vm_event_item_valid(enum vm_event_item idx);
 unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx);
+unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+					  enum node_stat_item idx);
 unsigned long lruvec_page_state_local(struct lruvec *lruvec,
 				      enum node_stat_item idx);
 	return node_page_state(lruvec_pgdat(lruvec), idx);
 }
+static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+							enum node_stat_item idx)
+{
+	return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);
+}
 static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
 						    enum node_stat_item idx)
 {

include/linux/mm.h+125−7

@@ -1556,7 +1556,7 @@ static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) · 1555 unmodified lines
1556	desc->vm_ops = NULL;	1556	desc->vm_ops = NULL;
1557	}	1557	}
1558		1558
1559	static inline bool vma_is_anonymous(struct vm_area_struct *vma)	1559	static inline bool vma_is_anonymous(const struct vm_area_struct *vma)
1560	{	1560	{
1561	return !vma->vm_ops;	1561	return !vma->vm_ops;
1562	}	1562	}
@@ -2271,17 +2271,76 @@ void unpin_user_pages(struct page **pages, unsigned long npages); · 708 unmodified lines
2271	void unpin_user_folio(struct folio *folio, unsigned long npages);	2271	void unpin_user_folio(struct folio *folio, unsigned long npages);
2272	void unpin_folios(struct folio **folios, unsigned long nfolios);	2272	void unpin_folios(struct folio **folios, unsigned long nfolios);
2273		2273
2274	static inline bool is_cow_mapping(vm_flags_t flags)	2274	/**
		2275	* vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping?
		2276	* @flags: The VMA flags to check.
		2277	*
		2278	* Mappings which could be CoW'd (subject to Copy-On-Write faults) are
		2279	* described as CoW mappings.
		2280	*
		2281	* All mappings backed by anonymous folios (all anonymous mappings and most
		2282	* MAP_PRIVATE-file backed ranges) are CoW mappings.
		2283	*
		2284	* All other mappings (including all MAP_SHARED mappings) are non-CoW.
		2285	*
		2286	* The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT.
		2287	*
		2288	* VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both
		2289	* future mprotect() calls which can render a read-only mapping writable, and
		2290	* GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping.
		2291	*
		2292	* - No anonymous mapping can ever clear VMA_MAYWRITE_BIT.
		2293	*
		2294	* - Writes to anonymous mappings do not immediately result in CoW faults but
		2295	* may do so after the process is forked or if a read is followed by a
		2296	* write.
		2297	*
		2298	* - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may
		2299	* do so again after fork.
		2300	*
		2301	* - MAP_SHARED mappings of a file opened read-only are transformed into
		2302	* VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain
		2303	* non-CoW.
		2304	*
		2305	* - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot
		2306	* mark themselves anonymous. Having cleared this flag it is not valid for
		2307	* them to leave the VMA_WRITE_BIT flag set.
		2308	*
		2309	* As a consequence, the anonymous reverse mapping only tracks CoW mappings.
		2310	*
		2311	* Returns: true if the flags indicate a CoW mapping, otherwise false.
		2312	*/
		2313	static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags)
2275	{	2314	{
2276	return (flags & (VM_SHARED \| VM_MAYWRITE)) == VM_MAYWRITE;	2315	return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&
		2316	!vma_flags_test(flags, VMA_SHARED_BIT);
2277	}	2317	}
2278		2318
2279	static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc)	2319	/**
		2320	* vma_is_cow_mapping() - Is this VMA a CoW mapping?
		2321	* @vma: The VMA to check.
		2322	*
		2323	* See vma_flags_is_cow_mapping() for details.
		2324	*
		2325	* Returns: true if the VMA is a CoW mapping, otherwise false.
		2326	*/
		2327	static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma)
2280	{	2328	{
2281	const vma_flags_t *flags = &desc->vma_flags;	2329	return vma_flags_is_cow_mapping(&vma->flags);
		2330	}
2282		2331
2283	return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&	2332	/**
2284	!vma_flags_test(flags, VMA_SHARED_BIT);	2333	* vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping?
		2334	* @desc: The VMA descriptor to check.
		2335	*
		2336	* See vma_flags_is_cow_mapping() for details.
		2337	*
		2338	* Returns: true if the VMA descriptor describes a CoW mapping, otherwise
		2339	* false.
		2340	*/
		2341	static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc)
		2342	{
		2343	return vma_flags_is_cow_mapping(&desc->vma_flags);
2285	}	2344	}
2286		2345
2287	#ifndef CONFIG_MMU	2346	#ifndef CONFIG_MMU
@@ -4393,6 +4452,65 @@ static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) · 2105 unmodified lines
4393	return vma_end_pgoff(vma) - 1;	4452	return vma_end_pgoff(vma) - 1;
4394	}	4453	}
4395		4454
		4455	/**
		4456	* vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma
		4457	* @vma: The VMA whose anonymous page offset is required.
		4458	*
		4459	* If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the
		4460	* anonymous page offset at the time of first fault.
		4461	*
		4462	* If the VMA is anonymous, this returns the same value as vma_start_pgoff().
		4463	*
		4464	* This value is used for tracking MAP_PRIVATE file-backed mappings by their
		4465	* anonymous page offset.
		4466	*
		4467	* Returns: The anonymous page offset of the start of @vma.
		4468	*/
		4469	static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma)
		4470	{
		4471	pgoff_t pgoff = 0;
		4472
		4473	#ifdef CONFIG_64BIT
		4474	pgoff += vma->__vm_anon_pgoff_hi;
		4475	pgoff <<= 32;
		4476	#endif
		4477	pgoff += vma->__vm_anon_pgoff_lo;
		4478	return pgoff;
		4479	}
		4480
		4481	/**
		4482	* vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of
		4483	* @vma.
		4484	* @vma: The VMA whose end anonymous page offset is required.
		4485	*
		4486	* This returns the anonymous exclusive end page offset of @vma, which is useful
		4487	* for expressing page offset ranges.
		4488	*
		4489	* See the description of vma_start_anon_pgoff() for a description of VMA
		4490	* anonymous page offsets.
		4491	*
		4492	* Returns: The exclusive end anonymous page offset of @vma.
		4493	*/
		4494	static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma)
		4495	{
		4496	return vma_start_anon_pgoff(vma) + vma_pages(vma);
		4497	}
		4498
		4499	/**
		4500	* vma_last_anon_pgoff() - Get the anonymous page offset of the last page in
		4501	* @vma.
		4502	* @vma: The VMA whose last anonymous page offset is required.
		4503	*
		4504	* See the description of vma_start_anon_pgoff() for a description of VMA
		4505	* anonymous page offsets.
		4506	*
		4507	* Returns: The last anonymous page offset of @vma.
		4508	*/
		4509	static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma)
		4510	{
		4511	return vma_end_anon_pgoff(vma) - 1;
		4512	}
		4513
4396	static inline unsigned long vma_desc_size(const struct vm_area_desc *desc)	4514	static inline unsigned long vma_desc_size(const struct vm_area_desc *desc)
4397	{	4515	{
4398	return desc->end - desc->start;	4516	return desc->end - desc->start;

include/linux/mm_types.h+26−14

 	 */
 	unsigned int vm_lock_seq;
 #endif
+	/*
+	 * Low 32-bits of anonymous page offset.
+	 * See vma_start_anon_pgoff() comment for details.
+	 */
+	unsigned int __vm_anon_pgoff_lo;
 	/*
 	 * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
 	 * list, after a COW of one of the file pages.	A MAP_SHARED vma
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	struct lockdep_map vmlock_dep_map;
 #endif
+#endif
+#ifdef CONFIG_64BIT
+	/*
+	 * High 32-bits of anonymous page offset.
+	 * See vma_start_anon_pgoff() comment for details.
+	 */
+	unsigned int __vm_anon_pgoff_hi;
 #endif
 	/*
 	 * For areas with an address space and backing store,
 			VM_FAULT_SIGSEGV | VM_FAULT_HWPOISON |	\
 			VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK)
-#define VM_FAULT_RESULT_TRACE \
+#define VM_FAULT_RESULT_TRACE						\
-	{ VM_FAULT_OOM,                 "OOM" },	\
+	{ (__force u32)VM_FAULT_OOM,                 "OOM" },		\
-	{ VM_FAULT_SIGBUS,              "SIGBUS" },	\
+	{ (__force u32)VM_FAULT_SIGBUS,              "SIGBUS" },	\
-	{ VM_FAULT_MAJOR,               "MAJOR" },	\
+	{ (__force u32)VM_FAULT_MAJOR,               "MAJOR" },		\
-	{ VM_FAULT_HWPOISON,            "HWPOISON" },	\
+	{ (__force u32)VM_FAULT_HWPOISON,            "HWPOISON" },	\
-	{ VM_FAULT_HWPOISON_LARGE,      "HWPOISON_LARGE" },	\
+	{ (__force u32)VM_FAULT_HWPOISON_LARGE,      "HWPOISON_LARGE" }, \
-	{ VM_FAULT_SIGSEGV,             "SIGSEGV" },	\
+	{ (__force u32)VM_FAULT_SIGSEGV,             "SIGSEGV" },	\
-	{ VM_FAULT_NOPAGE,              "NOPAGE" },	\
+	{ (__force u32)VM_FAULT_NOPAGE,              "NOPAGE" },	\
-	{ VM_FAULT_LOCKED,              "LOCKED" },	\
+	{ (__force u32)VM_FAULT_LOCKED,              "LOCKED" },	\
-	{ VM_FAULT_RETRY,               "RETRY" },	\
+	{ (__force u32)VM_FAULT_RETRY,               "RETRY" },		\
-	{ VM_FAULT_FALLBACK,            "FALLBACK" },	\
+	{ (__force u32)VM_FAULT_FALLBACK,            "FALLBACK" },	\
-	{ VM_FAULT_DONE_COW,            "DONE_COW" },	\
+	{ (__force u32)VM_FAULT_DONE_COW,            "DONE_COW" },	\
-	{ VM_FAULT_NEEDDSYNC,           "NEEDDSYNC" },	\
+	{ (__force u32)VM_FAULT_NEEDDSYNC,           "NEEDDSYNC" },	\
-	{ VM_FAULT_COMPLETED,           "COMPLETED" }
+	{ (__force u32)VM_FAULT_COMPLETED,           "COMPLETED" }
 struct vm_special_mapping {
 	const char *name;	/* The name, e.g. "[vdso]". */

include/linux/mmzone.h+13−3

 	PGSCAN_PROACTIVE,
 	PGSCAN_ANON,
 	PGSCAN_FILE,
+	PGROTATE_ANON,
+	PGROTATE_FILE,
 	PGREFILL,
 #ifdef CONFIG_HUGETLB_PAGE
 	NR_HUGETLB,
 #endif /* CONFIG_LRU_GEN */
+struct lru_cost {
+	unsigned long		count;
+	unsigned long		last_rotated;
+	unsigned long		last_io;
+};
 struct lruvec {
 	struct list_head		lists[NR_LRU_LISTS];
 	/* per lruvec lru_lock for memcg */
 	 * These track the cost of reclaiming one LRU - file or anon -
 	 * over the other. As the observed cost of reclaiming one LRU
 	 * increases, the reclaim scan balance tips toward the other.
+	 * Updated and decayed at prepare_scan_control() time; cost_lock
+	 * serialises that update.
 	 */
-	unsigned long			anon_cost;
+	struct lru_cost			cost[ANON_AND_FILE];
-	unsigned long			file_cost;
+	/* Protects cost[]. */
+	spinlock_t			cost_lock;
 	/* Non-resident age, driven by LRU movement */
 	atomic_long_t			nonresident_age;
 	/* Refaults at the time of last reclaim cycle */
 #endif
 	return &mem_section[root][nr & SECTION_ROOT_MASK];
 }
-extern size_t mem_section_usage_size(void);
 /*
  * We use the lower bits of the mem_map pointer to store a little bit of

include/linux/nfs_fs.h+0−1

@@ -552,7 +552,6 @@ static inline const struct cred nfs_file_cred(struct file file) · 551 unmodified lines
552	/*	552	/*
553	* linux/fs/nfs/direct.c	553	* linux/fs/nfs/direct.c
554	*/	554	*/
555	int nfs_swap_rw(struct kiocb iocb, struct iov_iter iter);
556	ssize_t nfs_file_direct_read(struct kiocb *iocb,	555	ssize_t nfs_file_direct_read(struct kiocb *iocb,
557	struct iov_iter *iter, bool swap);	556	struct iov_iter *iter, bool swap);
558	ssize_t nfs_file_direct_write(struct kiocb *iocb,	557	ssize_t nfs_file_direct_write(struct kiocb *iocb,

include/linux/page_ext.h+0−5

 extern bool early_page_ext;
 extern unsigned long page_ext_size;
-extern void pgdat_page_ext_init(struct pglist_data *pgdat);
 static inline bool early_page_ext_enabled(void)
 {
 	return false;
 }
-static inline void pgdat_page_ext_init(struct pglist_data *pgdat)
-{
-}
 static inline void page_ext_init(void)
 {
 }

include/linux/pagemap.h+37−3

@@ -1094,10 +1094,44 @@ static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, · 1093 unmodified lines
1094	static inline pgoff_t linear_page_index(const struct vm_area_struct *vma,	1094	static inline pgoff_t linear_page_index(const struct vm_area_struct *vma,
1095	const unsigned long address)	1095	const unsigned long address)
1096	{	1096	{
1097	pgoff_t pgoff;	1097	return linear_page_delta(vma, address) + vma_start_pgoff(vma);
		1098	}
		1099
		1100	static inline pgoff_t __linear_anon_page_index(const struct vm_area_struct *vma,
		1101	const unsigned long address)
		1102	{
		1103	return linear_page_delta(vma, address) + vma_start_anon_pgoff(vma);
		1104	}
		1105
		1106	/**
		1107	* linear_anon_page_index() - Determine the absolute anonymous page offset of
		1108	* @address within @vma.
		1109	* @vma: An anonymous or MAP_PRIVATE file-backed VMA in which @address resides.
		1110	* @address: The address whose absolute page offset is required.
		1111	*
		1112	* This returns the anonymous page offset of @address, which is the page offset
		1113	* the address possessed at the time the VMA was first faulted.
		1114	*
		1115	* For anonymous mappings, this returns the same value as linear_page_index().
		1116	*
		1117	* For MAP_PRIVATE file-backed mappings, this returns the anonymous page offset
		1118	* of @address, which is the page offset the address possessed at the time the
		1119	* VMA was first faulted.
		1120	*
		1121	* It is not valid to call this function for shared file-backed mappings.
		1122	*
		1123	* Returns: The absolute anonymous page offset of @address within @vma.
		1124	*/
		1125	static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma,
		1126	const unsigned long address)
		1127	{
		1128	const pgoff_t pgoff = __linear_anon_page_index(vma, address);
		1129
		1130	VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma));
		1131	/* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */
		1132	if (vma_is_anonymous(vma) && !vma->vm_file)
		1133	VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address));
1098		1134
1099	pgoff = linear_page_delta(vma, address);
1100	pgoff += vma_start_pgoff(vma);
1101	return pgoff;	1135	return pgoff;
1102	}	1136	}
1103		1137

include/linux/percpu-defs.h+1−6

  *
  * Archs which need weak percpu definitions should set
  * CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU when necessary.
- *
- * To ensure that the generic code observes the above two
- * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
- * definition is used for all cases.
  */
-#if (defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)) || \
+#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)
-	defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
 /*
  * __pcpu_scope_* dummy variable is used to enforce scope.  It
  * receives the static modifier when it's used in front of

include/linux/rmap.h+6−5

  * Called from mm/vmscan.c to handle paging out
  */
 int folio_referenced(struct folio *, int is_locked,
-			struct mem_cgroup *memcg, vm_flags_t *vm_flags);
+		struct mem_cgroup *memcg, vma_flags_t *vma_flags);
 void try_to_migrate(struct folio *folio, enum ttu_flags flags);
 void try_to_unmap(struct folio *, enum ttu_flags flags);
 struct page_vma_mapped_walk {
 	unsigned long pfn;
 	unsigned long nr_pages;
-	pgoff_t pgoff;
+	pgoff_t pgoff;	/* Only meaningful if nr_pages > 1 and not a KSM walk */
 	struct vm_area_struct *vma;
 	unsigned long address;
 	pmd_t *pmd;
 	pte_t *pte;
 	spinlock_t *ptl;
 	unsigned int flags;
+	bool pgoff_is_anon : 1;
 };
 #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags)	\
 		.vma = _vma,						\
 		.address = _address,					\
 		.flags = _flags,					\
+		.pgoff_is_anon = folio_test_anon(_folio),		\
 	}
 static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
 #define anon_vma_prepare(vma)	(0)
 static inline int folio_referenced(struct folio *folio, int is_locked,
-				  struct mem_cgroup *memcg,
+		struct mem_cgroup *memcg, vma_flags_t *vma_flags)
-				  vm_flags_t *vm_flags)
 {
-	*vm_flags = 0;
+	vma_flags_clear_all(vma_flags);
 	return 0;
 }

include/linux/sched.h+1−0

@@ -1298,6 +1298,7 @@ struct task_struct { · 1297 unmodified lines
1298	u64 curr_chain_key;	1298	u64 curr_chain_key;
1299	int lockdep_depth;	1299	int lockdep_depth;
1300	unsigned int lockdep_recursion;	1300	unsigned int lockdep_recursion;
		1301	unsigned int lockdep_seq;
1301	struct held_lock held_locks[MAX_LOCK_DEPTH];	1302	struct held_lock held_locks[MAX_LOCK_DEPTH];
1302	#endif	1303	#endif
1303		1304

include/linux/shmem_fs.h+1−4

 #include <linux/userfaultfd_k.h>
 #include <linux/bits.h>
-struct swap_iocb;
 /* inode in-kernel data */
 #ifdef CONFIG_TMPFS_QUOTA
 void shmem_unlock_mapping(struct address_space *mapping);
 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
 					pgoff_t index, gfp_t gfp_mask);
-int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
+int shmem_write_folio(struct folio *folio);
-		struct list_head *folio_list);
 void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end);
 int shmem_unuse(unsigned int type);

include/linux/swap.h+9−21

 	SWP_SOLIDSTATE	= (1 << 4),	/* blkdev seeks are cheap */
 	SWP_BLKDEV	= (1 << 6),	/* its a block device */
 	SWP_ACTIVATED	= (1 << 7),	/* set after swap_activate success */
-	SWP_FS_OPS	= (1 << 8),	/* swapfile operations go through fs */
 	SWP_AREA_DISCARD = (1 << 9),	/* single-time swap area discards */
 	SWP_PAGE_DISCARD = (1 << 10),	/* freed swap page-cluster discards */
 	SWP_STABLE_WRITES = (1 << 11),	/* no overwrite PG_writeback pages */
 	struct work_struct reclaim_work; /* reclaim worker */
 	struct list_head discard_clusters; /* discard clusters list */
 	struct plist_node avail_list;   /* entry in swap_avail_head */
+	const struct swap_ops *ops;
 };
 static inline swp_entry_t page_swap_entry(struct page *page)
 void folio_mark_accessed(struct folio *folio);
 void lru_add_drain_all(void);
+enum lru_cache_drained {
+	LRU_CACHE_NOT_DRAINED,
+	LRU_CACHE_DRAINED,
+	LRU_CACHE_DRAINED_ALL,
+};
+void lru_cache_drain_for_folio(const struct folio *folio,
+		unsigned int extra_refs, enum lru_cache_drained *drained);
 /* linux/mm/folio-compat.c */
 void mark_page_accessed(struct page *page);
 }
 extern unsigned long shrink_all_memory(unsigned long nr_pages);
-extern int vm_swappiness;
 long remove_mapping(struct address_space *mapping, struct folio *folio);
 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
 extern void __meminit kswapd_stop(int nid);
 #ifdef CONFIG_SWAP
 int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
 		unsigned long nr_pages, sector_t start_block);
 int generic_swapfile_activate(struct swap_info_struct *, struct file *,
 }
 #endif /* CONFIG_SWAP */
 #ifdef CONFIG_MEMCG
-static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
-{
-	/* Cgroup2 doesn't have per-cgroup swappiness */
-	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
-		return READ_ONCE(vm_swappiness);
-	/* root ? */
-	if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg))
-		return READ_ONCE(vm_swappiness);
-	return READ_ONCE(memcg->swappiness);
-}
 void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
-#else
-static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
-{
-	return READ_ONCE(vm_swappiness);
-}
 #endif
 #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)

include/linux/swap_ops.h+44−0

@@ -0,0 +1,44 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _MM_SWAP_OPS_H
	3	#define _MM_SWAP_OPS_H
	4
	5	#include <linux/swap.h> /* for SWAP_CLUSTER_MAX */
	6
	7	struct swap_iocb {
	8	union {
	9	struct kiocb iocb;
	10	struct bio bio;
	11	};
	12	struct bio_vec bvecs[SWAP_CLUSTER_MAX];
	13	int nr_bvecs;
	14	int len;
	15	};
	16
	17	struct swap_io_ctx {
	18	struct swap_iocb *sio;
	19	struct swap_info_struct *sis;
	20	};
	21
	22	/*
	23	* SWAP_OPS_F_REQUIRE_NOFS:
	24	* When set, all reclaim operations must operated as GFS_NOFS and not
	25	* just GFP_NOIO, as GFP_NOIO allocations could recourse into the
	26	* file system backing this swap file.
	27	*/
	28	#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0)
	29
	30	struct swap_ops {
	31	unsigned int flags;
	32
	33	bool (can_merge)(struct folio folio, struct folio *prev_folio,
	34	size_t prev_folio_size, int rw);
	35	void (submit_write)(struct swap_io_ctx ctx);
	36	void (submit_read)(struct swap_io_ctx ctx);
	37	};
	38
	39	void swap_fs_prepare_rw(struct swap_io_ctx ctx, int rw, struct iov_iter iter);
	40	bool swap_fs_can_merge(struct folio folio, struct folio prev_folio,
	41	size_t prev_folio_size, int rw);
	42	int swap_fs_activate(struct swap_info_struct sis, const struct swap_ops ops);
	43
	44	#endif /* _MM_SWAP_OPS_H */

include/linux/swapops.h+4−3

 #include <linux/radix-tree.h>
 #include <linux/bug.h>
 #include <linux/mm_types.h>
+#include <linux/swap.h>
 #ifdef CONFIG_MMU
 extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
 		struct page *page);
-extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
+void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
-		struct page *new);
+		struct folio *folio);
 extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd);
 }
 static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
-		struct page *new)
+		struct folio *folio)
 {
 	BUILD_BUG();
 }

include/linux/vm_event_item.h+4−0

 		KSTACK_REST,
 #endif
 #endif /* CONFIG_DEBUG_STACK_USAGE */
+#ifdef CONFIG_SWAP
+		NRSWPIN,
+		NRSWPOUT,
+#endif /* CONFIG_SWAP */
 		NR_VM_EVENT_ITEMS
 };

include/linux/vmalloc.h+1−1

@@ -62,7 +62,7 @@ struct vm_struct { · 61 unmodified lines
62	#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC	62	#ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
63	unsigned int page_order;	63	unsigned int page_order;
64	#endif	64	#endif
65	unsigned int nr_pages;	65	unsigned long nr_pages;
66	phys_addr_t phys_addr;	66	phys_addr_t phys_addr;
67	const void *caller;	67	const void *caller;
68	unsigned long requested_size;	68	unsigned long requested_size;

include/linux/vmstat.h+16−1

 	unsigned nr_congested;
 	unsigned nr_writeback;
 	unsigned nr_immediate;
-	unsigned nr_pageout;
 	unsigned nr_activate[ANON_AND_FILE];
 	unsigned nr_ref_keep;
 	unsigned nr_unmap_fail;
 	return x;
 }
+/*
+ * Non-clamping variant of global_node_page_state() intended for callers that
+ * snapshot a monotonically-incremented counter and subtract two samples.
+ * Returns the raw wrapping value so that unsigned modular subtraction stays
+ * correct across a signed-long overflow (a real hazard on 32-bit) that the
+ * clamp in global_node_page_state() would otherwise turn into a huge spurious
+ * delta. Do NOT use for non-monotonic page-count reads.
+ */
+static inline unsigned long global_node_page_state_monotonic(enum node_stat_item item)
+{
+	return (unsigned long)atomic_long_read(&vm_node_stat[item]);
+}
 static inline unsigned long global_node_page_state(enum node_stat_item item)
 {
 	VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
 						enum node_stat_item item);
 extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
 					   enum node_stat_item item);
+extern unsigned long node_page_state_monotonic(struct pglist_data *pgdat,
+					       enum node_stat_item item);
 extern void fold_vm_numa_events(void);
 #else
 #define sum_zone_node_page_state(node, item) global_zone_page_state(item)
 #define node_page_state(node, item) global_node_page_state(item)
 #define node_page_state_pages(node, item) global_node_page_state_pages(item)
+#define node_page_state_monotonic(node, item) global_node_page_state_monotonic(item)
 static inline void fold_vm_numa_events(void)
 {
 }

include/rv/da_monitor.h+1−1

@@ -25,7 +25,7 @@ · 24 unmodified lines
25		25
26	/*	26	/*
27	* Per-cpu variables require a unique name although static in some	27	* Per-cpu variables require a unique name although static in some
28	* configurations (e.g. CONFIG_DEBUG_FORCE_WEAK_PER_CPU or alpha modules).	28	* configurations (e.g. alpha modules).
29	*/	29	*/
30	#define DA_MON_NAME CONCATENATE(da_mon_, MONITOR_NAME)	30	#define DA_MON_NAME CONCATENATE(da_mon_, MONITOR_NAME)
31		31

include/uapi/linux/alloc_tag.h+99−0

@@ -0,0 +1,99 @@
	1	/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
	2	/*
	3	* alloc_tag IOCTL API definition
	4	*
	5	* Copyright (C) 2026 Google, LLC. All rights reserved.
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*/
	11
	12	#ifndef _UAPI_ALLOC_TAG_H
	13	#define _UAPI_ALLOC_TAG_H
	14
	15	#include <linux/types.h>
	16
	17	/*
	18	* Function, file and module names often have the same prefixes, therefore
	19	* when filtering by these criteria, we compare the last 64 characters to
	20	* minimize the chances of name collisions
	21	*/
	22	#define ALLOCINFO_STR_SIZE 64
	23
	24	struct allocinfo_content_id {
	25	__u64 id;
	26	};
	27
	28	struct allocinfo_tag {
	29	/* Longer names are trimmed */
	30	char modname[ALLOCINFO_STR_SIZE];
	31	char function[ALLOCINFO_STR_SIZE];
	32	char filename[ALLOCINFO_STR_SIZE];
	33	__u64 lineno;
	34	};
	35
	36	/* The alignment ensures 32-bit compatible interfaces are not broken */
	37	struct allocinfo_counter {
	38	__u64 bytes;
	39	__u64 calls;
	40	__u8 accurate;
	41	} __attribute__((aligned(8)));
	42
	43	struct allocinfo_tag_data {
	44	struct allocinfo_tag tag;
	45	struct allocinfo_counter counter;
	46	};
	47
	48	enum {
	49	ALLOCINFO_FILTER_MODNAME,
	50	ALLOCINFO_FILTER_FUNCTION,
	51	ALLOCINFO_FILTER_FILENAME,
	52	ALLOCINFO_FILTER_LINENO,
	53	ALLOCINFO_FILTER_INACCURATE,
	54	ALLOCINFO_FILTER_MIN_SIZE,
	55	ALLOCINFO_FILTER_MAX_SIZE,
	56	__ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE
	57	};
	58
	59	#define ALLOCINFO_FILTER_MASK_MODNAME (1 << ALLOCINFO_FILTER_MODNAME)
	60	#define ALLOCINFO_FILTER_MASK_FUNCTION (1 << ALLOCINFO_FILTER_FUNCTION)
	61	#define ALLOCINFO_FILTER_MASK_FILENAME (1 << ALLOCINFO_FILTER_FILENAME)
	62	#define ALLOCINFO_FILTER_MASK_LINENO (1 << ALLOCINFO_FILTER_LINENO)
	63	#define ALLOCINFO_FILTER_MASK_INACCURATE (1 << ALLOCINFO_FILTER_INACCURATE)
	64	#define ALLOCINFO_FILTER_MASK_MIN_SIZE (1 << ALLOCINFO_FILTER_MIN_SIZE)
	65	#define ALLOCINFO_FILTER_MASK_MAX_SIZE (1 << ALLOCINFO_FILTER_MAX_SIZE)
	66
	67	#define ALLOCINFO_FILTER_MASKS \
	68	((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1)
	69
	70	struct allocinfo_filter {
	71	__u64 mask; /* bitmask of the filter fields used */
	72	struct allocinfo_tag fields;
	73	__u64 min_size;
	74	__u64 max_size;
	75	/* filter criteria only; see allocinfo_counter.accurate for actual accuracy */
	76	__u64 inaccurate;
	77	};
	78
	79	struct allocinfo_get_at {
	80	/* inputs */
	81	__u64 pos;
	82	struct allocinfo_filter filter;
	83	/* output */
	84	struct allocinfo_tag_data data;
	85	};
	86
	87	#define _ALLOCINFO_IOC_CONTENT_ID 0
	88	#define _ALLOCINFO_IOC_GET_AT 1
	89	#define _ALLOCINFO_IOC_GET_NEXT 2
	90
	91	#define ALLOCINFO_IOC_BASE 0xA6
	92	#define ALLOCINFO_IOC_CONTENT_ID _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_CONTENT_ID, \
	93	struct allocinfo_content_id)
	94	#define ALLOCINFO_IOC_GET_AT _IOWR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_AT, \
	95	struct allocinfo_get_at)
	96	#define ALLOCINFO_IOC_GET_NEXT _IOR(ALLOCINFO_IOC_BASE, _ALLOCINFO_IOC_GET_NEXT, \
	97	struct allocinfo_tag_data)
	98
	99	#endif /* _UAPI_ALLOC_TAG_H */

kernel/events/uprobes.c+1−1

@@ -513,7 +513,7 @@ int uprobe_write(struct arch_uprobe auprobe, struct vm_area_struct vma, · 512 unmodified lines
513		513
514	uprobe = container_of(auprobe, struct uprobe, arch);	514	uprobe = container_of(auprobe, struct uprobe, arch);
515		515
516	if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags)))	516	if (WARN_ON_ONCE(!vma_is_cow_mapping(vma)))
517	return -EINVAL;	517	return -EINVAL;
518		518
519	/*	519	/*

kernel/locking/lockdep.c+51−7

 static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 			  int trylock, int read, int check, int hardirqs_off,
 			  struct lockdep_map *nest_lock, unsigned long ip,
-			  int references, int pin_count, int sync)
+			  int references, int pin_count, int sync, int seq)
 {
 	struct task_struct *curr = current;
 	struct lock_class *class = NULL;
 	hlock->holdtime_stamp = lockstat_clock();
 #endif
 	hlock->pin_count = pin_count;
+	hlock->seq_count = seq;
 	if (check_wait_context(curr, hlock))
 		return 0;
 				    hlock->read, hlock->check,
 				    hlock->hardirqs_off,
 				    hlock->nest_lock, hlock->acquire_ip,
-				    hlock->references, hlock->pin_count, 0)) {
+				    hlock->references, hlock->pin_count, 0, hlock->seq_count)) {
 		case 0:
 			return 1;
 		case 1:
 		struct held_lock *hlock = curr->held_locks + i;
 		if (match_held_lock(hlock, lock)) {
+			int pin_count;
 			if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))
 				return;
-			hlock->pin_count -= cookie.val;
+			pin_count = hlock->pin_count - cookie.val;
-			if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))
+			if (WARN(pin_count < 0, "pin count corrupted\n"))
-				hlock->pin_count = 0;
+				pin_count = 0;
+			hlock->pin_count = pin_count;
 			return;
 		}
 	}
 	WARN(1, "unpinning an unheld lock\n");
 }
+static u32 __lock_sequence(struct lockdep_map *lock)
+{
+	struct task_struct *curr = current;
+	int i;
+	if (unlikely(!debug_locks))
+		return ~0;
+	for (i = 0; i < curr->lockdep_depth; i++) {
+		struct held_lock *hlock = curr->held_locks + i;
+		if (match_held_lock(hlock, lock))
+			return hlock->seq_count;
+	}
+	return ~0;
+}
 /*
  * Check whether we follow the irq-flags state precisely:
  */
 	lockdep_recursion_inc();
 	__lock_acquire(lock, subclass, trylock, read, check,
-		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0);
+		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0,
+		       ++current->lockdep_seq);
 	lockdep_recursion_finish();
 	raw_local_irq_restore(flags);
 }
 	lockdep_recursion_inc();
 	__lock_acquire(lock, subclass, 0, read, check,
-		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1);
+		       irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1,
+		       ++current->lockdep_seq);
 	check_chain_key(current);
 	lockdep_recursion_finish();
 	raw_local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(lock_unpin_lock);
+u32 lock_sequence(struct lockdep_map *lock)
+{
+	unsigned long flags;
+	u32 seq = ~0;
+	if (unlikely(!lockdep_enabled()))
+		return seq;
+	raw_local_irq_save(flags);
+	check_flags(flags);
+	lockdep_recursion_inc();
+	seq = __lock_sequence(lock);
+	lockdep_recursion_finish();
+	raw_local_irq_restore(flags);
+	return seq;
+}
+EXPORT_SYMBOL_GPL(lock_sequence);
 #ifdef CONFIG_LOCK_STAT
 static void print_lock_contention_bug(struct task_struct *curr,
 				      struct lockdep_map *lock,

lib/Kconfig.debug+0−15

@@ -613,21 +613,6 @@ config BUILTIN_MODULE_RANGES · 612 unmodified lines
613	It also records an anchor symbol to determine the load address of the	613	It also records an anchor symbol to determine the load address of the
614	section.	614	section.
615		615
616	config DEBUG_FORCE_WEAK_PER_CPU
617	bool "Force weak per-cpu definitions"
618	depends on DEBUG_KERNEL
619	help
620	s390 and alpha require percpu variables in modules to be
621	defined weak to work around addressing range issue which
622	puts the following two restrictions on percpu variable
623	definitions.
624
625	1. percpu symbols must be unique whether static or not
626	2. percpu variables can't be defined inside a function
627
628	To ensure that generic code follows the above rules, this
629	option forces all percpu variables to be defined as weak.
630
631	config WARN_CONTEXT_ANALYSIS	616	config WARN_CONTEXT_ANALYSIS
632	bool "Compiler context-analysis warnings"	617	bool "Compiler context-analysis warnings"
633	depends on CC_IS_CLANG && CLANG_VERSION >= 230000	618	depends on CC_IS_CLANG && CLANG_VERSION >= 230000

lib/codetag.c+18−0

 	struct codetag_type_desc desc;
 	/* generates unique sequence number for module load */
 	unsigned long next_mod_seq;
+	/* bumped on every module load and unload */
+	unsigned long content_id;
 };
 struct codetag_range {
 	up_read(&cttype->mod_lock);
 }
+unsigned long codetag_get_content_id(struct codetag_type *cttype)
+{
+	lockdep_assert_held(&cttype->mod_lock);
+	return cttype->content_id;
+}
+unsigned int codetag_get_count(struct codetag_type *cttype)
+{
+	lockdep_assert_held(&cttype->mod_lock);
+	return cttype->count;
+}
 struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype)
 {
 	struct codetag_iterator iter = {
 	down_write(&cttype->mod_lock);
 	cmod->mod_seq = ++cttype->next_mod_seq;
+	++cttype->content_id;
 	mod_id = idr_alloc(&cttype->mod_idr, cmod, 0, 0, GFP_KERNEL);
 	if (mod_id >= 0) {
 		if (cttype->desc.module_load) {
 			cttype->count -= range_size(cttype, &cmod->range);
 			idr_remove(&cttype->mod_idr, mod_id);
 			kfree(cmod);
+			++cttype->content_id;
 		}
 		up_write(&cttype->mod_lock);
 		if (found && cttype->desc.free_section_mem)

lib/maple_tree.c+280−121

@@ -261,6 +261,12 @@ static inline bool mas_is_underflow(struct ma_state *mas) · 260 unmodified lines
261	return mas->status == ma_underflow;	261	return mas->status == ma_underflow;
262	}	262	}
263		263
		264	static inline void mas_make_walkable(struct ma_state *mas)
		265	{
		266	if (!mas_is_active(mas) && !mas_is_start(mas))
		267	mas->status = ma_start;
		268	}
		269
264	static __always_inline struct maple_node *mte_to_node(	270	static __always_inline struct maple_node *mte_to_node(
265	const struct maple_enode *entry)	271	const struct maple_enode *entry)
266	{	272	{
@@ -356,11 +362,6 @@ static __always_inline bool mte_is_root(const struct maple_enode *node) · 89 unmodified lines
356	return ma_is_root(mte_to_node(node));	362	return ma_is_root(mte_to_node(node));
357	}	363	}
358		364
359	static inline bool mas_is_root_limits(const struct ma_state *mas)
360	{
361	return !mas->min && mas->max == ULONG_MAX;
362	}
363
364	static __always_inline bool mt_is_alloc(struct maple_tree *mt)	365	static __always_inline bool mt_is_alloc(struct maple_tree *mt)
365	{	366	{
366	return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE);	367	return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE);
@@ -455,46 +456,6 @@ enum maple_type mas_parent_type(struct ma_state mas, struct maple_enode enode) · 88 unmodified lines
455	return 0;	456	return 0;
456	}	457	}
457		458
458	/*
459	* mas_set_parent() - Set the parent node and encode the slot
460	* @mas: The maple state
461	* @enode: The encoded maple node.
462	* @parent: The encoded maple node that is the parent of @enode.
463	* @slot: The slot that @enode resides in @parent.
464	*
465	* Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the
466	* parent type.
467	*/
468	static inline
469	void mas_set_parent(struct ma_state mas, struct maple_enode enode,
470	const struct maple_enode *parent, unsigned char slot)
471	{
472	unsigned long val = (unsigned long)parent;
473	unsigned long shift;
474	unsigned long type;
475	enum maple_type p_type = mte_node_type(parent);
476
477	MAS_BUG_ON(mas, p_type == maple_dense);
478	MAS_BUG_ON(mas, p_type == maple_leaf_64);
479
480	switch (p_type) {
481	case maple_range_64:
482	case maple_arange_64:
483	shift = MAPLE_PARENT_SLOT_SHIFT;
484	type = MAPLE_PARENT_RANGE64;
485	break;
486	default:
487	case maple_dense:
488	case maple_leaf_64:
489	shift = type = 0;
490	break;
491	}
492
493	val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */
494	val \|= (slot << shift) \| type;
495	mte_to_node(enode)->parent = ma_parent_ptr(val);
496	}
497
498	/*	459	/*
499	* mte_parent_slot() - get the parent slot of @enode.	460	* mte_parent_slot() - get the parent slot of @enode.
500	* @enode: The encoded maple node.	461	* @enode: The encoded maple node.
@@ -876,6 +837,42 @@ static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, · 375 unmodified lines
876	meta->gap = offset;	837	meta->gap = offset;
877	}	838	}
878		839
		840	/*
		841	* mas_set_parent_slots() - Bulk operation to set many slot parent pointers
		842	* @mas: The maple state
		843	* @parent: The encoded maple node that is the parent of @enode.
		844	* @slot: The slot that of the @enode.
		845	* @start_slot: The offset into @slot
		846	* @count: The number of slots to set (eg: exclusive)
		847	*/
		848	static inline
		849	void mas_set_parent_slots(struct ma_state mas, struct maple_enode parent,
		850	void __rcu **slots, unsigned char start_slot, unsigned char count)
		851	{
		852	unsigned long val;
		853	unsigned long shift;
		854	unsigned long type;
		855	enum maple_type p_type = mte_node_type(parent);
		856	unsigned char i;
		857
		858	MAS_BUG_ON(mas, p_type != maple_range_64 &&
		859	p_type != maple_arange_64);
		860
		861	shift = MAPLE_PARENT_SLOT_SHIFT;
		862	type = MAPLE_PARENT_RANGE64;
		863
		864	val = (unsigned long)parent;
		865	val &= ~MAPLE_NODE_MASK;
		866
		867	for (i = 0; i < count; i++) {
		868	unsigned long pval = val \| ((start_slot + i) << shift) \| type;
		869	struct maple_enode *child;
		870
		871	child = mt_slot_locked(mas->tree, slots, i);
		872	mte_to_node(child)->parent = ma_parent_ptr(pval);
		873	}
		874	}
		875
879	/*	876	/*
880	* mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes.	877	* mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes.
881	* @mat: the ma_topiary, a linked list of dead nodes.	878	* @mat: the ma_topiary, a linked list of dead nodes.
@@ -1153,6 +1150,79 @@ static inline void mas_free(struct ma_state mas, struct maple_enode used) · 271 unmodified lines
1153	ma_free_rcu(mte_to_node(used));	1150	ma_free_rcu(mte_to_node(used));
1154	}	1151	}
1155		1152
		1153
		1154	#ifdef CONFIG_LOCKDEP
		1155	static struct lockdep_map mas_lockdep_map(struct ma_state mas)
		1156	{
		1157	struct maple_tree *mt = mas->tree;
		1158
		1159	if (mt_external_lock(mt))
		1160	return mt->ma_external_lock;
		1161
		1162	return &(mt->ma_lock).dep_map;
		1163	}
		1164
		1165	#endif
		1166
		1167	static void mas_lock_check(struct ma_state *mas)
		1168	{
		1169	#ifdef CONFIG_LOCKDEP
		1170	struct lockdep_map *map;
		1171	u32 seq;
		1172
		1173	if (!mas_is_active(mas))
		1174	return;
		1175
		1176	#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
		1177	if (!mt_locked(mas->tree)) {
		1178	if (mt_in_rcu(mas->tree))
		1179	WARN_ON_ONCE(poll_state_synchronize_rcu(mas->rcu_gp));
		1180	}
		1181	#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
		1182
		1183	map = mas_lockdep_map(mas);
		1184	if (!map)
		1185	return;
		1186
		1187	seq = lock_sequence(map);
		1188	if (seq != UINT_MAX && mas->ld_seq != UINT_MAX)
		1189	WARN_ON_ONCE(mas->ld_seq != seq);
		1190	#endif /* CONFIG_LOCKDEP */
		1191
		1192	}
		1193
		1194	static void mas_init_lock_check(struct ma_state *mas)
		1195	{
		1196	#ifdef CONFIG_LOCKDEP
		1197	struct lockdep_map *map;
		1198	#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
		1199	if (!mt_locked(mas->tree)) {
		1200	if (mt_in_rcu(mas->tree))
		1201	mas->rcu_gp = get_state_synchronize_rcu();
		1202	return;
		1203	}
		1204	#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
		1205
		1206	map = mas_lockdep_map(mas);
		1207	if (map) /* Update regardless of lock state */
		1208	mas->ld_seq = lock_sequence(map);
		1209	#endif /* CONFIG_LOCKDEP */
		1210
		1211	}
		1212
		1213	static void mas_may_init_lock_check(struct ma_state *mas)
		1214	{
		1215	#ifdef CONFIG_LOCKDEP
		1216	#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD
		1217	if (mas_is_start(mas) \|\| mas_is_paused(mas)) {
		1218	mas_init_lock_check(mas);
		1219	return;
		1220	}
		1221	#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */
		1222	mas_lock_check(mas);
		1223	#endif /* CONFIG_LOCKDEP */
		1224	}
		1225
1156	/*	1226	/*
1157	* mas_start() - Sets up maple state for operations.	1227	* mas_start() - Sets up maple state for operations.
1158	* @mas: The maple state.	1228	* @mas: The maple state.
@@ -1171,6 +1241,7 @@ static inline struct maple_enode mas_start(struct ma_state mas) · 12 unmodified lines
1171	if (likely(mas_is_start(mas))) {	1241	if (likely(mas_is_start(mas))) {
1172	struct maple_enode *root;	1242	struct maple_enode *root;
1173		1243
		1244	mas_init_lock_check(mas);
1174	mas->min = 0;	1245	mas->min = 0;
1175	mas->max = ULONG_MAX;	1246	mas->max = ULONG_MAX;
1176		1247
@@ -1501,14 +1572,26 @@ ascend: · 324 unmodified lines
1501	goto ascend;	1572	goto ascend;
1502	}	1573	}
1503		1574
		1575	static __always_inline void mas_update_gap_known(struct ma_state *mas,
		1576	unsigned long gap)
		1577	{
		1578	unsigned char pslot;
		1579	unsigned long p_gap;
		1580
		1581	pslot = mte_parent_slot(mas->node);
		1582	p_gap = ma_gaps(mte_parent(mas->node),
		1583	mas_parent_type(mas, mas->node))[pslot];
		1584
		1585	if (p_gap != gap)
		1586	mas_parent_gap(mas, pslot, gap);
		1587	}
		1588
1504	/*	1589	/*
1505	* mas_update_gap() - Update a nodes gaps and propagate up if necessary.	1590	* mas_update_gap() - Update a nodes gaps and propagate up if necessary.
1506	* @mas: the maple state.	1591	* @mas: the maple state.
1507	*/	1592	*/
1508	static inline void mas_update_gap(struct ma_state *mas)	1593	static inline void mas_update_gap(struct ma_state *mas)
1509	{	1594	{
1510	unsigned char pslot;
1511	unsigned long p_gap;
1512	unsigned long max_gap;	1595	unsigned long max_gap;
1513		1596
1514	if (!mt_is_alloc(mas->tree))	1597	if (!mt_is_alloc(mas->tree))
@@ -1518,13 +1601,7 @@ static inline void mas_update_gap(struct ma_state *mas) · 3 unmodified lines
1518	return;	1601	return;
1519		1602
1520	max_gap = mas_max_gap(mas);	1603	max_gap = mas_max_gap(mas);
1521		1604	mas_update_gap_known(mas, max_gap);
1522	pslot = mte_parent_slot(mas->node);
1523	p_gap = ma_gaps(mte_parent(mas->node),
1524	mas_parent_type(mas, mas->node))[pslot];
1525
1526	if (p_gap != max_gap)
1527	mas_parent_gap(mas, pslot, max_gap);
1528	}	1605	}
1529		1606
1530	/*	1607	/*
@@ -1540,14 +1617,10 @@ static inline void mas_adopt_children(struct ma_state *mas, · 9 unmodified lines
1540	struct maple_node *node = mte_to_node(parent);	1617	struct maple_node *node = mte_to_node(parent);
1541	void __rcu **slots = ma_slots(node, type);	1618	void __rcu **slots = ma_slots(node, type);
1542	unsigned long *pivots = ma_pivots(node, type);	1619	unsigned long *pivots = ma_pivots(node, type);
1543	struct maple_enode *child;	1620	unsigned char end;
1544	unsigned char offset;
1545		1621
1546	offset = ma_data_end(node, type, pivots, mas->max);	1622	end = ma_data_end(node, type, pivots, mas->max);
1547	do {	1623	mas_set_parent_slots(mas, parent, slots, 0, end + 1);
1548	child = mas_slot_locked(mas, slots, offset);
1549	mas_set_parent(mas, child, parent, offset);
1550	} while (offset--);
1551	}	1624	}
1552		1625
1553	/*	1626	/*
@@ -1929,15 +2002,10 @@ unsigned long node_copy(struct ma_state mas, struct maple_node src, · 375 unmodified lines
1929	s_slots = ma_slots(src, s_mt) + start;	2002	s_slots = ma_slots(src, s_mt) + start;
1930	s_pivots = ma_pivots(src, s_mt) + start;	2003	s_pivots = ma_pivots(src, s_mt) + start;
1931	memcpy(d_slots, s_slots, size * sizeof(void __rcu *));	2004	memcpy(d_slots, s_slots, size * sizeof(void __rcu *));
1932	if (!ma_is_leaf(d_mt) && s_mt == maple_copy) {
1933	struct maple_enode *edst = mt_mk_node(dst, d_mt);
1934
1935		2005
1936	for (int i = 0; i < size; i++)	2006	if (!ma_is_leaf(d_mt) && s_mt == maple_copy)
1937	mas_set_parent(mas,	2007	mas_set_parent_slots(mas, mt_mk_node(dst, d_mt),
1938	mt_slot_locked(mas->tree, d_slots, i),	2008	d_slots, d_start, size);
1939	edst, d_start + i);
1940	}
1941		2009
1942	d_gaps = ma_gaps(dst, d_mt);	2010	d_gaps = ma_gaps(dst, d_mt);
1943	if (d_gaps) {	2011	if (d_gaps) {
@@ -2081,8 +2149,8 @@ static inline void mas_wmb_replace(struct ma_state mas, struct maple_copy cp) · 137 unmodified lines
2081	mas->node = mt_slot_locked(mas->tree, cp->slot, 0);	2149	mas->node = mt_slot_locked(mas->tree, cp->slot, 0);
2082	/* Insert the new data in the tree */	2150	/* Insert the new data in the tree */
2083	mas_topiary_replace(mas, old_enode, cp->height);	2151	mas_topiary_replace(mas, old_enode, cp->height);
2084	if (!mte_is_leaf(mas->node))	2152	if (mt_is_alloc(mas->tree) && !mte_is_root(mas->node))
2085	mas_update_gap(mas);	2153	mas_update_gap_known(mas, cp->gap[0]);
2086		2154
2087	mtree_range_walk(mas);	2155	mtree_range_walk(mas);
2088	}	2156	}
@@ -2928,13 +2996,6 @@ static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, · 839 unmodified lines
2928	if (r_mas->last < r_wr_mas->r_max)	2996	if (r_mas->last < r_wr_mas->r_max)
2929	r_mas->last = r_wr_mas->r_max;	2997	r_mas->last = r_wr_mas->r_max;
2930	r_mas->offset++;	2998	r_mas->offset++;
2931	} else if ((r_mas->last == r_wr_mas->r_max) &&
2932	(r_mas->last < r_mas->max) &&
2933	!mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) {
2934	r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots,
2935	r_wr_mas->type, r_mas->offset + 1);
2936	r_mas->offset++;
2937	r_wr_mas->r_max = r_mas->last;
2938	}	2999	}
2939	}	3000	}
2940		3001
@@ -3125,7 +3186,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas) · 184 unmodified lines
3125	static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)	3186	static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
3126	{	3187	{
3127	unsigned char dst_offset, offset_end;	3188	unsigned char dst_offset, offset_end;
3128	unsigned char copy_size, node_pivots;	3189	unsigned char copy_size, node_pivots, node_slots;
3129	struct maple_node reuse, *newnode;	3190	struct maple_node reuse, *newnode;
3130	unsigned long *dst_pivots;	3191	unsigned long *dst_pivots;
3131	void __rcu **dst_slots;	3192	void __rcu **dst_slots;
@@ -3138,6 +3199,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) · 6 unmodified lines
3138	in_rcu = mt_in_rcu(mas->tree);	3199	in_rcu = mt_in_rcu(mas->tree);
3139	offset_end = wr_mas->offset_end;	3200	offset_end = wr_mas->offset_end;
3140	node_pivots = mt_pivots[wr_mas->type];	3201	node_pivots = mt_pivots[wr_mas->type];
		3202	node_slots = mt_slots[wr_mas->type];
3141	/* Assume last adds an entry */	3203	/* Assume last adds an entry */
3142	new_end = mas->end + 1 - offset_end + mas->offset;	3204	new_end = mas->end + 1 - offset_end + mas->offset;
3143	if (mas->last == wr_mas->end_piv) {	3205	if (mas->last == wr_mas->end_piv) {
@@ -3149,7 +3211,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) · 5 unmodified lines
3149	if (in_rcu) {	3211	if (in_rcu) {
3150	newnode = mas_pop_node(mas);	3212	newnode = mas_pop_node(mas);
3151	} else {	3213	} else {
3152	memset(&reuse, 0, sizeof(struct maple_node));
3153	newnode = &reuse;	3214	newnode = &reuse;
3154	}	3215	}
3155		3216
@@ -3193,7 +3254,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas) · 37 unmodified lines
3193	dst_pivots[new_end] = mas->max;	3254	dst_pivots[new_end] = mas->max;
3194		3255
3195	done:	3256	done:
3196	mas_leaf_set_meta(newnode, maple_leaf_64, new_end);	3257	if (!in_rcu && new_end + 2 < node_slots) {
		3258	unsigned char clear_from = new_end + 1;
		3259
		3260	/*
		3261	* Note that the last slot is never cleared, since the metadata
		3262	* will be stored there or it has a value.
		3263	*/
		3264	memset(dst_slots + clear_from, 0,
		3265	sizeof(void __rcu ) (node_slots - clear_from));
		3266	if (clear_from < node_pivots)
		3267	memset(dst_pivots + clear_from, 0,
		3268	sizeof(unsigned long) * (node_pivots - clear_from));
		3269	}
		3270
		3271	mas_leaf_set_meta(newnode, wr_mas->type, new_end);
3197	if (in_rcu) {	3272	if (in_rcu) {
3198	struct maple_enode *old_enode = mas->node;	3273	struct maple_enode *old_enode = mas->node;
3199		3274
@@ -3218,7 +3293,7 @@ static inline void mas_wr_slot_store(struct ma_wr_state *wr_mas) · 18 unmodified lines
3218	void __rcu **slots = wr_mas->slots;	3293	void __rcu **slots = wr_mas->slots;
3219	bool gap = false;	3294	bool gap = false;
3220		3295
3221	gap \|= !mt_slot_locked(mas->tree, slots, offset);	3296	gap \|= !wr_mas->content;
3222	gap \|= !mt_slot_locked(mas->tree, slots, offset + 1);	3297	gap \|= !mt_slot_locked(mas->tree, slots, offset + 1);
3223		3298
3224	if (wr_mas->offset_end - offset == 1) {	3299	if (wr_mas->offset_end - offset == 1) {
@@ -3595,7 +3670,7 @@ set_content: · 370 unmodified lines
3595		3670
3596	/**	3671	/**
3597	* mas_prealloc_calc() - Calculate number of nodes needed for a	3672	* mas_prealloc_calc() - Calculate number of nodes needed for a
3598	* given store oepration	3673	* given store operation
3599	* @wr_mas: The maple write state	3674	* @wr_mas: The maple write state
3600	* @entry: The entry to store into the tree	3675	* @entry: The entry to store into the tree
3601	*	3676	*
@@ -3661,6 +3736,9 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) · 59 unmodified lines
3661	{	3736	{
3662	struct ma_state *mas = wr_mas->mas;	3737	struct ma_state *mas = wr_mas->mas;
3663	unsigned char new_end;	3738	unsigned char new_end;
		3739	bool appending;
		3740	bool one_slot;
		3741	bool in_rcu;
3664		3742
3665	if (unlikely(mas_is_none(mas) \|\| mas_is_ptr(mas)))	3743	if (unlikely(mas_is_none(mas) \|\| mas_is_ptr(mas)))
3666	return wr_store_root;	3744	return wr_store_root;
@@ -3680,21 +3758,30 @@ static inline enum store_type mas_wr_store_type(struct ma_wr_state *wr_mas) · 13 unmodified lines
3680	return wr_new_root;	3758	return wr_new_root;
3681		3759
3682	new_end = mas_wr_new_end(wr_mas);	3760	new_end = mas_wr_new_end(wr_mas);
		3761	in_rcu = mt_in_rcu(mas->tree);
		3762	appending = mas->offset == mas->end;
		3763	one_slot = wr_mas->offset_end - mas->offset == 1;
		3764
3683	/* Potential spanning rebalance collapsing a node */	3765	/* Potential spanning rebalance collapsing a node */
3684	if (new_end < mt_min_slots[wr_mas->type]) {	3766	if (new_end < mt_min_slots[wr_mas->type]) {
3685	if (!mte_is_root(mas->node))	3767	if (!mte_is_root(mas->node))
3686	return wr_rebalance;	3768	return wr_rebalance;
		3769	if (!in_rcu) {
		3770	if (appending)
		3771	return wr_append;
		3772	else if (mas->end == new_end && one_slot)
		3773	return wr_slot_store;
		3774	}
3687	return wr_node_store;	3775	return wr_node_store;
3688	}	3776	}
3689		3777
3690	if (new_end >= mt_slots[wr_mas->type])	3778	if (new_end >= mt_slots[wr_mas->type])
3691	return wr_split_store;	3779	return wr_split_store;
3692		3780
3693	if (!mt_in_rcu(mas->tree) && (mas->offset == mas->end))	3781	if (!in_rcu && appending)
3694	return wr_append;	3782	return wr_append;
3695		3783
3696	if ((new_end == mas->end) && (!mt_in_rcu(mas->tree) \|\|	3784	if (new_end == mas->end && (!in_rcu \|\| one_slot))
3697	(wr_mas->offset_end - mas->offset == 1)))
3698	return wr_slot_store;	3785	return wr_slot_store;
3699		3786
3700	return wr_node_store;	3787	return wr_node_store;
@@ -3793,35 +3880,40 @@ int mas_alloc_cyclic(struct ma_state mas, unsigned long startp, · 92 unmodified lines
3793	void *entry, unsigned long range_lo, unsigned long range_hi,	3880	void *entry, unsigned long range_lo, unsigned long range_hi,
3794	unsigned long *next, gfp_t gfp)	3881	unsigned long *next, gfp_t gfp)
3795	{	3882	{
3796	unsigned long min = range_lo;	3883	int ret;
3797	int ret = 0;	3884	unsigned long min;
3798
3799	range_lo = max(min, *next);
3800	ret = mas_empty_area(mas, range_lo, range_hi, 1);
3801	if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) {
3802	mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED;
3803	ret = 1;
3804	}
3805	if (ret < 0 && range_lo > min) {
3806	mas_reset(mas);
3807	ret = mas_empty_area(mas, min, range_hi, 1);
3808	if (ret == 0)
3809	ret = 1;
3810	}
3811	if (ret < 0)
3812	return ret;
3813		3885
		3886	min = range_lo;
3814	do {	3887	do {
		3888	range_lo = max(min, *next);
		3889	ret = mas_empty_area(mas, range_lo, range_hi, 1);
		3890	if (ret < 0 && range_lo > min) {
		3891	mas_reset(mas);
		3892	ret = mas_empty_area(mas, min, range_hi, 1);
		3893	if (ret == 0)
		3894	ret = 1;
		3895	}
		3896	if (ret < 0)
		3897	goto out;
		3898
3815	mas_insert(mas, entry);	3899	mas_insert(mas, entry);
3816	} while (mas_nomem(mas, gfp));	3900	} while (mas_nomem(mas, gfp));
3817	if (mas_is_err(mas))
3818	return xa_err(mas->node);
3819		3901
		3902	if (mas_is_err(mas)) {
		3903	ret = xa_err(mas->node);
		3904	goto out;
		3905	}
		3906
		3907	if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) {
		3908	mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED;
		3909	ret = 1;
		3910	}
3820	*startp = mas->index;	3911	*startp = mas->index;
3821	next = startp + 1;	3912	next = startp + 1;
3822	if (*next == 0)	3913	if (*next == 0)
3823	mas->tree->ma_flags \|= MT_FLAGS_ALLOC_WRAPPED;	3914	mas->tree->ma_flags \|= MT_FLAGS_ALLOC_WRAPPED;
3824		3915
		3916	out:
3825	mas_destroy(mas);	3917	mas_destroy(mas);
3826	return ret;	3918	return ret;
3827	}	3919	}
@@ -4360,8 +4452,8 @@ void mas_walk(struct ma_state mas) · 532 unmodified lines
4360	{	4452	{
4361	void *entry;	4453	void *entry;
4362		4454
4363	if (!mas_is_active(mas) && !mas_is_start(mas))	4455	mas_may_init_lock_check(mas);
4364	mas->status = ma_start;	4456	mas_make_walkable(mas);
4365	retry:	4457	retry:
4366	entry = mas_state_walk(mas);	4458	entry = mas_state_walk(mas);
4367	if (mas_is_start(mas)) {	4459	if (mas_is_start(mas)) {
@@ -4831,6 +4923,7 @@ void mas_store(struct ma_state mas, void *entry) · 463 unmodified lines
4831	{	4923	{
4832	MA_WR_STATE(wr_mas, mas, entry);	4924	MA_WR_STATE(wr_mas, mas, entry);
4833		4925
		4926	mas_may_init_lock_check(mas);
4834	trace_ma_write(TP_FCT, mas, 0, entry);	4927	trace_ma_write(TP_FCT, mas, 0, entry);
4835	#ifdef CONFIG_DEBUG_MAPLE_TREE	4928	#ifdef CONFIG_DEBUG_MAPLE_TREE
4836	if (MAS_WARN_ON(mas, mas->index > mas->last))	4929	if (MAS_WARN_ON(mas, mas->index > mas->last))
@@ -4889,6 +4982,7 @@ int mas_store_gfp(struct ma_state mas, void entry, gfp_t gfp) · 52 unmodified lines
4889	MA_WR_STATE(wr_mas, mas, entry);	4982	MA_WR_STATE(wr_mas, mas, entry);
4890	int ret = 0;	4983	int ret = 0;
4891		4984
		4985	mas_may_init_lock_check(mas);
4892	retry:	4986	retry:
4893	mas_wr_preallocate(&wr_mas, entry);	4987	mas_wr_preallocate(&wr_mas, entry);
4894	if (unlikely(mas_nomem(mas, gfp))) {	4988	if (unlikely(mas_nomem(mas, gfp))) {
@@ -4919,6 +5013,7 @@ void mas_store_prealloc(struct ma_state mas, void entry) · 24 unmodified lines
4919	{	5013	{
4920	MA_WR_STATE(wr_mas, mas, entry);	5014	MA_WR_STATE(wr_mas, mas, entry);
4921		5015
		5016	mas_lock_check(mas);
4922	if (mas->store_type == wr_store_root) {	5017	if (mas->store_type == wr_store_root) {
4923	mas_wr_prealloc_setup(&wr_mas);	5018	mas_wr_prealloc_setup(&wr_mas);
4924	goto store;	5019	goto store;
@@ -4951,6 +5046,7 @@ int mas_preallocate(struct ma_state mas, void entry, gfp_t gfp) · 26 unmodified lines
4951	{	5046	{
4952	MA_WR_STATE(wr_mas, mas, entry);	5047	MA_WR_STATE(wr_mas, mas, entry);
4953		5048
		5049	mas_may_init_lock_check(mas);
4954	mas_wr_prealloc_setup(&wr_mas);	5050	mas_wr_prealloc_setup(&wr_mas);
4955	mas->store_type = mas_wr_store_type(&wr_mas);	5051	mas->store_type = mas_wr_store_type(&wr_mas);
4956	mas_prealloc_calc(&wr_mas, entry);	5052	mas_prealloc_calc(&wr_mas, entry);
@@ -4997,6 +5093,7 @@ static void mas_may_activate(struct ma_state *mas) · 40 unmodified lines
4997	mas->status = ma_start;	5093	mas->status = ma_start;
4998	} else {	5094	} else {
4999	mas->status = ma_active;	5095	mas->status = ma_active;
		5096	mas_lock_check(mas);
5000	}	5097	}
5001	}	5098	}
5002		5099
@@ -5074,6 +5171,7 @@ void mas_next(struct ma_state mas, unsigned long max) · 71 unmodified lines
5074	{	5171	{
5075	void *entry = NULL;	5172	void *entry = NULL;
5076		5173
		5174	mas_may_init_lock_check(mas);
5077	if (mas_next_setup(mas, max, &entry))	5175	if (mas_next_setup(mas, max, &entry))
5078	return entry;	5176	return entry;
5079		5177
@@ -5097,6 +5195,7 @@ void mas_next_range(struct ma_state mas, unsigned long max) · 17 unmodified lines
5097	{	5195	{
5098	void *entry = NULL;	5196	void *entry = NULL;
5099		5197
		5198	mas_may_init_lock_check(mas);
5100	if (mas_next_setup(mas, max, &entry))	5199	if (mas_next_setup(mas, max, &entry))
5101	return entry;	5200	return entry;
5102		5201
@@ -5205,6 +5304,7 @@ void mas_prev(struct ma_state mas, unsigned long min) · 102 unmodified lines
5205	{	5304	{
5206	void *entry = NULL;	5305	void *entry = NULL;
5207		5306
		5307	mas_may_init_lock_check(mas);
5208	if (mas_prev_setup(mas, min, &entry))	5308	if (mas_prev_setup(mas, min, &entry))
5209	return entry;	5309	return entry;
5210		5310
@@ -5228,6 +5328,7 @@ void mas_prev_range(struct ma_state mas, unsigned long min) · 17 unmodified lines
5228	{	5328	{
5229	void *entry = NULL;	5329	void *entry = NULL;
5230		5330
		5331	mas_may_init_lock_check(mas);
5231	if (mas_prev_setup(mas, min, &entry))	5332	if (mas_prev_setup(mas, min, &entry))
5232	return entry;	5333	return entry;
5233		5334
@@ -5274,6 +5375,7 @@ EXPORT_SYMBOL_GPL(mt_prev); · 40 unmodified lines
5274	*/	5375	*/
5275	void mas_pause(struct ma_state *mas)	5376	void mas_pause(struct ma_state *mas)
5276	{	5377	{
		5378	mas_lock_check(mas);
5277	mas->status = ma_pause;	5379	mas->status = ma_pause;
5278	mas->node = NULL;	5380	mas->node = NULL;
5279	}	5381	}
@@ -5382,6 +5484,7 @@ void mas_find(struct ma_state mas, unsigned long max) · 102 unmodified lines
5382	{	5484	{
5383	void *entry = NULL;	5485	void *entry = NULL;
5384		5486
		5487	mas_may_init_lock_check(mas);
5385	if (mas_find_setup(mas, max, &entry))	5488	if (mas_find_setup(mas, max, &entry))
5386	return entry;	5489	return entry;
5387		5490
@@ -5409,6 +5512,7 @@ void mas_find_range(struct ma_state mas, unsigned long max) · 21 unmodified lines
5409	{	5512	{
5410	void *entry = NULL;	5513	void *entry = NULL;
5411		5514
		5515	mas_may_init_lock_check(mas);
5412	if (mas_find_setup(mas, max, &entry))	5516	if (mas_find_setup(mas, max, &entry))
5413	return entry;	5517	return entry;
5414		5518
@@ -5428,7 +5532,6 @@ EXPORT_SYMBOL_GPL(mas_find_range); · 13 unmodified lines
5428	static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,	5532	static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
5429	void **entry)	5533	void **entry)
5430	{	5534	{
5431
5432	switch (mas->status) {	5535	switch (mas->status) {
5433	case ma_active:	5536	case ma_active:
5434	goto active;	5537	goto active;
@@ -5521,6 +5624,7 @@ void mas_find_rev(struct ma_state mas, unsigned long min) · 86 unmodified lines
5521	{	5624	{
5522	void *entry = NULL;	5625	void *entry = NULL;
5523		5626
		5627	mas_may_init_lock_check(mas);
5524	if (mas_find_rev_setup(mas, min, &entry))	5628	if (mas_find_rev_setup(mas, min, &entry))
5525	return entry;	5629	return entry;
5526		5630
@@ -5547,6 +5651,7 @@ void mas_find_range_rev(struct ma_state mas, unsigned long min) · 20 unmodified lines
5547	{	5651	{
5548	void *entry = NULL;	5652	void *entry = NULL;
5549		5653
		5654	mas_may_init_lock_check(mas);
5550	if (mas_find_rev_setup(mas, min, &entry))	5655	if (mas_find_rev_setup(mas, min, &entry))
5551	return entry;	5656	return entry;
5552		5657
@@ -5564,6 +5669,11 @@ EXPORT_SYMBOL_GPL(mas_find_range_rev); · 11 unmodified lines
5564	* Searches for @mas->index, sets @mas->index and @mas->last to the range and	5669	* Searches for @mas->index, sets @mas->index and @mas->last to the range and
5565	* erases that range.	5670	* erases that range.
5566	*	5671	*
		5672	* Note that erase requires allocations and will use GFP_KERNEL \| __GFP_NOFAIL
		5673	* to do so if necessary. If the allocation fails, the internal lock will be
		5674	* dropped to retry. An externally locked tree must be protected by a lock that
		5675	* allows blocking for this API.
		5676	*
5567	* Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated.	5677	* Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated.
5568	*/	5678	*/
5569	void mas_erase(struct ma_state mas)	5679	void mas_erase(struct ma_state mas)
@@ -5572,25 +5682,25 @@ void mas_erase(struct ma_state mas) · 2 unmodified lines
5572	unsigned long index = mas->index;	5682	unsigned long index = mas->index;
5573	MA_WR_STATE(wr_mas, mas, NULL);	5683	MA_WR_STATE(wr_mas, mas, NULL);
5574		5684
5575	if (!mas_is_active(mas) \|\| !mas_is_start(mas))	5685	/*
5576	mas->status = ma_start;	5686	* In low memory situations, the allocation is retried with the gfp flag
		5687	* GFP_KERNEL \| __GFP_NOFAIL. The internal spinlock is dropped in
		5688	* mas_nomem_nofail(), however the external lock is not dropped.
		5689	*/
		5690	if (mt_external_lock(mas->tree))
		5691	might_alloc(GFP_KERNEL);
5577		5692
		5693	mas_make_walkable(mas);
5578	write_retry:	5694	write_retry:
5579	entry = mas_state_walk(mas);	5695	entry = mas_state_walk(mas);
5580	if (!entry)	5696	if (!entry)
5581	return NULL;	5697	goto out;
5582		5698
5583	/* Must reset to ensure spanning writes of last slot are detected */	5699	/* Must reset to ensure spanning writes of last slot are detected */
5584	mas_reset(mas);	5700	mas_reset(mas);
5585	mas_wr_preallocate(&wr_mas, NULL);	5701	mas_wr_preallocate(&wr_mas, NULL);
5586	if (mas_nomem(mas, GFP_KERNEL)) {	5702	if (mas_nomem_nofail(mas, index, index))
5587	/* in case the range of entry changed when unlocked */
5588	mas->index = mas->last = index;
5589	goto write_retry;	5703	goto write_retry;
5590	}
5591
5592	if (mas_is_err(mas))
5593	goto out;
5594		5704
5595	mas_wr_store_entry(&wr_mas);	5705	mas_wr_store_entry(&wr_mas);
5596	out:	5706	out:
@@ -5601,10 +5711,11 @@ EXPORT_SYMBOL_GPL(mas_erase); · 4 unmodified lines
5601		5711
5602	/**	5712	/**
5603	* mas_nomem() - Check if there was an error allocating and do the allocation	5713	* mas_nomem() - Check if there was an error allocating and do the allocation
5604	* if necessary If there are allocations, then free them.	5714	* if necessary.
		5715	*
5605	* @mas: The maple state	5716	* @mas: The maple state
5606	* @gfp: The GFP_FLAGS to use for allocations	5717	* @gfp: The GFP_FLAGS to use for allocations
5607	* Return: true on allocation, false otherwise.	5718	* Return: False on no memory. True otherwise (partial success as well)
5608	*/	5719	*/
5609	bool mas_nomem(struct ma_state *mas, gfp_t gfp)	5720	bool mas_nomem(struct ma_state *mas, gfp_t gfp)
5610	__must_hold(mas->tree->ma_lock)	5721	__must_hold(mas->tree->ma_lock)
@@ -5612,6 +5723,10 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) · 1 unmodified line
5612	if (likely(mas->node != MA_ERROR(-ENOMEM)))	5723	if (likely(mas->node != MA_ERROR(-ENOMEM)))
5613	return false;	5724	return false;
5614		5725
		5726	/* Allocations can fail, don't do this. */
		5727	WARN_ON_ONCE(!gfpflags_allow_blocking(gfp) &&
		5728	mt_external_lock(mas->tree));
		5729
5615	if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) {	5730	if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) {
5616	mtree_unlock(mas->tree);	5731	mtree_unlock(mas->tree);
5617	mas_alloc_nodes(mas, gfp);	5732	mas_alloc_nodes(mas, gfp);
@@ -5620,10 +5735,49 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp) · 2 unmodified lines
5620	mas_alloc_nodes(mas, gfp);	5735	mas_alloc_nodes(mas, gfp);
5621	}	5736	}
5622		5737
		5738	/*
		5739	* Return false on zero forward progress. Partial allocations are kept
		5740	* so the retry path will attempt to get the rest. The failure should
		5741	* not happen as we try our best to reclaim. The user would need an
		5742	* external lock with a non-blocking gfp in a low memory situation.
		5743	*/
5623	if (!mas->sheaf && !mas->alloc)	5744	if (!mas->sheaf && !mas->alloc)
5624	return false;	5745	return false;
5625		5746
5626	mas->status = ma_start;	5747	mas_reset(mas);
		5748	return true;
		5749	}
		5750
		5751	/**
		5752	* mas_nomem_nofail() - Retry allocations with __GFP_NOFAIL, if the maple state
		5753	* has stored the -ENOMEM error.
		5754	* @mas: The maple state
		5755	* @index: The start of the range for the @mas reset
		5756	* @last: The end of the range for the @mas reset
		5757	*
		5758	* Return: false if @mas isn't in an -ENOMEM state. True if the allocation
		5759	* happens, the state is reset. The internal lock will be dropped and external
		5760	* locks must allow blocking.
		5761	*/
		5762	bool mas_nomem_nofail(struct ma_state *mas, unsigned long index,
		5763	unsigned long last)
		5764	__must_hold(mas->tree->ma_lock)
		5765	{
		5766	gfp_t gfp;
		5767
		5768	if (likely(mas->node != MA_ERROR(-ENOMEM)))
		5769	return false;
		5770
		5771	gfp = GFP_KERNEL \| __GFP_NOFAIL;
		5772	if (!mt_external_lock(mas->tree)) {
		5773	mtree_unlock(mas->tree);
		5774	mas_alloc_nodes(mas, gfp);
		5775	mtree_lock(mas->tree);
		5776	} else {
		5777	mas_alloc_nodes(mas, gfp);
		5778	}
		5779
		5780	mas_set_range(mas, index, last);
5627	return true;	5781	return true;
5628	}	5782	}
5629		5783
@@ -5913,6 +6067,10 @@ EXPORT_SYMBOL(mtree_alloc_rrange); · 283 unmodified lines
5913	* Erasing is the same as a walk to an entry then a store of a NULL to that	6067	* Erasing is the same as a walk to an entry then a store of a NULL to that
5914	* ENTIRE range. In fact, it is implemented as such using the advanced API.	6068	* ENTIRE range. In fact, it is implemented as such using the advanced API.
5915	*	6069	*
		6070	* Note that erase requires allocations and will use GFP_KERNEL \| __GFP_NOFAIL
		6071	* to do so if necessary. If the allocation fails, the internal lock will be
		6072	* dropped to retry.
		6073	*
5916	* Return: The entry stored at the @index or %NULL	6074	* Return: The entry stored at the @index or %NULL
5917	*/	6075	*/
5918	void mtree_erase(struct maple_tree mt, unsigned long index)	6076	void mtree_erase(struct maple_tree mt, unsigned long index)
@@ -5922,6 +6080,7 @@ void mtree_erase(struct maple_tree mt, unsigned long index) · 3 unmodified lines
5922	MA_STATE(mas, mt, index, index);	6080	MA_STATE(mas, mt, index, index);
5923	trace_ma_op(TP_FCT, &mas);	6081	trace_ma_op(TP_FCT, &mas);
5924		6082
		6083	might_alloc(GFP_KERNEL);
5925	mtree_lock(mt);	6084	mtree_lock(mt);
5926	entry = mas_erase(&mas);	6085	entry = mas_erase(&mas);
5927	mtree_unlock(mt);	6086	mtree_unlock(mt);

lib/test_hmm.c+2−3

 		if (!dpage && !order)
 			return VM_FAULT_OOM;
-		pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
-				page_to_pfn(spage), page_to_pfn(dpage));
 		if (dpage) {
+			pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n",
+					page_to_pfn(spage), page_to_pfn(dpage));
 			lock_page(dpage);
 			*dst |= migrate_pfn(page_to_pfn(dpage));
 		}

lib/xarray.c+3−0

@@ -1053,6 +1053,9 @@ void xas_split_alloc(struct xa_state xas, void entry, unsigned int order, · 1052 unmodified lines
1053	if (xas->xa_shift + XA_CHUNK_SHIFT > order)	1053	if (xas->xa_shift + XA_CHUNK_SHIFT > order)
1054	return;	1054	return;
1055		1055
		1056	if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT)
		1057	gfp \|= __GFP_ACCOUNT;
		1058
1056	do {	1059	do {
1057	struct xa_node *node;	1060	struct xa_node *node;
1058		1061

mm/Kconfig+11−11

 config ZSMALLOC
 	tristate
-if ZSMALLOC
 menu "Zsmalloc allocator options"
 	depends on ZSMALLOC
 endmenu
-endif
 menu "Slab allocator options"
 config SLUB
 config FLATMEM_MANUAL
 	bool "Flat Memory"
+	depends on !NUMA
 	depends on !ARCH_SPARSEMEM_ENABLE || ARCH_FLATMEM_ENABLE
 	help
 	  This option is best suited for non-NUMA systems with
 config FLATMEM
 	def_bool y
+	depends on !NUMA
 	depends on !SPARSEMEM || FLATMEM_MANUAL
 #
 config MEMORY_HOTREMOVE
 	bool "Allow for memory hot remove"
-	depends on MEMORY_HOTPLUG
 	select MIGRATION
 config MHP_MEMMAP_ON_MEMORY
 	def_bool y
-	depends on MEMORY_HOTPLUG && SPARSEMEM_VMEMMAP
 	depends on ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE
 endif # MEMORY_HOTPLUG
 	depends on ARCH_SUPPORTS_MEMORY_FAILURE
 	bool "Enable recovery from hardware memory errors"
 	select INTERVAL_TREE
+	select MIGRATION
 	help
 	  Enables code to recover from some memory failures on systems
 	  with MCA recovery. This allows a system to continue running
 config ZONE_DEVICE
 	bool "Device memory (pmem, HMM, etc...) hotplug support"
-	depends on MEMORY_HOTPLUG
 	depends on MEMORY_HOTREMOVE
-	depends on SPARSEMEM_VMEMMAP
 	select XARRAY_MULTI
 	help
 	  Enable the userfaultfd() system call that allows to intercept and
 	  handle page faults in userland.
-if USERFAULTFD
 config PTE_MARKER_UFFD_WP
 	bool "Userfaultfd write protection support for shmem/hugetlbfs"
 	default y
-	depends on HAVE_ARCH_USERFAULTFD_WP
+	depends on USERFAULTFD && HAVE_ARCH_USERFAULTFD_WP
 	help
 	  Allows to create marker PTEs for userfaultfd write protection
 	  purposes.  It is required to enable userfaultfd write protection on
 	  file-backed memory types like shmem and hugetlbfs.
-endif # USERFAULTFD
 # multi-gen LRU {
 config LRU_GEN
 	  into virtual nodes when booted with "numa=fake=N", where N is the
 	  number of nodes. This is only useful for debugging.
+config GENERIC_ARCH_NUMA
+	bool
+	select NUMA_MEMBLKS
+	help
+	  Enable support for generic NUMA implementation. Currently, RISC-V
+	  and ARM64 use it.
 config ARCH_HAS_USER_SHADOW_STACK
 	bool
 	help

mm/Kconfig.debug+0−1

@@ -326,7 +326,6 @@ config MEM_ALLOC_PROFILING · 325 unmodified lines
326	default n	326	default n
327	depends on MMU	327	depends on MMU
328	depends on PROC_FS	328	depends on PROC_FS
329	depends on !DEBUG_FORCE_WEAK_PER_CPU
330	select CODE_TAGGING	329	select CODE_TAGGING
331	select PAGE_EXTENSION	330	select PAGE_EXTENSION
332	select SLAB_OBJ_EXT	331	select SLAB_OBJ_EXT

mm/Makefile+1−0

@@ -122,6 +122,7 @@ obj-$(CONFIG_CMA) += cma.o · 121 unmodified lines
122	obj-$(CONFIG_NUMA) += numa.o	122	obj-$(CONFIG_NUMA) += numa.o
123	obj-$(CONFIG_NUMA_MEMBLKS) += numa_memblks.o	123	obj-$(CONFIG_NUMA_MEMBLKS) += numa_memblks.o
124	obj-$(CONFIG_NUMA_EMU) += numa_emulation.o	124	obj-$(CONFIG_NUMA_EMU) += numa_emulation.o
		125	obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o
125	obj-$(CONFIG_BALLOON) += balloon.o	126	obj-$(CONFIG_BALLOON) += balloon.o
126	obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o	127	obj-$(CONFIG_PAGE_EXTENSION) += page_ext.o
127	obj-$(CONFIG_PAGE_TABLE_CHECK) += page_table_check.o	128	obj-$(CONFIG_PAGE_TABLE_CHECK) += page_table_check.o

mm/alloc_tag.c+349−2

@@ -5,6 +5,8 @@ · 4 unmodified lines
5	#include <linux/gfp.h>	5	#include <linux/gfp.h>
6	#include <linux/kallsyms.h>	6	#include <linux/kallsyms.h>
7	#include <linux/module.h>	7	#include <linux/module.h>
		8	#include <linux/mutex.h>
		9	#include <linux/compat.h>
8	#include <linux/page_ext.h>	10	#include <linux/page_ext.h>
9	#include <linux/pgalloc_tag.h>	11	#include <linux/pgalloc_tag.h>
10	#include <linux/proc_fs.h>	12	#include <linux/proc_fs.h>
@@ -14,6 +16,7 @@ · 3 unmodified lines
14	#include <linux/string_choices.h>	16	#include <linux/string_choices.h>
15	#include <linux/vmalloc.h>	17	#include <linux/vmalloc.h>
16	#include <linux/kmemleak.h>	18	#include <linux/kmemleak.h>
		19	#include <uapi/linux/alloc_tag.h>
17		20
18	#include "internal.h"	21	#include "internal.h"
19	#include "page_alloc.h"	22	#include "page_alloc.h"
@@ -59,6 +62,11 @@ struct allocinfo_private { · 39 unmodified lines
59	struct codetag_iterator iter;	62	struct codetag_iterator iter;
60	struct codetag_iterator reported_iter;	63	struct codetag_iterator reported_iter;
61	bool print_header;	64	bool print_header;
		65	struct allocinfo_filter filter;
		66	/* ioctl uses a separate iterator not to interfere with reads */
		67	struct codetag_iterator ioctl_iter;
		68	bool positioned; /* seq_open_private() sets to 0 */
		69	struct mutex ioctl_lock;
62	};	70	};
63		71
64	static void allocinfo_start(struct seq_file m, loff_t *pos)	72	static void allocinfo_start(struct seq_file m, loff_t *pos)
@@ -142,6 +150,340 @@ static const struct seq_operations allocinfo_seq_op = { · 77 unmodified lines
142	.show = allocinfo_show,	150	.show = allocinfo_show,
143	};	151	};
144		152
		153	/*
		154	* Initializes seq_file operations and allocates private state when opening
		155	* the /proc/allocinfo procfs entry.
		156	*/
		157	static int allocinfo_open(struct inode inode, struct file file)
		158	{
		159	int ret;
		160
		161	ret = seq_open_private(file, &allocinfo_seq_op,
		162	sizeof(struct allocinfo_private));
		163	if (!ret) {
		164	struct seq_file *m = file->private_data;
		165	struct allocinfo_private *priv = m->private;
		166
		167	mutex_init(&priv->ioctl_lock);
		168	}
		169	return ret;
		170	}
		171
		172	/*
		173	* Cleans up the seq_file state and frees up the private state allocated in
		174	* allocinfo_open() when closing the /proc/allocinfo file descriptor.
		175	*/
		176	static int allocinfo_release(struct inode inode, struct file file)
		177	{
		178	struct seq_file *m = file->private_data;
		179	struct allocinfo_private *priv = m->private;
		180
		181	mutex_destroy(&priv->ioctl_lock);
		182	return seq_release_private(inode, file);
		183	}
		184
		185	/*
		186	* Returns a pointer to the suffix of a string so that its length fits within
		187	* ALLOCINFO_STR_SIZE, preserving the trailing characters.
		188	* Function, file and module names often have the same prefixes, therefore
		189	* when filtering by these criteria, we compare the last 64 characters to
		190	* minimize the chances of name collisions
		191	*/
		192	static const char allocinfo_str(const char str)
		193	{
		194	size_t len = strlen(str);
		195
		196	/* Keep an extra space for the trailing NULL. */
		197	if (len >= ALLOCINFO_STR_SIZE)
		198	str += (len - ALLOCINFO_STR_SIZE) + 1;
		199	return str;
		200	}
		201
		202	/* Copy a string and trim from the beginning if it's too long */
		203	static void allocinfo_copy_str(char dest, const char src)
		204	{
		205	strscpy_pad(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE);
		206	}
		207
		208	/* Compare two strings and only consider the trimmed suffix if s1 is too long */
		209	static int allocinfo_cmp_str(const char str, const char template)
		210	{
		211	return strncmp(allocinfo_str(str), template, ALLOCINFO_STR_SIZE);
		212	}
		213
		214	/* Fetch the per-CPU counters */
		215	static inline struct alloc_tag_counters allocinfo_prefetch_counters(struct codetag *ct)
		216	{
		217	return alloc_tag_read(ct_to_alloc_tag(ct));
		218	}
		219
		220	/*
		221	* Populates the UAPI allocinfo_tag_data structure with active runtime
		222	* profiling counters extracted from the given kernel codetag.
		223	*/
		224	static void allocinfo_to_params(struct codetag *ct,
		225	struct allocinfo_tag_data *data,
		226	struct alloc_tag_counters *counters)
		227	{
		228	if (ct->modname)
		229	allocinfo_copy_str(data->tag.modname, ct->modname);
		230	else
		231	data->tag.modname[0] = '\0';
		232	allocinfo_copy_str(data->tag.function, ct->function);
		233	allocinfo_copy_str(data->tag.filename, ct->filename);
		234	data->tag.lineno = ct->lineno;
		235	data->counter.bytes = counters->bytes;
		236	data->counter.calls = counters->calls;
		237	data->counter.accurate = !alloc_tag_is_inaccurate(ct_to_alloc_tag(ct));
		238	}
		239
		240	/*
		241	* Retrieves the unique content ID representing the current allocation tag module
		242	* layout, allowing userspace to detect if modules were loaded / unloaded.
		243	*/
		244	static int allocinfo_ioctl_get_content_id(struct seq_file m, void __user arg)
		245	{
		246	struct allocinfo_content_id params;
		247
		248	codetag_lock_module_list(alloc_tag_cttype);
		249	params.id = codetag_get_content_id(alloc_tag_cttype);
		250	codetag_unlock_module_list(alloc_tag_cttype);
		251	if (copy_to_user(arg, &params, sizeof(params)))
		252	return -EFAULT;
		253
		254	return 0;
		255	}
		256
		257	/*
		258	* Verifies whether a given codetag satisfies the active filtering criteria by
		259	* matching its characteristics against the specified filter.
		260	*/
		261	static bool matches_filter(struct codetag ct, struct allocinfo_filter filter,
		262	struct alloc_tag_counters *counters,
		263	bool *fetched_counters)
		264	{
		265	bool inaccurate;
		266
		267	if (!filter \|\| !filter->mask)
		268	return true;
		269
		270	if (filter->mask & ALLOCINFO_FILTER_MASK_MODNAME) {
		271	/* user wants to filter by modname but ct->modname is NULL */
		272	if (!ct->modname) {
		273	/* validate if user was attempting to filter for built-in allocations */
		274	if (filter->fields.modname[0] != '\0')
		275	return false;
		276	} else if (allocinfo_cmp_str(ct->modname, filter->fields.modname))
		277	return false;
		278	}
		279
		280	if ((filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) &&
		281	ct->function && allocinfo_cmp_str(ct->function, filter->fields.function))
		282	return false;
		283
		284	if ((filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) &&
		285	ct->filename && allocinfo_cmp_str(ct->filename, filter->fields.filename))
		286	return false;
		287
		288	if ((filter->mask & ALLOCINFO_FILTER_MASK_LINENO) &&
		289	ct->lineno != filter->fields.lineno)
		290	return false;
		291
		292	if (filter->mask & ALLOCINFO_FILTER_MASK_INACCURATE) {
		293	inaccurate = !!(ct->flags & CODETAG_FLAG_INACCURATE);
		294	if (inaccurate != !!(filter->inaccurate))
		295	return false;
		296	}
		297
		298	if (filter->mask & (ALLOCINFO_FILTER_MASK_MIN_SIZE \| ALLOCINFO_FILTER_MASK_MAX_SIZE)) {
		299	if (!*fetched_counters) {
		300	*counters = allocinfo_prefetch_counters(ct);
		301	*fetched_counters = true;
		302	}
		303	if ((filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
		304	counters->bytes < filter->min_size)
		305	return false;
		306	if ((filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
		307	counters->bytes > filter->max_size)
		308	return false;
		309	}
		310
		311	return true;
		312	}
		313
		314	/*
		315	* Seeks the ioctl iterator to the specified 0-indexed tag position, reads its
		316	* profiling data and returns it to userspace.
		317	*/
		318	static int allocinfo_ioctl_get_at(struct seq_file m, void __user arg)
		319	{
		320	struct allocinfo_private *priv;
		321	struct codetag *ct;
		322	struct allocinfo_get_at params = {0};
		323	__u64 skip_count;
		324	struct alloc_tag_counters counters;
		325	bool fetched_counters;
		326
		327	if (copy_from_user(&params, arg, sizeof(params)))
		328	return -EFAULT;
		329
		330	if (params.filter.mask & ~ALLOCINFO_FILTER_MASKS)
		331	return -EINVAL;
		332
		333	if ((params.filter.mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
		334	(params.filter.mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
		335	params.filter.min_size > params.filter.max_size)
		336	return -EINVAL;
		337
		338	priv = m->private;
		339
		340	mutex_lock(&priv->ioctl_lock);
		341	codetag_lock_module_list(alloc_tag_cttype);
		342
		343	if (params.pos >= codetag_get_count(alloc_tag_cttype)) {
		344	codetag_unlock_module_list(alloc_tag_cttype);
		345	mutex_unlock(&priv->ioctl_lock);
		346	return -ENOENT;
		347	}
		348
		349	skip_count = params.pos;
		350
		351	if (params.filter.mask)
		352	priv->filter = params.filter;
		353	else
		354	priv->filter.mask = 0;
		355
		356	/* Find the codetag */
		357	priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype);
		358	ct = codetag_next_ct(&priv->ioctl_iter);
		359
		360	while (ct) {
		361	fetched_counters = false;
		362	if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) {
		363	if (skip_count == 0)
		364	break;
		365	skip_count--;
		366	}
		367	ct = codetag_next_ct(&priv->ioctl_iter);
		368	}
		369
		370	if (ct) {
		371	if (!fetched_counters)
		372	counters = allocinfo_prefetch_counters(ct);
		373	allocinfo_to_params(ct, &params.data, &counters);
		374	priv->positioned = true;
		375	}
		376
		377	codetag_unlock_module_list(alloc_tag_cttype);
		378	mutex_unlock(&priv->ioctl_lock);
		379
		380	if (!ct)
		381	return -ENOENT;
		382
		383	if (copy_to_user(arg, &params, sizeof(params)))
		384	return -EFAULT;
		385
		386	return 0;
		387	}
		388
		389	/*
		390	* Advances the ioctl iterator to the next allocation tag in the sequence and
		391	* returns its profiling data to userspace.
		392	*/
		393	static int allocinfo_ioctl_get_next(struct seq_file m, void __user arg)
		394	{
		395	struct allocinfo_private *priv;
		396	struct codetag *ct;
		397	struct allocinfo_tag_data params;
		398	int ret = 0;
		399	struct alloc_tag_counters counters;
		400	bool fetched_counters;
		401
		402	memset(&params, 0, sizeof(params));
		403	priv = m->private;
		404
		405	mutex_lock(&priv->ioctl_lock);
		406	codetag_lock_module_list(alloc_tag_cttype);
		407
		408	if (!priv->positioned) {
		409	priv->ioctl_iter = codetag_get_ct_iter(alloc_tag_cttype);
		410	priv->positioned = true;
		411	}
		412
		413	ct = codetag_next_ct(&priv->ioctl_iter);
		414	while (ct) {
		415	fetched_counters = false;
		416	if (matches_filter(ct, &priv->filter, &counters, &fetched_counters))
		417	break;
		418	ct = codetag_next_ct(&priv->ioctl_iter);
		419	}
		420
		421	if (ct) {
		422	if (!fetched_counters)
		423	counters = allocinfo_prefetch_counters(ct);
		424	allocinfo_to_params(ct, &params, &counters);
		425	}
		426	if (!ct) {
		427	priv->positioned = false;
		428	ret = -ENOENT;
		429	}
		430	codetag_unlock_module_list(alloc_tag_cttype);
		431	mutex_unlock(&priv->ioctl_lock);
		432
		433	if (ret == 0) {
		434	if (copy_to_user(arg, &params, sizeof(params)))
		435	return -EFAULT;
		436	}
		437	return ret;
		438	}
		439
		440	/*
		441	* Entry point ioctl function for /proc/allocinfo routing requests to fetch the
		442	* layout content ID, seek to a specific tag, or read sequential tags.
		443	*/
		444	static long allocinfo_ioctl(struct file *file, unsigned int cmd,
		445	unsigned long __arg)
		446	{
		447	void __user arg = (void __user )__arg;
		448	int ret;
		449
		450	switch (cmd) {
		451	case ALLOCINFO_IOC_CONTENT_ID:
		452	ret = allocinfo_ioctl_get_content_id(file->private_data, arg);
		453	break;
		454	case ALLOCINFO_IOC_GET_AT:
		455	ret = allocinfo_ioctl_get_at(file->private_data, arg);
		456	break;
		457	case ALLOCINFO_IOC_GET_NEXT:
		458	ret = allocinfo_ioctl_get_next(file->private_data, arg);
		459	break;
		460	default:
		461	ret = -ENOIOCTLCMD;
		462	break;
		463	}
		464
		465	return ret;
		466	}
		467
		468	#ifdef CONFIG_COMPAT
		469	static long allocinfo_compat_ioctl(struct file *file, unsigned int cmd,
		470	unsigned long arg)
		471	{
		472	return allocinfo_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
		473	}
		474	#endif
		475
		476	static const struct proc_ops allocinfo_proc_ops = {
		477	.proc_open = allocinfo_open,
		478	.proc_read_iter = seq_read_iter,
		479	.proc_lseek = seq_lseek,
		480	.proc_release = allocinfo_release,
		481	.proc_ioctl = allocinfo_ioctl,
		482	#ifdef CONFIG_COMPAT
		483	.proc_compat_ioctl = allocinfo_compat_ioctl,
		484	#endif
		485	};
		486
145	size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)	487	size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
146	{	488	{
147	struct codetag_iterator iter;	489	struct codetag_iterator iter;
@@ -961,6 +1303,12 @@ static const struct ctl_table memory_allocation_profiling_sysctls[] = { · 813 unmodified lines
961	.mode = 0644,	1303	.mode = 0644,
962	.proc_handler = proc_mem_profiling_handler,	1304	.proc_handler = proc_mem_profiling_handler,
963	},	1305	},
		1306	{
		1307	.procname = "mem_profiling_compressed",
		1308	.data = &mem_profiling_compressed,
		1309	.mode = 0444,
		1310	.proc_handler = proc_do_static_key,
		1311	},
964	};	1312	};
965		1313
966	static void __init sysctl_init(void)	1314	static void __init sysctl_init(void)
@@ -993,8 +1341,7 @@ static int __init alloc_tag_init(void) · 26 unmodified lines
993	return 0;	1341	return 0;
994	}	1342	}
995		1343
996	if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op,	1344	if (!proc_create(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_proc_ops)) {
997	sizeof(struct allocinfo_private), NULL)) {
998	pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);	1345	pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
999	shutdown_mem_profiling(false);	1346	shutdown_mem_profiling(false);
1000	return -ENOMEM;	1347	return -ENOMEM;

mm/arch_numa.crenamed from drivers/base/arch_numa.c+12−0

@@ -105,6 +105,18 @@ static void __init setup_node_to_cpumask_map(void) · 104 unmodified lines
105	if (nr_node_ids == MAX_NUMNODES)	105	if (nr_node_ids == MAX_NUMNODES)
106	setup_nr_node_ids();	106	setup_nr_node_ids();
107		107
		108	/*
		109	* This check should never be true but it makes it clear to compilers
		110	* that node_to_cpumask_map is bound by nr_node_ids, avoiding false
		111	* positive fortify warnings when accessing node_to_cpumask_map in the
		112	* for loop below.
		113	*/
		114	if (unlikely(nr_node_ids > MAX_NUMNODES)) {
		115	pr_err("nr_node_ids (%u) is larger than MAX_NUMNODES (%u)\n",
		116	nr_node_ids, MAX_NUMNODES);
		117	return;
		118	}
		119
108	/* allocate and clear the mapping */	120	/* allocate and clear the mapping */
109	for (node = 0; node < nr_node_ids; node++) {	121	for (node = 0; node < nr_node_ids; node++) {
110	alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);	122	alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);

mm/cma.c+1−1

@@ -242,7 +242,7 @@ static int __init cma_new_area(const char *name, phys_addr_t size, · 241 unmodified lines
242	if (name)	242	if (name)
243	strscpy(cma->name, name);	243	strscpy(cma->name, name);
244	else	244	else
245	snprintf(cma->name, CMA_MAX_NAME, "cma%d\n", cma_area_count);	245	snprintf(cma->name, CMA_MAX_NAME, "cma%d", cma_area_count);
246		246
247	cma->available_count = cma->count = size >> PAGE_SHIFT;	247	cma->available_count = cma->count = size >> PAGE_SHIFT;
248	cma->order_per_bit = order_per_bit;	248	cma->order_per_bit = order_per_bit;

mm/cma.h+1−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	#ifndef __MM_CMA_H__	2	#ifndef __MM_CMA_H__
3	#define __MM_CMA_H__	3	#define __MM_CMA_H__
4		4
		5	#include <linux/cma.h>
5	#include <linux/debugfs.h>	6	#include <linux/debugfs.h>
6	#include <linux/kobject.h>	7	#include <linux/kobject.h>
7		8

mm/debug_page_alloc.c+4−4

@@ -20,14 +20,14 @@ early_param("debug_pagealloc", early_debug_pagealloc); · 19 unmodified lines
20		20
21	static int __init debug_guardpage_minorder_setup(char *buf)	21	static int __init debug_guardpage_minorder_setup(char *buf)
22	{	22	{
23	unsigned long res;	23	unsigned int res;
24		24
25	if (kstrtoul(buf, 10, &res) < 0 \|\| res > MAX_PAGE_ORDER / 2) {	25	if (!buf \|\| kstrtouint(buf, 10, &res) < 0 \|\| res > MAX_PAGE_ORDER / 2) {
26	pr_err("Bad debug_guardpage_minorder value: %s\n", buf);	26	pr_err("Bad debug_guardpage_minorder value: %s\n", buf ?: "(missing)");
27	return 0;	27	return 0;
28	}	28	}
29	_debug_guardpage_minorder = res;	29	_debug_guardpage_minorder = res;
30	pr_info("Setting debug_guardpage_minorder to %lu\n", res);	30	pr_info("Setting debug_guardpage_minorder to %u\n", res);
31	return 0;	31	return 0;
32	}	32	}
33	early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);	33	early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);

mm/folio-compat.c+1−0

@@ -41,6 +41,7 @@ void set_page_writeback(struct page *page) · 40 unmodified lines
41	}	41	}
42	EXPORT_SYMBOL(set_page_writeback);	42	EXPORT_SYMBOL(set_page_writeback);
43		43
		44	/* Read the comment above folio_mark_dirty() regarding required locks! */
44	bool set_page_dirty(struct page *page)	45	bool set_page_dirty(struct page *page)
45	{	46	{
46	return folio_mark_dirty(page_folio(page));	47	return folio_mark_dirty(page_folio(page));

mm/folio.c+55−69

 	folio_batch_add_and_move(folio, lru_move_tail);
 }
-void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
-		unsigned int nr_io, unsigned int nr_rotated)
-		__releases(lruvec->lru_lock)
-		__releases(rcu)
-{
-	unsigned long cost;
-	/*
-	 * Reflect the relative cost of incurring IO and spending CPU
-	 * time on rotations. This doesn't attempt to make a precise
-	 * comparison, it just says: if reloads are about comparable
-	 * between the LRU lists, or rotations are overwhelmingly
-	 * different between them, adjust scan balance for CPU work.
-	 */
-	cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated;
-	if (!cost) {
-		spin_unlock_irq(&lruvec->lru_lock);
-		rcu_read_unlock();
-		return;
-	}
-	for (;;) {
-		unsigned long lrusize;
-		/* Record cost event */
-		if (file)
-			lruvec->file_cost += cost;
-		else
-			lruvec->anon_cost += cost;
-		/*
-		 * Decay previous events
-		 *
-		 * Because workloads change over time (and to avoid
-		 * overflow) we keep these statistics as a floating
-		 * average, which ends up weighing recent refaults
-		 * more than old ones.
-		 */
-		lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
-			  lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
-			  lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
-			  lruvec_page_state(lruvec, NR_ACTIVE_FILE);
-		if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
-			lruvec->file_cost /= 2;
-			lruvec->anon_cost /= 2;
-		}
-		spin_unlock_irq(&lruvec->lru_lock);
-		lruvec = parent_lruvec(lruvec);
-		if (!lruvec) {
-			rcu_read_unlock();
-			break;
-		}
-		spin_lock_irq(&lruvec->lru_lock);
-	}
-}
-void lru_note_cost_refault(struct folio *folio)
-{
-	struct lruvec *lruvec;
-	lruvec = folio_lruvec_lock_irq(folio);
-	lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio),
-				folio_nr_pages(folio), 0);
-}
 static void lru_activate(struct lruvec *lruvec, struct folio *folio)
 {
 	long nr_pages = folio_nr_pages(folio);
 }
 #endif /* CONFIG_SMP */
+/**
+ * lru_cache_drain_for_folio() - drain LRU caches if the caches might hold
+ *				 folio references
+ * @folio: The folio.
+ * @extra_refs: Extra folio references held by the caller.
+ * @drained: Drain status for batch folio processing.
+ *
+ * Drain LRU caches if the caches might hold folio references. Start
+ * with a local LRU cache drain, to then drain LRU caches on all CPUs if
+ * local draining was insufficient.
+ *
+ * This function detects LRU cache references by comparing the folio refcount
+ * with the sum of the expected folio refcount + extra references held by the
+ * caller. Note that we cannot rely on PG_lru to reliably detect all LRU
+ * cache references, and there are rare scenarios (concurrent folio (un)mapping)
+ * where this function might miss detecting LRU cache references.
+ *
+ * If @drained is not NULL, the function will avoid re-draining LRU caches
+ * when processing multiple folios in a row. In that case, the variable
+ * @drained points at must be initialized to LRU_CACHE_NOT_DRAINED before
+ * the first invocation by the caller.
+ */
+void lru_cache_drain_for_folio(const struct folio *folio,
+		unsigned int extra_refs, enum lru_cache_drained *drained)
+{
+	if (!folio_may_be_lru_cached(folio))
+		return;
+	if (!drained || *drained == LRU_CACHE_NOT_DRAINED) {
+		if (folio_ref_count(folio) ==
+		    folio_expected_ref_count(folio) + extra_refs)
+			return;
+		lru_add_drain();
+		if (drained)
+			*drained = LRU_CACHE_DRAINED;
+	}
+	if (!drained || *drained == LRU_CACHE_DRAINED) {
+		if (folio_ref_count(folio) ==
+		    folio_expected_ref_count(folio) + extra_refs)
+			return;
+		lru_add_drain_all();
+		if (drained)
+			*drained = LRU_CACHE_DRAINED_ALL;
+	}
+}
 atomic_t lru_disable_count = ATOMIC_INIT(0);
 /*
 	for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) {
 		unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid);
+		if (!size)
+			continue;
+		/*
+		 * The folios are accounted to the parent from now on, so the
+		 * size has to be moved, not just copied. Leaving it behind
+		 * makes the dying child describe folios it no longer owns.
+		 */
 		mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size);
+		mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size);
 	}
 }
 	child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
 	parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid));
-	parent_lruvec->anon_cost += child_lruvec->anon_cost;
-	parent_lruvec->file_cost += child_lruvec->file_cost;
 	for_each_lru(lru)
 		lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid);

mm/gup.c+9−14

 			 * Anon pages in shared mappings are surprising: now
 			 * just reject it.
 			 */
-			if (!is_cow_mapping(vm_flags))
+			if (!vma_is_cow_mapping(vma))
 				return -EFAULT;
 		}
 	} else if (!(vm_flags & VM_READ)) {
 		struct list_head *movable_folio_list,
 		struct pages_or_folios *pofs)
 {
+	enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED;
 	unsigned long collected = 0;
 	struct folio *folio;
-	int drained = 0;
 	long i = 0;
 	for (folio = pofs_get_folio(pofs, i); folio;
 	     folio = pofs_next_folio(folio, pofs, &i)) {
+		const int pin_refs = folio_has_pincount(folio) ? 1 : GUP_PIN_COUNTING_BIAS;
 		if (folio_is_longterm_pinnable(folio))
 			continue;
 			continue;
 		}
-		if (drained == 0 && folio_may_be_lru_cached(folio) &&
+		/*
-				folio_ref_count(folio) !=
+		 * We drain not only to make the folio_isolate_lru() succeed,
-				folio_expected_ref_count(folio) + 1) {
+		 * but also to remove any other folio references from LRU
-			lru_add_drain();
+		 * caches.
-			drained = 1;
+		 */
-		}
+		lru_cache_drain_for_folio(folio, pin_refs, &drained);
-		if (drained == 1 && folio_may_be_lru_cached(folio) &&
-				folio_ref_count(folio) !=
-				folio_expected_ref_count(folio) + 1) {
-			lru_add_drain_all();
-			drained = 2;
-		}
 		if (!folio_isolate_lru(folio))
 			continue;

mm/gup_test.c+62−32

 #include <linux/highmem.h>
 #include "gup_test.h"
+struct gup_test_data {
+	struct mutex longterm_mutex;
+	struct page **longterm_pages;
+	unsigned long longterm_nr_pages;
+};
 static void put_back_pages(unsigned int cmd, struct page **pages,
 			   unsigned long nr_pages, unsigned int gup_test_flags)
 {
 	return ret;
 }
-static DEFINE_MUTEX(pin_longterm_test_mutex);
+static inline void pin_longterm_test_stop(struct gup_test_data *data)
-static struct page **pin_longterm_test_pages;
-static unsigned long pin_longterm_test_nr_pages;
-static inline void pin_longterm_test_stop(void)
 {
-	if (pin_longterm_test_pages) {
+	if (data->longterm_pages) {
-		if (pin_longterm_test_nr_pages)
+		if (data->longterm_nr_pages)
-			unpin_user_pages(pin_longterm_test_pages,
+			unpin_user_pages(data->longterm_pages,
-					 pin_longterm_test_nr_pages);
+					 data->longterm_nr_pages);
-		kvfree(pin_longterm_test_pages);
+		kvfree(data->longterm_pages);
-		pin_longterm_test_pages = NULL;
+		data->longterm_pages = NULL;
-		pin_longterm_test_nr_pages = 0;
+		data->longterm_nr_pages = 0;
 	}
 }
-static inline int pin_longterm_test_start(unsigned long arg)
+static inline int pin_longterm_test_start(struct gup_test_data *data,
+		unsigned long arg)
 {
 	long nr_pages, cur_pages, addr, remaining_pages;
 	int gup_flags = FOLL_LONGTERM;
 	int ret = 0;
 	bool fast;
-	if (pin_longterm_test_pages)
+	if (data->longterm_pages)
 		return -EINVAL;
 	if (copy_from_user(&args, (void __user *)arg, sizeof(args)))
 		return -EINTR;
 	}
-	pin_longterm_test_pages = pages;
+	data->longterm_pages = pages;
-	pin_longterm_test_nr_pages = 0;
+	data->longterm_nr_pages = 0;
-	while (nr_pages - pin_longterm_test_nr_pages) {
+	while (nr_pages - data->longterm_nr_pages) {
-		remaining_pages = nr_pages - pin_longterm_test_nr_pages;
+		remaining_pages = nr_pages - data->longterm_nr_pages;
-		addr = args.addr + pin_longterm_test_nr_pages * PAGE_SIZE;
+		addr = args.addr + data->longterm_nr_pages * PAGE_SIZE;
 		if (fast)
 			cur_pages = pin_user_pages_fast(addr, remaining_pages,
 			cur_pages = pin_user_pages(addr, remaining_pages,
 						   gup_flags, pages);
 		if (cur_pages < 0) {
-			pin_longterm_test_stop();
+			pin_longterm_test_stop(data);
 			ret = cur_pages;
 			break;
 		}
-		pin_longterm_test_nr_pages += cur_pages;
+		data->longterm_nr_pages += cur_pages;
 		pages += cur_pages;
 	}
 	return ret;
 }
-static inline int pin_longterm_test_read(unsigned long arg)
+static inline int pin_longterm_test_read(struct gup_test_data *data,
+		unsigned long arg)
 {
 	__u64 user_addr;
 	unsigned long i;
-	if (!pin_longterm_test_pages)
+	if (!data->longterm_pages)
 		return -EINVAL;
 	if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr)))
 		return -EFAULT;
-	for (i = 0; i < pin_longterm_test_nr_pages; i++) {
+	for (i = 0; i < data->longterm_nr_pages; i++) {
-		void *addr = kmap_local_page(pin_longterm_test_pages[i]);
+		void *addr = kmap_local_page(data->longterm_pages[i]);
 		unsigned long ret;
 		ret = copy_to_user((void __user *)(unsigned long)user_addr, addr,
 static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd,
 				    unsigned long arg)
 {
+	struct gup_test_data *data = filep->private_data;
 	int ret = -EINVAL;
-	if (mutex_lock_killable(&pin_longterm_test_mutex))
+	if (mutex_lock_killable(&data->longterm_mutex))
 		return -EINTR;
 	switch (cmd) {
 	case PIN_LONGTERM_TEST_START:
-		ret = pin_longterm_test_start(arg);
+		ret = pin_longterm_test_start(data, arg);
 		break;
 	case PIN_LONGTERM_TEST_STOP:
-		pin_longterm_test_stop();
+		pin_longterm_test_stop(data);
 		ret = 0;
 		break;
 	case PIN_LONGTERM_TEST_READ:
-		ret = pin_longterm_test_read(arg);
+		ret = pin_longterm_test_read(data, arg);
 		break;
 	}
-	mutex_unlock(&pin_longterm_test_mutex);
+	mutex_unlock(&data->longterm_mutex);
 	return ret;
 }
 	return 0;
 }
+static int gup_test_open(struct inode *inode, struct file *file)
+{
+	struct gup_test_data *data;
+	int ret;
+	data = kzalloc_obj(*data);
+	if (!data)
+		return -ENOMEM;
+	ret = nonseekable_open(inode, file);
+	if (ret) {
+		kfree(data);
+		return ret;
+	}
+	mutex_init(&data->longterm_mutex);
+	file->private_data = data;
+	return 0;
+}
 static int gup_test_release(struct inode *inode, struct file *file)
 {
-	pin_longterm_test_stop();
+	struct gup_test_data *data = file->private_data;
+	pin_longterm_test_stop(data);
+	mutex_destroy(&data->longterm_mutex);
+	kfree(data);
+	file->private_data = NULL;
 	return 0;
 }
 static const struct file_operations gup_test_fops = {
-	.open = nonseekable_open,
+	.open = gup_test_open,
 	.unlocked_ioctl = gup_test_ioctl,
 	.compat_ioctl = compat_ptr_ioctl,
 	.release = gup_test_release,

mm/hmm.c+4−1

 		ret = handle_mm_fault(vma, addr, fault_flags, NULL);
 		if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) {
-			*hmm_vma_walk->locked = false;
+			if (hmm_vma_walk->locked)    /* needed by sparse */
+				*hmm_vma_walk->locked = false;
+			else
+				WARN_ON_ONCE(1);    /* broken fault handler */
 			return HMM_FAULT_UNLOCKED;
 		}

mm/huge_memory.c+28−18

 	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
 	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
 						(VM_PFNMAP|VM_MIXEDMAP));
-	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
+	BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma));
 	pfnmap_setup_cachemode_pfn(pfn, &pgprot);
 	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
 	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
 						(VM_PFNMAP|VM_MIXEDMAP));
-	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
+	BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma));
 	pfnmap_setup_cachemode_pfn(pfn, &pgprot);
 		 * applied special bit, or we made the PRIVATE mapping be
 		 * able to wrongly write to the backend MMIO.
 		 */
-		VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd));
+		VM_WARN_ON_ONCE(vma_is_cow_mapping(src_vma) && pmd_write(pmd));
 		goto set_pmd;
 	}
 	 * TODO: once we support anonymous pages, use
 	 * folio_try_dup_anon_rmap_*() and split if duplicating fails.
 	 */
-	if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) {
+	if (vma_is_cow_mapping(vma) && pud_write(pud)) {
 		pudp_set_wrprotect(src_mm, addr, src_pud);
 		pud = pud_wrprotect(pud);
 	}
 		}
 		folio_move_anon_rmap(src_folio, dst_vma);
-		src_folio->index = linear_page_index(dst_vma, dst_addr);
+		src_folio->index = linear_anon_page_index(dst_vma, dst_addr);
 		_dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot);
 		/* Follow mremap() behavior and treat the entry dirty after the move */
 	XA_STATE(xas, &folio->mapping->i_pages, folio->index);
 	struct folio *end_folio = folio_next(folio);
 	bool is_anon = folio_test_anon(folio);
+	struct mem_cgroup *memcg, *old_memcg;
 	struct address_space *mapping = NULL;
 	struct anon_vma *anon_vma = NULL;
 	int old_order = folio_order(folio);
 	struct folio *new_folio, *next;
 	int nr_shmem_dropped = 0;
 	enum ttu_flags ttu_flags = 0;
-	int ret;
 	pgoff_t end = 0;
+	int ret;
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio);
 	if (folio != page_folio(split_at) || folio != page_folio(lock_at)) {
 		ret = -EINVAL;
-		goto out;
+		goto out_no_memcg;
 	}
 	if (new_order >= old_order) {
 		ret = -EINVAL;
-		goto out;
+		goto out_no_memcg;
 	}
 	ret = folio_check_splittable(folio, new_order, split_type);
 	if (ret) {
 		VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio");
-		goto out;
+		goto out_no_memcg;
 	}
+	/*
+	 * switch to folio's memcg as xarray node allocation can happen and
+	 * needs to charge to it.
+	 */
+	memcg = get_mem_cgroup_from_folio(folio);
+	old_memcg = set_active_memcg(memcg);
 	if (is_anon) {
 		/*
 		 * The caller does not necessarily hold an mmap_lock that would
 	if (mapping)
 		i_mmap_unlock_read(mapping);
 out:
+	/* restore to caller's old_memcg */
+	set_active_memcg(old_memcg);
+	mem_cgroup_put(memcg);
+out_no_memcg:
 	xas_destroy(&xas);
 	if (is_pmd_order(old_order))
 		count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
 	return 0;
 }
-void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
+void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct folio *folio)
 {
-	struct folio *folio = page_folio(new);
 	struct vm_area_struct *vma = pvmw->vma;
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long address = pvmw->address;
 		swp_entry_t entry;
 		if (pmd_write(pmde))
-			entry = make_writable_device_private_entry(
+			entry = make_writable_device_private_entry(folio_pfn(folio));
-							page_to_pfn(new));
 		else
-			entry = make_readable_device_private_entry(
+			entry = make_readable_device_private_entry(folio_pfn(folio));
-							page_to_pfn(new));
 		pmde = softleaf_to_pmd(entry);
 		if (pmd_swp_soft_dirty(*pvmw->pmd))
 		if (!softleaf_is_migration_read(entry))
 			rmap_flags |= RMAP_EXCLUSIVE;
-		folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags);
+		folio_add_anon_rmap_pmd(folio, &folio->page, vma, haddr, rmap_flags);
 	} else {
-		folio_add_file_rmap_pmd(folio, new, vma);
+		folio_add_file_rmap_pmd(folio, &folio->page, vma);
 	}
-	VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new));
+	VM_WARN_ON_ONCE(pmd_write(pmde) && folio_test_anon(folio) &&
+			!PageAnonExclusive(&folio->page));
 	set_pmd_at(mm, haddr, pvmw->pmd, pmde);
 	/* No need to invalidate - it was non-present before */

mm/hugetlb.c+7−3

 	pte_t *src_pte, *dst_pte, entry;
 	struct folio *pte_folio;
 	unsigned long addr;
-	bool cow = is_cow_mapping(src_vma->vm_flags);
+	bool cow = vma_is_cow_mapping(src_vma);
 	struct hstate *h = hstate_vma(src_vma);
 	unsigned long sz = huge_page_size(h);
 	unsigned long npages = pages_per_huge_page(h);
 	bool adjust_reservation;
 	unsigned long last_addr_mask;
+	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
 	WARN_ON(!is_vm_hugetlb_page(vma));
 	BUG_ON(start & ~huge_page_mask(h));
 	BUG_ON(end & ~huge_page_mask(h));
 		/*
 		 * Restore the reservation for anonymous page, otherwise the
-		 * backing page could be stolen by someone.
+		 * backing page could be stolen by someone. Restore only on the
+		 * last unmap, otherwise the owner could empty its resv map
+		 * while the folio is still mapped by a child. Note that holding
+		 * i_mmap_lock_write is needed to check the number of mappings.
 		 * If there we are freeing a surplus, do not set the restore
 		 * reservation bit.
 		 */
 		spin_lock_irq(&hugetlb_lock);
 		if (!h->surplus_huge_pages && __vma_private_lock(vma) &&
-		    folio_test_anon(folio)) {
+		    !folio_mapped(folio) && folio_test_anon(folio)) {
 			folio_set_hugetlb_restore_reserve(folio);
 			/* Reservation to be adjusted after the spin lock */
 			adjust_reservation = true;

mm/hugetlb_cma.c+134−8

@@ -9,6 +9,9 @@ · 8 unmodified lines
9	#include <asm/setup.h>	9	#include <asm/setup.h>
10		10
11	#include <linux/hugetlb.h>	11	#include <linux/hugetlb.h>
		12	#include <linux/memblock.h>
		13	#include <linux/math.h>
		14	#include <linux/math64.h>
12	#include "internal.h"	15	#include "internal.h"
13	#include "hugetlb_cma.h"	16	#include "hugetlb_cma.h"
14		17
@@ -18,6 +21,28 @@ static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata; · 3 unmodified lines
18	static bool hugetlb_cma_only __ro_after_init;	21	static bool hugetlb_cma_only __ro_after_init;
19	static unsigned long hugetlb_cma_size __ro_after_init;	22	static unsigned long hugetlb_cma_size __ro_after_init;
20		23
		24	static unsigned int hugetlb_cma_percent __initdata;
		25	static unsigned int hugetlb_cma_percent_in_node[MAX_NUMNODES] __initdata;
		26
		27	#ifdef CONFIG_NUMA
		28	static phys_addr_t __init memblock_node_memory_size(int nid)
		29	{
		30	struct memblock_region *reg;
		31	phys_addr_t size = 0;
		32
		33	for_each_mem_region(reg) {
		34	if (reg->nid == nid)
		35	size += reg->size;
		36	}
		37	return size;
		38	}
		39	#else
		40	static phys_addr_t __init memblock_node_memory_size(int nid)
		41	{
		42	return memblock_phys_mem_size();
		43	}
		44	#endif
		45
21	void hugetlb_cma_free_frozen_folio(struct folio *folio)	46	void hugetlb_cma_free_frozen_folio(struct folio *folio)
22	{	47	{
23	WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)],	48	WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)],
@@ -90,14 +115,31 @@ static int __init cmdline_parse_hugetlb_cma(char *p) · 66 unmodified lines
90	break;	115	break;
91		116
92	if (s[count] == ':') {	117	if (s[count] == ':') {
		118	char *next;
		119
93	if (tmp >= MAX_NUMNODES)	120	if (tmp >= MAX_NUMNODES)
94	break;	121	break;
95	nid = array_index_nospec(tmp, MAX_NUMNODES);	122	nid = array_index_nospec(tmp, MAX_NUMNODES);
96		123
		124	hugetlb_cma_size = 0;
		125	hugetlb_cma_percent = 0;
		126
97	s += count + 1;	127	s += count + 1;
98	tmp = memparse(s, &s);	128	tmp = memparse(s, &next);
99	hugetlb_cma_size_in_node[nid] = tmp;	129	if (*next == '%') {
100	hugetlb_cma_size += tmp;	130	if (tmp > 100) {
		131	pr_warn("hugetlb_cma: invalid percentage %lu for node %d\n",
		132	tmp, nid);
		133	break;
		134	}
		135	hugetlb_cma_percent_in_node[nid] = tmp;
		136	hugetlb_cma_size_in_node[nid] = 0;
		137	s = next + 1;
		138	} else {
		139	hugetlb_cma_size_in_node[nid] = tmp;
		140	hugetlb_cma_percent_in_node[nid] = 0;
		141	s = next;
		142	}
101		143
102	/*	144	/*
103	* Skip the separator if have one, otherwise	145	* Skip the separator if have one, otherwise
@@ -108,7 +150,28 @@ static int __init cmdline_parse_hugetlb_cma(char *p) · 4 unmodified lines
108	else	150	else
109	break;	151	break;
110	} else {	152	} else {
111	hugetlb_cma_size = memparse(p, &p);	153	char *next;
		154
		155	tmp = memparse(p, &next);
		156	if (*next == '%') {
		157	if (tmp > 100) {
		158	pr_warn("hugetlb_cma: invalid percentage %lu\n", tmp);
		159	} else {
		160	hugetlb_cma_percent = tmp;
		161	hugetlb_cma_size = 0;
		162	for (nid = 0; nid < MAX_NUMNODES; nid++) {
		163	hugetlb_cma_size_in_node[nid] = 0;
		164	hugetlb_cma_percent_in_node[nid] = 0;
		165	}
		166	}
		167	} else {
		168	hugetlb_cma_size = tmp;
		169	hugetlb_cma_percent = 0;
		170	for (nid = 0; nid < MAX_NUMNODES; nid++) {
		171	hugetlb_cma_size_in_node[nid] = 0;
		172	hugetlb_cma_percent_in_node[nid] = 0;
		173	}
		174	}
112	break;	175	break;
113	}	176	}
114	}	177	}
@@ -134,8 +197,36 @@ void __init hugetlb_cma_reserve(void) · 19 unmodified lines
134	{	197	{
135	unsigned long size, reserved, per_node, order, gigantic_page_size;	198	unsigned long size, reserved, per_node, order, gigantic_page_size;
136	bool node_specific_cma_alloc = false;	199	bool node_specific_cma_alloc = false;
		200	bool has_node_specific_param = false;
137	int nid;	201	int nid;
138		202
		203	for (nid = 0; nid < MAX_NUMNODES; nid++) {
		204	if (hugetlb_cma_size_in_node[nid] \|\| hugetlb_cma_percent_in_node[nid]) {
		205	has_node_specific_param = true;
		206	break;
		207	}
		208	}
		209
		210	if (has_node_specific_param) {
		211	hugetlb_cma_size = 0;
		212	for (nid = 0; nid < MAX_NUMNODES; nid++) {
		213	if (hugetlb_cma_percent_in_node[nid]) {
		214	phys_addr_t node_gfp_mem = memblock_node_memory_size(nid);
		215	u64 s;
		216
		217	s = mul_u64_u32_div((u64)node_gfp_mem,
		218	hugetlb_cma_percent_in_node[nid],
		219	100);
		220
		221	hugetlb_cma_size_in_node[nid] = s;
		222	}
		223	hugetlb_cma_size += hugetlb_cma_size_in_node[nid];
		224	}
		225	} else if (hugetlb_cma_percent) {
		226	hugetlb_cma_size = mul_u64_u32_div((u64)memblock_phys_mem_size(),
		227	hugetlb_cma_percent, 100);
		228	}
		229
139	if (!hugetlb_cma_size)	230	if (!hugetlb_cma_size)
140	return;	231	return;
141		232
@@ -154,6 +245,32 @@ void __init hugetlb_cma_reserve(void) · 12 unmodified lines
154	VM_WARN_ON(order <= MAX_PAGE_ORDER);	245	VM_WARN_ON(order <= MAX_PAGE_ORDER);
155	gigantic_page_size = PAGE_SIZE << order;	246	gigantic_page_size = PAGE_SIZE << order;
156		247
		248	if (hugetlb_cma_percent) {
		249	unsigned long orig_size = hugetlb_cma_size;
		250
		251	hugetlb_cma_size = ALIGN_DOWN(hugetlb_cma_size, PAGE_SIZE << order);
		252	if (orig_size && !hugetlb_cma_size)
		253	pr_warn("hugetlb_cma: reservation size rounded down to 0 from %lu MiB (%u%%)\n",
		254	orig_size / SZ_1M, hugetlb_cma_percent);
		255	} else if (has_node_specific_param) {
		256	hugetlb_cma_size = 0;
		257	for (nid = 0; nid < MAX_NUMNODES; nid++) {
		258	if (hugetlb_cma_percent_in_node[nid]) {
		259	unsigned long orig_size = hugetlb_cma_size_in_node[nid];
		260
		261	hugetlb_cma_size_in_node[nid] =
		262	ALIGN_DOWN(hugetlb_cma_size_in_node[nid],
		263	PAGE_SIZE << order);
		264	if (orig_size && !hugetlb_cma_size_in_node[nid])
		265	pr_warn("hugetlb_cma: reservation size rounded down to 0 from %lu MiB (%u%%) on node %d\n",
		266	orig_size / SZ_1M,
		267	hugetlb_cma_percent_in_node[nid],
		268	nid);
		269	}
		270	hugetlb_cma_size += hugetlb_cma_size_in_node[nid];
		271	}
		272	}
		273
157	hugetlb_bootmem_set_nodes();	274	hugetlb_bootmem_set_nodes();
158		275
159	for (nid = 0; nid < MAX_NUMNODES; nid++) {	276	for (nid = 0; nid < MAX_NUMNODES; nid++) {
@@ -194,8 +311,13 @@ void __init hugetlb_cma_reserve(void) · 34 unmodified lines
194	per_node = DIV_ROUND_UP(hugetlb_cma_size,	311	per_node = DIV_ROUND_UP(hugetlb_cma_size,
195	nodes_weight(hugetlb_bootmem_nodes));	312	nodes_weight(hugetlb_bootmem_nodes));
196	per_node = round_up(per_node, gigantic_page_size);	313	per_node = round_up(per_node, gigantic_page_size);
197	pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",	314	if (hugetlb_cma_percent)
198	hugetlb_cma_size / SZ_1M, per_node / SZ_1M);	315	pr_info("hugetlb_cma: reserve %lu MiB (%u%%), up to %lu MiB per node\n",
		316	hugetlb_cma_size / SZ_1M, hugetlb_cma_percent,
		317	per_node / SZ_1M);
		318	else
		319	pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
		320	hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
199	}	321	}
200		322
201	reserved = 0;	323	reserved = 0;
@@ -230,8 +352,12 @@ void __init hugetlb_cma_reserve(void) · 28 unmodified lines
230	}	352	}
231		353
232	reserved += size;	354	reserved += size;
233	pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",	355	if (hugetlb_cma_percent_in_node[nid])
234	size / SZ_1M, nid);	356	pr_info("hugetlb_cma: reserved %lu MiB (%u%%) on node %d\n",
		357	size / SZ_1M, hugetlb_cma_percent_in_node[nid], nid);
		358	else
		359	pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
		360	size / SZ_1M, nid);
235		361
236	if (reserved >= hugetlb_cma_size)	362	if (reserved >= hugetlb_cma_size)
237	break;	363	break;

mm/hugetlb_cma.h+2−0

@@ -2,6 +2,8 @@ · 1 unmodified line
2	#ifndef _LINUX_HUGETLB_CMA_H	2	#ifndef _LINUX_HUGETLB_CMA_H
3	#define _LINUX_HUGETLB_CMA_H	3	#define _LINUX_HUGETLB_CMA_H
4		4
		5	#include <linux/hugetlb.h>
		6
5	#ifdef CONFIG_CMA	7	#ifdef CONFIG_CMA
6	void hugetlb_cma_free_frozen_folio(struct folio *folio);	8	void hugetlb_cma_free_frozen_folio(struct folio *folio);
7	struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,	9	struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,

mm/internal.h+54−23

 void workingset_activation(struct folio *folio);
 /* mm/folio.c */
-void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
-		unsigned int nr_io, unsigned int nr_rotated);
-void lru_note_cost_refault(struct folio *folio);
 void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma);
-static inline bool folio_may_be_lru_cached(struct folio *folio)
+static inline bool folio_may_be_lru_cached(const struct folio *folio)
 {
 	/*
 	 * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting.
 		return false;
 	pgoff_folio = folio_pgoff(folio);
-	pgoff_vma_start = vma_start_pgoff(vma);
+	pgoff_vma_start = folio_test_anon(folio) ?
+		vma_start_anon_pgoff(vma) : vma_start_pgoff(vma);
 	if (start < vma->vm_start)
 		start = vma->vm_start;
 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
-/**
+static inline unsigned long __vma_address(const struct vm_area_struct *vma,
- * vma_address - Find the virtual address a page range is mapped at
+		pgoff_t pgoff, pgoff_t pgoff_start, unsigned long nr_pages)
- * @vma: The vma which maps this object.
- * @pgoff: The page offset within its object.
- * @nr_pages: The number of pages to consider.
- *
- * If any page in this range is mapped by this VMA, return the first address
- * where any of these pages appear.  Otherwise, return -EFAULT.
- */
-static inline unsigned long vma_address(const struct vm_area_struct *vma,
-		pgoff_t pgoff, unsigned long nr_pages)
 {
-	const pgoff_t pgoff_start = vma_start_pgoff(vma);
 	unsigned long address;
 	if (pgoff >= pgoff_start) {
 	return address;
 }
+/**
+ * vma_filebacked_address - Find the virtual address a file-backed page range is
+ * mapped at.
+ * @vma: The vma which maps this object.
+ * @pgoff: The page offset within its object.
+ * @nr_pages: The number of pages to consider.
+ *
+ * Returns: If any page in this range is mapped by this VMA, return the first
+ * address where any of these pages appear.  Otherwise, return -EFAULT.
+ */
+static inline unsigned long vma_filebacked_address(const struct vm_area_struct *vma,
+		pgoff_t pgoff, unsigned long nr_pages)
+{
+	VM_WARN_ON_ONCE(vma_is_anonymous(vma));
+	return __vma_address(vma, pgoff, vma_start_pgoff(vma), nr_pages);
+}
+/**
+ * vma_anon_address - Find the virtual address an anonymous page range is mapped
+ * at.
+ * @vma: The vma which maps this object.
+ * @pgoff_anon: The anonymous page index belonging to the folio.
+ * @nr_pages: The number of pages to consider.
+ *
+ * This is only valid for anonymous or MAP_PRIVATE-mapped file-backed VMAs.
+ *
+ * Returns: If any page in this range is mapped by this VMA, return the first
+ * address where any of these pages appear. Otherwise, return -EFAULT.
+ */
+static inline unsigned long vma_anon_address(const struct vm_area_struct *vma,
+		pgoff_t pgoff_anon, unsigned long nr_pages)
+{
+	VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma));
+	return __vma_address(vma, pgoff_anon, vma_start_anon_pgoff(vma), nr_pages);
+}
 /*
- * Then at what user virtual address will none of the range be found in vma?
+ * At what user virtual address will none of the range be found in vma?
  * Assumes that vma_address() already returned a good starting address.
  */
 static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw)
 {
-	struct vm_area_struct *vma = pvmw->vma;
+	const pgoff_t pgoff_end = pvmw->pgoff + pvmw->nr_pages;
-	pgoff_t pgoff;
+	const struct vm_area_struct *vma = pvmw->vma;
+	pgoff_t pgoff_vma_start;
 	unsigned long address;
 	/* Common case, plus ->pgoff is invalid for KSM */
 	if (pvmw->nr_pages == 1)
 		return pvmw->address + PAGE_SIZE;
-	pgoff = pvmw->pgoff + pvmw->nr_pages;
+	if (pvmw->pgoff_is_anon)
+		pgoff_vma_start = vma_start_anon_pgoff(vma);
+	else
+		pgoff_vma_start = vma_start_pgoff(vma);
 	address = vma->vm_start +
-		((pgoff - vma_start_pgoff(vma)) << PAGE_SHIFT);
+		((pgoff_end - pgoff_vma_start) << PAGE_SHIFT);
 	/* Check for address beyond vma (or wrapped through 0?) */
 	if (address < vma->vm_start || address > vma->vm_end)
 		address = vma->vm_end;
 		 * ... because we only care about writable private ("COW")
 		 * mappings where we have to break COW early.
 		 */
-		return is_cow_mapping(vma->vm_flags);
+		return vma_is_cow_mapping(vma);
 	}
 	/* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */

mm/interval_tree.c+2−2

@@ -83,12 +83,12 @@ mapping_rmap_tree_iter_next(struct vm_area_struct *vma, · 82 unmodified lines
83		83
84	static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc)	84	static pgoff_t avc_start_pgoff(struct anon_vma_chain *avc)
85	{	85	{
86	return vma_start_pgoff(avc->vma);	86	return vma_start_anon_pgoff(avc->vma);
87	}	87	}
88		88
89	static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc)	89	static pgoff_t avc_last_pgoff(struct anon_vma_chain *avc)
90	{	90	{
91	return vma_last_pgoff(avc->vma);	91	return vma_last_anon_pgoff(avc->vma);
92	}	92	}
93		93
94	INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last,	94	INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, pgoff_t, rb_subtree_last,

mm/kasan/quarantine.c+11−1

 	 */
 	on_each_cpu(per_cpu_remove_cache, cache, 1);
-	for_each_online_cpu(cpu) {
+	/*
+	 * A CPU can go offline after on_each_cpu() returns, leaving cache
+	 * objects on that CPU's shrink list. Scan all possible CPUs to
+	 * drain those lists.
+	 */
+	for_each_possible_cpu(cpu) {
 		sq = per_cpu_ptr(&shrink_qlist, cpu);
 		raw_spin_lock_irqsave(&sq->lock, flags);
 		qlist_move_cache(&sq->qlist, &to_free, cache);
 	raw_spin_lock_irqsave(&quarantine_lock, flags);
 	for (i = 0; i < QUARANTINE_BATCHES; i++) {
+		size_t old_bytes;
 		if (qlist_empty(&global_quarantine[i]))
 			continue;
+		old_bytes = global_quarantine[i].bytes;
 		qlist_move_cache(&global_quarantine[i], &to_free, cache);
+		WRITE_ONCE(quarantine_size, quarantine_size -
+			   (old_bytes - global_quarantine[i].bytes));
 		/* Scanning whole quarantine can take a while. */
 		raw_spin_unlock_irqrestore(&quarantine_lock, flags);
 		cond_resched();

mm/khugepaged.c+65−65

 		/*
 		 * This is required to serialize against
 		 * collapse_test_exit() (which is guaranteed to run
-		 * under mmap sem read mode). Stop here (after we return all
+		 * under mmap_lock read mode). Stop here (after we return all
 		 * pagetables will be destroyed) until khugepaged has finished
 		 * working on the pagetables under the mmap_lock.
 		 */
 	}
 }
+static void collapse_control_init_scan(struct collapse_control *cc)
+{
+	memset(cc->node_load, 0, sizeof(cc->node_load));
+	nodes_clear(cc->alloc_nmask);
+	bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE);
+}
 static void release_pte_folio(struct folio *folio)
 {
 	node_stat_mod_folio(folio,
 	}
 }
+static bool folio_pte_referenced(struct folio *folio,
+		struct vm_area_struct *vma, unsigned long addr, pte_t pteval)
+{
+	/* The folio was referenced previously ... */
+	if (folio_test_young(folio) || folio_test_referenced(folio))
+		return true;
+	/* ... or the PTE mapping was recently used */
+	return pte_young(pteval) || mmu_notifier_test_young(vma->vm_mm, addr);
+}
+static void count_collapse_event(unsigned int order, enum vm_event_item vm_event,
+		enum mthp_stat_item mthp_event)
+{
+	if (is_pmd_order(order))
+		count_vm_event(vm_event);
+	count_mthp_stat(order, mthp_event);
+}
 static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
 		unsigned long start_addr, pte_t *pte, struct collapse_control *cc,
 		unsigned int order, struct list_head *compound_pagelist)
 		if (pte_none_or_zero(pteval)) {
 			if (++none_or_zero > max_ptes_none) {
 				result = SCAN_EXCEED_NONE_PTE;
-				if (is_pmd_order(order))
+				count_collapse_event(order, THP_SCAN_EXCEED_NONE_PTE,
-					count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+						     MTHP_STAT_COLLAPSE_EXCEED_NONE);
-				count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_NONE);
 				goto out;
 			}
 			continue;
 			 */
 			if (++shared > max_ptes_shared) {
 				result = SCAN_EXCEED_SHARED_PTE;
-				if (is_pmd_order(order))
+				count_collapse_event(order, THP_SCAN_EXCEED_SHARED_PTE,
-					count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+						     MTHP_STAT_COLLAPSE_EXCEED_SHARED);
-				count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SHARED);
 				goto out;
 			}
 		}
 		/*
 		 * We can do it before folio_isolate_lru because the
-		 * folio can't be freed from under us. NOTE: PG_lock
+		 * folio can't be freed from under us. NOTE: folio lock
-		 * is needed to serialize against split_huge_page
+		 * is needed to serialize against split_huge_page()
 		 * when invoked from the VM.
 		 */
 		if (!folio_trylock(folio)) {
 		}
 		/*
-		 * Isolate the page to avoid collapsing an hugepage
+		 * Isolate the folio to avoid collapsing a hugepage
 		 * currently in use by the VM.
 		 */
 		if (!folio_isolate_lru(folio)) {
 		if (folio_test_large(folio))
 			list_add_tail(&folio->lru, compound_pagelist);
 next:
-		/*
-		 * If collapse was initiated by khugepaged, check that there is
-		 * enough young pte to justify collapsing the page
-		 */
 		if (cc->is_khugepaged &&
-		    (pte_young(pteval) || folio_test_young(folio) ||
+		    folio_pte_referenced(folio, vma, addr, pteval))
-		     folio_test_referenced(folio) ||
-		     mmu_notifier_test_young(vma->vm_mm, addr)))
 			referenced++;
 	}
 	 * Re-establish the PMD to point to the original page table
 	 * entry. Restoring PMD needs to be done prior to releasing
 	 * pages. Since pages are still isolated and locked here,
-	 * acquiring anon_vma_lock_write is unnecessary.
+	 * acquiring anon_vma_lock_write() is unnecessary.
 	 */
 	pmd_ptl = pmd_lock(vma->vm_mm, pmd);
 	pmd_populate(vma->vm_mm, pmd, pmd_pgtable(orig_pmd));
 		return SCAN_VMA_CHECK;
 	/*
 	 * Anon VMA expected, the address may be unmapped then
-	 * remapped to file after khugepaged reaquired the mmap_lock.
+	 * remapped to file after khugepaged reacquired the mmap_lock.
 	 *
-	 * thp_vma_allowable_orders may return true for qualified file
+	 * thp_vma_allowable_orders() may return true for qualified file
 	 * vmas.
 	 */
 	if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
 /*
  * Bring missing pages in from swap, to complete THP collapse.
- * Only done if khugepaged_scan_pmd believes it is worthwhile.
+ * Only done if collapse_scan_pmd() believes it is worthwhile.
  *
  * For mTHP orders the function bails on the first swap entry, because
  * faulting pages back in during collapse could re-populate PTEs that
 		pte = NULL;
 		/*
-		 * do_swap_page returns VM_FAULT_RETRY with released mmap_lock.
+		 * do_swap_page() returns VM_FAULT_RETRY with released mmap_lock.
 		 * Note we treat VM_FAULT_RETRY as VM_FAULT_ERROR here because
 		 * we do not retry here and swap entry will remain in pagetable
 		 * resulting in later failure.
 	folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask);
 	if (!folio) {
 		*foliop = NULL;
-		if (is_pmd_order(order))
+		count_collapse_event(order, THP_COLLAPSE_ALLOC_FAILED,
-			count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
+				     MTHP_STAT_COLLAPSE_ALLOC_FAILED);
-		count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC_FAILED);
 		return SCAN_ALLOC_HUGE_PAGE_FAIL;
 	}
-	if (is_pmd_order(order))
+	count_collapse_event(order, THP_COLLAPSE_ALLOC, MTHP_STAT_COLLAPSE_ALLOC);
-		count_vm_event(THP_COLLAPSE_ALLOC);
-	count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC);
 	if (unlikely(mem_cgroup_charge(folio, mm, gfp))) {
 		folio_put(folio);
 }
 /*
- * collapse_huge_page expects the mmap_lock to be unlocked before entering and
+ * collapse_huge_page() expects the mmap_lock to be unlocked before entering and
  * will always return with the lock unlocked, to avoid holding the mmap_lock
  * while allocating a THP, as that could trigger direct reclaim/compaction.
  * Note that the VMA must be rechecked after grabbing the mmap_lock again.
 	if (unmapped) {
 		/*
-		 * __collapse_huge_page_swapin will return with mmap_lock
+		 * __collapse_huge_page_swapin() will return with mmap_lock
 		 * released when it fails. So we jump out_nolock directly in
 		 * that case.  Continuing to collapse causes inconsistency.
 		 */
 	mmap_read_unlock(mm);
 	/*
 	 * Prevent all access to pagetables with the exception of
-	 * gup_fast later handled by the ptep_clear_flush and the VM
+	 * gup_fast later handled by the pmdp_collapse_flush() and the VM
-	 * handled by the anon_vma lock + PG_lock.
+	 * handled by the anon_vma lock + folio lock.
 	 *
 	 * UFFDIO_MOVE is prevented to race as well thanks to the
 	 * mmap_lock.
 		spin_lock(pmd_ptl);
 		VM_WARN_ON_ONCE(!pmd_none(*pmd));
 		/*
-		 * We can only use set_pmd_at when establishing
+		 * We can only use set_pmd_at() when establishing
 		 * hugepmds and never for establishing regular pmds that
-		 * points to regular pagetables. Use pmd_populate for that
+		 * points to regular pagetables. Use pmd_populate() for that
 		 */
 		pmd_populate(mm, pmd, pmd_pgtable(_pmd));
 		spin_unlock(pmd_ptl);
 	result = SCAN_SUCCEED;
 out_up_write:
-	if (anon_vma_locked)
-		anon_vma_unlock_write(vma->anon_vma);
 	if (pte)
 		pte_unmap(pte);
+	if (anon_vma_locked)
+		anon_vma_unlock_write(vma->anon_vma);
 	mmap_write_unlock(mm);
 out_nolock:
 	if (folio)
 		goto out;
 	}
-	bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE);
+	collapse_control_init_scan(cc);
-	memset(cc->node_load, 0, sizeof(cc->node_load));
-	nodes_clear(cc->alloc_nmask);
 	enabled_orders = collapse_possible_orders(vma, vma->vm_flags, tva_flags);
 	/*
 	 * If PMD is the only enabled order, enforce max_ptes_none, otherwise
-	 * scan all pages to populate the bitmap for mTHP collapse.
+	 * scan all pages to populate the bitmap for mTHP collapse. The bitmap
+	 * is then checked again in mthp_collapse() for each attempted order.
 	 */
 	if (enabled_orders != BIT(HPAGE_PMD_ORDER))
 		max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
 		if (pte_none_or_zero(pteval)) {
 			if (++none_or_zero > max_ptes_none) {
 				result = SCAN_EXCEED_NONE_PTE;
-				count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+				count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_NONE_PTE,
-				count_mthp_stat(HPAGE_PMD_ORDER,
+						     MTHP_STAT_COLLAPSE_EXCEED_NONE);
-						MTHP_STAT_COLLAPSE_EXCEED_NONE);
 				goto out_unmap;
 			}
 			continue;
 		if (!pte_present(pteval)) {
 			if (++unmapped > max_ptes_swap) {
 				result = SCAN_EXCEED_SWAP_PTE;
-				count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
+				count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_SWAP_PTE,
-				count_mthp_stat(HPAGE_PMD_ORDER,
+						     MTHP_STAT_COLLAPSE_EXCEED_SWAP);
-						MTHP_STAT_COLLAPSE_EXCEED_SWAP);
 				goto out_unmap;
 			}
 			/*
 		if (folio_maybe_mapped_shared(folio)) {
 			if (++shared > max_ptes_shared) {
 				result = SCAN_EXCEED_SHARED_PTE;
-				count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+				count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_SHARED_PTE,
-				count_mthp_stat(HPAGE_PMD_ORDER,
+						     MTHP_STAT_COLLAPSE_EXCEED_SHARED);
-						MTHP_STAT_COLLAPSE_EXCEED_SHARED);
 				goto out_unmap;
 			}
 		}
 		/*
 		 * Check if the page has any GUP (or other external) pins.
 		 *
-		 * Here the check may be racy:
+		 * Here the check is racy, but such cases are ephemeral and
-		 * it may see folio_mapcount() > folio_ref_count().
+		 * we can always retry collapse later. Anyway the same
-		 * But such case is ephemeral we could always retry collapse
+		 * check will be done again later, so the risk seems to be low.
-		 * later.  However it may report false positive if the page
-		 * has excessive GUP pins (i.e. 512).  Anyway the same check
-		 * will be done again later the risk seems low.
 		 */
 		if (folio_expected_ref_count(folio) != folio_ref_count(folio)) {
 			result = SCAN_PAGE_COUNT;
 			goto out_unmap;
 		}
-		/*
-		 * If collapse was initiated by khugepaged, check that there is
-		 * enough young pte to justify collapsing the page
-		 */
 		if (cc->is_khugepaged &&
-		    (pte_young(pteval) || folio_test_young(folio) ||
+		    folio_pte_referenced(folio, vma, addr, pteval))
-		     folio_test_referenced(folio) ||
-		     mmu_notifier_test_young(vma->vm_mm, addr)))
 			referenced++;
 	}
 	if (cc->is_khugepaged &&
 out_unmap:
 	pte_unmap_unlock(pte, ptl);
 	if (result == SCAN_SUCCEED) {
-		/* collapse_huge_page expects the lock to be dropped before calling */
+		/* collapse_huge_page() expects the lock to be dropped before calling */
 		mmap_read_unlock(mm);
 		result = mthp_collapse(mm, start_addr, referenced,
 				       unmapped, cc, enabled_orders);
 	present = 0;
 	swap = 0;
-	memset(cc->node_load, 0, sizeof(cc->node_load));
+	collapse_control_init_scan(cc);
-	nodes_clear(cc->alloc_nmask);
 	rcu_read_lock();
 	xas_for_each(&xas, folio, start + HPAGE_PMD_NR - 1) {
 		if (xas_retry(&xas, folio))

mm/kmemleak.c+92−13

@@ -151,6 +151,8 @@ struct kmemleak_object { · 150 unmodified lines
151	int min_count;	151	int min_count;
152	/* the total number of pointers found pointing to this object */	152	/* the total number of pointers found pointing to this object */
153	int count;	153	int count;
		154	/* consecutive scans the object has been seen unreferenced */
		155	unsigned int unref_scans;
154	/* checksum for detecting modified objects */	156	/* checksum for detecting modified objects */
155	u32 checksum;	157	u32 checksum;
156	depot_stack_handle_t trace_handle;	158	depot_stack_handle_t trace_handle;
@@ -175,6 +177,8 @@ struct kmemleak_object { · 18 unmodified lines
175	#define OBJECT_PHYS (1 << 4)	177	#define OBJECT_PHYS (1 << 4)
176	/* flag set for per-CPU pointers */	178	/* flag set for per-CPU pointers */
177	#define OBJECT_PERCPU (1 << 5)	179	#define OBJECT_PERCPU (1 << 5)
		180	/* flag set on an object left unreferenced by the full scan, pending confirmation */
		181	#define OBJECT_SUSPECT (1 << 6)
178		182
179	/* set when __remove_object() called */	183	/* set when __remove_object() called */
180	#define DELSTATE_REMOVED (1 << 0)	184	#define DELSTATE_REMOVED (1 << 0)
@@ -232,9 +236,15 @@ static unsigned long max_percpu_addr; · 51 unmodified lines
232	static struct task_struct *scan_thread;	236	static struct task_struct *scan_thread;
233	/* used to avoid reporting of recently allocated objects */	237	/* used to avoid reporting of recently allocated objects */
234	static unsigned long jiffies_min_age;	238	static unsigned long jiffies_min_age;
		239	/* consecutive scans an object must stay unreferenced before reporting */
		240	static unsigned int min_unref_scans =
		241	IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 2 : 1;
		242	module_param(min_unref_scans, uint, 0644);
235	static unsigned long jiffies_last_scan;	243	static unsigned long jiffies_last_scan;
236	/* delay between automatic memory scannings */	244	/* delay between automatic memory scannings */
237	static unsigned long jiffies_scan_wait;	245	static unsigned long jiffies_scan_wait;
		246	/* number of objects flagged OBJECT_SUSPECT during the current scan */
		247	static int nr_suspects;
238	/* enables or disables the task stacks scanning */	248	/* enables or disables the task stacks scanning */
239	static int kmemleak_stack_scan = 1;	249	static int kmemleak_stack_scan = 1;
240	/* protects the memory scanning, parameters and debug/kmemleak file access */	250	/* protects the memory scanning, parameters and debug/kmemleak file access */
@@ -688,6 +698,7 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp) · 447 unmodified lines
688	object->excess_ref = 0;	698	object->excess_ref = 0;
689	object->count = 0; /* white color initially */	699	object->count = 0; /* white color initially */
690	object->checksum = ~0;	700	object->checksum = ~0;
		701	object->unref_scans = 0;
691	object->del_state = 0;	702	object->del_state = 0;
692		703
693	/* task information */	704	/* task information */
@@ -1440,6 +1451,11 @@ static void update_refs(struct kmemleak_object *object) · 746 unmodified lines
1440	*/	1451	*/
1441	object->count++;	1452	object->count++;
1442	if (color_gray(object)) {	1453	if (color_gray(object)) {
		1454	/* referenced after all, no longer a suspect */
		1455	if (object->flags & OBJECT_SUSPECT) {
		1456	object->flags &= ~OBJECT_SUSPECT;
		1457	nr_suspects--;
		1458	}
1443	/* put_object() called when removing from gray_list */	1459	/* put_object() called when removing from gray_list */
1444	WARN_ON(!get_object(object));	1460	WARN_ON(!get_object(object));
1445	list_add_tail(&object->gray_list, &gray_list);	1461	list_add_tail(&object->gray_list, &gray_list);
@@ -1571,7 +1587,7 @@ static int scan_large_block(void start, void end) · 125 unmodified lines
1571	if (scan_block(start, next, NULL))	1587	if (scan_block(start, next, NULL))
1572	return 1;	1588	return 1;
1573	start = next;	1589	start = next;
1574	cond_resched();	1590	cond_resched_tasks_rcu_qs();
1575	}	1591	}
1576		1592
1577	return 0;	1593	return 0;
@@ -1608,7 +1624,7 @@ static void scan_object(struct kmemleak_object *object) · 30 unmodified lines
1608	scan_block(start, end, object);	1624	scan_block(start, end, object);
1609		1625
1610	raw_spin_unlock_irqrestore(&object->lock, flags);	1626	raw_spin_unlock_irqrestore(&object->lock, flags);
1611	cond_resched();	1627	cond_resched_tasks_rcu_qs();
1612	raw_spin_lock_irqsave(&object->lock, flags);	1628	raw_spin_lock_irqsave(&object->lock, flags);
1613	if (!(object->flags & OBJECT_ALLOCATED))	1629	if (!(object->flags & OBJECT_ALLOCATED))
1614	break;	1630	break;
@@ -1630,7 +1646,7 @@ static void scan_object(struct kmemleak_object *object) · 15 unmodified lines
1630	break;	1646	break;
1631		1647
1632	raw_spin_unlock_irqrestore(&object->lock, flags);	1648	raw_spin_unlock_irqrestore(&object->lock, flags);
1633	cond_resched();	1649	cond_resched_tasks_rcu_qs();
1634	raw_spin_lock_irqsave(&object->lock, flags);	1650	raw_spin_lock_irqsave(&object->lock, flags);
1635	} while (object->flags & OBJECT_ALLOCATED);	1651	} while (object->flags & OBJECT_ALLOCATED);
1636	} else {	1652	} else {
@@ -1658,7 +1674,7 @@ static void scan_gray_list(void) · 21 unmodified lines
1658	*/	1674	*/
1659	object = list_entry(gray_list.next, typeof(*object), gray_list);	1675	object = list_entry(gray_list.next, typeof(*object), gray_list);
1660	while (&object->gray_list != &gray_list) {	1676	while (&object->gray_list != &gray_list) {
1661	cond_resched();	1677	cond_resched_tasks_rcu_qs();
1662		1678
1663	/* may add new objects to the list */	1679	/* may add new objects to the list */
1664	if (!scan_should_stop())	1680	if (!scan_should_stop())
@@ -1693,7 +1709,7 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) · 28 unmodified lines
1693	raw_spin_unlock_irq(&kmemleak_lock);	1709	raw_spin_unlock_irq(&kmemleak_lock);
1694		1710
1695	rcu_read_unlock();	1711	rcu_read_unlock();
1696	cond_resched();	1712	cond_resched_tasks_rcu_qs();
1697	rcu_read_lock();	1713	rcu_read_lock();
1698		1714
1699	raw_spin_lock_irq(&kmemleak_lock);	1715	raw_spin_lock_irq(&kmemleak_lock);
@@ -1738,7 +1754,7 @@ static void kmemleak_scan_task_stacks(void) · 38 unmodified lines
1738	}	1754	}
1739	put_task_struct(p);	1755	put_task_struct(p);
1740	}	1756	}
1741	cond_resched();	1757	cond_resched_tasks_rcu_qs();
1742	} while (pid && !stop);	1758	} while (pid && !stop);
1743	}	1759	}
1744		1760
@@ -1844,16 +1860,16 @@ static void dedup_flush(struct xarray *dedup) · 99 unmodified lines
1844	* kernel's standard allocators. This function must be called with the	1860	* kernel's standard allocators. This function must be called with the
1845	* scan_mutex held.	1861	* scan_mutex held.
1846	*/	1862	*/
1847	static void kmemleak_scan(void)	1863	static int __kmemleak_scan(bool full)
1848	{	1864	{
1849	struct kmemleak_object *object;	1865	struct kmemleak_object *object;
1850	struct zone *zone;	1866	struct zone *zone;
1851	int __maybe_unused i;	1867	int __maybe_unused i;
1852	struct xarray dedup;
1853	int new_leaks = 0;
1854	int stop = 0;	1868	int stop = 0;
1855		1869
1856	jiffies_last_scan = jiffies;	1870	jiffies_last_scan = jiffies;
		1871	if (full)
		1872	nr_suspects = 0;
1857		1873
1858	/* prepare the kmemleak_object's */	1874	/* prepare the kmemleak_object's */
1859	rcu_read_lock();	1875	rcu_read_lock();
@@ -1881,8 +1897,13 @@ static void kmemleak_scan(void) · 21 unmodified lines
1881	__paint_it(object, KMEMLEAK_BLACK);	1897	__paint_it(object, KMEMLEAK_BLACK);
1882	}	1898	}
1883		1899
		1900	/* referenced last scan: restart the unreferenced run */
		1901	if (!color_white(object))
		1902	object->unref_scans = 0;
1884	/* reset the reference count (whiten the object) */	1903	/* reset the reference count (whiten the object) */
1885	object->count = 0;	1904	object->count = 0;
		1905	if (full)
		1906	object->flags &= ~OBJECT_SUSPECT;
1886	if (color_gray(object) && get_object(object))	1907	if (color_gray(object) && get_object(object))
1887	list_add_tail(&object->gray_list, &gray_list);	1908	list_add_tail(&object->gray_list, &gray_list);
1888		1909
@@ -1915,7 +1936,7 @@ static void kmemleak_scan(void) · 26 unmodified lines
1915	struct page *page = pfn_to_online_page(pfn);	1936	struct page *page = pfn_to_online_page(pfn);
1916		1937
1917	if (!(pfn & 63))	1938	if (!(pfn & 63))
1918	cond_resched();	1939	cond_resched_tasks_rcu_qs();
1919		1940
1920	if (!page)	1941	if (!page)
1921	continue;	1942	continue;
@@ -1950,6 +1971,10 @@ static void kmemleak_scan(void) · 28 unmodified lines
1950	scan_gray:	1971	scan_gray:
1951	scan_gray_list();	1972	scan_gray_list();
1952		1973
		1974	/* a confirmation scan does not look for modified objects */
		1975	if (!full)
		1976	return nr_suspects;
		1977
1953	/*	1978	/*
1954	* Check for new or unreferenced objects modified since the previous	1979	* Check for new or unreferenced objects modified since the previous
1955	* scan and color them gray until the next scan.	1980	* scan and color them gray until the next scan.
@@ -1972,6 +1997,11 @@ scan_gray: · 16 unmodified lines
1972	/* color it gray temporarily */	1997	/* color it gray temporarily */
1973	object->count = object->min_count;	1998	object->count = object->min_count;
1974	list_add_tail(&object->gray_list, &gray_list);	1999	list_add_tail(&object->gray_list, &gray_list);
		2000	} else if (unreferenced_object(object) &&
		2001	!(object->flags & OBJECT_REPORTED)) {
		2002	/* flag the objects left unreferenced by this scan */
		2003	object->flags \|= OBJECT_SUSPECT;
		2004	nr_suspects++;
1975	}	2005	}
1976	raw_spin_unlock_irq(&object->lock);	2006	raw_spin_unlock_irq(&object->lock);
1977	}	2007	}
@@ -1982,12 +2012,62 @@ scan_gray: · 4 unmodified lines
1982	*/	2012	*/
1983	scan_gray_list();	2013	scan_gray_list();
1984		2014
		2015	return nr_suspects;
		2016	}
		2017
		2018	/*
		2019	* Promote a suspected object to a reported leak once it has stayed
		2020	* unreferenced for min_unref_scans consecutive scans. Called with
		2021	* object->lock held; returns true when the object is newly reported.
		2022	*/
		2023	static bool confirm_leak(struct kmemleak_object *object)
		2024	{
		2025	if (!unreferenced_object(object) \|\|
		2026	!(object->flags & OBJECT_SUSPECT) \|\|
		2027	(object->flags & OBJECT_REPORTED))
		2028	return false;
		2029
		2030	object->unref_scans += 1;
		2031	if (object->unref_scans < min_unref_scans)
		2032	return false;
		2033
		2034	object->flags \|= OBJECT_REPORTED;
		2035	return true;
		2036	}
		2037
		2038	/*
		2039	* Scan the memory and report the unreferenced objects as leaks. Must be
		2040	* called with the scan_mutex held.
		2041	*/
		2042	static void kmemleak_scan(void)
		2043	{
		2044	struct kmemleak_object *object;
		2045	struct xarray dedup;
		2046	int new_leaks = 0;
		2047
		2048	/*
		2049	* Full scan. Objects left unreferenced are flagged OBJECT_SUSPECT and
		2050	* counted in the return value; nothing to confirm or report otherwise.
		2051	*/
		2052	if (!__kmemleak_scan(true))
		2053	return;
		2054
1985	/*	2055	/*
1986	* If scanning was stopped do not report any new unreferenced objects.	2056	* If scanning was stopped do not report any new unreferenced objects.
1987	*/	2057	*/
1988	if (scan_should_stop())	2058	if (scan_should_stop())
1989	return;	2059	return;
1990		2060
		2061	/*
		2062	* A live object whose only reference is moved by, for example, a
		2063	* concurrent RCU update can be missed for one scan and reported as a
		2064	* transient false positive. Scan again and only report the objects
		2065	* left unreferenced (still flagged OBJECT_SUSPECT) by both scans.
		2066	*/
		2067	__kmemleak_scan(false);
		2068	if (scan_should_stop())
		2069	return;
		2070
1991	/*	2071	/*
1992	* Scanning result reporting. When verbose printing is enabled, dedupe	2072	* Scanning result reporting. When verbose printing is enabled, dedupe
1993	* by stackdepot trace_handle so each unique backtrace is logged once	2073	* by stackdepot trace_handle so each unique backtrace is logged once
@@ -2014,9 +2094,8 @@ scan_gray: · 20 unmodified lines
2014	raw_spin_lock_irq(&object->lock);	2094	raw_spin_lock_irq(&object->lock);
2015	trace_handle = 0;	2095	trace_handle = 0;
2016	dedup_print = false;	2096	dedup_print = false;
2017	if (unreferenced_object(object) &&	2097
2018	!(object->flags & OBJECT_REPORTED)) {	2098	if (confirm_leak(object)) {
2019	object->flags \|= OBJECT_REPORTED;
2020	if (kmemleak_verbose) {	2099	if (kmemleak_verbose) {
2021	trace_handle = object->trace_handle;	2100	trace_handle = object->trace_handle;
2022	dedup_print = true;	2101	dedup_print = true;

mm/ksm.c+7−9

  * seconds or even minutes: much too unresponsive.  So instead we use a
  * "keyhole reference": access to the ksm page from the stable node peeps
  * out through its keyhole to see if that page still holds the right key,
- * pointing back to this stable node.  This relies on freeing a PageAnon
+ * pointing back to this stable node.  This relies on freeing an anon
- * page to reset its page->mapping to NULL, and relies on no other use of
+ * folio to reset its mapping to NULL, and relies on no other use of a
- * a page to put something that might look like our key in page->mapping.
+ * folio to put something that might look like our key in its mapping.
- * is on its way to being freed; but it is an anomaly to bear in mind.
  */
 static struct folio *ksm_get_folio(struct ksm_stable_node *stable_node,
 				 enum ksm_get_folio_flags flags)
 	 * stable_tree, break_cow() will clean it up.
 	 */
 	rmap_item->anon_vma = vma->anon_vma;
-	rmap_item->linear_page_index = linear_page_index(vma, rmap_item->address);
+	rmap_item->linear_page_index = linear_anon_page_index(vma, rmap_item->address);
 	get_anon_vma(vma->anon_vma);
 out:
 	mmap_read_unlock(mm);
 	slot = &mm_slot->slot;
+	spin_lock(&ksm_mmlist_lock);
 	/* Check ksm_run too?  Would need tighter locking */
 	needs_wakeup = list_empty(&ksm_mm_head.slot.mm_node);
-	spin_lock(&ksm_mmlist_lock);
 	mm_slot_insert(mm_slots_hash, mm, slot);
 	/*
 	 * When KSM_RUN_MERGE (or KSM_RUN_STOP),
 			return folio;	/* no need to copy it */
 	} else if (!anon_vma) {
 		return folio;		/* no need to copy it */
-	} else if (folio->index == linear_page_index(vma, addr) &&
+	} else if (folio->index == linear_anon_page_index(vma, addr) &&
 			anon_vma->root == vma->anon_vma->root) {
 		return folio;		/* still no need to copy it */
 	}
 		/*
 		 * Currently, KSM folios are always small folios, so it's
 		 * sufficient to search for a single page. We can simply use
-		 * the linear_page_index of the original de-duplicate
+		 * the linear_anon_page_index of the original de-duplicate
 		 * anonymous page that we remembered in the rmap_item while
 		 * de-duplicating. Note that mremap() always de-duplicates KSM
 		 * folios: so if there was mremap() in our parent or our child,

mm/madvise.c+9−8

 #include <linux/leafops.h>
 #include <linux/shmem_fs.h>
 #include <linux/mmu_notifier.h>
+#include <linux/swap_ops.h>
 #include <asm/tlb.h>
 		unsigned long end, struct mm_walk *walk)
 {
 	struct vm_area_struct *vma = walk->private;
-	struct swap_iocb *splug = NULL;
+	struct swap_io_ctx ctx = {};
 	pte_t *ptep = NULL;
 	spinlock_t *ptl;
 	unsigned long addr;
 		pte_unmap_unlock(ptep, ptl);
 		ptep = NULL;
-		folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
+		folio = read_swap_cache_async(&ctx, entry, GFP_HIGHUSER_MOVABLE,
-					     vma, addr, &splug);
+					vma, addr);
 		if (folio)
 			folio_put(folio);
 	}
 	if (ptep)
 		pte_unmap_unlock(ptep, ptl);
-	swap_read_unplug(splug);
+	swap_read_submit(&ctx);
 	cond_resched();
 	return 0;
 	XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
 	pgoff_t end_index = linear_page_index(vma, end) - 1;
 	struct folio *folio;
-	struct swap_iocb *splug = NULL;
+	struct swap_io_ctx ctx = {};
 	rcu_read_lock();
 	xas_for_each(&xas, folio, end_index) {
 		xas_pause(&xas);
 		rcu_read_unlock();
-		folio = read_swap_cache_async(entry, mapping_gfp_mask(mapping),
+		folio = read_swap_cache_async(&ctx, entry,
-					     vma, addr, &splug);
+				mapping_gfp_mask(mapping), vma, addr);
 		if (folio)
 			folio_put(folio);
 		rcu_read_lock();
 	}
 	rcu_read_unlock();
-	swap_read_unplug(splug);
+	swap_read_submit(&ctx);
 }
 #endif		/* CONFIG_SWAP */

mm/memcontrol-v1.c+2−2

@@ -2287,8 +2287,8 @@ void memcg1_stat_format(struct mem_cgroup memcg, struct seq_buf s) · 2286 unmodified lines
2287	for_each_online_pgdat(pgdat) {	2287	for_each_online_pgdat(pgdat) {
2288	mz = memcg->nodeinfo[pgdat->node_id];	2288	mz = memcg->nodeinfo[pgdat->node_id];
2289		2289
2290	anon_cost += mz->lruvec.anon_cost;	2290	anon_cost += mz->lruvec.cost[WORKINGSET_ANON].count;
2291	file_cost += mz->lruvec.file_cost;	2291	file_cost += mz->lruvec.cost[WORKINGSET_FILE].count;
2292	}	2292	}
2293	seq_buf_printf(s, "anon_cost %lu\n", anon_cost);	2293	seq_buf_printf(s, "anon_cost %lu\n", anon_cost);
2294	seq_buf_printf(s, "file_cost %lu\n", file_cost);	2294	seq_buf_printf(s, "file_cost %lu\n", file_cost);

mm/memcontrol-v1.h+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	#define __MM_MEMCONTROL_V1_H	4	#define __MM_MEMCONTROL_V1_H
5		5
6	#include <linux/cgroup-defs.h>	6	#include <linux/cgroup-defs.h>
		7	#include <linux/memcontrol.h>
7		8
8	/* Cgroup v1 and v2 common declarations */	9	/* Cgroup v1 and v2 common declarations */
9		10

mm/memcontrol.c+66−3

@@ -63,6 +63,7 @@ · 62 unmodified lines
63	#include <linux/sched/isolation.h>	63	#include <linux/sched/isolation.h>
64	#include <linux/kmemleak.h>	64	#include <linux/kmemleak.h>
65	#include "internal.h"	65	#include "internal.h"
		66	#include "swap.h"
66	#include "swap_table.h"	67	#include "swap_table.h"
67	#include <net/sock.h>	68	#include <net/sock.h>
68	#include <net/ip.h>	69	#include <net/ip.h>
@@ -398,6 +399,7 @@ static const unsigned int memcg_node_stat_items[] = { · 329 unmodified lines
398	NR_SHMEM_THPS,	399	NR_SHMEM_THPS,
399	NR_FILE_THPS,	400	NR_FILE_THPS,
400	NR_ANON_THPS,	401	NR_ANON_THPS,
		402	NR_VMSCAN_WRITE,
401	NR_VMALLOC,	403	NR_VMALLOC,
402	NR_KERNEL_STACK_KB,	404	NR_KERNEL_STACK_KB,
403	NR_PAGETABLE,	405	NR_PAGETABLE,
@@ -424,6 +426,8 @@ static const unsigned int memcg_node_stat_items[] = { · 20 unmodified lines
424	PGSCAN_PROACTIVE,	426	PGSCAN_PROACTIVE,
425	PGSCAN_ANON,	427	PGSCAN_ANON,
426	PGSCAN_FILE,	428	PGSCAN_FILE,
		429	PGROTATE_ANON,
		430	PGROTATE_FILE,
427	PGREFILL,	431	PGREFILL,
428	#ifdef CONFIG_HUGETLB_PAGE	432	#ifdef CONFIG_HUGETLB_PAGE
429	NR_HUGETLB,	433	NR_HUGETLB,
@@ -507,6 +511,42 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) · 77 unmodified lines
507	return x;	511	return x;
508	}	512	}
509		513
		514	/**
		515	* lruvec_page_state_monotonic - non-clamping lruvec stat read for delta sampling
		516	* @lruvec: the LRU vector to read from
		517	* @idx: the node_stat_item to read
		518	*
		519	* Returns the raw state[idx] value cast to unsigned long, skipping the
		520	* clamp-negative-to-zero step in lruvec_page_state(). Intended for callers
		521	* that snapshot a monotonically-incremented counter and subtract two
		522	* samples: unsigned modular arithmetic then yields the correct delta across
		523	* a signed-long wraparound (a real hazard on 32-bit) that the clamp would
		524	* otherwise turn into a huge spurious delta.
		525	*
		526	* Do NOT use for non-monotonic page-count reads where a transient negative
		527	* reading from per-CPU delta skew must present as zero.
		528	*
		529	* XXX: This helper (and its node/global peers) exists because some
		530	* monotonically-incremented event counters are stored in
		531	* enum node_stat_item.
		532	*/
		533	unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
		534	enum node_stat_item idx)
		535	{
		536	struct mem_cgroup_per_node *pn;
		537	int i;
		538
		539	if (mem_cgroup_disabled())
		540	return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);
		541
		542	i = memcg_stats_index(idx);
		543	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
		544	return 0;
		545
		546	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
		547	return (unsigned long)READ_ONCE(pn->lruvec_stats->state[i]);
		548	}
		549
510	unsigned long lruvec_page_state_local(struct lruvec *lruvec,	550	unsigned long lruvec_page_state_local(struct lruvec *lruvec,
511	enum node_stat_item idx)	551	enum node_stat_item idx)
512	{	552	{
@@ -2100,7 +2140,12 @@ static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages) · 1587 unmodified lines
2100		2140
2101	stock_pages = READ_ONCE(stock->nr_pages[i]);	2141	stock_pages = READ_ONCE(stock->nr_pages[i]);
2102	if (stock_pages >= nr_pages) {	2142	if (stock_pages >= nr_pages) {
2103	WRITE_ONCE(stock->nr_pages[i], stock_pages - nr_pages);	2143	stock_pages -= nr_pages;
		2144	WRITE_ONCE(stock->nr_pages[i], stock_pages);
		2145	if (!stock_pages) {
		2146	css_put(&memcg->css);
		2147	WRITE_ONCE(stock->cached[i], NULL);
		2148	}
2104	ret = true;	2149	ret = true;
2105	}	2150	}
2106	break;	2151	break;
@@ -2653,6 +2698,19 @@ retry: · 546 unmodified lines
2653	if (!gfpflags_allow_blocking(gfp_mask))	2698	if (!gfpflags_allow_blocking(gfp_mask))
2654	goto nomem;	2699	goto nomem;
2655		2700
		2701	/*
		2702	* OOM victim still needs to charge memory to exit. OOM reaper should
		2703	* help but it might fail on mmap_lock contention. If the victim is a
		2704	* large thread group then all exiting threads might compete on oom_lock
		2705	* just to learn that there is nothing really killable anymore. Bail
		2706	* out early and fail the charge to expedite their exit. They are
		2707	* considered fully reclaimed by the oom reaper and they shouldn't
		2708	* contribute further charges.
		2709	*/
		2710	if (tsk_is_oom_victim(current) &&
		2711	mm_flags_test(MMF_OOM_SKIP, current->signal->oom_mm))
		2712	goto nomem;
		2713
2656	__memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);	2714	__memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);
2657	raised_max_event = true;	2715	raised_max_event = true;
2658		2716
@@ -4177,11 +4235,10 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) · 1518 unmodified lines
4177	#endif	4235	#endif
4178	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);	4236	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
4179	if (parent) {	4237	if (parent) {
4180	WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
4181
4182	page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);	4238	page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);
4183	page_counter_init(&memcg->swap, &parent->swap, false);	4239	page_counter_init(&memcg->swap, &parent->swap, false);
4184	#ifdef CONFIG_MEMCG_V1	4240	#ifdef CONFIG_MEMCG_V1
		4241	WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
4185	memcg->memory.track_failcnt = !memcg_on_dfl;	4242	memcg->memory.track_failcnt = !memcg_on_dfl;
4186	memcg->memsw.track_failcnt = !memcg_on_dfl;	4243	memcg->memsw.track_failcnt = !memcg_on_dfl;
4187	WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));	4244	WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
@@ -4801,6 +4858,9 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 613 unmodified lines
4801	unsigned long nr_pages = page_counter_read(&memcg->memory);	4858	unsigned long nr_pages = page_counter_read(&memcg->memory);
4802	unsigned long reclaimed;	4859	unsigned long reclaimed;
4803		4860
		4861	if (high != READ_ONCE(memcg->memory.high))
		4862	break;
		4863
4804	if (nr_pages <= high)	4864	if (nr_pages <= high)
4805	break;	4865	break;
4806		4866
@@ -4856,6 +4916,9 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, · 49 unmodified lines
4856	for (;;) {	4916	for (;;) {
4857	unsigned long nr_pages = page_counter_read(&memcg->memory);	4917	unsigned long nr_pages = page_counter_read(&memcg->memory);
4858		4918
		4919	if (max != READ_ONCE(memcg->memory.max))
		4920	break;
		4921
4859	if (nr_pages <= max)	4922	if (nr_pages <= max)
4860	break;	4923	break;
4861		4924

mm/memory-failure.c+2−2

@@ -620,7 +620,7 @@ static void add_to_kill_fsdax(struct task_struct tsk, const struct page p, · 619 unmodified lines
620	struct vm_area_struct *vma,	620	struct vm_area_struct *vma,
621	struct list_head *to_kill, pgoff_t pgoff)	621	struct list_head *to_kill, pgoff_t pgoff)
622	{	622	{
623	unsigned long addr = vma_address(vma, pgoff, 1);	623	unsigned long addr = vma_filebacked_address(vma, pgoff, 1);
624	__add_to_kill(tsk, p, vma, to_kill, addr);	624	__add_to_kill(tsk, p, vma, to_kill, addr);
625	}	625	}
626		626
@@ -2265,7 +2265,7 @@ static void add_to_kill_pgoff(struct task_struct *tsk, · 1638 unmodified lines
2265	}	2265	}
2266		2266
2267	/* Check for pgoff not backed by struct page */	2267	/* Check for pgoff not backed by struct page */
2268	tk->addr = vma_address(vma, pgoff, 1);	2268	tk->addr = vma_filebacked_address(vma, pgoff, 1);
2269	tk->size_shift = PAGE_SHIFT;	2269	tk->size_shift = PAGE_SHIFT;
2270		2270
2271	if (tk->addr == -EFAULT)	2271	if (tk->addr == -EFAULT)

mm/memory.c+101−39

 	return false;
 }
+static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size)
+{
+	if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) {
+		snprintf(buf, buf_size, "overflow");
+		return;
+	}
+	switch (entry_size) {
+	case sizeof(u32):
+		snprintf(buf, buf_size, "%08x", *(const u32 *)entry);
+		break;
+	case sizeof(u64):
+		snprintf(buf, buf_size, "%016llx", *(const u64 *)entry);
+		break;
+#if defined(__SIZEOF_INT128__)
+	case sizeof(u128):
+		snprintf(buf, buf_size, "%016llx%016llx",
+			 (unsigned long long)(*(const u128 *)entry >> 64),
+			 (unsigned long long)*(const u128 *)entry);
+		break;
+#endif
+	default:
+		snprintf(buf, buf_size, "unsupported");
+		break;
+	}
+}
+#define ptval_to_str(buf, val)								\
+	do {										\
+		auto __val = (val);							\
+											\
+		ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val));	\
+	} while (0)
+#if defined(__SIZEOF_INT128__)
+#define PTVAL_STR_MAX	(32 + 1) /* Max 128-bit value in hex + NUL */
+#else
+#define PTVAL_STR_MAX	(16 + 1) /* Max 64-bit value in hex + NUL */
+#endif
 static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr)
 {
-	unsigned long long pgdv, p4dv, pudv, pmdv;
+	char pgd_str[PTVAL_STR_MAX];
+	char p4d_str[PTVAL_STR_MAX];
+	char pud_str[PTVAL_STR_MAX];
+	char pmd_str[PTVAL_STR_MAX];
 	p4d_t p4d, *p4dp;
 	pud_t pud, *pudp;
 	pmd_t pmd, *pmdp;
 	 * see locking requirements for print_bad_page_map().
 	 */
 	pgdp = pgd_offset(mm, addr);
-	pgdv = pgd_val(*pgdp);
+	ptval_to_str(pgd_str, pgd_val(*pgdp));
 	if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) {
-		pr_alert("pgd:%08llx\n", pgdv);
+		pr_alert("pgd:%s\n", pgd_str);
 		return;
 	}
 	p4dp = p4d_offset(pgdp, addr);
 	p4d = p4dp_get(p4dp);
-	p4dv = p4d_val(p4d);
+	ptval_to_str(p4d_str, p4d_val(p4d));
 	if (!p4d_present(p4d) || p4d_leaf(p4d)) {
-		pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv);
+		pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str);
 		return;
 	}
 	pudp = pud_offset(p4dp, addr);
 	pud = pudp_get(pudp);
-	pudv = pud_val(pud);
+	ptval_to_str(pud_str, pud_val(pud));
 	if (!pud_present(pud) || pud_leaf(pud)) {
-		pr_alert("pgd:%08llx p4d:%08llx pud:%08llx\n", pgdv, p4dv, pudv);
+		pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str);
 		return;
 	}
 	pmdp = pmd_offset(pudp, addr);
 	pmd = pmdp_get(pmdp);
-	pmdv = pmd_val(pmd);
+	ptval_to_str(pmd_str, pmd_val(pmd));
 	/*
 	 * Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE,
 	 * doing another map would be bad. print_bad_page_map() should
 	 * already take care of printing the PTE.
 	 */
-	pr_alert("pgd:%08llx p4d:%08llx pud:%08llx pmd:%08llx\n", pgdv,
+	pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str);
-		 p4dv, pudv, pmdv);
 }
 /*
  * page table lock.
  */
 static void print_bad_page_map(struct vm_area_struct *vma,
-		unsigned long addr, unsigned long long entry, struct page *page,
+		unsigned long addr, const void *entry, size_t entry_size,
-		enum pgtable_level level)
+		struct page *page, enum pgtable_level level)
 {
 	struct address_space *mapping;
-	pgoff_t index;
+	char entry_str[PTVAL_STR_MAX];
+	pgoff_t index, anon_index;
 	if (is_bad_page_map_ratelimited())
 		return;
 	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
 	index = linear_page_index(vma, addr);
+	anon_index = __linear_anon_page_index(vma, addr);
-	pr_alert("BUG: Bad page map in process %s  %s:%08llx", current->comm,
+	ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size);
-		 pgtable_level_to_str(level), entry);
+	pr_alert("BUG: Bad page map in process %s  %s:%s", current->comm,
+		 pgtable_level_to_str(level), entry_str);
 	__print_bad_page_map_pgtable(vma->vm_mm, addr);
 	if (page)
 		dump_page(page, "bad page map");
-	pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
+	pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px",
-		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
+		 (void *)addr, vma->vm_flags, vma->anon_vma, mapping);
+	if (!vma_is_cow_mapping(vma) || index == anon_index) {
+		pr_cont(" index:%lx\n", index);
+	} else {
+		pr_cont(" index:%lx (file) %lx (anon)\n", index, anon_index);
+	}
 	pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n",
 		 vma->vm_file,
 		 vma->vm_ops ? vma->vm_ops->fault : NULL,
 	}
 }
-#define print_bad_pte(vma, addr, pte, page) \
+static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
-	print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE)
+			  pte_t pte, struct page *page)
+{
+	auto entry = pte_val(pte);
+	print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE);
+}
 /**
  * __vm_normal_page() - Get the "struct page" associated with a page table entry.
  * @addr: The address where the page table entry is mapped.
  * @pfn: The PFN stored in the page table entry.
  * @special: Whether the page table entry is marked "special".
- * @level: The page table level for error reporting purposes only.
  * @entry: The page table entry value for error reporting purposes only.
+ * @entry_size: The size of @entry.
+ * @level: The page table level for error reporting purposes only.
  *
  * "Special" mappings do not wish to be associated with a "struct page" (either
  * it doesn't exist, or it exists but they don't want to touch it). In this
  */
 static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 		unsigned long addr, unsigned long pfn, bool special,
-		unsigned long long entry, enum pgtable_level level)
+		const void *entry, size_t entry_size, enum pgtable_level level)
 {
 	if (pgtable_level_has_pxx_special(level)) {
 		if (unlikely(special)) {
 			if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn))
 				return NULL;
-			print_bad_page_map(vma, addr, entry, NULL, level);
+			print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
 			return NULL;
 		}
 		/*
 				/* Only CoW'ed anon folios are "normal". */
 				if (pfn == index)
 					return NULL;
-				if (!is_cow_mapping(vma->vm_flags))
+				if (!vma_is_cow_mapping(vma))
 					return NULL;
 			}
 		}
 	if (unlikely(pfn > highest_memmap_pfn)) {
 		/* Corrupted page table entry. */
-		print_bad_page_map(vma, addr, entry, NULL, level);
+		print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
 		return NULL;
 	}
 	/*
 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
 			    pte_t pte)
 {
+	auto entry = pte_val(pte);
 	return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte),
-				pte_val(pte), PGTABLE_LEVEL_PTE);
+				&entry, sizeof(entry), PGTABLE_LEVEL_PTE);
 }
 /**
 struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 				pmd_t pmd)
 {
+	auto entry = pmd_val(pmd);
 	return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd),
-				pmd_val(pmd), PGTABLE_LEVEL_PMD);
+				&entry, sizeof(entry), PGTABLE_LEVEL_PMD);
 }
 /**
 struct page *vm_normal_page_pud(struct vm_area_struct *vma,
 		unsigned long addr, pud_t pud)
 {
+	auto entry = pud_val(pud);
 	return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud),
-				pud_val(pud), PGTABLE_LEVEL_PUD);
+				&entry, sizeof(entry), PGTABLE_LEVEL_PUD);
 }
 #endif
 		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
 		struct vm_area_struct *src_vma, unsigned long addr, int *rss)
 {
-	vm_flags_t vm_flags = dst_vma->vm_flags;
 	pte_t orig_pte = ptep_get(src_pte);
 	softleaf_t entry = softleaf_from_pte(orig_pte);
 	pte_t pte = orig_pte;
 		rss[mm_counter(folio)]++;
 		if (!softleaf_is_migration_read(entry) &&
-				is_cow_mapping(vm_flags)) {
+				vma_is_cow_mapping(dst_vma)) {
 			/*
 			 * COW mappings require pages in both parent and child
 			 * to be set to read. A previously exclusive entry is
 		 * save and restore device driver state).
 		 */
 		if (softleaf_is_device_private_write(entry) &&
-		    is_cow_mapping(vm_flags)) {
+		    vma_is_cow_mapping(dst_vma)) {
 			entry = make_readable_device_private_entry(
 							swp_offset(entry));
 			pte = swp_entry_to_pte(entry);
 		 * exclusive entries currently only support private writable
 		 * (ie. COW) mappings.
 		 */
-		VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
+		VM_BUG_ON(!vma_is_cow_mapping(src_vma));
 		if (try_restore_exclusive_pte(src_vma, addr, src_pte, orig_pte))
 			return -EBUSY;
 		return -ENOENT;
 	}
 	/* If it's a COW mapping, write protect it both processes. */
-	if (is_cow_mapping(src_vma->vm_flags) && writable) {
+	if (vma_is_cow_mapping(src_vma) && writable) {
 		wrprotect_ptes(src_mm, addr, src_pte, nr);
 		pte = pte_wrprotect(pte);
 	}
 	 * We need to invalidate the secondary MMU mappings only when
 	 * there could be a permission downgrade on the ptes of the
 	 * parent mm. And a permission downgrade will only happen if
-	 * is_cow_mapping() returns true.
+	 * vma_is_cow_mapping() returns true.
 	 */
-	is_cow = is_cow_mapping(src_vma->vm_flags);
+	is_cow = vma_is_cow_mapping(src_vma);
 	if (is_cow) {
 		mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
 	if (mm_forbids_zeropage(vma->vm_mm))
 		return false;
 	/* zeropages in COW mappings are common and unproblematic. */
-	if (is_cow_mapping(vma->vm_flags))
+	if (vma_is_cow_mapping(vma))
 		return true;
 	/* Mappings that do not allow for writable PTEs are unproblematic. */
 	if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)))
 	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
 	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
 						(VM_PFNMAP|VM_MIXEDMAP));
-	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
+	BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma));
 	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
 	if (addr < vma->vm_start || addr >= vma->vm_end)
 				       unsigned long size)
 {
 	const unsigned long end = addr + PAGE_ALIGN(size);
-	const bool is_cow = is_cow_mapping(vma->vm_flags);
+	const bool is_cow = vma_is_cow_mapping(vma);
 	int err;
 	err = get_remap_pgoff(is_cow, addr, end, vma->vm_start, vma->vm_end,
 		 * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
 		 * just treat it like an ordinary read-fault otherwise.
 		 */
-		if (!is_cow_mapping(vma->vm_flags))
+		if (!vma_is_cow_mapping(vma))
 			*flags &= ~FAULT_FLAG_UNSHARE;
 	} else if (*flags & FAULT_FLAG_WRITE) {
 		/* Write faults on read-only mappings are impossible ... */
 			return VM_FAULT_SIGSEGV;
 		/* ... and FOLL_FORCE only applies to COW mappings. */
 		if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
-				 !is_cow_mapping(vma->vm_flags)))
+				 !vma_is_cow_mapping(vma)))
 			return VM_FAULT_SIGSEGV;
 	}
 #ifdef CONFIG_PER_VMA_LOCK

mm/mempolicy.c+7−16

@@ -844,7 +844,7 @@ bool folio_can_map_prot_numa(struct folio folio, struct vm_area_struct vma, · 843 unmodified lines
844	return false;	844	return false;
845		845
846	/* Also skip shared copy-on-write folios */	846	/* Also skip shared copy-on-write folios */
847	if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio))	847	if (vma_is_cow_mapping(vma) && folio_maybe_mapped_shared(folio))
848	return false;	848	return false;
849		849
850	/* Folios are pinned and can't be migrated */	850	/* Folios are pinned and can't be migrated */
@@ -2060,24 +2060,15 @@ struct mempolicy get_vma_policy(struct vm_area_struct vma, · 1209 unmodified lines
2060	bool vma_policy_mof(struct vm_area_struct *vma)	2060	bool vma_policy_mof(struct vm_area_struct *vma)
2061	{	2061	{
2062	struct mempolicy *pol;	2062	struct mempolicy *pol;
		2063	pgoff_t ilx;
		2064	bool mof;
2063		2065
2064	if (vma->vm_ops && vma->vm_ops->get_policy) {	2066	pol = __get_vma_policy(vma, vma->vm_start, &ilx);
2065	bool ret = false;
2066	pgoff_t ilx; /* ignored here */
2067
2068	pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx);
2069	if (pol && (pol->flags & MPOL_F_MOF))
2070	ret = true;
2071	mpol_cond_put(pol);
2072
2073	return ret;
2074	}
2075
2076	pol = vma->vm_policy;
2077	if (!pol)	2067	if (!pol)
2078	pol = get_task_policy(current);	2068	pol = get_task_policy(current);
2079		2069	mof = pol->flags & MPOL_F_MOF;
2080	return pol->flags & MPOL_F_MOF;	2070	mpol_cond_put(pol);
		2071	return mof;
2081	}	2072	}
2082		2073
2083	bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)	2074	bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)

mm/migrate.c+10−11

@@ -356,23 +356,18 @@ static bool remove_migration_pte(struct folio *folio, · 355 unmodified lines
356		356
357	while (page_vma_mapped_walk(&pvmw)) {	357	while (page_vma_mapped_walk(&pvmw)) {
358	rmap_t rmap_flags = RMAP_NONE;	358	rmap_t rmap_flags = RMAP_NONE;
359	pte_t old_pte;	359	unsigned long idx = 0;
360	pte_t pte;
361	softleaf_t entry;	360	softleaf_t entry;
362	struct page *new;	361	struct page *new;
363	unsigned long idx = 0;	362	pte_t old_pte;
364		363	pte_t pte;
365	/* pgoff is invalid for ksm pages, but they are never large */
366	if (folio_test_large(folio) && !folio_test_hugetlb(folio))
367	idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
368	new = folio_page(folio, idx);
369		364
370	#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES	365	#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
371	/* PMD-mapped THP migration entry */	366	/* PMD-mapped THP migration entry */
372	if (!pvmw.pte) {	367	if (!pvmw.pte) {
373	VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) \|\|	368	VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) \|\|
374	!folio_test_pmd_mappable(folio), folio);	369	!folio_test_pmd_mappable(folio), folio);
375	remove_migration_pmd(&pvmw, new);	370	remove_migration_pmd(&pvmw, folio);
376	continue;	371	continue;
377	}	372	}
378	#endif	373	#endif
@@ -381,14 +376,18 @@ static bool remove_migration_pte(struct folio *folio, · 2 unmodified lines
381	pvmw.pte);	376	pvmw.pte);
382	else	377	else
383	old_pte = ptep_get(pvmw.pte);	378	old_pte = ptep_get(pvmw.pte);
		379
		380	entry = softleaf_from_pte(old_pte);
		381	if (folio_test_large(folio) && !folio_test_hugetlb(folio))
		382	idx = softleaf_to_pfn(entry) - pvmw.pfn;
		383
384	if (rmap_walk_arg->map_unused_to_zeropage &&	384	if (rmap_walk_arg->map_unused_to_zeropage &&
385	try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))	385	try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx))
386	continue;	386	continue;
387		387
388	folio_get(folio);	388	folio_get(folio);
		389	new = folio_page(folio, idx);
389	pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));	390	pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
390
391	entry = softleaf_from_pte(old_pte);
392	if (!softleaf_is_migration_young(entry))	391	if (!softleaf_is_migration_young(entry))
393	pte = pte_mkold(pte);	392	pte = pte_mkold(pte);
394	if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry))	393	if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry))

mm/migrate_device.c+14−1

 	if (flush) {
 		pte_free(vma->vm_mm, pgtable);
-		flush_cache_page(vma, addr, addr + HPAGE_PMD_SIZE);
+		flush_cache_range(vma, addr, addr + HPAGE_PMD_SIZE);
 		pmdp_invalidate(vma, addr, pmdp);
 	} else {
 		pgtable_trans_huge_deposit(vma->vm_mm, pmdp, pgtable);
 							 MIGRATE_PFN_COMPOUND);
 					goto next;
 				}
+				/*
+				 * reset nr so that only first after-split folio
+				 * is processed below
+				 */
+				VM_WARN_ON_ONCE(folio_test_large(folio));
+				nr = 1;
 			} else if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) &&
 				(dst_pfns[i] & MIGRATE_PFN_COMPOUND) &&
 				!(src_pfns[i] & MIGRATE_PFN_COMPOUND)) {
 			folio = page_folio(migrate_pfn_to_page(src_pfns[i+j]));
 			newfolio = page_folio(migrate_pfn_to_page(dst_pfns[i+j]));
+			/*
+			 * folio_free_swap() removed the folio from the swap
+			 * cache. Refresh the saved mapping before migration.
+			 */
+			mapping = folio_mapping(folio);
 			r = folio_migrate_mapping(mapping, newfolio, folio, extra_cnt);
 			if (r)
 				src_pfns[i+j] &= ~MIGRATE_PFN_MIGRATE;

mm/mm_init.c+0−1

@@ -1339,7 +1339,6 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) · 1338 unmodified lines
1339	for (i = 0; i < NR_VMSCAN_THROTTLE; i++)	1339	for (i = 0; i < NR_VMSCAN_THROTTLE; i++)
1340	init_waitqueue_head(&pgdat->reclaim_wait[i]);	1340	init_waitqueue_head(&pgdat->reclaim_wait[i]);
1341		1341
1342	pgdat_page_ext_init(pgdat);
1343	lruvec_init(&pgdat->__lruvec);	1342	lruvec_init(&pgdat->__lruvec);
1344	}	1343	}
1345		1344

mm/mmzone.c+1−0

@@ -79,6 +79,7 @@ void lruvec_init(struct lruvec *lruvec) · 78 unmodified lines
79		79
80	memset(lruvec, 0, sizeof(struct lruvec));	80	memset(lruvec, 0, sizeof(struct lruvec));
81	spin_lock_init(&lruvec->lru_lock);	81	spin_lock_init(&lruvec->lru_lock);
		82	spin_lock_init(&lruvec->cost_lock);
82	zswap_lruvec_state_init(lruvec);	83	zswap_lruvec_state_init(lruvec);
83		84
84	for_each_lru(lru)	85	for_each_lru(lru)

mm/mremap.c+5−3

 	for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
 		new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
-		VM_WARN_ON_ONCE(!pte_none(*new_ptep));
+		VM_WARN_ON_ONCE(!pte_none(ptep_get(new_ptep)));
 		nr_ptes = 1;
 		max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
 static int copy_vma_and_data(struct vma_remap_struct *vrm,
 			     struct vm_area_struct **new_vma_ptr)
 {
-	const unsigned long new_pgoff = linear_page_index(vrm->vma, vrm->addr);
+	const pgoff_t new_pgoff = linear_page_index(vrm->vma, vrm->addr);
+	const pgoff_t new_anon_pgoff =
+		__linear_anon_page_index(vrm->vma, vrm->addr);
 	struct vm_area_struct *vma = vrm->vma;
 	struct vm_area_struct *new_vma;
 	unsigned long moved_len;
 	PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len);
 	new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff,
-			   &pmc.need_rmap_locks);
+			   new_anon_pgoff, &pmc.need_rmap_locks);
 	if (!new_vma) {
 		vrm_uncharge(vrm);
 		*new_vma_ptr = NULL;

mm/nommu.c+4−0

 	setup_vma_to_mm(vma, mm);
 	setup_vma_to_mm(new, mm);
 	vma_iter_store_new(vmi, new);
+	/* vmi should point lower address */
+	if (new_below)
+		vma_next(vmi);
 	mm->map_count++;
 	return 0;

mm/page-writeback.c+5−0

  * in this folio.  Truncation will block on the page table lock as it
  * unmaps pages before removing the folio from its mapping.
  *
+ * .. DANGER::
+ *    Do not use this on a folio obtained from a function like
+ *    get_user_pages_fast() without holding appropriate locks; you might want to
+ *    use set_page_dirty_lock() or folio_mark_dirty_lock() instead.
+ *
  * Return: True if the folio was newly dirtied, false if it was already dirty.
  */
 bool folio_mark_dirty(struct folio *folio)

mm/page_alloc.c+20−8

 /*
  * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
- *	proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
+ *	proc_dointvec_minmax() so that we can call
- *	whenever sysctl_lowmem_reserve_ratio changes.
+ *	setup_per_zone_lowmem_reserve() when the sysctl is written.
  *
  * The reserve ratio obviously has absolutely no relation with the
  * minimum watermarks. The lowmem reserve ratio can only make sense
 static int lowmem_reserve_ratio_sysctl_handler(const struct ctl_table *table,
 		int write, void *buffer, size_t *length, loff_t *ppos)
 {
-	int i;
+	struct ctl_table tmp = *table;
+	int ratio[ARRAY_SIZE(sysctl_lowmem_reserve_ratio)];
+	int rc;
-	proc_dointvec_minmax(table, write, buffer, length, ppos);
+	if (!write)
+		return proc_dointvec_minmax(table, write, buffer, length, ppos);
-	for (i = 0; i < MAX_NR_ZONES; i++) {
+	/*
-		if (sysctl_lowmem_reserve_ratio[i] < 1)
+	 * proc_dointvec_max() works incrementally. Use a buffer and only set
-			sysctl_lowmem_reserve_ratio[i] = 0;
+	 * the values if all of them parse cleanly.
-	}
+	 */
+	memcpy(ratio, sysctl_lowmem_reserve_ratio, sizeof(ratio));
+	tmp.data = ratio;
+	rc = proc_dointvec_minmax(&tmp, write, buffer, length, ppos);
+	if (rc)
+		return rc;
+	memcpy(sysctl_lowmem_reserve_ratio, ratio, sizeof(ratio));
 	setup_per_zone_lowmem_reserve();
 	return 0;
 }
 		.maxlen		= sizeof(sysctl_lowmem_reserve_ratio),
 		.mode		= 0644,
 		.proc_handler	= lowmem_reserve_ratio_sysctl_handler,
+		.extra1		= SYSCTL_ZERO,
 	},
 #ifdef CONFIG_NUMA
 	{

mm/page_ext.c+0−9

 	invoke_init_callbacks();
 }
-void __meminit pgdat_page_ext_init(struct pglist_data *pgdat)
-{
-	pgdat->node_page_ext = NULL;
-}
 static struct page_ext *lookup_page_ext(const struct page *page)
 {
 	unsigned long pfn = page_to_pfn(page);
 	panic("Out of memory");
 }
-void __meminit pgdat_page_ext_init(struct pglist_data *pgdat)
-{
-}
 #endif
 /**

mm/page_io.c+281−300

@@ -25,57 +25,10 @@ · 24 unmodified lines
25	#include <linux/sched/task.h>	25	#include <linux/sched/task.h>
26	#include <linux/delayacct.h>	26	#include <linux/delayacct.h>
27	#include <linux/zswap.h>	27	#include <linux/zswap.h>
		28	#include <linux/swap_ops.h>
28	#include "swap.h"	29	#include "swap.h"
29	#include "swap_table.h"	30	#include "swap_table.h"
30		31
31	static void __end_swap_bio_write(struct bio *bio)
32	{
33	struct folio *folio = bio_first_folio_all(bio);
34
35	if (bio->bi_status) {
36	/*
37	* We failed to write the page out to swap-space.
38	* Re-dirty the page in order to avoid it being reclaimed.
39	* Also print a dire warning that things will go BAD (tm)
40	* very quickly.
41	*
42	* Also clear PG_reclaim to avoid folio_rotate_reclaimable()
43	*/
44	folio_mark_dirty(folio);
45	pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
46	MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
47	(unsigned long long)bio->bi_iter.bi_sector);
48	folio_clear_reclaim(folio);
49	}
50	folio_end_writeback(folio);
51	}
52
53	static void end_swap_bio_write(struct bio *bio)
54	{
55	__end_swap_bio_write(bio);
56	bio_put(bio);
57	}
58
59	static void __end_swap_bio_read(struct bio *bio)
60	{
61	struct folio *folio = bio_first_folio_all(bio);
62
63	if (bio->bi_status) {
64	pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
65	MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
66	(unsigned long long)bio->bi_iter.bi_sector);
67	} else {
68	folio_mark_uptodate(folio);
69	}
70	folio_unlock(folio);
71	}
72
73	static void end_swap_bio_read(struct bio *bio)
74	{
75	__end_swap_bio_read(bio);
76	bio_put(bio);
77	}
78
79	int generic_swapfile_activate(struct swap_info_struct *sis,	32	int generic_swapfile_activate(struct swap_info_struct *sis,
80	struct file *swap_file,	33	struct file *swap_file,
81	sector_t *span)	34	sector_t *span)
@@ -248,7 +201,7 @@ static void swap_zeromap_folio_clear(struct folio *folio) · 166 unmodified lines
248	* We may have stale swap cache pages in memory: notice	201	* We may have stale swap cache pages in memory: notice
249	* them here and get rid of the unnecessary final write.	202	* them here and get rid of the unnecessary final write.
250	*/	203	*/
251	int swap_writeout(struct folio folio, struct swap_iocb *swap_plug)	204	int swap_writeout(struct swap_io_ctx ctx, struct folio folio)
252	{	205	{
253	int ret = 0;	206	int ret = 0;
254		207
@@ -295,39 +248,44 @@ int swap_writeout(struct folio folio, struct swap_iocb *swap_plug) · 40 unmodified lines
295	}	248	}
296	rcu_read_unlock();	249	rcu_read_unlock();
297		250
298	__swap_writepage(folio, swap_plug);	251	__swap_writepage(ctx, folio);
299	return 0;	252	return 0;
300	out_unlock:	253	out_unlock:
301	folio_unlock(folio);	254	folio_unlock(folio);
302	return ret;	255	return ret;
303	}	256	}
304		257
305	static inline void count_swpout_vm_event(struct folio *folio)	258	#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
		259	static struct cgroup_subsys_state folio_memcg_blkg_css(struct folio folio)
306	{	260	{
307	#ifdef CONFIG_TRANSPARENT_HUGEPAGE	261	return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys);
308	if (unlikely(folio_test_pmd_mappable(folio))) {	262	}
309	count_memcg_folio_events(folio, THP_SWPOUT, 1);	263
310	count_vm_event(THP_SWPOUT);	264	static bool folio_blkg_can_merge(struct folio folio, struct folio prev_folio)
		265	{
		266	bool can_merge = true;
		267
		268	if (folio_memcg_charged(folio) != folio_memcg_charged(prev_folio))
		269	return false;
		270	if (folio_memcg_charged(folio)) {
		271	rcu_read_lock();
		272	if (folio_memcg_blkg_css(folio) !=
		273	folio_memcg_blkg_css(prev_folio))
		274	can_merge = false;
		275	rcu_read_unlock();
311	}	276	}
312	#endif	277	return can_merge;
313	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT);
314	count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio));
315	count_vm_events(PSWPOUT, folio_nr_pages(folio));
316	}	278	}
317		279
318	#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
319	static void bio_associate_blkg_from_page(struct bio bio, struct folio folio)	280	static void bio_associate_blkg_from_page(struct bio bio, struct folio folio)
320	{	281	{
321	struct cgroup_subsys_state *css;	282	struct cgroup_subsys_state *css;
322	struct mem_cgroup *memcg;
323		283
324	if (!folio_memcg_charged(folio))	284	if (!folio_memcg_charged(folio))
325	return;	285	return;
326
327	rcu_read_lock();	286	rcu_read_lock();
328	memcg = folio_memcg(folio);	287	css = folio_memcg_blkg_css(folio);
329	css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys);	288	if (css && !css_tryget(css))
330	if (!css \|\| !css_tryget(css))
331	css = NULL;	289	css = NULL;
332	rcu_read_unlock();	290	rcu_read_unlock();
333		291
@@ -336,15 +294,13 @@ static void bio_associate_blkg_from_page(struct bio bio, struct folio folio) · 2 unmodified lines
336	css_put(css);	294	css_put(css);
337	}	295	}
338	#else	296	#else
		297	static bool folio_blkg_can_merge(struct folio folio, struct folio prev_folio)
		298	{
		299	return true;
		300	}
339	#define bio_associate_blkg_from_page(bio, folio) do { } while (0)	301	#define bio_associate_blkg_from_page(bio, folio) do { } while (0)
340	#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */	302	#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
341		303
342	struct swap_iocb {
343	struct kiocb iocb;
344	struct bio_vec bvecs[SWAP_CLUSTER_MAX];
345	int nr_bvecs;
346	int len;
347	};
348	static mempool_t *sio_pool;	304	static mempool_t *sio_pool;
349		305
350	int sio_pool_init(void)	306	int sio_pool_init(void)
@@ -360,168 +316,74 @@ int sio_pool_init(void) · 9 unmodified lines
360	return 0;	316	return 0;
361	}	317	}
362		318
363	static void sio_write_complete(struct kiocb *iocb, long ret)	319	static bool swap_can_merge(struct swap_io_ctx ctx, struct folio folio,
		320	int rw)
364	{	321	{
365	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);	322	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
366	struct page *page = sio->bvecs[0].bv_page;	323	struct bio_vec *last_bv = &ctx->sio->bvecs[ctx->sio->nr_bvecs - 1];
367	int p;	324	struct folio *prev_folio = bvec_folio(last_bv);
368		325	size_t prev_folio_size = folio_size(prev_folio);
369	if (ret != sio->len) {
370	/*
371	* In the case of swap-over-nfs, this can be a
372	* temporary failure if the system has limited
373	* memory for allocating transmit buffers.
374	* Mark the page dirty and avoid
375	* folio_rotate_reclaimable but rate-limit the
376	* messages.
377	*/
378	pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
379	ret, swap_dev_pos(page_swap_entry(page)));
380	for (p = 0; p < sio->nr_bvecs; p++) {
381	page = sio->bvecs[p].bv_page;
382	set_page_dirty(page);
383	ClearPageReclaim(page);
384	}
385	}
386
387	for (p = 0; p < sio->nr_bvecs; p++)
388	end_page_writeback(sio->bvecs[p].bv_page);
389		326
390	mempool_free(sio, sio_pool);	327	if (ctx->sis != sis)
		328	return false;
		329	return sis->ops->can_merge(folio, prev_folio, prev_folio_size, rw);
391	}	330	}
392		331
393	static void swap_writepage_fs(struct folio folio, struct swap_iocb *swap_plug)	332	static void swap_add_folio(struct swap_io_ctx ctx, struct folio folio, int rw)
394	{	333	{
395	struct swap_iocb sio = swap_plug ? swap_plug : NULL;
396	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);	334	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
397	struct file *swap_file = sis->swap_file;	335	struct swap_iocb *sio = ctx->sio;
398	loff_t pos = swap_dev_pos(folio->swap);	336
399		337	if (sio && !swap_can_merge(ctx, folio, rw)) {
400	count_swpout_vm_event(folio);	338	if (rw == WRITE)
401	folio_start_writeback(folio);	339	swap_write_submit(ctx);
402	folio_unlock(folio);	340	else
403	if (sio) {	341	swap_read_submit(ctx);
404	if (sio->iocb.ki_filp != swap_file \|\|	342	sio = ctx->sio;
405	sio->iocb.ki_pos + sio->len != pos) {
406	swap_write_unplug(sio);
407	sio = NULL;
408	}
409	}	343	}
		344
410	if (!sio) {	345	if (!sio) {
411	sio = mempool_alloc(sio_pool, GFP_NOIO);	346	ctx->sis = sis;
412	init_sync_kiocb(&sio->iocb, swap_file);	347	ctx->sio = sio = mempool_alloc(sio_pool, GFP_NOIO);
413	sio->iocb.ki_complete = sio_write_complete;
414	sio->iocb.ki_pos = pos;
415	sio->nr_bvecs = 0;	348	sio->nr_bvecs = 0;
416	sio->len = 0;	349	sio->len = 0;
417	}	350	}
418	bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);	351	bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
419	sio->len += folio_size(folio);	352	sio->len += folio_size(folio);
420	sio->nr_bvecs += 1;
421	if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) \|\| !swap_plug) {
422	swap_write_unplug(sio);
423	sio = NULL;
424	}
425	if (swap_plug)
426	*swap_plug = sio;
427	}
428
429	static void swap_writepage_bdev_sync(struct folio *folio,
430	struct swap_info_struct *sis)
431	{
432	struct bio_vec bv;
433	struct bio bio;
434
435	bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_WRITE \| REQ_SWAP);
436	bio.bi_iter.bi_sector = swap_folio_sector(folio);
437	bio_add_folio_nofail(&bio, folio, folio_size(folio), 0);
438
439	bio_associate_blkg_from_page(&bio, folio);
440	count_swpout_vm_event(folio);
441
442	folio_start_writeback(folio);
443	folio_unlock(folio);
444		353
445	submit_bio_wait(&bio);
446	__end_swap_bio_write(&bio);
447	}
448
449	static void swap_writepage_bdev_async(struct folio *folio,
450	struct swap_info_struct *sis)
451	{
452	struct bio *bio;
453
454	bio = bio_alloc(sis->bdev, 1, REQ_OP_WRITE \| REQ_SWAP, GFP_NOIO);
455	bio->bi_iter.bi_sector = swap_folio_sector(folio);
456	bio->bi_end_io = end_swap_bio_write;
457	bio_add_folio_nofail(bio, folio, folio_size(folio), 0);
458
459	bio_associate_blkg_from_page(bio, folio);
460	count_swpout_vm_event(folio);
461	folio_start_writeback(folio);
462	folio_unlock(folio);
463	submit_bio(bio);
464	}
465
466	void __swap_writepage(struct folio folio, struct swap_iocb *swap_plug)
467	{
468	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
469
470	VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
471	/*
472	* ->flags can be updated non-atomically,
473	* but that will never affect SWP_FS_OPS, so the data_race
474	* is safe.
475	*/
476	if (data_race(sis->flags & SWP_FS_OPS))
477	swap_writepage_fs(folio, swap_plug);
478	/*	354	/*
479	* ->flags can be updated non-atomically,	355	* Write out the iocb if we filled it, or if the device is synchronous.
480	* but that will never affect SWP_SYNCHRONOUS_IO, so the data_race	356	*
481	* is safe.	357	* The latter is to work around expectations in the classic LRU code
		358	* which make synchronous clearing of the folio writeback flag in the
		359	* reclaim path beneficial.
482	*/	360	*/
483	else if (data_race(sis->flags & SWP_SYNCHRONOUS_IO))	361	if (++sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) \|\|
484	swap_writepage_bdev_sync(folio, sis);	362	(rw == WRITE && (sis->flags & SWP_SYNCHRONOUS_IO))) {
485	else	363	if (rw == WRITE)
486	swap_writepage_bdev_async(folio, sis);	364	swap_write_submit(ctx);
487	}	365	else
488		366	swap_read_submit(ctx);
489	void swap_write_unplug(struct swap_iocb *sio)	367	}
490	{
491	struct iov_iter from;
492	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
493	int ret;
494
495	iov_iter_bvec(&from, ITER_SOURCE, sio->bvecs, sio->nr_bvecs, sio->len);
496	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
497	if (ret != -EIOCBQUEUED)
498	sio_write_complete(&sio->iocb, ret);
499	}	368	}
500		369
501	static void sio_read_complete(struct kiocb *iocb, long ret)	370	void __swap_writepage(struct swap_io_ctx ctx, struct folio folio)
502	{	371	{
503	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);	372	VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
504	int p;
505
506	if (ret == sio->len) {
507	for (p = 0; p < sio->nr_bvecs; p++) {
508	struct folio *folio = bvec_folio(&sio->bvecs[p]);
509
510	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
511	count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
512	folio_mark_uptodate(folio);
513	folio_unlock(folio);
514	}
515	count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
516	} else {
517	for (p = 0; p < sio->nr_bvecs; p++) {
518	struct folio *folio = bvec_folio(&sio->bvecs[p]);
519		373
520	folio_unlock(folio);	374	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
521	}	375	if (unlikely(folio_test_pmd_mappable(folio))) {
522	pr_alert_ratelimited("Read-error on swap-device\n");	376	count_memcg_folio_events(folio, THP_SWPOUT, 1);
		377	count_vm_event(THP_SWPOUT);
523	}	378	}
524	mempool_free(sio, sio_pool);	379	#endif
		380	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPOUT);
		381	count_memcg_folio_events(folio, PSWPOUT, folio_nr_pages(folio));
		382	count_vm_events(PSWPOUT, folio_nr_pages(folio));
		383
		384	folio_start_writeback(folio);
		385	folio_unlock(folio);
		386	swap_add_folio(ctx, folio, WRITE);
525	}	387	}
526		388
527	/*	389	/*
@@ -587,78 +449,7 @@ static bool swap_read_folio_zeromap(struct folio *folio) · 59 unmodified lines
587	return true;	449	return true;
588	}	450	}
589		451
590	static void swap_read_folio_fs(struct folio folio, struct swap_iocb *plug)	452	void swap_read_folio(struct swap_io_ctx ctx, struct folio folio)
591	{
592	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
593	struct swap_iocb *sio = NULL;
594	loff_t pos = swap_dev_pos(folio->swap);
595
596	if (plug)
597	sio = *plug;
598	if (sio) {
599	if (sio->iocb.ki_filp != sis->swap_file \|\|
600	sio->iocb.ki_pos + sio->len != pos) {
601	swap_read_unplug(sio);
602	sio = NULL;
603	}
604	}
605	if (!sio) {
606	sio = mempool_alloc(sio_pool, GFP_KERNEL);
607	init_sync_kiocb(&sio->iocb, sis->swap_file);
608	sio->iocb.ki_pos = pos;
609	sio->iocb.ki_complete = sio_read_complete;
610	sio->nr_bvecs = 0;
611	sio->len = 0;
612	}
613	bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
614	sio->len += folio_size(folio);
615	sio->nr_bvecs += 1;
616	if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) \|\| !plug) {
617	swap_read_unplug(sio);
618	sio = NULL;
619	}
620	if (plug)
621	*plug = sio;
622	}
623
624	static void swap_read_folio_bdev_sync(struct folio *folio,
625	struct swap_info_struct *sis)
626	{
627	struct bio_vec bv;
628	struct bio bio;
629
630	bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ);
631	bio.bi_iter.bi_sector = swap_folio_sector(folio);
632	bio_add_folio_nofail(&bio, folio, folio_size(folio), 0);
633	/*
634	* Keep this task valid during swap readpage because the oom killer may
635	* attempt to access it in the page fault retry time check.
636	*/
637	get_task_struct(current);
638	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
639	count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
640	count_vm_events(PSWPIN, folio_nr_pages(folio));
641	submit_bio_wait(&bio);
642	__end_swap_bio_read(&bio);
643	put_task_struct(current);
644	}
645
646	static void swap_read_folio_bdev_async(struct folio *folio,
647	struct swap_info_struct *sis)
648	{
649	struct bio *bio;
650
651	bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL);
652	bio->bi_iter.bi_sector = swap_folio_sector(folio);
653	bio->bi_end_io = end_swap_bio_read;
654	bio_add_folio_nofail(bio, folio, folio_size(folio), 0);
655	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
656	count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
657	count_vm_events(PSWPIN, folio_nr_pages(folio));
658	submit_bio(bio);
659	}
660
661	void swap_read_folio(struct folio folio, struct swap_iocb *plug)
662	{	453	{
663	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);	454	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
664	bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;	455	bool synchronous = sis->flags & SWP_SYNCHRONOUS_IO;
@@ -691,14 +482,7 @@ void swap_read_folio(struct folio folio, struct swap_iocb *plug) · 26 unmodified lines
691		482
692	/* We have to read from slower devices. Increase zswap protection. */	483	/* We have to read from slower devices. Increase zswap protection. */
693	zswap_folio_swapin(folio);	484	zswap_folio_swapin(folio);
694		485	swap_add_folio(ctx, folio, READ);
695	if (data_race(sis->flags & SWP_FS_OPS)) {
696	swap_read_folio_fs(folio, plug);
697	} else if (synchronous) {
698	swap_read_folio_bdev_sync(folio, sis);
699	} else {
700	swap_read_folio_bdev_async(folio, sis);
701	}
702		486
703	finish:	487	finish:
704	if (workingset) {	488	if (workingset) {
@@ -708,14 +492,211 @@ finish: · 3 unmodified lines
708	delayacct_swapin_end();	492	delayacct_swapin_end();
709	}	493	}
710		494
711	void __swap_read_unplug(struct swap_iocb *sio)	495	static void swap_write_end(struct swap_iocb *sio, bool failed)
712	{	496	{
713	struct iov_iter from;	497	int p;
714	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;	498
715	int ret;	499	for (p = 0; p < sio->nr_bvecs; p++) {
		500	struct page *page = sio->bvecs[p].bv_page;
		501
		502	if (failed) {
		503	set_page_dirty(page);
		504	ClearPageReclaim(page);
		505	}
		506	end_page_writeback(page);
		507	}
		508	mempool_free(sio, sio_pool);
		509	}
		510
		511	static void swap_fs_write_complete(struct kiocb *iocb, long ret)
		512	{
		513	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
		514	bool failed = ret != sio->len;
		515
		516	if (failed) {
		517	struct page *page = sio->bvecs[0].bv_page;
		518
		519	/*
		520	* In the case of swap-over-nfs, this can be a temporary failure
		521	* if the system has limited memory for allocating transmit
		522	* buffers. Mark the page dirty and avoid
		523	* folio_rotate_reclaimable but rate-limit the messages.
		524	*/
		525	pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
		526	ret, swap_dev_pos(page_swap_entry(page)));
		527	}
		528
		529	swap_write_end(sio, failed);
		530	}
716		531
717	iov_iter_bvec(&from, ITER_DEST, sio->bvecs, sio->nr_bvecs, sio->len);	532	static void end_swap_bio_write(struct bio *bio)
718	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);	533	{
719	if (ret != -EIOCBQUEUED)	534	struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
720	sio_read_complete(&sio->iocb, ret);	535	bool failed = !!bio->bi_status;
		536
		537	if (failed)
		538	pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
		539	MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
		540	(unsigned long long)bio->bi_iter.bi_sector);
		541	bio_uninit(bio);
		542	swap_write_end(sio, failed);
		543	}
		544
		545	static void swap_read_end(struct swap_iocb *sio, bool failed)
		546	{
		547	int p;
		548
		549	for (p = 0; p < sio->nr_bvecs; p++) {
		550	struct folio *folio = bvec_folio(&sio->bvecs[p]);
		551
		552	if (!failed) {
		553	count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
		554	count_memcg_folio_events(folio, PSWPIN,
		555	folio_nr_pages(folio));
		556	folio_mark_uptodate(folio);
		557	}
		558	folio_unlock(folio);
		559	}
		560
		561	if (!failed)
		562	count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
		563
		564	mempool_free(sio, sio_pool);
		565	}
		566
		567	static void swap_fs_read_complete(struct kiocb *iocb, long ret)
		568	{
		569	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
		570	bool failed = ret != sio->len;
		571
		572	if (failed)
		573	pr_alert_ratelimited("Read-error on swap-device\n");
		574	swap_read_end(sio, failed);
		575	}
		576
		577	static void swap_bio_read_end_io(struct bio *bio)
		578	{
		579	struct swap_iocb *sio = container_of(bio, struct swap_iocb, bio);
		580	bool failed = !!bio->bi_status;
		581
		582	if (failed)
		583	pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
		584	MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
		585	(unsigned long long)bio->bi_iter.bi_sector);
		586	bio_uninit(bio);
		587	swap_read_end(sio, failed);
		588	}
		589
		590	static void swap_bdev_submit_write(struct swap_io_ctx *ctx)
		591	{
		592	struct swap_iocb *sio = ctx->sio;
		593	struct bio *bio = &sio->bio;
		594
		595	bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
		596	REQ_OP_WRITE \| REQ_SWAP);
		597	bio->bi_iter.bi_size = sio->len;
		598	bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
		599	bio_associate_blkg_from_page(bio, bio_first_folio_all(bio));
		600
		601	if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
		602	submit_bio_wait(bio);
		603	end_swap_bio_write(bio);
		604	} else {
		605	bio->bi_end_io = end_swap_bio_write;
		606	submit_bio(bio);
		607	}
		608	}
		609
		610	static void swap_bdev_submit_read(struct swap_io_ctx *ctx)
		611	{
		612	struct swap_iocb *sio = ctx->sio;
		613	struct bio *bio = &sio->bio;
		614
		615	bio_init(bio, ctx->sis->bdev, sio->bvecs, ARRAY_SIZE(sio->bvecs),
		616	REQ_OP_READ);
		617	bio->bi_iter.bi_size = sio->len;
		618	bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
		619
		620	if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
		621	/*
		622	* Keep this task valid during swap readpage because the oom
		623	* killer may attempt to access it in the page fault retry
		624	* time check.
		625	*/
		626	get_task_struct(current);
		627	submit_bio_wait(bio);
		628	swap_bio_read_end_io(bio);
		629	put_task_struct(current);
		630	} else {
		631	bio->bi_end_io = swap_bio_read_end_io;
		632	submit_bio(bio);
		633	}
		634	}
		635
		636	static bool swap_bdev_can_merge(struct folio folio, struct folio prev_folio,
		637	size_t prev_folio_size, int rw)
		638	{
		639	if (swap_folio_sector(folio) !=
		640	swap_folio_sector(prev_folio) + (prev_folio_size >> SECTOR_SHIFT))
		641	return false;
		642	if (rw == WRITE && !folio_blkg_can_merge(folio, prev_folio))
		643	return false;
		644	return true;
		645	}
		646
		647	const struct swap_ops swap_bdev_ops = {
		648	.submit_write = swap_bdev_submit_write,
		649	.submit_read = swap_bdev_submit_read,
		650	.can_merge = swap_bdev_can_merge,
		651	};
		652
		653	void swap_fs_prepare_rw(struct swap_io_ctx ctx, int rw, struct iov_iter iter)
		654	{
		655	struct swap_iocb *sio = ctx->sio;
		656
		657	init_sync_kiocb(&sio->iocb, ctx->sis->swap_file);
		658	sio->iocb.ki_pos = swap_dev_pos(bvec_folio(&sio->bvecs[0])->swap);
		659	if (rw == WRITE)
		660	sio->iocb.ki_complete = swap_fs_write_complete;
		661	else
		662	sio->iocb.ki_complete = swap_fs_read_complete;
		663
		664	iov_iter_bvec(iter, rw == WRITE ? ITER_SOURCE : ITER_DEST,
		665	sio->bvecs, sio->nr_bvecs, sio->len);
		666	}
		667	EXPORT_SYMBOL_GPL(swap_fs_prepare_rw);
		668
		669	bool swap_fs_can_merge(struct folio folio, struct folio prev_folio,
		670	size_t prev_folio_size, int rw)
		671	{
		672	return swap_dev_pos(folio->swap) ==
		673	swap_dev_pos(prev_folio->swap) + prev_folio_size;
		674	}
		675	EXPORT_SYMBOL_GPL(swap_fs_can_merge);
		676
		677	int swap_fs_activate(struct swap_info_struct sis, const struct swap_ops ops)
		678	{
		679	sis->ops = ops;
		680	return add_swap_extent(sis, 0, sis->max, 0);
		681	}
		682	EXPORT_SYMBOL_GPL(swap_fs_activate);
		683
		684	void swap_write_submit(struct swap_io_ctx *ctx)
		685	{
		686	if (!ctx->sio)
		687	return;
		688	count_vm_events(NRSWPOUT, 1);
		689	ctx->sis->ops->submit_write(ctx);
		690	ctx->sio = NULL;
		691	ctx->sis = NULL;
		692	}
		693
		694	void swap_read_submit(struct swap_io_ctx *ctx)
		695	{
		696	if (!ctx->sio)
		697	return;
		698	count_vm_events(NRSWPIN, 1);
		699	ctx->sis->ops->submit_read(ctx);
		700	ctx->sio = NULL;
		701	ctx->sis = NULL;
721	}	702	}

mm/page_reporting.c+18−10

  */
 EXPORT_SYMBOL_GPL(page_reporting_order);
-#define PAGE_REPORTING_DELAY	(2 * HZ)
+static unsigned int page_reporting_delay_ms = 2 * MSEC_PER_SEC;
+module_param(page_reporting_delay_ms, uint, 0644);
+MODULE_PARM_DESC(page_reporting_delay_ms,
+		 "Set page reporting delay in milliseconds");
 static struct page_reporting_dev_info __rcu *pr_dev_info __read_mostly;
 enum {
 	PAGE_REPORTING_ACTIVE
 };
+/* schedule work for page reporting */
+static void page_reporting_schedule_work(struct page_reporting_dev_info *prdev)
+{
+	queue_delayed_work(system_freezable_wq, &prdev->work,
+			   msecs_to_jiffies(page_reporting_delay_ms));
+}
 /* request page reporting */
 static void
 __page_reporting_request(struct page_reporting_dev_info *prdev)
 		return;
 	/*
-	 * Delay the start of work to allow a sizable queue to build. For
+	 * Delay the start of work to allow a sizable queue to build.
-	 * now we are limiting this to running no more than once every
+	 * We limit this based on page_reporting_delay_ms.
-	 * couple of seconds.
 	 */
-	queue_delayed_work(system_freezable_wq, &prdev->work,
+	page_reporting_schedule_work(prdev);
-			   PAGE_REPORTING_DELAY);
 }
 /* notify prdev of free page reporting request */
 err_out:
 	/*
 	 * If the state has reverted back to requested then there may be
-	 * additional pages to be processed. We will defer for 2s to allow
+	 * additional pages to be processed. We will defer by
-	 * more pages to accumulate.
+	 * page_reporting_delay_ms to allow more pages to accumulate.
 	 */
 	state = atomic_cmpxchg(&prdev->state, state, PAGE_REPORTING_IDLE);
 	if (state == PAGE_REPORTING_REQUESTED)
-		queue_delayed_work(system_freezable_wq, &prdev->work,
+		page_reporting_schedule_work(prdev);
-				   PAGE_REPORTING_DELAY);
 }
 static DEFINE_MUTEX(page_reporting_mutex);

mm/page_vma_mapped.c+5−1

 		struct vm_area_struct *vma)
 {
 	const struct folio *folio = page_folio(page);
+	const pgoff_t pgoff = page_pgoff(folio, page);
 	struct page_vma_mapped_walk pvmw = {
 		.pfn = page_to_pfn(page),
 		.nr_pages = 1,
 		.flags = PVMW_SYNC,
 	};
-	pvmw.address = vma_address(vma, page_pgoff(folio, page), 1);
+	if (folio_test_anon(folio))
+		pvmw.address = vma_anon_address(vma, pgoff, 1);
+	else
+		pvmw.address = vma_filebacked_address(vma, pgoff, 1);
 	if (pvmw.address == -EFAULT)
 		goto out;
 	if (!page_vma_mapped_walk(&pvmw))

mm/pgalloc-track.h+3−0

@@ -2,6 +2,9 @@ · 1 unmodified line
2	#ifndef _LINUX_PGALLOC_TRACK_H	2	#ifndef _LINUX_PGALLOC_TRACK_H
3	#define _LINUX_PGALLOC_TRACK_H	3	#define _LINUX_PGALLOC_TRACK_H
4		4
		5	#include <linux/mm.h>
		6	#include <linux/pgtable.h>
		7
5	#if defined(CONFIG_MMU)	8	#if defined(CONFIG_MMU)
6	static inline p4d_t p4d_alloc_track(struct mm_struct mm, pgd_t *pgd,	9	static inline p4d_t p4d_alloc_track(struct mm_struct mm, pgd_t *pgd,
7	unsigned long address,	10	unsigned long address,

mm/rmap.c+25−18

 	 * Now link the anon_vma's back to the newly inserted AVCs.
 	 * Note that all anon_vma's share the same root.
 	 */
-	anon_vma_lock_write(src->anon_vma);
+	anon_vma_lock_write(active_anon_vma);
 	list_for_each_entry_reverse(avc, &dst->anon_vma_chain, same_vma) {
 		struct anon_vma *anon_vma = avc->anon_vma;
 		if (!vma->anon_vma || !anon_vma ||
 		    vma->anon_vma->root != anon_vma->root)
 			return -EFAULT;
+		/* KSM folios don't reach here because of the !anon_vma check */
+		return vma_anon_address(vma, page_pgoff(folio, page), 1);
 	} else if (!vma->vm_file) {
 		return -EFAULT;
 	} else if (vma->vm_file->f_mapping != folio->mapping) {
 		return -EFAULT;
 	}
-	/* KSM folios don't reach here because of the !anon_vma check */
+	return vma_filebacked_address(vma, page_pgoff(folio, page), 1);
-	return vma_address(vma, page_pgoff(folio, page), 1);
 }
 /*
 struct folio_referenced_arg {
 	int mapcount;
 	int referenced;
-	vm_flags_t vm_flags;
+	vma_flags_t vma_flags;
 	struct mem_cgroup *memcg;
 };
 		address = pvmw.address;
 		nr = 1;
-		if (vma->vm_flags & VM_LOCKED) {
+		if (vma_test(vma, VMA_LOCKED_BIT)) {
 			ptes++;
 			pra->mapcount--;
 			/* Restore the mlock which got missed */
 			mlock_vma_folio(folio, vma);
 			page_vma_mapped_walk_done(&pvmw);
-			pra->vm_flags |= VM_LOCKED;
+			vma_flags_set(&pra->vma_flags, VMA_LOCKED_BIT);
 			return false; /* To break the loop */
 		}
 		referenced++;
 	if (referenced) {
+		vma_flags_t vma_flags = vma->flags;
 		pra->referenced++;
-		pra->vm_flags |= vma->vm_flags & ~VM_LOCKED;
+		vma_flags_clear(&vma_flags, VMA_LOCKED_BIT);
+		vma_flags_set_mask(&pra->vma_flags, vma_flags);
 	}
 	if (!pra->mapcount)
  * @folio: The folio to test.
  * @is_locked: Caller holds lock on the folio.
  * @memcg: target memory cgroup
- * @vm_flags: A combination of all the vma->vm_flags which referenced the folio.
+ * @vma_flags: A combination of all the vma->flags which referenced the folio.
  *
  * Quick test_and_clear_referenced for all mappings of a folio,
  *
  * the function bailed out due to rmap lock contention.
  */
 int folio_referenced(struct folio *folio, int is_locked,
-		     struct mem_cgroup *memcg, vm_flags_t *vm_flags)
+		struct mem_cgroup *memcg, vma_flags_t *vma_flags)
 {
 	bool we_locked = false;
 	struct folio_referenced_arg pra = {
 	};
 	VM_WARN_ON_ONCE_FOLIO(folio_is_zone_device(folio), folio);
-	*vm_flags = 0;
+	vma_flags_clear_all(vma_flags);
 	if (!pra.mapcount)
 		return 0;
 	}
 	rmap_walk(folio, &rwc);
-	*vm_flags = pra.vm_flags;
+	vma_flags_set_mask(vma_flags, pra.vma_flags);
 	if (we_locked)
 		folio_unlock(folio);
 		.vma		= vma,
 		.address	= address,
 		.flags		= PVMW_SYNC,
+		.pgoff_is_anon	= false,
 	};
 	state->cleaned += page_vma_mkclean_one(&pvmw);
 		.pgoff		= pgoff,
 		.vma		= vma,
 		.flags		= PVMW_SYNC,
+		.pgoff_is_anon	= false,
 	};
 	if (invalid_mkclean_vma(vma, NULL))
 		return 0;
-	pvmw.address = vma_address(vma, pgoff, nr_pages);
+	pvmw.address = vma_filebacked_address(vma, pgoff, nr_pages);
 	VM_BUG_ON_VMA(pvmw.address == -EFAULT, vma);
 	return page_vma_mkclean_one(&pvmw);
 	 */
 	anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON;
 	WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);
-	folio->index = linear_page_index(vma, address);
+	folio->index = linear_anon_page_index(vma, address);
 }
 /**
 	 */
 	VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
 			folio);
-	VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address),
+	VM_BUG_ON_PAGE(page_pgoff(folio, page) !=
-		       page);
+		       linear_anon_page_index(vma, address), page);
 }
 static __always_inline void __folio_add_anon_rmap(struct folio *folio,
 	pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
 	anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_end) {
 		struct vm_area_struct *vma = avc->vma;
-		unsigned long address = vma_address(vma, pgoff_start,
+		const unsigned long address = vma_anon_address(vma, pgoff_start,
 				folio_nr_pages(folio));
-		VM_BUG_ON_VMA(address == -EFAULT, vma);
+		VM_WARN_ON_ONCE_VMA(address == -EFAULT, vma);
 		cond_resched();
 		if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
 	}
 lookup:
 	mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_end) {
-		unsigned long address = vma_address(vma, pgoff_start, nr_pages);
+		unsigned long address = vma_filebacked_address(vma, pgoff_start,
+							       nr_pages);
 		VM_BUG_ON_VMA(address == -EFAULT, vma);
 		cond_resched();

mm/shmem.c+24−5

@@ -41,6 +41,7 @@ · 40 unmodified lines
41	#include <linux/swapfile.h>	41	#include <linux/swapfile.h>
42	#include <linux/iversion.h>	42	#include <linux/iversion.h>
43	#include <linux/unicode.h>	43	#include <linux/unicode.h>
		44	#include <linux/swap_ops.h>
44	#include "swap.h"	45	#include "swap.h"
45		46
46	static struct vfsmount *shm_mnt __ro_after_init;	47	static struct vfsmount *shm_mnt __ro_after_init;
@@ -1597,13 +1598,13 @@ start_over: · 1550 unmodified lines
1597		1598
1598	/**	1599	/**
1599	* shmem_writeout - Write the folio to swap	1600	* shmem_writeout - Write the folio to swap
		1601	* @ctx: swap I/O context
1600	* @folio: The folio to write	1602	* @folio: The folio to write
1601	* @plug: swap plug
1602	* @folio_list: list to put back folios on split	1603	* @folio_list: list to put back folios on split
1603	*	1604	*
1604	* Move the folio from the page cache to the swap cache.	1605	* Move the folio from the page cache to the swap cache.
1605	*/	1606	*/
1606	int shmem_writeout(struct folio folio, struct swap_iocb *plug,	1607	int shmem_writeout(struct swap_io_ctx ctx, struct folio folio,
1607	struct list_head *folio_list)	1608	struct list_head *folio_list)
1608	{	1609	{
1609	struct address_space *mapping = folio->mapping;	1610	struct address_space *mapping = folio->mapping;
@@ -1715,7 +1716,7 @@ try_split: · 105 unmodified lines
1715	shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));	1716	shmem_delete_from_page_cache(folio, swp_to_radix_entry(folio->swap));
1716		1717
1717	BUG_ON(folio_mapped(folio));	1718	BUG_ON(folio_mapped(folio));
1718	error = swap_writeout(folio, plug);	1719	error = swap_writeout(ctx, folio);
1719	if (error != AOP_WRITEPAGE_ACTIVATE) {	1720	if (error != AOP_WRITEPAGE_ACTIVATE) {
1720	/* folio has been unlocked */	1721	/* folio has been unlocked */
1721	return error;	1722	return error;
@@ -1751,7 +1752,17 @@ redirty: · 29 unmodified lines
1751	folio_mark_dirty(folio);	1752	folio_mark_dirty(folio);
1752	return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */	1753	return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */
1753	}	1754	}
1754	EXPORT_SYMBOL_GPL(shmem_writeout);	1755
		1756	int shmem_write_folio(struct folio *folio)
		1757	{
		1758	struct swap_io_ctx ctx = {};
		1759	int err;
		1760
		1761	err = shmem_writeout(&ctx, folio, NULL);
		1762	swap_write_submit(&ctx);
		1763	return err;
		1764	}
		1765	EXPORT_SYMBOL_GPL(shmem_write_folio);
1755		1766
1756	#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)	1767	#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1757	static void shmem_show_mpol(struct seq_file seq, struct mempolicy mpol)	1768	static void shmem_show_mpol(struct seq_file seq, struct mempolicy mpol)
@@ -3617,6 +3628,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, · 1859 unmodified lines
3617	struct shmem_inode_info *info = SHMEM_I(inode);	3628	struct shmem_inode_info *info = SHMEM_I(inode);
3618	struct shmem_falloc shmem_falloc;	3629	struct shmem_falloc shmem_falloc;
3619	pgoff_t start, index, end, undo_fallocend;	3630	pgoff_t start, index, end, undo_fallocend;
		3631	loff_t aligned_end;
3620	int error;	3632	int error;
3621		3633
3622	if (mode & ~(FALLOC_FL_KEEP_SIZE \| FALLOC_FL_PUNCH_HOLE))	3634	if (mode & ~(FALLOC_FL_KEEP_SIZE \| FALLOC_FL_PUNCH_HOLE))
@@ -3673,8 +3685,15 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, · 50 unmodified lines
3673	goto out;	3685	goto out;
3674	}	3686	}
3675		3687
		3688	/* Check for wraparound */
		3689	if (check_add_overflow(offset + len, (loff_t)PAGE_SIZE - 1,
		3690	&aligned_end)) {
		3691	error = -EFBIG;
		3692	goto out;
		3693	}
		3694
3676	start = offset >> PAGE_SHIFT;	3695	start = offset >> PAGE_SHIFT;
3677	end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;	3696	end = aligned_end >> PAGE_SHIFT;
3678	/* Try to avoid a swapstorm if len is impossible to satisfy */	3697	/* Try to avoid a swapstorm if len is impossible to satisfy */
3679	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {	3698	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
3680	error = -ENOSPC;	3699	error = -ENOSPC;

mm/show_mem.c+6−6

@@ -309,8 +309,8 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, · 308 unmodified lines
309	" min:%lukB"	309	" min:%lukB"
310	" low:%lukB"	310	" low:%lukB"
311	" high:%lukB"	311	" high:%lukB"
312	" reserved_highatomic:%luKB"	312	" reserved_highatomic:%lukB"
313	" free_highatomic:%luKB"	313	" free_highatomic:%lukB"
314	" active_anon:%lukB"	314	" active_anon:%lukB"
315	" inactive_anon:%lukB"	315	" inactive_anon:%lukB"
316	" active_file:%lukB"	316	" active_file:%lukB"
@@ -323,7 +323,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, · 6 unmodified lines
323	" mlocked:%lukB"	323	" mlocked:%lukB"
324	" bounce:%lukB"	324	" bounce:%lukB"
325	" free_pcp:%lukB"	325	" free_pcp:%lukB"
326	" local_pcp:%ukB"	326	" local_pcp:%lukB"
327	" free_cma:%lukB"	327	" free_cma:%lukB"
328	"\n",	328	"\n",
329	zone->name,	329	zone->name,
@@ -350,7 +350,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, · 20 unmodified lines
350	K(zone_page_state(zone, NR_MLOCK)),	350	K(zone_page_state(zone, NR_MLOCK)),
351	0UL,	351	0UL,
352	K(free_pcp),	352	K(free_pcp),
353	K(this_cpu_read(zone->per_cpu_pageset->count)),	353	K((unsigned long)this_cpu_read(zone->per_cpu_pageset->count)),
354	K(zone_page_state(zone, NR_FREE_CMA_PAGES)));	354	K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
355	printk("lowmem_reserve[]:");	355	printk("lowmem_reserve[]:");
356	for (i = 0; i < MAX_NR_ZONES; i++)	356	for (i = 0; i < MAX_NR_ZONES; i++)
@@ -400,7 +400,7 @@ static void show_free_areas(unsigned int filter, const nodemask_t *nodemask, · 43 unmodified lines
400	hugetlb_show_meminfo_node(nid);	400	hugetlb_show_meminfo_node(nid);
401	}	401	}
402		402
403	printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));	403	printk("%lu total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
404		404
405	show_swap_cache_info();	405	show_swap_cache_info();
406	}	406	}
@@ -430,7 +430,7 @@ void __show_mem(unsigned int filter, const nodemask_t *nodemask, · 23 unmodified lines
430	printk("%lu pages cma reserved\n", totalcma_pages);	430	printk("%lu pages cma reserved\n", totalcma_pages);
431	#endif	431	#endif
432	#ifdef CONFIG_MEMORY_FAILURE	432	#ifdef CONFIG_MEMORY_FAILURE
433	printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages));	433	printk("%ld pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages));
434	#endif	434	#endif
435	#ifdef CONFIG_MEM_ALLOC_PROFILING	435	#ifdef CONFIG_MEM_ALLOC_PROFILING
436	static DEFINE_SPINLOCK(mem_alloc_profiling_spinlock);	436	static DEFINE_SPINLOCK(mem_alloc_profiling_spinlock);

mm/shuffle.h+2−0

@@ -2,7 +2,9 @@ · 1 unmodified line
2	// Copyright(c) 2018 Intel Corporation. All rights reserved.	2	// Copyright(c) 2018 Intel Corporation. All rights reserved.
3	#ifndef _MM_SHUFFLE_H	3	#ifndef _MM_SHUFFLE_H
4	#define _MM_SHUFFLE_H	4	#define _MM_SHUFFLE_H
		5
5	#include <linux/jump_label.h>	6	#include <linux/jump_label.h>
		7	#include <linux/mmzone.h>
6		8
7	#define SHUFFLE_ORDER MAX_PAGE_ORDER	9	#define SHUFFLE_ORDER MAX_PAGE_ORDER
8		10

mm/sparse.c+4−14

 	return 0;
 }
 #else /* !SPARSEMEM_EXTREME */
-int sparse_index_init(unsigned long section_nr, int nid)
+int __meminit sparse_index_init(unsigned long section_nr, int nid)
 {
 	return 0;
 }
 }
 /* Validate the physical addressing limitations of the model */
-static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
+static void __init mminit_validate_memmodel_limits(unsigned long *start_pfn,
 						unsigned long *end_pfn)
 {
 	unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT;
 		memory_present(nid, start, end);
 }
-static unsigned long usemap_size(void)
-{
-	return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);
-}
-size_t mem_section_usage_size(void)
-{
-	return sizeof(struct mem_section_usage) + usemap_size();
-}
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 unsigned long __init section_map_size(void)
 {
 {
 }
-static void *sparse_usagebuf __meminitdata;
+static void *sparse_usagebuf __initdata;
-static void *sparse_usagebuf_end __meminitdata;
+static void *sparse_usagebuf_end __initdata;
 /*
  * Helper function that is used for generic section initialization, and

mm/sparse.h+6−0

 	ms->section_mem_map |= SECTION_MARKED_PRESENT;
 }
+static inline size_t mem_section_usage_size(void)
+{
+	return struct_size_t(struct mem_section_usage, pageblock_flags,
+			     BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS));
+}
 #else
 static inline void sparse_init(void) {}
 #endif /* CONFIG_SPARSEMEM */

mm/swap.h+31−34

@@ -4,9 +4,13 @@ · 3 unmodified lines
4		4
5	#include <linux/atomic.h> /* for atomic_long_t */	5	#include <linux/atomic.h> /* for atomic_long_t */
6	#include <linux/mm.h> /* for PAGE_SHIFT */	6	#include <linux/mm.h> /* for PAGE_SHIFT */
		7	#include <linux/memcontrol.h> /* for mem_cgroup_swappiness() */
		8	#include <linux/swap.h> /* for MAX_SWAPFILES_SHIFT, struct swap_info_struct */
		9
7	struct mempolicy;	10	struct mempolicy;
8	struct swap_iocb;	11	struct swap_iocb;
9	struct swap_memcg_table;	12	struct swap_memcg_table;
		13	struct swap_io_ctx;
10		14
11	#if defined(MAX_POSSIBLE_PHYSMEM_BITS)	15	#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
12	#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)	16	#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
@@ -76,6 +80,18 @@ enum swap_cluster_flags { · 63 unmodified lines
76	CLUSTER_FLAG_MAX,	80	CLUSTER_FLAG_MAX,
77	};	81	};
78		82
		83	extern int vm_swappiness;
		84
		85	static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
		86	{
		87	#ifdef CONFIG_MEMCG_V1
		88	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
		89	!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
		90	return READ_ONCE(memcg->swappiness);
		91	#endif
		92	return READ_ONCE(vm_swappiness);
		93	}
		94
79	#ifdef CONFIG_SWAP	95	#ifdef CONFIG_SWAP
80	#include <linux/swapops.h> /* for swp_offset */	96	#include <linux/swapops.h> /* for swp_offset */
81	#include <linux/blk_types.h> /* for bio_end_io_t */	97	#include <linux/blk_types.h> /* for bio_end_io_t */
@@ -238,17 +254,11 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, · 156 unmodified lines
238		254
239	/* linux/mm/page_io.c */	255	/* linux/mm/page_io.c */
240	int sio_pool_init(void);	256	int sio_pool_init(void);
241	struct swap_iocb;	257	void swap_read_folio(struct swap_io_ctx ctx, struct folio folio);
242	void swap_read_folio(struct folio folio, struct swap_iocb *plug);	258	void swap_read_submit(struct swap_io_ctx *ctx);
243	void __swap_read_unplug(struct swap_iocb *plug);	259	void swap_write_submit(struct swap_io_ctx *ctx);
244	static inline void swap_read_unplug(struct swap_iocb *plug)	260	int swap_writeout(struct swap_io_ctx ctx, struct folio folio);
245	{	261	void __swap_writepage(struct swap_io_ctx ctx, struct folio folio);
246	if (unlikely(plug))
247	__swap_read_unplug(plug);
248	}
249	void swap_write_unplug(struct swap_iocb *sio);
250	int swap_writeout(struct folio folio, struct swap_iocb *swap_plug);
251	void __swap_writepage(struct folio folio, struct swap_iocb *swap_plug);
252		262
253	/* linux/mm/swap_state.c */	263	/* linux/mm/swap_state.c */
254	extern struct address_space swap_space __read_mostly;	264	extern struct address_space swap_space __read_mostly;
@@ -315,9 +325,8 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci, · 60 unmodified lines
315		325
316	void show_swap_cache_info(void);	326	void show_swap_cache_info(void);
317	void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);	327	void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
318	struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,	328	struct folio read_swap_cache_async(struct swap_io_ctx ctx, swp_entry_t entry,
319	struct vm_area_struct *vma, unsigned long addr,	329	gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr);
320	struct swap_iocb **plug);
321	struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,	330	struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
322	struct mempolicy *mpol, pgoff_t ilx);	331	struct mempolicy *mpol, pgoff_t ilx);
323	struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,	332	struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
@@ -327,18 +336,7 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders, · 3 unmodified lines
327	void swap_update_readahead(struct folio folio, struct vm_area_struct vma,	336	void swap_update_readahead(struct folio folio, struct vm_area_struct vma,
328	unsigned long addr);	337	unsigned long addr);
329		338
330	static inline unsigned int folio_swap_flags(struct folio *folio)
331	{
332	return __swap_entry_to_info(folio->swap)->flags;
333	}
334
335	#else /* CONFIG_SWAP */	339	#else /* CONFIG_SWAP */
336	struct swap_iocb;
337	static inline struct swap_cluster_info *swap_cluster_lock(
338	struct swap_info_struct *si, pgoff_t offset, bool irq)
339	{
340	return NULL;
341	}
342		340
343	static inline struct swap_cluster_info *swap_cluster_get_and_lock(	341	static inline struct swap_cluster_info *swap_cluster_get_and_lock(
344	struct folio *folio)	342	struct folio *folio)
@@ -379,11 +377,11 @@ static inline void folio_put_swap(struct folio folio, struct page page) · 34 unmodified lines
379	{	377	{
380	}	378	}
381		379
382	static inline void swap_read_folio(struct folio folio, struct swap_iocb *plug)	380	static inline void swap_read_folio(struct swap_io_ctx ctx, struct folio folio)
383	{	381	{
384	}	382	}
385		383
386	static inline void swap_write_unplug(struct swap_iocb *sio)	384	static inline void swap_write_submit(struct swap_io_ctx *ctx)
387	{	385	{
388	}	386	}
389		387
@@ -425,8 +423,7 @@ static inline void swap_update_readahead(struct folio *folio, · 35 unmodified lines
425	{	423	{
426	}	424	}
427		425
428	static inline int swap_writeout(struct folio *folio,	426	static inline int swap_writeout(struct swap_io_ctx ctx, struct folio folio)
429	struct swap_iocb **swap_plug)
430	{	427	{
431	return 0;	428	return 0;
432	}	429	}
@@ -464,11 +461,11 @@ static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, · 31 unmodified lines
464	struct folio old, struct folio new)	461	struct folio old, struct folio new)
465	{	462	{
466	}	463	}
		464	#endif /* CONFIG_SWAP */
467		465
468	static inline unsigned int folio_swap_flags(struct folio *folio)	466	extern const struct swap_ops swap_bdev_ops;
469	{	467
470	return 0;	468	int shmem_writeout(struct swap_io_ctx ctx, struct folio folio,
471	}	469	struct list_head *folio_list);
472		470
473	#endif /* CONFIG_SWAP */
474	#endif /* _MM_SWAP_H */	471	#endif /* _MM_SWAP_H */

mm/swap_state.c+32−22

 #include <linux/huge_mm.h>
 #include <linux/shmem_fs.h>
 #include <linux/sysctl.h>
+#include <linux/swap_ops.h>
 #include "internal.h"
 #include "swap_table.h"
 #include "swap.h"
 	}
 }
-static struct folio *swap_cache_read_folio(swp_entry_t entry, gfp_t gfp,
+static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx,
-					   struct mempolicy *mpol, pgoff_t ilx,
+		swp_entry_t entry, gfp_t gfp, struct mempolicy *mpol,
-					   struct swap_iocb **plug, bool readahead)
+		pgoff_t ilx, bool readahead)
 {
 	struct folio *folio;
 	if (IS_ERR_OR_NULL(folio))
 		return NULL;
-	swap_read_folio(folio, plug);
+	swap_read_folio(ctx, folio);
 	if (readahead) {
 		folio_set_readahead(folio);
 		count_vm_event(SWAP_RA);
 struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
 			   struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx)
 {
+	struct swap_io_ctx ctx = {};
 	struct folio *folio;
 	do {
 	if (IS_ERR(folio))
 		return folio;
-	swap_read_folio(folio, NULL);
+	swap_read_folio(&ctx, folio);
+	swap_read_submit(&ctx);
 	return folio;
 }
  * A failure return means that either the page allocation failed or that
  * the swap entry is no longer in use.
  */
-struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
+struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
-		struct vm_area_struct *vma, unsigned long addr,
+		gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr)
-		struct swap_iocb **plug)
 {
 	struct swap_info_struct *si;
 	struct mempolicy *mpol;
 		return NULL;
 	mpol = get_vma_policy(vma, addr, 0, &ilx);
-	folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx, plug, false);
+	folio = swap_cache_read_folio(ctx, entry, gfp_mask, mpol, ilx, false);
 	mpol_cond_put(mpol);
 	put_swap_device(si);
 	return folio;
 }
+static struct folio *swap_cache_read_folio_sync(swp_entry_t entry, gfp_t gfp,
+		struct mempolicy *mpol, pgoff_t ilx)
+{
+	struct swap_io_ctx ctx = {};
+	struct folio *folio;
+	folio = swap_cache_read_folio(&ctx, entry, gfp, mpol, ilx, false);
+	swap_read_submit(&ctx);
+	return folio;
+}
 static unsigned int __swapin_nr_pages(unsigned long prev_offset,
 				      unsigned long offset,
 				      int hits,
 	unsigned long start_offset, end_offset;
 	unsigned long mask;
 	struct swap_info_struct *si = __swap_entry_to_info(entry);
+	struct swap_io_ctx ctx = {};
 	struct blk_plug plug;
-	struct swap_iocb *splug = NULL;
 	swp_entry_t ra_entry;
 	mask = swapin_nr_pages(offset) - 1;
 	for (offset = start_offset; offset <= end_offset ; offset++) {
 		/* Ok, do the async read-ahead now */
 		ra_entry = swp_entry(swp_type(entry), offset);
-		folio = swap_cache_read_folio(ra_entry, gfp_mask, mpol, ilx,
+		folio = swap_cache_read_folio(&ctx, ra_entry, gfp_mask, mpol,
-					      &splug, offset != entry_offset);
+				ilx, offset != entry_offset);
 		if (!folio)
 			continue;
 		folio_put(folio);
 	}
 	blk_finish_plug(&plug);
-	swap_read_unplug(splug);
+	swap_read_submit(&ctx);
 skip:
-	/* The page was likely read above, so no need for plugging here */
+	return swap_cache_read_folio_sync(entry, gfp_mask, mpol, ilx);
-	return swap_cache_read_folio(entry, gfp_mask, mpol, ilx, NULL, false);
 }
 static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
 static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
 		struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf)
 {
+	struct swap_io_ctx ctx = {};
 	struct blk_plug plug;
-	struct swap_iocb *splug = NULL;
 	struct folio *folio;
 	pte_t *pte = NULL, pentry;
 	int win;
 			if (!si)
 				continue;
 		}
-		folio = swap_cache_read_folio(entry, gfp_mask, mpol, ilx,
+		folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx,
-					      &splug, addr != vmf->address);
+					      addr != vmf->address);
 		if (si)
 			put_swap_device(si);
 		if (!folio)
 	if (pte)
 		pte_unmap(pte);
 	blk_finish_plug(&plug);
-	swap_read_unplug(splug);
+	swap_read_submit(&ctx);
 skip:
 	/* The folio was likely read above, so no need for plugging here */
-	folio = swap_cache_read_folio(targ_entry, gfp_mask, mpol, targ_ilx,
+	return swap_cache_read_folio_sync(targ_entry, gfp_mask, mpol, targ_ilx);
-				      NULL, false);
-	return folio;
 }
 /**

mm/swapfile.c+16−8

 	return si;
 bad_nofile:
-	pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
+	pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val);
 out:
 	return NULL;
 put_out:
-	pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
+	pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
 	percpu_ref_put(&si->users);
 	return NULL;
 }
 	struct inode *inode = mapping->host;
 	int ret;
+	ret = sio_pool_init();
+	if (ret)
+		return ret;
+	sis->ops = &swap_bdev_ops;
 	if (S_ISBLK(inode->i_mode)) {
 		ret = add_swap_extent(sis, 0, sis->max, 0);
 		*span = sis->pages;
 		if (ret < 0)
 			return ret;
 		sis->flags |= SWP_ACTIVATED;
-		if ((sis->flags & SWP_FS_OPS) &&
-		    sio_pool_init() != 0) {
-			destroy_swap_extents(sis, swap_file);
-			return -ENOMEM;
-		}
 		return ret;
 	}
 		error = -EBUSY;
 		goto bad_swap_unlock_inode;
 	}
+	if (IS_ENCRYPTED(inode)) {
+		pr_warn_once(
+			"Filesystem-level encrypted swapfile '%s' is unsupported. Create a loop device over it, or use dm-crypt\n",
+			name->name);
+		error = -EINVAL;
+		goto bad_swap_unlock_inode;
+	}
 	/*
 	 * The swap subsystem needs a major overhaul to support this.
 	si = swap_entry_to_info(entry);
 	if (WARN_ON_ONCE(!si)) {
-		pr_err("%s%08lx\n", Bad_file, entry.val);
+		pr_err_ratelimited("%s%08lx\n", Bad_file, entry.val);
 		return -EINVAL;
 	}

mm/userfaultfd.c+2−2

@@ -1352,7 +1352,7 @@ static long move_present_ptes(struct mm_struct *mm, · 1351 unmodified lines
1352	}	1352	}
1353		1353
1354	folio_move_anon_rmap(src_folio, dst_vma);	1354	folio_move_anon_rmap(src_folio, dst_vma);
1355	src_folio->index = linear_page_index(dst_vma, dst_addr);	1355	src_folio->index = linear_anon_page_index(dst_vma, dst_addr);
1356		1356
1357	orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot);	1357	orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot);
1358	/* Set soft dirty bit so userspace can notice the pte was moved */	1358	/* Set soft dirty bit so userspace can notice the pte was moved */
@@ -1428,7 +1428,7 @@ static int move_swap_pte(struct mm_struct mm, struct vm_area_struct dst_vma, · 69 unmodified lines
1428	*/	1428	*/
1429	if (src_folio) {	1429	if (src_folio) {
1430	folio_move_anon_rmap(src_folio, dst_vma);	1430	folio_move_anon_rmap(src_folio, dst_vma);
1431	src_folio->index = linear_page_index(dst_vma, dst_addr);	1431	src_folio->index = linear_anon_page_index(dst_vma, dst_addr);
1432	} else {	1432	} else {
1433	/*	1433	/*
1434	* Check if the swap entry is cached after acquiring the src_pte	1434	* Check if the swap entry is cached after acquiring the src_pte

mm/vma.c+79−34

@@ -18,6 +18,7 @@ struct mmap_state { · 17 unmodified lines
18	unsigned long addr;	18	unsigned long addr;
19	unsigned long end;	19	unsigned long end;
20	pgoff_t pgoff;	20	pgoff_t pgoff;
		21	pgoff_t anon_pgoff;
21	unsigned long pglen;	22	unsigned long pglen;
22	union {	23	union {
23	vm_flags_t vm_flags;	24	vm_flags_t vm_flags;
@@ -46,13 +47,14 @@ struct mmap_state { · 22 unmodified lines
46	bool file_doesnt_need_get :1;	47	bool file_doesnt_need_get :1;
47	};	48	};
48		49
49	#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \	50	#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, anon_pgoff_, vma_flags_, file_) \
50	struct mmap_state name = { \	51	struct mmap_state name = { \
51	.mm = mm_, \	52	.mm = mm_, \
52	.vmi = vmi_, \	53	.vmi = vmi_, \
53	.addr = addr_, \	54	.addr = addr_, \
54	.end = (addr_) + (len_), \	55	.end = (addr_) + (len_), \
55	.pgoff = pgoff_, \	56	.pgoff = pgoff_, \
		57	.anon_pgoff = anon_pgoff_, \
56	.pglen = PHYS_PFN(len_), \	58	.pglen = PHYS_PFN(len_), \
57	.vma_flags = vma_flags_, \	59	.vma_flags = vma_flags_, \
58	.file = file_, \	60	.file = file_, \
@@ -67,6 +69,7 @@ struct mmap_state { · 8 unmodified lines
67	.end = (map_)->end, \	69	.end = (map_)->end, \
68	.vma_flags = (map_)->vma_flags, \	70	.vma_flags = (map_)->vma_flags, \
69	.pgoff = (map_)->pgoff, \	71	.pgoff = (map_)->pgoff, \
		72	.anon_pgoff = (map_)->anon_pgoff, \
70	.file = (map_)->file, \	73	.file = (map_)->file, \
71	.prev = (map_)->prev, \	74	.prev = (map_)->prev, \
72	.middle = vma_, \	75	.middle = vma_, \
@@ -82,10 +85,11 @@ static void __vma_set_range(struct vm_area_struct *vma, unsigned long start, · 9 unmodified lines
82	}	85	}
83		86
84	static void vma_set_range(struct vm_area_struct *vma, unsigned long start,	87	static void vma_set_range(struct vm_area_struct *vma, unsigned long start,
85	unsigned long end, pgoff_t pgoff)	88	unsigned long end, pgoff_t pgoff, pgoff_t anon_pgoff)
86	{	89	{
87	__vma_set_range(vma, start, end);	90	__vma_set_range(vma, start, end);
88	vma_set_pgoff(vma, pgoff);	91	vma_set_pgoff(vma, pgoff);
		92	vma_set_anon_pgoff(vma, anon_pgoff);
89	}	93	}
90		94
91	/* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */	95	/* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */
@@ -200,6 +204,25 @@ static void init_multi_vma_prep(struct vma_prepare *vp, · 108 unmodified lines
200	vp->skip_vma_uprobe = true;	204	vp->skip_vma_uprobe = true;
201	}	205	}
202		206
		207	/*
		208	* Does this merge require that adjacent VMAs must have adjacent anonymous page
		209	* offsets in addition to having adjacent vma->vm_pgoff?
		210	*
		211	* This is only required for MAP_PRIVATE-file backed mappings as the page offset
		212	* for pure anonymous VMAs is equal to the anonymous page offset.
		213	*
		214	* Read-only shared mappings (with VMA_SHARED_BIT cleared) are always unfaulted
		215	* so automatically have correct anonymous page offset (as it is always updated
		216	* on remap).
		217	*
		218	* 'Special' mappings in the sense of VDSO, VVAR etc. have !file but would in
		219	* any case not be candidates for merge nor be mergeable.
		220	*/
		221	static bool needs_adjacent_anon_pgoff(const struct vma_merge_struct *vmg)
		222	{
		223	return vmg->file && vma_flags_is_cow_mapping(&vmg->vma_flags);
		224	}
		225
203	/*	226	/*
204	* Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff)	227	* Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff)
205	* in front of (at a lower virtual address and file offset than) the vma.	228	* in front of (at a lower virtual address and file offset than) the vma.
@@ -221,6 +244,9 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg) · 15 unmodified lines
221	return false;	244	return false;
222	if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next))	245	if (vmg_end_pgoff(vmg) != vma_start_pgoff(vmg->next))
223	return false;	246	return false;
		247	if (needs_adjacent_anon_pgoff(vmg) &&
		248	vmg_end_anon_pgoff(vmg) != vma_start_anon_pgoff(vmg->next))
		249	return false;
224	return true;	250	return true;
225	}	251	}
226		252
@@ -241,6 +267,9 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) · 14 unmodified lines
241	return false;	267	return false;
242	if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg))	268	if (vma_end_pgoff(vmg->prev) != vmg_start_pgoff(vmg))
243	return false;	269	return false;
		270	if (needs_adjacent_anon_pgoff(vmg) &&
		271	vma_end_anon_pgoff(vmg->prev) != vmg_start_anon_pgoff(vmg))
		272	return false;
244	return true;	273	return true;
245	}	274	}
246		275
@@ -812,7 +841,8 @@ static int commit_merge(struct vma_merge_struct *vmg) · 565 unmodified lines
812	*/	841	*/
813	vma_adjust_trans_huge(vma, vmg->start, vmg->end,	842	vma_adjust_trans_huge(vma, vmg->start, vmg->end,
814	vmg->__adjust_middle_start ? vmg->middle : NULL);	843	vmg->__adjust_middle_start ? vmg->middle : NULL);
815	vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg));	844	vma_set_range(vma, vmg->start, vmg->end, vmg_start_pgoff(vmg),
		845	vmg_start_anon_pgoff(vmg));
816	vmg_adjust_set_range(vmg);	846	vmg_adjust_set_range(vmg);
817	vma_iter_store_overwrite(vmg->vmi, vmg->target);	847	vma_iter_store_overwrite(vmg->vmi, vmg->target);
818		848
@@ -982,6 +1012,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( · 163 unmodified lines
982	vmg->start = prev->vm_start;	1012	vmg->start = prev->vm_start;
983	vmg->end = next->vm_end;	1013	vmg->end = next->vm_end;
984	vmg->pgoff = vma_start_pgoff(prev);	1014	vmg->pgoff = vma_start_pgoff(prev);
		1015	vmg->anon_pgoff = vma_start_anon_pgoff(prev);
985		1016
986	/*	1017	/*
987	* We already ensured anon_vma compatibility above, so now it's	1018	* We already ensured anon_vma compatibility above, so now it's
@@ -1000,6 +1031,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( · 12 unmodified lines
1000	*/	1031	*/
1001	vmg->start = prev->vm_start;	1032	vmg->start = prev->vm_start;
1002	vmg->pgoff = vma_start_pgoff(prev);	1033	vmg->pgoff = vma_start_pgoff(prev);
		1034	vmg->anon_pgoff = vma_start_anon_pgoff(prev);
1003		1035
1004	if (!vmg->__remove_middle)	1036	if (!vmg->__remove_middle)
1005	vmg->__adjust_middle_start = true;	1037	vmg->__adjust_middle_start = true;
@@ -1022,12 +1054,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( · 16 unmodified lines
1022	if (vmg->__remove_middle) {	1054	if (vmg->__remove_middle) {
1023	vmg->end = next->vm_end;	1055	vmg->end = next->vm_end;
1024	vmg->pgoff = vma_start_pgoff(next) - pglen;	1056	vmg->pgoff = vma_start_pgoff(next) - pglen;
		1057	vmg->anon_pgoff = vma_start_anon_pgoff(next) - pglen;
1025	} else {	1058	} else {
1026	/* We shrink middle and expand next. */	1059	/* We shrink middle and expand next. */
1027	vmg->__adjust_next_start = true;	1060	vmg->__adjust_next_start = true;
1028	vmg->start = middle->vm_start;	1061	vmg->start = middle->vm_start;
1029	vmg->end = start;	1062	vmg->end = start;
1030	vmg->pgoff = vma_start_pgoff(middle);	1063	vmg->pgoff = vma_start_pgoff(middle);
		1064	vmg->anon_pgoff = vma_start_anon_pgoff(middle);
1031	}	1065	}
1032		1066
1033	err = dup_anon_vma(next, middle, &anon_dup);	1067	err = dup_anon_vma(next, middle, &anon_dup);
@@ -1137,6 +1171,7 @@ struct vm_area_struct vma_merge_new_range(struct vma_merge_struct vmg) · 103 unmodified lines
1137	vmg->start = prev->vm_start;	1171	vmg->start = prev->vm_start;
1138	vmg->target = prev;	1172	vmg->target = prev;
1139	vmg->pgoff = vma_start_pgoff(prev);	1173	vmg->pgoff = vma_start_pgoff(prev);
		1174	vmg->anon_pgoff = vma_start_anon_pgoff(prev);
1140		1175
1141	/*	1176	/*
1142	* If this merge would result in removal of the next VMA but we	1177	* If this merge would result in removal of the next VMA but we
@@ -1908,23 +1943,27 @@ static int vma_link(struct mm_struct mm, struct vm_area_struct vma) · 765 unmodified lines
1908	*/	1943	*/
1909	struct vm_area_struct copy_vma(struct vm_area_struct *vmap,	1944	struct vm_area_struct copy_vma(struct vm_area_struct *vmap,
1910	unsigned long addr, unsigned long len, pgoff_t pgoff,	1945	unsigned long addr, unsigned long len, pgoff_t pgoff,
1911	bool *need_rmap_locks)	1946	pgoff_t anon_pgoff, bool *need_rmap_locks)
1912	{	1947	{
1913	struct vm_area_struct vma = vmap;	1948	struct vm_area_struct vma = vmap;
1914	unsigned long vma_start = vma->vm_start;	1949	unsigned long old_vma_start = vma->vm_start;
1915	struct mm_struct *mm = vma->vm_mm;	1950	struct mm_struct *mm = vma->vm_mm;
1916	struct vm_area_struct *new_vma;	1951	struct vm_area_struct *new_vma;
1917	bool faulted_in_anon_vma = true;	1952	bool can_self_merge = false;
1918	VMA_ITERATOR(vmi, mm, addr);	1953	VMA_ITERATOR(vmi, mm, addr);
1919	VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len);	1954	VMG_VMA_STATE(vmg, &vmi, NULL, vma, addr, addr + len);
1920		1955
1921	/*	1956	/*
1922	* If anonymous vma has not yet been faulted, update new pgoff	1957	* If a vma has not yet been faulted, update its anonymous pgoff to
1923	* to match new location, to increase its chance of merging.	1958	* match the new location to increase its chance of merging.
1924	*/	1959	*/
1925	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {	1960	if (!vma->anon_vma) {
1926	pgoff = addr >> PAGE_SHIFT;	1961	anon_pgoff = addr >> PAGE_SHIFT;
1927	faulted_in_anon_vma = false;	1962
		1963	if (vma_is_anonymous(vma)) {
		1964	pgoff = anon_pgoff;
		1965	can_self_merge = true;
		1966	}
1928	}	1967	}
1929		1968
1930	/*	1969	/*
@@ -1940,28 +1979,26 @@ struct vm_area_struct copy_vma(struct vm_area_struct *vmap, · 9 unmodified lines
1940	return NULL; /* should never get here */	1979	return NULL; /* should never get here */
1941		1980
1942	vmg.pgoff = pgoff;	1981	vmg.pgoff = pgoff;
		1982	vmg.anon_pgoff = anon_pgoff;
1943	vmg.next = vma_iter_next_rewind(&vmi, NULL);	1983	vmg.next = vma_iter_next_rewind(&vmi, NULL);
1944	new_vma = vma_merge_copied_range(&vmg);	1984	new_vma = vma_merge_copied_range(&vmg);
1945		1985
1946	if (new_vma) {	1986	if (new_vma) {
1947	/*	1987	/* Self-merged and VMA replaced. */
1948	* Source vma may have been merged into new_vma	1988	if (unlikely(new_vma->vm_start < old_vma_start &&
1949	*/	1989	new_vma->vm_end > old_vma_start)) {
1950	if (unlikely(vma_start >= new_vma->vm_start &&
1951	vma_start < new_vma->vm_end)) {
1952	/*	1990	/*
1953	* The only way we can get a vma_merge with	1991	* The only way a VMA can both self-merge and be
1954	* self during an mremap is if the vma hasn't	1992	* replaced is if the remap places the new VMA
1955	* been faulted in yet and we were allowed to	1993	* immediately prior to its old self ('next') and
1956	* reset the dst vma->vm_pgoff to the	1994	* immediately after another VMA ('prev') causing the
1957	* destination address of the mremap to allow	1995	* next to be removed and prev to be expanded to cover
1958	* the merge to happen. mremap must change the	1996	* the entire range.
1959	* vm_pgoff linearity between src and dst vmas	1997	*
1960	* (in turn preventing a vma_merge) to be	1998	* This should only be possible if the anonymous page
1961	* safe. It is only safe to keep the vm_pgoff	1999	* offset was updated, i.e. the VMA is unfaulted.
1962	* linear if there are no pages mapped yet.
1963	*/	2000	*/
1964	VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);	2001	VM_WARN_ON_ONCE_VMA(!can_self_merge, new_vma);
1965	*vmap = vma = new_vma;	2002	*vmap = vma = new_vma;
1966	}	2003	}
1967	*need_rmap_locks =	2004	*need_rmap_locks =
@@ -1970,7 +2007,7 @@ struct vm_area_struct copy_vma(struct vm_area_struct *vmap, · 2 unmodified lines
1970	new_vma = vm_area_dup(vma);	2007	new_vma = vm_area_dup(vma);
1971	if (!new_vma)	2008	if (!new_vma)
1972	goto out;	2009	goto out;
1973	vma_set_range(new_vma, addr, addr + len, pgoff);	2010	vma_set_range(new_vma, addr, addr + len, pgoff, anon_pgoff);
1974	if (vma_dup_policy(vma, new_vma))	2011	if (vma_dup_policy(vma, new_vma))
1975	goto out_free_vma;	2012	goto out_free_vma;
1976	if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP))	2013	if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP))
@@ -2036,7 +2073,12 @@ static int anon_vma_compatible(struct vm_area_struct a, struct vm_area_struct · 59 unmodified lines
2036	if (!vma_flags_empty(&diff))	2073	if (!vma_flags_empty(&diff))
2037	return false;	2074	return false;
2038	/* Page offset must align. */	2075	/* Page offset must align. */
2039	return vma_end_pgoff(a) == vma_start_pgoff(b);	2076	if (vma_end_pgoff(a) != vma_start_pgoff(b))
		2077	return false;
		2078	/* Only reached from anon path, so either MAP_PRIVATE file or anon. */
		2079	if (vma_end_anon_pgoff(a) != vma_start_anon_pgoff(b))
		2080	return false;
		2081	return true;
2040	}	2082	}
2041		2083
2042	/*	2084	/*
@@ -2612,7 +2654,7 @@ static int __mmap_new_vma(struct mmap_state map, struct vm_area_struct *vmap, · 569 unmodified lines
2612	if (is_anon)	2654	if (is_anon)
2613	vma_set_anonymous(vma);	2655	vma_set_anonymous(vma);
2614		2656
2615	vma_set_range(vma, map->addr, map->end, map->pgoff);	2657	vma_set_range(vma, map->addr, map->end, map->pgoff, map->anon_pgoff);
2616	vma->flags = map->vma_flags;	2658	vma->flags = map->vma_flags;
2617	vma->vm_page_prot = map->page_prot;	2659	vma->vm_page_prot = map->page_prot;
2618		2660
@@ -2801,7 +2843,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, · 182 unmodified lines
2801	struct vm_area_struct *vma = NULL;	2843	struct vm_area_struct *vma = NULL;
2802	bool have_mmap_prepare = file && file->f_op->mmap_prepare;	2844	bool have_mmap_prepare = file && file->f_op->mmap_prepare;
2803	VMA_ITERATOR(vmi, mm, addr);	2845	VMA_ITERATOR(vmi, mm, addr);
2804	MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file);	2846	const pgoff_t anon_pgoff = addr >> PAGE_SHIFT;
		2847	MMAP_STATE(map, mm, &vmi, addr, len, pgoff, anon_pgoff, vma_flags, file);
2805	struct vm_area_desc desc = {	2848	struct vm_area_desc desc = {
2806	.mm = mm,	2849	.mm = mm,
2807	.file = file,	2850	.file = file,
@@ -2946,6 +2989,7 @@ int do_brk_flags(struct vma_iterator vmi, struct vm_area_struct vma, · 138 unmodified lines
2946	unsigned long addr, unsigned long len, vma_flags_t vma_flags)	2989	unsigned long addr, unsigned long len, vma_flags_t vma_flags)
2947	{	2990	{
2948	struct mm_struct *mm = current->mm;	2991	struct mm_struct *mm = current->mm;
		2992	const pgoff_t pgoff = addr >> PAGE_SHIFT;
2949		2993
2950	/*	2994	/*
2951	* Check against address space limits by the changed size	2995	* Check against address space limits by the changed size
@@ -2970,7 +3014,7 @@ int do_brk_flags(struct vma_iterator vmi, struct vm_area_struct vma, · 18 unmodified lines
2970	* occur after forking, so the expand will only happen on new VMAs.	3014	* occur after forking, so the expand will only happen on new VMAs.
2971	*/	3015	*/
2972	if (vma && vma->vm_end == addr) {	3016	if (vma && vma->vm_end == addr) {
2973	VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr));	3017	VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, pgoff, pgoff);
2974		3018
2975	vmg.prev = vma;	3019	vmg.prev = vma;
2976	/* vmi is positioned at prev, which this mode expects. */	3020	/* vmi is positioned at prev, which this mode expects. */
@@ -2990,7 +3034,7 @@ int do_brk_flags(struct vma_iterator vmi, struct vm_area_struct vma, · 13 unmodified lines
2990	goto unacct_fail;	3034	goto unacct_fail;
2991		3035
2992	vma_set_anonymous(vma);	3036	vma_set_anonymous(vma);
2993	vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT);	3037	vma_set_range(vma, addr, addr + len, pgoff, pgoff);
2994	vma->flags = vma_flags;	3038	vma->flags = vma_flags;
2995	vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags));	3039	vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags));
2996	vma_start_write(vma);	3040	vma_start_write(vma);
@@ -3382,6 +3426,7 @@ int insert_vm_struct(struct mm_struct mm, struct vm_area_struct vma) · 385 unmodified lines
3382	WARN_ON_ONCE(vma->anon_vma);	3426	WARN_ON_ONCE(vma->anon_vma);
3383	vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT);	3427	vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT);
3384	}	3428	}
		3429	vma_set_anon_pgoff(vma, vma->vm_start >> PAGE_SHIFT);
3385		3430
3386	if (vma_link(mm, vma)) {	3431	if (vma_link(mm, vma)) {
3387	if (vma_test(vma, VMA_ACCOUNT_BIT))	3432	if (vma_test(vma, VMA_ACCOUNT_BIT))
@@ -3437,7 +3482,7 @@ struct vm_area_struct *__install_special_mapping( · 49 unmodified lines
3437		3482
3438	vma->vm_ops = ops;	3483	vma->vm_ops = ops;
3439	vma->vm_private_data = priv;	3484	vma->vm_private_data = priv;
3440	vma_set_range(vma, addr, addr + len, 0);	3485	vma_set_range(vma, addr, addr + len, 0, addr >> PAGE_SHIFT);
3441		3486
3442	ret = insert_vm_struct(mm, vma);	3487	ret = insert_vm_struct(mm, vma);
3443	if (ret)	3488	if (ret)

mm/vma.h+59−30

 	unsigned long start;
 	unsigned long end;
 	pgoff_t pgoff;
+	pgoff_t anon_pgoff;
 	union {
 		/* Temporary while VMA flags are being converted. */
 	return vmg->state == VMA_MERGE_ERROR_NOMEM;
 }
-static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg)
-{
-	return vmg->pgoff;
-}
 static inline pgoff_t vmg_pages(const struct vma_merge_struct *vmg)
 {
 	const unsigned long size = vmg->end - vmg->start;
 	return size >> PAGE_SHIFT;
 }
+static inline pgoff_t vmg_start_pgoff(const struct vma_merge_struct *vmg)
+{
+	return vmg->pgoff;
+}
 static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg)
 {
 	return vmg_start_pgoff(vmg) + vmg_pages(vmg);
 	vma->vm_pgoff = pgoff;
 }
+static inline pgoff_t vmg_start_anon_pgoff(const struct vma_merge_struct *vmg)
+{
+	return vmg->anon_pgoff;
+}
+static inline pgoff_t vmg_end_anon_pgoff(const struct vma_merge_struct *vmg)
+{
+	return vmg_start_anon_pgoff(vmg) + vmg_pages(vmg);
+}
+static inline void __vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
+{
+#ifdef CONFIG_64BIT
+	vma->__vm_anon_pgoff_hi = pgoff >> 32;
+#endif
+	vma->__vm_anon_pgoff_lo = pgoff & GENMASK(31, 0);
+}
+static inline void vma_set_anon_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
+{
+	vma_assert_can_modify(vma);
+	__vma_set_anon_pgoff(vma, pgoff);
+}
 static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta)
 {
 	vma_assert_can_modify(vma);
 	vma_set_pgoff(vma, vma_start_pgoff(vma) + delta);
+	vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) + delta);
 }
 static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta)
 {
 	vma_assert_can_modify(vma);
 	vma_set_pgoff(vma, vma_start_pgoff(vma) - delta);
-}
+	vma_set_anon_pgoff(vma, vma_start_anon_pgoff(vma) - delta);
+}
+#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_, anon_pgoff_) \
+	struct vma_merge_struct name = {					  \
+		.mm = mm_,							  \
+		.vmi = vmi_,							  \
+		.start = start_,						  \
+		.end = end_,							  \
+		.vma_flags = vma_flags_,					  \
+		.pgoff = pgoff_,						  \
+		.anon_pgoff = anon_pgoff_,					  \
+		.state = VMA_MERGE_START,					  \
+	}
-#define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_)	\
+#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_)		\
 	struct vma_merge_struct name = {				\
-		.mm = mm_,						\
+		.mm = vma_->vm_mm,					\
 		.vmi = vmi_,						\
+		.prev = prev_,						\
+		.middle = vma_,						\
+		.next = NULL,						\
 		.start = start_,					\
 		.end = end_,						\
-		.vma_flags = vma_flags_,				\
+		.vm_flags = vma_->vm_flags,				\
-		.pgoff = pgoff_,					\
+		.pgoff = linear_page_index(vma_, start_),		\
+		.anon_pgoff = __linear_anon_page_index(vma_, start_),	\
+		.file = vma_->vm_file,					\
+		.anon_vma = vma_->anon_vma,				\
+		.policy = vma_policy(vma_),				\
+		.uffd_ctx = vma_->vm_userfaultfd_ctx,			\
+		.anon_name = anon_vma_name(vma_),			\
 		.state = VMA_MERGE_START,				\
 	}
-#define VMG_VMA_STATE(name, vmi_, prev_, vma_, start_, end_)	\
-	struct vma_merge_struct name = {			\
-		.mm = vma_->vm_mm,				\
-		.vmi = vmi_,					\
-		.prev = prev_,					\
-		.middle = vma_,					\
-		.next = NULL,					\
-		.start = start_,				\
-		.end = end_,					\
-		.vm_flags = vma_->vm_flags,			\
-		.pgoff = linear_page_index(vma_, start_),	\
-		.file = vma_->vm_file,				\
-		.anon_vma = vma_->anon_vma,			\
-		.policy = vma_policy(vma_),			\
-		.uffd_ctx = vma_->vm_userfaultfd_ctx,		\
-		.anon_name = anon_vma_name(vma_),		\
-		.state = VMA_MERGE_START,			\
-	}
 #ifdef CONFIG_DEBUG_VM_MAPLE_TREE
 void validate_mm(struct mm_struct *mm);
 #else
 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 	unsigned long addr, unsigned long len, pgoff_t pgoff,
-	bool *need_rmap_locks);
+	pgoff_t anon_pgoff, bool *need_rmap_locks);
 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);

mm/vma_exec.c+1−1

@@ -41,7 +41,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) · 40 unmodified lines
41	unsigned long new_end = old_end - shift;	41	unsigned long new_end = old_end - shift;
42	VMA_ITERATOR(vmi, mm, new_start);	42	VMA_ITERATOR(vmi, mm, new_start);
43	VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS,	43	VMG_STATE(vmg, mm, &vmi, new_start, old_end, EMPTY_VMA_FLAGS,
44	vma_start_pgoff(vma));	44	vma_start_pgoff(vma), vma_start_anon_pgoff(vma));
45	struct vm_area_struct *next;	45	struct vm_area_struct *next;
46	struct mmu_gather tlb;	46	struct mmu_gather tlb;
47	PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length);	47	PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length);

mm/vma_init.c+1−0

@@ -51,6 +51,7 @@ static void vm_area_init_from(const struct vm_area_struct *src, · 50 unmodified lines
51	dest->vm_end = src->vm_end;	51	dest->vm_end = src->vm_end;
52	dest->anon_vma = src->anon_vma;	52	dest->anon_vma = src->anon_vma;
53	dest->vm_pgoff = vma_start_pgoff(src);	53	dest->vm_pgoff = vma_start_pgoff(src);
		54	__vma_set_anon_pgoff(dest, vma_start_anon_pgoff(src));
54	dest->vm_file = src->vm_file;	55	dest->vm_file = src->vm_file;
55	dest->vm_private_data = src->vm_private_data;	56	dest->vm_private_data = src->vm_private_data;
56	vm_flags_init(dest, src->vm_flags);	57	vm_flags_init(dest, src->vm_flags);

mm/vmalloc.c+34−37

 	/* Update the free vmap_area. */
 	ret = va_clip(root, head, va, nva_start_addr, size);
-	if (WARN_ON_ONCE(ret))
+	if (ret) {
+		WARN_ON_ONCE(ret != -ENOMEM);
 		return ret;
+	}
 	return nva_start_addr;
 }
 	/*
 	 * Preload this CPU with one extra vmap_area object. It is used
-	 * when fit type of free area is NE_FIT_TYPE. It guarantees that
+	 * when fit type of free area is NE_FIT_TYPE. It is best effort
-	 * a CPU that does an allocation is preloaded.
+	 * pre-loading. If it fails va_clip() may return -ENOMEM from its
-	 *
+	 * GFP_NOWAIT fallback.
-	 * We do it in non-atomic context, thus it allows us to use more
-	 * permissive allocation masks to be more stable under low memory
-	 * condition and high memory pressure.
 	 */
 	if (!this_cpu_read(ne_fit_preload_node))
 		va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
 static inline void set_area_direct_map(const struct vm_struct *area,
 				       int (*set_direct_map)(struct page *page))
 {
-	int i;
+	unsigned long i;
 	/* HUGE_VMALLOC passes small pages to set_direct_map */
 	for (i = 0; i < area->nr_pages; i++)
 	unsigned long start = ULONG_MAX, end = 0;
 	unsigned int page_order = vm_area_page_order(area);
 	int flush_dmap = 0;
-	int i;
+	unsigned long i;
 	/*
 	 * Find the start and end range of the direct mappings to make sure that
  * Caller is responsible for unmapping (vunmap_range) and KASAN
  * poisoning before calling this.
  */
-static void vm_area_free_pages(struct vm_struct *vm, unsigned int start_idx,
+static void vm_area_free_pages(struct vm_struct *vm, unsigned long start_idx,
-			       unsigned int end_idx)
+			       unsigned long end_idx)
 {
-	unsigned int i;
+	unsigned long i;
 	if (!(vm->flags & VM_MAP_PUT_PAGES)) {
 		for (i = start_idx; i < end_idx; i++)
 	return flags;
 }
-static inline unsigned int
+static inline unsigned long
 vm_area_alloc_pages(gfp_t gfp, int nid,
-		unsigned int order, unsigned int nr_pages, struct page **pages)
+		unsigned int order, unsigned long nr_pages, struct page **pages)
 {
-	unsigned int nr_allocated = 0;
+	unsigned long nr_allocated = 0;
-	unsigned int nr_remaining = nr_pages;
+	unsigned long nr_remaining = nr_pages;
 	unsigned int max_attempt_order = MAX_PAGE_ORDER;
 	struct page *page;
 	int i;
 	if (!order) {
 		while (nr_allocated < nr_pages) {
 			unsigned int nr, nr_pages_request;
-			int i;
+			unsigned long i;
 			/*
 			 * A maximum allowed request is hard-coded and is 100
 			 * long preemption off scenario in the bulk-allocator
 			 * so the range is [1:100].
 			 */
-			nr_pages_request = min(100U, nr_pages - nr_allocated);
+			nr_pages_request = min(100UL, nr_pages - nr_allocated);
 			/* memory allocation should consider mempolicy, we can't
 			 * wrongly use nearest node when nid == NUMA_NO_NODE,
 	unsigned long addr = (unsigned long)area->addr;
 	unsigned long size = get_vm_area_size(area);
 	unsigned long array_size;
-	unsigned int nr_small_pages = size >> PAGE_SHIFT;
+	unsigned long nr_small_pages = size >> PAGE_SHIFT;
 	unsigned int page_order;
 	unsigned int flags;
 	int ret;
-	array_size = (unsigned long)nr_small_pages * sizeof(struct page *);
+	array_size = nr_small_pages * sizeof(struct page *);
 	/* __GFP_NOFAIL and "noblock" flags are mutually exclusive. */
 	if (!gfpflags_allow_blocking(gfp_mask))
 	}
 	if (size <= old_size) {
-		unsigned int new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
+		unsigned long new_nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
 		/* Zero out "freed" memory, potentially for future realloc. */
 		if (want_init_on_free() || want_init_on_alloc(flags))
 		    !(vm->flags & (VM_FLUSH_RESET_PERMS | VM_USERMAP)) &&
 		    gfp_has_io_fs(flags)) {
 			unsigned long addr = (unsigned long)kasan_reset_tag(p);
-			unsigned int old_nr_pages = vm->nr_pages;
+			unsigned long old_nr_pages = vm->nr_pages;
 			/*
 			 * Use the node lock to synchronize with concurrent
 			spin_unlock(&vn->busy.lock);
 			/* Notify kmemleak of the reduced allocation size before unmapping. */
-			kmemleak_free_part(
+			kmemleak_free_part((void *)addr +
-				(void *)addr + ((unsigned long)new_nr_pages
+					   (new_nr_pages << PAGE_SHIFT),
-						<< PAGE_SHIFT),
+					   (old_nr_pages - new_nr_pages)
-				(unsigned long)(old_nr_pages - new_nr_pages)
+						<< PAGE_SHIFT);
-					<< PAGE_SHIFT);
-			vunmap_range(addr + ((unsigned long)new_nr_pages
+			vunmap_range(addr + (new_nr_pages << PAGE_SHIFT),
-					     << PAGE_SHIFT),
+				     addr + (old_nr_pages << PAGE_SHIFT));
-				     addr + ((unsigned long)old_nr_pages
-					     << PAGE_SHIFT));
 			vm_area_free_pages(vm, new_nr_pages, old_nr_pages);
 		}
 	struct vmap_area *va;
 	struct vmap_node *vn;
 	unsigned long addr;
-	unsigned int nr_pages;
+	unsigned long nr_pages;
 	addr = PAGE_ALIGN((unsigned long) object);
 	vn = addr_to_node(addr);
 	nr_pages = vm->nr_pages;
 	spin_unlock(&vn->busy.lock);
-	pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n",
+	pr_cont(" %lu-page vmalloc region starting at %#lx allocated at %pS\n",
 		nr_pages, addr, caller);
 	return true;
 static void show_numa_info(struct seq_file *m, struct vm_struct *v,
 				 unsigned int *counters)
 {
-	unsigned int nr;
 	unsigned int step = 1U << vm_area_page_order(v);
+	unsigned long i;
+	unsigned int nr;
 	if (!counters)
 		return;
 	memset(counters, 0, nr_node_ids * sizeof(unsigned int));
-	for (nr = 0; nr < v->nr_pages; nr += step)
+	for (i = 0; i < v->nr_pages; i += step)
-		counters[page_to_nid(v->pages[nr])] += step;
+		counters[page_to_nid(v->pages[i])] += step;
 	for_each_node_state(nr, N_HIGH_MEMORY)
 		if (counters[nr])
 			seq_printf(m, " N%u=%u", nr, counters[nr]);
 				seq_printf(m, " %pS", v->caller);
 			if (v->nr_pages)
-				seq_printf(m, " pages=%d", v->nr_pages);
+				seq_printf(m, " pages=%lu", v->nr_pages);
 			if (v->phys_addr)
 				seq_printf(m, " phys=%pa", &v->phys_addr);

mm/vmalloc.h+2−0

@@ -5,6 +5,8 @@ · 4 unmodified lines
5	#ifndef __MM_VMALLOC_H	5	#ifndef __MM_VMALLOC_H
6	#define __MM_VMALLOC_H	6	#define __MM_VMALLOC_H
7		7
		8	#include <linux/vmalloc.h>
		9
8	#ifdef CONFIG_MMU	10	#ifdef CONFIG_MMU
9	void __init vmalloc_init(void);	11	void __init vmalloc_init(void);
10	int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,	12	int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,

mm/vmscan.c+151−79

@@ -58,6 +58,7 @@ · 57 unmodified lines
58	#include <linux/random.h>	58	#include <linux/random.h>
59	#include <linux/mmu_notifier.h>	59	#include <linux/mmu_notifier.h>
60	#include <linux/parser.h>	60	#include <linux/parser.h>
		61	#include <linux/swap_ops.h>
61		62
62	#include <asm/tlbflush.h>	63	#include <asm/tlbflush.h>
63	#include <asm/div64.h>	64	#include <asm/div64.h>
@@ -199,6 +200,13 @@ struct scan_control { · 135 unmodified lines
199	*/	200	*/
200	int vm_swappiness = 60;	201	int vm_swappiness = 60;
201		202
		203	static int sc_swappiness(struct scan_control sc, struct mem_cgroup memcg)
		204	{
		205	if (sc->proactive && sc->proactive_swappiness)
		206	return *sc->proactive_swappiness;
		207	return mem_cgroup_swappiness(memcg);
		208	}
		209
202	#ifdef CONFIG_MEMCG	210	#ifdef CONFIG_MEMCG
203		211
204	/* Returns true for reclaim through cgroup limits or cgroup interfaces. */	212	/* Returns true for reclaim through cgroup limits or cgroup interfaces. */
@@ -239,13 +247,6 @@ static bool writeback_throttling_sane(struct scan_control *sc) · 34 unmodified lines
239	#endif	247	#endif
240	return false;	248	return false;
241	}	249	}
242
243	static int sc_swappiness(struct scan_control sc, struct mem_cgroup memcg)
244	{
245	if (sc->proactive && sc->proactive_swappiness)
246	return *sc->proactive_swappiness;
247	return mem_cgroup_swappiness(memcg);
248	}
249	#else	250	#else
250	static bool cgroup_reclaim(struct scan_control *sc)	251	static bool cgroup_reclaim(struct scan_control *sc)
251	{	252	{
@@ -261,12 +262,13 @@ static bool writeback_throttling_sane(struct scan_control *sc) · 9 unmodified lines
261	{	262	{
262	return true;	263	return true;
263	}	264	}
		265	#endif
264		266
265	static int sc_swappiness(struct scan_control sc, struct mem_cgroup memcg)	267	static inline bool is_exec_file_folio(const struct folio *folio,
		268	const vma_flags_t *vma_flags)
266	{	269	{
267	return READ_ONCE(vm_swappiness);	270	return vma_flags_test(vma_flags, VMA_EXEC_BIT) && folio_is_file_lru(folio);
268	}	271	}
269	#endif
270		272
271	static void set_task_reclaim_state(struct task_struct *task,	273	static void set_task_reclaim_state(struct task_struct *task,
272	struct reclaim_state *rs)	274	struct reclaim_state *rs)
@@ -616,8 +618,8 @@ typedef enum { · 343 unmodified lines
616	/*	618	/*
617	* pageout is called by shrink_folio_list() for each dirty folio.	619	* pageout is called by shrink_folio_list() for each dirty folio.
618	*/	620	*/
619	static pageout_t pageout(struct folio folio, struct address_space mapping,	621	static pageout_t pageout(struct swap_io_ctx ctx, struct address_space mapping,
620	struct swap_iocb *plug, struct list_head folio_list)	622	struct folio folio, struct list_head folio_list)
621	{	623	{
622	int res;	624	int res;
623		625
@@ -653,9 +655,9 @@ static pageout_t pageout(struct folio folio, struct address_space mapping, · 29 unmodified lines
653	* the split out folios get added back to folio_list.	655	* the split out folios get added back to folio_list.
654	*/	656	*/
655	if (shmem_mapping(mapping))	657	if (shmem_mapping(mapping))
656	res = shmem_writeout(folio, plug, folio_list);	658	res = shmem_writeout(ctx, folio, folio_list);
657	else	659	else
658	res = swap_writeout(folio, plug);	660	res = swap_writeout(ctx, folio);
659		661
660	if (res < 0)	662	if (res < 0)
661	handle_write_error(mapping, folio, res);	663	handle_write_error(mapping, folio, res);
@@ -669,7 +671,7 @@ static pageout_t pageout(struct folio folio, struct address_space mapping, · 7 unmodified lines
669	folio_clear_reclaim(folio);	671	folio_clear_reclaim(folio);
670		672
671	trace_mm_vmscan_write_folio(folio);	673	trace_mm_vmscan_write_folio(folio);
672	node_stat_add_folio(folio, NR_VMSCAN_WRITE);	674	lruvec_stat_mod_folio(folio, NR_VMSCAN_WRITE, folio_nr_pages(folio));
673	return PAGE_SUCCESS;	675	return PAGE_SUCCESS;
674	}	676	}
675		677
@@ -835,10 +837,16 @@ enum folio_references { · 159 unmodified lines
835	* with PG_active set. In contrast, the aging (page table walk) path uses	837	* with PG_active set. In contrast, the aging (page table walk) path uses
836	* folio_update_gen().	838	* folio_update_gen().
837	*/	839	*/
838	static bool lru_gen_set_refs(struct folio *folio)	840	static bool lru_gen_set_refs(struct folio folio, const vma_flags_t vma_flags)
839	{	841	{
840	/* see the comment on LRU_REFS_FLAGS */	842	/* see the comment on LRU_REFS_FLAGS */
841	if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {	843	if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
		844	/* Activate file-backed executable folios after first usage. */
		845	if (is_exec_file_folio(folio, vma_flags)) {
		846	set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
		847	return true;
		848	}
		849
842	set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));	850	set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
843	return false;	851	return false;
844	}	852	}
@@ -851,7 +859,7 @@ static bool lru_gen_set_refs(struct folio *folio) · 6 unmodified lines
851	return true;	859	return true;
852	}	860	}
853	#else	861	#else
854	static bool lru_gen_set_refs(struct folio *folio)	862	static bool lru_gen_set_refs(struct folio folio, const vma_flags_t vma_flags)
855	{	863	{
856	return false;	864	return false;
857	}	865	}
@@ -861,16 +869,16 @@ static enum folio_references folio_check_references(struct folio *folio, · 3 unmodified lines
861	struct scan_control *sc)	869	struct scan_control *sc)
862	{	870	{
863	int referenced_ptes, referenced_folio;	871	int referenced_ptes, referenced_folio;
864	vm_flags_t vm_flags;	872	vma_flags_t vma_flags;
865		873
866	referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,	874	referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
867	&vm_flags);	875	&vma_flags);
868		876
869	/*	877	/*
870	* The supposedly reclaimable folio was found to be in a VM_LOCKED vma.	878	* The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
871	* Let the folio, now marked Mlocked, be moved to the unevictable list.	879	* Let the folio, now marked Mlocked, be moved to the unevictable list.
872	*/	880	*/
873	if (vm_flags & VM_LOCKED)	881	if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT))
874	return FOLIOREF_ACTIVATE;	882	return FOLIOREF_ACTIVATE;
875		883
876	/*	884	/*
@@ -886,7 +894,7 @@ static enum folio_references folio_check_references(struct folio *folio, · 9 unmodified lines
886	if (!referenced_ptes)	894	if (!referenced_ptes)
887	return FOLIOREF_RECLAIM;	895	return FOLIOREF_RECLAIM;
888		896
889	return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP;	897	return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
890	}	898	}
891		899
892	referenced_folio = folio_test_clear_referenced(folio);	900	referenced_folio = folio_test_clear_referenced(folio);
@@ -914,7 +922,7 @@ static enum folio_references folio_check_references(struct folio *folio, · 21 unmodified lines
914	/*	922	/*
915	* Activate file-backed executable folios after first usage.	923	* Activate file-backed executable folios after first usage.
916	*/	924	*/
917	if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio))	925	if (is_exec_file_folio(folio, &vma_flags))
918	return FOLIOREF_ACTIVATE;	926	return FOLIOREF_ACTIVATE;
919		927
920	return FOLIOREF_KEEP;	928	return FOLIOREF_KEEP;
@@ -1033,16 +1041,15 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) · 112 unmodified lines
1033	{	1041	{
1034	if (gfp_mask & __GFP_FS)	1042	if (gfp_mask & __GFP_FS)
1035	return true;	1043	return true;
1036	if (!folio_test_swapcache(folio) \|\| !(gfp_mask & __GFP_IO))
1037	return false;
1038	/*	1044	/*
1039	* We can "enter_fs" for swap-cache with only __GFP_IO	1045	* We can "enter_fs" for swap-cache with only __GFP_IO unless backed by
1040	* providing this isn't SWP_FS_OPS.	1046	* a swapfile that requires GFP_NOFS I/O.
1041	* ->flags can be updated non-atomically,
1042	* but that will never affect SWP_FS_OPS, so the data_race
1043	* is safe.
1044	*/	1047	*/
1045	return !data_race(folio_swap_flags(folio) & SWP_FS_OPS);	1048	if (folio_test_swapcache(folio) && (gfp_mask & __GFP_IO) &&
		1049	!(__swap_entry_to_info(folio->swap)->ops->flags &
		1050	SWAP_OPS_F_REQUIRE_NOFS))
		1051	return true;
		1052	return false;
1046	}	1053	}
1047		1054
1048	/*	1055	/*
@@ -1059,7 +1066,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, · 10 unmodified lines
1059	unsigned int nr_reclaimed = 0, nr_demoted = 0;	1066	unsigned int nr_reclaimed = 0, nr_demoted = 0;
1060	unsigned int pgactivate = 0;	1067	unsigned int pgactivate = 0;
1061	bool do_demote_pass;	1068	bool do_demote_pass;
1062	struct swap_iocb *plug = NULL;	1069	struct swap_io_ctx ctx = {};
1063		1070
1064	folio_batch_init(&free_folios);	1071	folio_batch_init(&free_folios);
1065	memset(stat, 0, sizeof(*stat));	1072	memset(stat, 0, sizeof(*stat));
@@ -1387,7 +1394,7 @@ retry: · 321 unmodified lines
1387	* starts and then write it out here.	1394	* starts and then write it out here.
1388	*/	1395	*/
1389	try_to_unmap_flush_dirty();	1396	try_to_unmap_flush_dirty();
1390	switch (pageout(folio, mapping, &plug, folio_list)) {	1397	switch (pageout(&ctx, mapping, folio, folio_list)) {
1391	case PAGE_KEEP:	1398	case PAGE_KEEP:
1392	goto keep_locked;	1399	goto keep_locked;
1393	case PAGE_ACTIVATE:	1400	case PAGE_ACTIVATE:
@@ -1406,8 +1413,6 @@ retry: · 12 unmodified lines
1406	sc->nr_scanned -= (nr_pages - 1);	1413	sc->nr_scanned -= (nr_pages - 1);
1407	nr_pages = 1;	1414	nr_pages = 1;
1408	}	1415	}
1409	stat->nr_pageout += nr_pages;
1410
1411	if (folio_test_writeback(folio))	1416	if (folio_test_writeback(folio))
1412	goto keep;	1417	goto keep;
1413	if (folio_test_dirty(folio))	1418	if (folio_test_dirty(folio))
@@ -1577,8 +1582,7 @@ keep: · 163 unmodified lines
1577	list_splice(&ret_folios, folio_list);	1582	list_splice(&ret_folios, folio_list);
1578	count_vm_events(PGACTIVATE, pgactivate);	1583	count_vm_events(PGACTIVATE, pgactivate);
1579		1584
1580	if (plug)	1585	swap_write_submit(&ctx);
1581	swap_write_unplug(plug);
1582	return nr_reclaimed;	1586	return nr_reclaimed;
1583	}	1587	}
1584		1588
@@ -2031,10 +2035,10 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, · 446 unmodified lines
2031	item = PGSTEAL_KSWAPD + reclaimer_offset(sc);	2035	item = PGSTEAL_KSWAPD + reclaimer_offset(sc);
2032	mod_lruvec_state(lruvec, item, nr_reclaimed);	2036	mod_lruvec_state(lruvec, item, nr_reclaimed);
2033	mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);	2037	mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);
		2038	if (nr_scanned > nr_reclaimed)
		2039	mod_lruvec_state(lruvec, PGROTATE_ANON + file,
		2040	nr_scanned - nr_reclaimed);
2034		2041
2035	lruvec_lock_irq(lruvec);
2036	lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout,
2037	nr_scanned - nr_reclaimed);
2038	handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);	2042	handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
2039	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,	2043	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2040	nr_scanned, nr_reclaimed, &stat, sc->priority, file);	2044	nr_scanned, nr_reclaimed, &stat, sc->priority, file);
@@ -2065,7 +2069,7 @@ static void shrink_active_list(unsigned long nr_to_scan, · 24 unmodified lines
2065	{	2069	{
2066	unsigned long nr_taken;	2070	unsigned long nr_taken;
2067	unsigned long nr_scanned;	2071	unsigned long nr_scanned;
2068	vm_flags_t vm_flags;	2072	vma_flags_t vma_flags;
2069	LIST_HEAD(l_hold); /* The folios which were snipped off */	2073	LIST_HEAD(l_hold); /* The folios which were snipped off */
2070	LIST_HEAD(l_active);	2074	LIST_HEAD(l_active);
2071	LIST_HEAD(l_inactive);	2075	LIST_HEAD(l_inactive);
@@ -2109,7 +2113,7 @@ static void shrink_active_list(unsigned long nr_to_scan, · 37 unmodified lines
2109		2113
2110	/* Referenced or rmap lock contention: rotate */	2114	/* Referenced or rmap lock contention: rotate */
2111	if (folio_referenced(folio, 0, sc->target_mem_cgroup,	2115	if (folio_referenced(folio, 0, sc->target_mem_cgroup,
2112	&vm_flags) != 0) {	2116	&vma_flags) != 0) {
2113	/*	2117	/*
2114	* Identify referenced, file-backed active folios and	2118	* Identify referenced, file-backed active folios and
2115	* give them one more trip around the active list. So	2119	* give them one more trip around the active list. So
@@ -2119,7 +2123,7 @@ static void shrink_active_list(unsigned long nr_to_scan, · 3 unmodified lines
2119	* IO, plus JVM can create lots of anon VM_EXEC folios,	2123	* IO, plus JVM can create lots of anon VM_EXEC folios,
2120	* so we ignore them here.	2124	* so we ignore them here.
2121	*/	2125	*/
2122	if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) {	2126	if (is_exec_file_folio(folio, &vma_flags)) {
2123	nr_rotated += folio_nr_pages(folio);	2127	nr_rotated += folio_nr_pages(folio);
2124	list_add(&folio->lru, &l_active);	2128	list_add(&folio->lru, &l_active);
2125	continue;	2129	continue;
@@ -2140,9 +2144,9 @@ static void shrink_active_list(unsigned long nr_to_scan, · 14 unmodified lines
2140	count_vm_events(PGDEACTIVATE, nr_deactivate);	2144	count_vm_events(PGDEACTIVATE, nr_deactivate);
2141	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);	2145	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
2142	mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);	2146	mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
		2147	if (nr_rotated)
		2148	mod_lruvec_state(lruvec, PGROTATE_ANON + file, nr_rotated);
2143		2149
2144	lruvec_lock_irq(lruvec);
2145	lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated);
2146	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,	2150	trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2147	nr_deactivate, nr_rotated, sc->priority, file);	2151	nr_deactivate, nr_rotated, sc->priority, file);
2148	}	2152	}
@@ -2275,8 +2279,10 @@ enum scan_balance { · 126 unmodified lines
2275		2279
2276	static void prepare_scan_control(pg_data_t pgdat, struct scan_control sc)	2280	static void prepare_scan_control(pg_data_t pgdat, struct scan_control sc)
2277	{	2281	{
2278	unsigned long file;	2282	struct lru_cost anon_cost, file_cost;
2279	struct lruvec *target_lruvec;	2283	struct lruvec *target_lruvec;
		2284	unsigned long lrusize;
		2285	unsigned long file;
2280		2286
2281	if (lru_gen_enabled() && !lru_gen_switching())	2287	if (lru_gen_enabled() && !lru_gen_switching())
2282	return;	2288	return;
@@ -2292,11 +2298,69 @@ static void prepare_scan_control(pg_data_t pgdat, struct scan_control sc) · 9 unmodified lines
2292		2298
2293	/*	2299	/*
2294	* Determine the scan balance between anon and file LRUs.	2300	* Determine the scan balance between anon and file LRUs.
		2301	*
		2302	* The cost model is based on rotations, refaults and
		2303	* reclaim-driven writes (anon only) on each side.
		2304	*
		2305	* These event counters are monotonic, so each reclaim cycle
		2306	* the delta since the last scan is extracted and incorporated
		2307	* into a decaying average. This ensures currency, as workloads
		2308	* change over time, and avoids overflow in the calculations.
		2309	*
		2310	* Use lruvec_page_state_monotonic() so unsigned subtraction
		2311	* yields the correct delta across a signed-long wraparound of
		2312	* the underlying counter (a real hazard on 32-bit that the
		2313	* clamp in lruvec_page_state() would otherwise turn into a huge
		2314	* spurious delta).
2295	*/	2315	*/
2296	spin_lock_irq(&target_lruvec->lru_lock);	2316	spin_lock(&target_lruvec->cost_lock);
2297	sc->anon_cost = target_lruvec->anon_cost;	2317
2298	sc->file_cost = target_lruvec->file_cost;	2318	for (int f = 0; f <= 1; f++) {
2299	spin_unlock_irq(&target_lruvec->lru_lock);	2319	struct lru_cost *cost = &target_lruvec->cost[f];
		2320	unsigned long rotated, io, nr_rotated, nr_io;
		2321
		2322	rotated = lruvec_page_state_monotonic(target_lruvec,
		2323	PGROTATE_ANON + f);
		2324	io = lruvec_page_state_monotonic(target_lruvec,
		2325	WORKINGSET_RESTORE_BASE + f);
		2326	if (f == WORKINGSET_ANON)
		2327	io += lruvec_page_state_monotonic(target_lruvec,
		2328	NR_VMSCAN_WRITE);
		2329
		2330	nr_rotated = rotated - cost->last_rotated;
		2331	nr_io = io - cost->last_io;
		2332
		2333	/*
		2334	* Reflect the relative cost of incurring IO and spending
		2335	* CPU time on rotations. This doesn't attempt to make a
		2336	* precise comparison, it just says: if reloads are about
		2337	* comparable between the LRU lists, or rotations are
		2338	* overwhelmingly different between them, adjust scan
		2339	* balance for CPU work.
		2340	*/
		2341	cost->count += nr_io * SWAP_CLUSTER_MAX + nr_rotated;
		2342
		2343	cost->last_rotated = rotated;
		2344	cost->last_io = io;
		2345	}
		2346
		2347	anon_cost = &target_lruvec->cost[WORKINGSET_ANON];
		2348	file_cost = &target_lruvec->cost[WORKINGSET_FILE];
		2349
		2350	lrusize = lruvec_page_state(target_lruvec, NR_INACTIVE_ANON) +
		2351	lruvec_page_state(target_lruvec, NR_ACTIVE_ANON) +
		2352	lruvec_page_state(target_lruvec, NR_INACTIVE_FILE) +
		2353	lruvec_page_state(target_lruvec, NR_ACTIVE_FILE);
		2354
		2355	while (anon_cost->count + file_cost->count > lrusize / 4) {
		2356	anon_cost->count /= 2;
		2357	file_cost->count /= 2;
		2358	}
		2359
		2360	sc->anon_cost = anon_cost->count;
		2361	sc->file_cost = file_cost->count;
		2362
		2363	spin_unlock(&target_lruvec->cost_lock);
2300		2364
2301	/*	2365	/*
2302	* Target desirable inactive:active list ratios for the anon	2366	* Target desirable inactive:active list ratios for the anon
@@ -3202,14 +3266,19 @@ static bool positive_ctrl_err(struct ctrl_pos sp, struct ctrl_pos pv) · 899 unmodified lines
3202	******************************************************************************/	3266	******************************************************************************/
3203		3267
3204	/* promote pages accessed through page tables */	3268	/* promote pages accessed through page tables */
3205	static int folio_update_gen(struct folio *folio, int gen)	3269	static int folio_update_gen(struct folio folio, int gen, const vma_flags_t vma_flags)
3206	{	3270	{
3207	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);	3271	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
3208		3272
3209	VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);	3273	VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
3210		3274
3211	/* see the comment on LRU_REFS_FLAGS */	3275	/*
3212	if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {	3276	* See the comment on LRU_REFS_FLAGS, and activate file-backed
		3277	* executable folios after first usage to avoid typical IO
		3278	* thrashing from reclaiming.
		3279	*/
		3280	if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
		3281	!is_exec_file_folio(folio, vma_flags)) {
3213	set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));	3282	set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
3214	return -1;	3283	return -1;
3215	}	3284	}
@@ -3442,8 +3511,8 @@ static bool suitable_to_scan(int total, int young) · 226 unmodified lines
3442	return young * n >= total;	3511	return young * n >= total;
3443	}	3512	}
3444		3513
3445	static void walk_update_folio(struct lru_gen_mm_walk walk, struct folio folio,	3514	static void walk_update_folio(struct lru_gen_mm_walk walk, struct vm_area_struct vma,
3446	int new_gen, bool dirty)	3515	struct folio *folio, int new_gen, bool dirty)
3447	{	3516	{
3448	int old_gen;	3517	int old_gen;
3449		3518
@@ -3456,10 +3525,10 @@ static void walk_update_folio(struct lru_gen_mm_walk walk, struct folio folio, · 6 unmodified lines
3456	folio_mark_dirty(folio);	3525	folio_mark_dirty(folio);
3457		3526
3458	if (walk) {	3527	if (walk) {
3459	old_gen = folio_update_gen(folio, new_gen);	3528	old_gen = folio_update_gen(folio, new_gen, &vma->flags);
3460	if (old_gen >= 0 && old_gen != new_gen)	3529	if (old_gen >= 0 && old_gen != new_gen)
3461	update_batch_size(walk, folio, old_gen, new_gen);	3530	update_batch_size(walk, folio, old_gen, new_gen);
3462	} else if (lru_gen_set_refs(folio)) {	3531	} else if (lru_gen_set_refs(folio, &vma->flags)) {
3463	old_gen = folio_lru_gen(folio);	3532	old_gen = folio_lru_gen(folio);
3464	if (old_gen >= 0 && old_gen != new_gen)	3533	if (old_gen >= 0 && old_gen != new_gen)
3465	folio_activate(folio);	3534	folio_activate(folio);
@@ -3532,7 +3601,7 @@ restart: · 66 unmodified lines
3532	continue;	3601	continue;
3533		3602
3534	if (last != folio) {	3603	if (last != folio) {
3535	walk_update_folio(walk, last, gen, dirty);	3604	walk_update_folio(walk, args->vma, last, gen, dirty);
3536		3605
3537	last = folio;	3606	last = folio;
3538	dirty = false;	3607	dirty = false;
@@ -3545,7 +3614,7 @@ restart: · 6 unmodified lines
3545	walk->mm_stats[MM_LEAF_YOUNG] += nr;	3614	walk->mm_stats[MM_LEAF_YOUNG] += nr;
3546	}	3615	}
3547		3616
3548	walk_update_folio(walk, last, gen, dirty);	3617	walk_update_folio(walk, args->vma, last, gen, dirty);
3549	last = NULL;	3618	last = NULL;
3550		3619
3551	if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))	3620	if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
@@ -3623,7 +3692,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area · 71 unmodified lines
3623	goto next;	3692	goto next;
3624		3693
3625	if (last != folio) {	3694	if (last != folio) {
3626	walk_update_folio(walk, last, gen, dirty);	3695	walk_update_folio(walk, vma, last, gen, dirty);
3627		3696
3628	last = folio;	3697	last = folio;
3629	dirty = false;	3698	dirty = false;
@@ -3637,7 +3706,7 @@ next: · 7 unmodified lines
3637	i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;	3706	i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
3638	} while (i <= MIN_LRU_BATCH);	3707	} while (i <= MIN_LRU_BATCH);
3639		3708
3640	walk_update_folio(walk, last, gen, dirty);	3709	walk_update_folio(walk, vma, last, gen, dirty);
3641		3710
3642	lazy_mmu_mode_disable();	3711	lazy_mmu_mode_disable();
3643	spin_unlock(ptl);	3712	spin_unlock(ptl);
@@ -4192,7 +4261,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) · 548 unmodified lines
4192	unsigned long end;	4261	unsigned long end;
4193	struct lru_gen_mm_walk *walk;	4262	struct lru_gen_mm_walk *walk;
4194	struct folio *last = NULL;	4263	struct folio *last = NULL;
4195	int young = 1;	4264	int young = nr;
4196	pte_t *pte = pvmw->pte;	4265	pte_t *pte = pvmw->pte;
4197	unsigned long addr = pvmw->address;	4266	unsigned long addr = pvmw->address;
4198	struct vm_area_struct *vma = pvmw->vma;	4267	struct vm_area_struct *vma = pvmw->vma;
@@ -4272,7 +4341,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) · 73 unmodified lines
4272	continue;	4341	continue;
4273		4342
4274	if (last != folio) {	4343	if (last != folio) {
4275	walk_update_folio(walk, last, gen, dirty);	4344	walk_update_folio(walk, vma, last, gen, dirty);
4276		4345
4277	last = folio;	4346	last = folio;
4278	dirty = false;	4347	dirty = false;
@@ -4284,7 +4353,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) · 5 unmodified lines
4284	young += nr;	4353	young += nr;
4285	}	4354	}
4286		4355
4287	walk_update_folio(walk, last, gen, dirty);	4356	walk_update_folio(walk, vma, last, gen, dirty);
4288		4357
4289	lazy_mmu_mode_disable();	4358	lazy_mmu_mode_disable();
4290		4359
@@ -4566,7 +4635,12 @@ void lru_gen_reparent_memcg(struct mem_cgroup memcg, struct mem_cgroup parent, · 275 unmodified lines
4566	for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) {	4635	for_each_managed_zone_pgdat(zone, NODE_DATA(nid), zid, MAX_NR_ZONES - 1) {
4567	unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid);	4636	unsigned long size = mem_cgroup_get_zone_lru_size(child_lruvec, lru, zid);
4568		4637
		4638	if (!size)
		4639	continue;
		4640
		4641	/* Move the accounting, do not duplicate it. */
4569	mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size);	4642	mem_cgroup_update_lru_size(parent_lruvec, lru, zid, size);
		4643	mem_cgroup_update_lru_size(child_lruvec, lru, zid, -(long)size);
4570	}	4644	}
4571	}	4645	}
4572	}	4646	}
@@ -4580,7 +4654,6 @@ void lru_gen_reparent_memcg(struct mem_cgroup memcg, struct mem_cgroup parent, · 7 unmodified lines
4580	static bool sort_folio(struct lruvec lruvec, struct folio folio, struct scan_control *sc,	4654	static bool sort_folio(struct lruvec lruvec, struct folio folio, struct scan_control *sc,
4581	int tier_idx)	4655	int tier_idx)
4582	{	4656	{
4583	bool success;
4584	int gen = folio_lru_gen(folio);	4657	int gen = folio_lru_gen(folio);
4585	int type = folio_is_file_lru(folio);	4658	int type = folio_is_file_lru(folio);
4586	int zone = folio_zonenum(folio);	4659	int zone = folio_zonenum(folio);
@@ -4592,15 +4665,9 @@ static bool sort_folio(struct lruvec lruvec, struct folio folio, struct scan_c · 5 unmodified lines
4592		4665
4593	VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);	4666	VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
4594		4667
4595	/* unevictable */	4668	/* unevictable: let it through and the generic path will cull it */
4596	if (!folio_evictable(folio)) {	4669	if (!folio_evictable(folio))
4597	success = lru_gen_del_folio(lruvec, folio, true);	4670	return false;
4598	VM_WARN_ON_ONCE_FOLIO(!success, folio);
4599	folio_set_unevictable(folio);
4600	lruvec_add_folio(lruvec, folio);
4601	__count_vm_events(UNEVICTABLE_PGCULLED, delta);
4602	return true;
4603	}
4604		4671
4605	/* promoted */	4672	/* promoted */
4606	if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {	4673	if (gen != lru_gen_from_seq(lrugen->min_seq[type])) {
@@ -4816,7 +4883,8 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · 209 unmodified lines
4816	struct reclaim_stat stat;	4883	struct reclaim_stat stat;
4817	struct lru_gen_mm_walk *walk;	4884	struct lru_gen_mm_walk *walk;
4818	int scanned, reclaimed;	4885	int scanned, reclaimed;
4819	int isolated = 0, type, type_scanned;	4886	int isolated = 0, nr_isolated = 0, type, type_scanned;
		4887	unsigned long total_reclaimed = 0;
4820	bool skip_retry = false;	4888	bool skip_retry = false;
4821	struct mem_cgroup *memcg = lruvec_memcg(lruvec);	4889	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
4822	struct pglist_data *pgdat = lruvec_pgdat(lruvec);	4890	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
@@ -4828,6 +4896,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · 5 unmodified lines
4828		4896
4829	scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness,	4897	scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness,
4830	&list, &isolated, &type, &type_scanned);	4898	&list, &isolated, &type, &type_scanned);
		4899	nr_isolated = isolated;
4831		4900
4832	/* Scanning may have emptied the oldest gen, flush it */	4901	/* Scanning may have emptied the oldest gen, flush it */
4833	if (scanned)	4902	if (scanned)
@@ -4840,6 +4909,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, · 6 unmodified lines
4840	retry:	4909	retry:
4841	reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg);	4910	reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg);
4842	sc->nr_reclaimed += reclaimed;	4911	sc->nr_reclaimed += reclaimed;
		4912	total_reclaimed += reclaimed;
4843	/* Retry pass is only meant for clean folios without new isolation */	4913	/* Retry pass is only meant for clean folios without new isolation */
4844	if (isolated)	4914	if (isolated)
4845	handle_reclaim_writeback(isolated, pgdat, sc, &stat);	4915	handle_reclaim_writeback(isolated, pgdat, sc, &stat);
@@ -4850,11 +4920,9 @@ retry: · 4 unmodified lines
4850	list_for_each_entry_safe_reverse(folio, next, &list, lru) {	4920	list_for_each_entry_safe_reverse(folio, next, &list, lru) {
4851	DEFINE_MIN_SEQ(lruvec);	4921	DEFINE_MIN_SEQ(lruvec);
4852		4922
4853	if (!folio_evictable(folio)) {	4923	/* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */
4854	list_del(&folio->lru);	4924	if (!folio_evictable(folio))
4855	folio_putback_lru(folio);
4856	continue;	4925	continue;
4857	}
4858		4926
4859	/* retry folios that may have missed folio_rotate_reclaimable() */	4927	/* retry folios that may have missed folio_rotate_reclaimable() */
4860	if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&	4928	if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) &&
@@ -4891,6 +4959,10 @@ retry: · 30 unmodified lines
4891	goto retry;	4959	goto retry;
4892	}	4960	}
4893		4961
		4962	if (nr_isolated > total_reclaimed)
		4963	mod_lruvec_state(lruvec, PGROTATE_ANON + type,
		4964	nr_isolated - total_reclaimed);
		4965
4894	return scanned;	4966	return scanned;
4895	}	4967	}
4896		4968
@@ -7209,7 +7281,7 @@ restart: · 2312 unmodified lines
7209		7281
7210	/*	7282	/*
7211	* If the low watermark is met there is no need for processes	7283	* If the low watermark is met there is no need for processes
7212	* to be throttled on pfmemalloc_wait as they should not be	7284	* to be throttled on pfmemalloc_wait as they should now be
7213	* able to safely make forward progress. Wake them	7285	* able to safely make forward progress. Wake them
7214	*/	7286	*/
7215	if (waitqueue_active(&pgdat->pfmemalloc_wait) &&	7287	if (waitqueue_active(&pgdat->pfmemalloc_wait) &&

mm/vmstat.c+18−1

 	return node_page_state_pages(pgdat, item);
 }
+/*
+ * Non-clamping variant of node_page_state() intended for callers that
+ * snapshot a monotonically-incremented counter and subtract two samples.
+ * See global_node_page_state_monotonic() for the rationale.
+ */
+unsigned long node_page_state_monotonic(struct pglist_data *pgdat,
+					enum node_stat_item item)
+{
+	return (unsigned long)atomic_long_read(&pgdat->vm_stat[item]);
+}
 #endif
 /*
 	[I(PGSCAN_PROACTIVE)]			= "pgscan_proactive",
 	[I(PGSCAN_ANON)]			= "pgscan_anon",
 	[I(PGSCAN_FILE)]			= "pgscan_file",
+	[I(PGROTATE_ANON)]			= "pgrotate_anon",
+	[I(PGROTATE_FILE)]			= "pgrotate_file",
 	[I(PGREFILL)]				= "pgrefill",
 #ifdef CONFIG_HUGETLB_PAGE
 	[I(NR_HUGETLB)]				= "nr_hugetlb",
 #if THREAD_SIZE > 65536
 	[I(KSTACK_REST)]			= "kstack_rest",
 #endif
-#endif
+#endif /* CONFIG_DEBUG_STACK_USAGE */
+#ifdef CONFIG_SWAP
+	[I(NRSWPIN)]				= "nrswpin",
+	[I(NRSWPOUT)]				= "nrswpout",
+#endif /* CONFIG_SWAP */
 #undef I
 #endif /* CONFIG_VM_EVENT_COUNTERS */
 };

mm/workingset.c+0−5

 	/* Folio was active prior to eviction */
 	if (workingset) {
 		folio_set_workingset(folio);
-		/*
-		 * XXX: Move to folio_add_lru() when it supports new vs
-		 * putback
-		 */
-		lru_note_cost_refault(folio);
 		mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr);
 	}
 out:

mm/zsmalloc.c+7−4

 	return min_t(int, ZS_SIZE_CLASSES - 1, idx);
 }
+static struct size_class *lookup_size_class(struct zs_pool *pool, size_t size)
+{
+	return pool->size_class[get_size_class_index(size + ZS_HANDLE_SIZE)];
+}
 static inline void class_stat_add(struct size_class *class, int type,
 				  unsigned long cnt)
 {
 {
 	struct size_class *class;
-	class = pool->size_class[get_size_class_index(size)];
+	class = lookup_size_class(pool, size);
 	return class->index;
 }
 	if (!handle)
 		return (unsigned long)ERR_PTR(-ENOMEM);
-	/* extra space in chunk to keep the handle */
+	class = lookup_size_class(pool, size);
-	size += ZS_HANDLE_SIZE;
-	class = pool->size_class[get_size_class_index(size)];
 	/* class->lock effectively protects the zpage migration */
 	spin_lock(&class->lock);

mm/zswap.c+29−17

 #include <linux/spinlock.h>
 #include <linux/types.h>
 #include <linux/atomic.h>
-#include <linux/swap.h>
+#include <linux/swap_ops.h>
 #include <linux/crypto.h>
 #include <linux/scatterlist.h>
 #include <linux/mempolicy.h>
 	return page_to_nid(virt_to_page(entry));
 }
-static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry)
+static void zswap_lru_add(struct zswap_entry *entry)
 {
 	int nid = entry_to_nid(entry);
 	struct mem_cgroup *memcg;
 	rcu_read_lock();
 	memcg = mem_cgroup_from_entry(entry);
 	/* will always succeed */
-	list_lru_add(list_lru, &entry->lru, nid, memcg);
+	list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg);
 	rcu_read_unlock();
 }
-static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry)
+static void zswap_lru_del(struct zswap_entry *entry)
 {
 	int nid = entry_to_nid(entry);
 	struct mem_cgroup *memcg;
 	rcu_read_lock();
 	memcg = mem_cgroup_from_entry(entry);
 	/* will always succeed */
-	list_lru_del(list_lru, &entry->lru, nid, memcg);
+	list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg);
 	rcu_read_unlock();
 }
  */
 static void zswap_entry_free(struct zswap_entry *entry)
 {
-	zswap_lru_del(&zswap_list_lru, entry);
+	zswap_lru_del(entry);
 	zs_free(entry->pool->zs_pool, entry->handle);
 	zswap_pool_put(entry->pool);
 	if (entry->objcg) {
 	struct folio *folio;
 	struct mempolicy *mpol;
 	struct swap_info_struct *si;
+	struct swap_io_ctx ctx = {};
 	int ret = 0;
 	/* try to allocate swap cache folio */
 	folio_set_reclaim(folio);
 	/* start writeback */
-	__swap_writepage(folio, NULL);
+	__swap_writepage(&ctx, folio);
+	swap_write_submit(&ctx);
 out:
 	if (ret) {
 	 * Without memcg, use the zswap pool-wide metrics.
 	 */
 	if (!mem_cgroup_disabled()) {
-		mem_cgroup_flush_stats(memcg);
+		mem_cgroup_flush_stats_ratelimited(memcg);
 		nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT;
 		nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED);
 	} else {
 	return shrinker;
 }
+/*
+ * Scan up to SWAP_CLUSTER_MAX pages on each per-node zswap LRU of @memcg
+ * and write back the reclaimable ones.
+ *
+ * Return: 0 if at least one entry was written back, -EAGAIN if entries
+ * were scanned but none could be written back, or -ENOENT if @memcg has
+ * writeback disabled, is a zombie cgroup, or has empty zswap LRUs.
+ */
 static int shrink_memcg(struct mem_cgroup *memcg)
 {
 	int nid, shrunk = 0, scanned = 0;
 		return -ENOENT;
 	for_each_node_state(nid, N_NORMAL_MEMORY) {
-		unsigned long nr_to_walk = 1;
+		unsigned long nr_to_walk = SWAP_CLUSTER_MAX;
 		shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg,
 					    &shrink_memcg_cb, NULL, &nr_to_walk);
-		scanned += 1 - nr_to_walk;
+		scanned += SWAP_CLUSTER_MAX - nr_to_walk;
 	}
+	/* Nothing was scanned: every LRU under @memcg was empty. */
 	if (!scanned)
 		return -ENOENT;
 		} while (memcg && !mem_cgroup_tryget_online(memcg));
 		spin_unlock(&zswap_shrink_lock);
-		if (!memcg) {
+		/*
-			/*
+		 * A NULL memcg ends a full hierarchy pass (except when memcg is
-			 * Continue shrinking without incrementing failures if
+		 * disabled, where it is always NULL: fall through to the root LRU).
-			 * we found candidate memcgs in the last tree walk.
+		 * Count a failure only if the last pass found no candidates.
-			 */
+		 */
+		if (!memcg && !mem_cgroup_disabled()) {
 			if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
 				break;
 		 * and failures.
 		 */
 		if (ret == -ENOENT)
-			continue;
+			goto resched;
 		++attempts;
 		if (ret && ++failures == MAX_RECLAIM_RETRIES)
 	entry->referenced = true;
 	if (entry->length) {
 		INIT_LIST_HEAD(&entry->lru);
-		zswap_lru_add(&zswap_list_lru, entry);
+		zswap_lru_add(entry);
 	}
 	return true;

tools/testing/radix-tree/maple.c+2−2

@@ -35234,7 +35234,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) · 35233 unmodified lines
35234	mt_set_non_kernel(1);	35234	mt_set_non_kernel(1);
35235	/* Spanning store */	35235	/* Spanning store */
35236	mas_set_range(&mas, 1, 100);	35236	mas_set_range(&mas, 1, 100);
35237	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0);	35237	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0);
35238	allocated = mas_allocated(&mas);	35238	allocated = mas_allocated(&mas);
35239	height = mas_mt_height(&mas);	35239	height = mas_mt_height(&mas);
35240	MT_BUG_ON(mt, allocated != 0);	35240	MT_BUG_ON(mt, allocated != 0);
@@ -35257,7 +35257,7 @@ static noinline void __init check_prealloc(struct maple_tree *mt) · 16 unmodified lines
35257	MT_BUG_ON(mt, mas_allocated(&mas) != 0);	35257	MT_BUG_ON(mt, mas_allocated(&mas) != 0);
35258	mas_set_range(&mas, 0, 200);	35258	mas_set_range(&mas, 0, 200);
35259	mt_set_non_kernel(1);	35259	mt_set_non_kernel(1);
35260	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0);	35260	MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_NOWAIT) == 0);
35261	allocated = mas_allocated(&mas);	35261	allocated = mas_allocated(&mas);
35262	height = mas_mt_height(&mas);	35262	height = mas_mt_height(&mas);
35263	MT_BUG_ON(mt, allocated != 0);	35263	MT_BUG_ON(mt, allocated != 0);

tools/testing/selftests/Makefile+1−0

@@ -1,5 +1,6 @@
1	# SPDX-License-Identifier: GPL-2.0	1	# SPDX-License-Identifier: GPL-2.0
2	TARGETS += acct	2	TARGETS += acct
		3	TARGETS += alloc_tag
3	TARGETS += alsa	4	TARGETS += alsa
4	TARGETS += amd-pstate	5	TARGETS += amd-pstate
5	TARGETS += arm64	6	TARGETS += arm64

tools/testing/selftests/alloc_tag/Makefile+8−0

@@ -0,0 +1,8 @@
	1	# SPDX-License-Identifier: GPL-2.0
	2
	3	TEST_GEN_PROGS := allocinfo_ioctl_test
	4
	5	CFLAGS += -Wall
	6	CFLAGS += $(KHDR_INCLUDES)
	7
	8	include ../lib.mk

tools/testing/selftests/alloc_tag/allocinfo_ioctl_test.c+548−0

@@ -0,0 +1,548 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2
	3	/* kselftest for allocinfo ioctl
	4	* allocinfo ioctl retrieves allocinfo data through ioctl
	5	* Copyright (C) 2026 Google, Inc.
	6	*/
	7
	8	#include <errno.h>
	9	#include <fcntl.h>
	10	#include <stdio.h>
	11	#include <stdlib.h>
	12	#include <string.h>
	13	#include <stdbool.h>
	14	#include <unistd.h>
	15	#include <sys/ioctl.h>
	16	#include <linux/types.h>
	17	#include <linux/alloc_tag.h>
	18	#include "../kselftest.h"
	19
	20	#define MAX_LINE_LEN 512
	21	#define ALLOCINFO_PROC "/proc/allocinfo"
	22
	23	enum ioctl_ret {
	24	IOCTL_SUCCESS = 0,
	25	IOCTL_FAILURE = 1,
	26	IOCTL_INVALID_DATA = 2,
	27	};
	28
	29	#define VEC_MAX_ENTRIES 32
	30
	31	struct allocinfo_tag_data_vec {
	32	struct allocinfo_tag_data tag[VEC_MAX_ENTRIES];
	33	__u64 count;
	34	};
	35
	36	static inline int __allocinfo_get_content_id(int dev_fd, struct allocinfo_content_id *params)
	37	{
	38	return ioctl(dev_fd, ALLOCINFO_IOC_CONTENT_ID, params);
	39	}
	40
	41	static inline int __allocinfo_get_at(int dev_fd, struct allocinfo_get_at *params)
	42	{
	43	return ioctl(dev_fd, ALLOCINFO_IOC_GET_AT, params);
	44	}
	45
	46	static inline int __allocinfo_get_next(int dev_fd, struct allocinfo_tag_data *params)
	47	{
	48	return ioctl(dev_fd, ALLOCINFO_IOC_GET_NEXT, params);
	49	}
	50
	51	static bool match_entry(const struct allocinfo_tag_data *procfs_entry,
	52	const struct allocinfo_tag_data *tag_data,
	53	bool match_bytes, bool match_calls, bool match_lineno,
	54	bool match_function, bool match_filename)
	55	{
	56	if (match_bytes && tag_data->counter.bytes != procfs_entry->counter.bytes) {
	57	ksft_print_msg("size retrieved through ioctl does not match procfs\n");
	58	return false;
	59	}
	60
	61	if (match_calls && tag_data->counter.calls != procfs_entry->counter.calls) {
	62	ksft_print_msg("call count retrieved through ioctl does not match procfs\n");
	63	return false;
	64	}
	65
	66	if (match_lineno && tag_data->tag.lineno != procfs_entry->tag.lineno) {
	67	ksft_print_msg("lineno retrieved through ioctl does not match procfs\n");
	68	return false;
	69	}
	70
	71	if (match_function &&
	72	strncmp(tag_data->tag.function, procfs_entry->tag.function, ALLOCINFO_STR_SIZE)) {
	73	ksft_print_msg("function retrieved through ioctl does not match procfs\n");
	74	return false;
	75	}
	76
	77	if (match_filename &&
	78	strncmp(tag_data->tag.filename, procfs_entry->tag.filename, ALLOCINFO_STR_SIZE)) {
	79	ksft_print_msg("filename retrieved through ioctl does not match procfs\n");
	80	return false;
	81	}
	82	return true;
	83	}
	84
	85	static bool match_entries(const struct allocinfo_tag_data_vec *procfs_entries,
	86	const struct allocinfo_tag_data_vec *tags,
	87	bool match_bytes, bool match_calls, bool match_lineno,
	88	bool match_function, bool match_filename)
	89	{
	90	__u64 i;
	91
	92	if (procfs_entries->count != tags->count) {
	93	ksft_print_msg("Entry count mismatch. ioctl entries: %llu, proc entries: %llu\n",
	94	tags->count, procfs_entries->count);
	95	return false;
	96	}
	97	for (i = 0; i < procfs_entries->count; i++) {
	98	if (!match_entry(&procfs_entries->tag[i], &tags->tag[i],
	99	match_bytes, match_calls, match_lineno,
	100	match_function, match_filename)) {
	101	ksft_print_msg("%lluth entry does not match.\n", i);
	102	return false;
	103	}
	104	}
	105	return true;
	106	}
	107
	108	static const char allocinfo_str(const char str)
	109	{
	110	size_t len = strlen(str);
	111
	112	if (len >= ALLOCINFO_STR_SIZE)
	113	str += (len - ALLOCINFO_STR_SIZE) + 1;
	114	return str;
	115	}
	116
	117	static void allocinfo_copy_str(char dest, const char src)
	118	{
	119	strncpy(dest, allocinfo_str(src), ALLOCINFO_STR_SIZE - 1);
	120	dest[ALLOCINFO_STR_SIZE - 1] = '\0';
	121	}
	122
	123	static int get_filtered_procfs_entries(struct allocinfo_tag_data_vec *procfs_entries,
	124	const struct allocinfo_filter *filter)
	125	{
	126	FILE *fp = fopen(ALLOCINFO_PROC, "r");
	127	char line[MAX_LINE_LEN];
	128	int matches;
	129	struct allocinfo_tag_data procfs_entry;
	130
	131	if (!fp) {
	132	ksft_print_msg("Failed to open " ALLOCINFO_PROC " for reading\n");
	133	return 1;
	134	}
	135	memset(procfs_entries, 0, sizeof(*procfs_entries));
	136	while (fgets(line, sizeof(line), fp) && procfs_entries->count < VEC_MAX_ENTRIES) {
	137	char filename[MAX_LINE_LEN];
	138	char function[MAX_LINE_LEN];
	139
	140	memset(&procfs_entry, 0, sizeof(procfs_entry));
	141	matches = sscanf(line, "%llu %llu %[^:]:%llu func:%s",
	142	&procfs_entry.counter.bytes,
	143	&procfs_entry.counter.calls,
	144	filename,
	145	&procfs_entry.tag.lineno,
	146	function);
	147
	148	if (matches != 5)
	149	continue;
	150
	151	allocinfo_copy_str(procfs_entry.tag.filename, filename);
	152	allocinfo_copy_str(procfs_entry.tag.function, function);
	153
	154	if (filter->mask & ALLOCINFO_FILTER_MASK_FILENAME) {
	155	if (strncmp(procfs_entry.tag.filename,
	156	filter->fields.filename, ALLOCINFO_STR_SIZE))
	157	continue;
	158	}
	159	if (filter->mask & ALLOCINFO_FILTER_MASK_FUNCTION) {
	160	if (strncmp(procfs_entry.tag.function,
	161	filter->fields.function, ALLOCINFO_STR_SIZE))
	162	continue;
	163	}
	164	if (filter->mask & ALLOCINFO_FILTER_MASK_LINENO) {
	165	if (procfs_entry.tag.lineno != filter->fields.lineno)
	166	continue;
	167	}
	168	if (filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) {
	169	if (procfs_entry.counter.bytes < filter->min_size)
	170	continue;
	171	}
	172	if (filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) {
	173	if (procfs_entry.counter.bytes > filter->max_size)
	174	continue;
	175	}
	176
	177	memcpy(&procfs_entries->tag[procfs_entries->count++], &procfs_entry,
	178	sizeof(procfs_entry));
	179	}
	180	fclose(fp);
	181	return 0;
	182	}
	183
	184	static enum ioctl_ret get_filtered_ioctl_entries(struct allocinfo_tag_data_vec *tags,
	185	const struct allocinfo_filter *filter,
	186	__u64 start_pos)
	187	{
	188	int fd = open(ALLOCINFO_PROC, O_RDONLY);
	189
	190	if (fd < 0) {
	191	ksft_print_msg("Failed to open " ALLOCINFO_PROC " for IOCTL\n");
	192	return IOCTL_FAILURE;
	193	}
	194
	195	struct allocinfo_content_id start_cont_id, end_cont_id;
	196	struct allocinfo_get_at get_at_params;
	197	const int max_retries = 10;
	198	int retry_count = 0;
	199	int status;
	200
	201	/*
	202	* __allocinfo_get_content_id may return different values if a kernel module was loaded
	203	* between the two calls. If that happens, the data gathered cannot be considered consistent
	204	* and hence needs to be fetched again to avoid flakiness.
	205	*/
	206	do {
	207	if (__allocinfo_get_content_id(fd, &start_cont_id)) {
	208	ksft_print_msg("allocinfo_get_content_id failed\n");
	209	status = IOCTL_FAILURE;
	210	break;
	211	}
	212
	213	memset(tags, 0, sizeof(*tags));
	214	memset(&get_at_params, 0, sizeof(get_at_params));
	215	memcpy(&get_at_params.filter, filter, sizeof(*filter));
	216	get_at_params.pos = start_pos;
	217	if (__allocinfo_get_at(fd, &get_at_params)) {
	218	ksft_print_msg("allocinfo_get_at failed\n");
	219	status = IOCTL_FAILURE;
	220	break;
	221	}
	222	memcpy(&tags->tag[tags->count++], &get_at_params.data, sizeof(get_at_params.data));
	223
	224	while (tags->count < VEC_MAX_ENTRIES &&
	225	__allocinfo_get_next(fd, &tags->tag[tags->count]) == 0)
	226	tags->count++;
	227
	228	if (__allocinfo_get_content_id(fd, &end_cont_id)) {
	229	ksft_print_msg("allocinfo_get_content_id failed\n");
	230	status = IOCTL_FAILURE;
	231	break;
	232	}
	233
	234	if (start_cont_id.id == end_cont_id.id) {
	235	status = IOCTL_SUCCESS;
	236	} else {
	237	ksft_print_msg("allocinfo_get_content_id mismatch, retrying...\n");
	238	status = IOCTL_INVALID_DATA;
	239	}
	240	} while (status == IOCTL_INVALID_DATA && retry_count++ < max_retries);
	241
	242	close(fd);
	243	return status;
	244	}
	245
	246	static int run_filter_test(const struct allocinfo_filter *filter)
	247	{
	248	struct allocinfo_tag_data_vec tags = malloc(sizeof(tags));
	249	struct allocinfo_tag_data_vec procfs_entries = malloc(sizeof(procfs_entries));
	250	int ioctl_status;
	251	int ret = KSFT_PASS;
	252
	253	if (!tags \|\| !procfs_entries) {
	254	ksft_print_msg("Memory allocation failed.\n");
	255	ret = KSFT_FAIL;
	256	goto exit;
	257	}
	258
	259	if (get_filtered_procfs_entries(procfs_entries, filter)) {
	260	ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n");
	261	ret = KSFT_SKIP;
	262	goto exit;
	263	}
	264
	265	if (procfs_entries->count == 0) {
	266	ksft_print_msg("No entries found in " ALLOCINFO_PROC ", skipping test\n");
	267	ret = KSFT_SKIP;
	268	goto exit;
	269	}
	270
	271	ioctl_status = get_filtered_ioctl_entries(tags, filter, 0);
	272	if (ioctl_status == IOCTL_INVALID_DATA) {
	273	ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n");
	274	ret = KSFT_SKIP;
	275	goto exit;
	276	}
	277	if (ioctl_status == IOCTL_FAILURE) {
	278	ksft_print_msg("Error retrieving IOCTL entries.\n");
	279	ret = KSFT_FAIL;
	280	goto exit;
	281	}
	282
	283	if (!match_entries(procfs_entries, tags, false, false, true, true, true))
	284	ret = KSFT_FAIL;
	285
	286	exit:
	287	free(tags);
	288	free(procfs_entries);
	289	return ret;
	290	}
	291
	292	static int test_filename_filter(void)
	293	{
	294	struct allocinfo_filter filter;
	295	const char *target_filename = "mm/memory.c";
	296
	297	memset(&filter, 0, sizeof(filter));
	298	filter.mask \|= ALLOCINFO_FILTER_MASK_FILENAME;
	299	strncpy(filter.fields.filename, target_filename, ALLOCINFO_STR_SIZE);
	300
	301	return run_filter_test(&filter);
	302	}
	303
	304	static int test_function_filter(void)
	305	{
	306	struct allocinfo_filter filter;
	307	const char *target_function = "dup_mm";
	308
	309	memset(&filter, 0, sizeof(filter));
	310	filter.mask \|= ALLOCINFO_FILTER_MASK_FUNCTION;
	311	strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE);
	312
	313	return run_filter_test(&filter);
	314	}
	315
	316	static int test_size_filter(void)
	317	{
	318	int fd;
	319	struct allocinfo_tag_data_vec tags = malloc(sizeof(tags));
	320	struct allocinfo_tag_data_vec procfs_entries = malloc(sizeof(procfs_entries));
	321	struct allocinfo_filter filter;
	322	int ret = KSFT_PASS;
	323	__u64 target_size, i, pos;
	324	struct allocinfo_tag_data *found_tag = NULL;
	325	const char *target_function = "do_init_module";
	326	struct allocinfo_content_id start_cont_id, end_cont_id;
	327	int retry = 0;
	328	const int max_retries = 10;
	329
	330	if (!tags \|\| !procfs_entries) {
	331	ksft_print_msg("Memory allocation failed.\n");
	332	ret = KSFT_FAIL;
	333	goto freemem;
	334	}
	335
	336	fd = open(ALLOCINFO_PROC, O_RDONLY);
	337	if (fd < 0) {
	338	ksft_print_msg("Failed to open " ALLOCINFO_PROC ": %s\n", strerror(errno));
	339	ret = KSFT_SKIP;
	340	goto freemem;
	341	}
	342
	343	do {
	344	found_tag = NULL;
	345	pos = 0;
	346
	347	if (__allocinfo_get_content_id(fd, &start_cont_id)) {
	348	ksft_print_msg("allocinfo_get_content_id failed\n");
	349	ret = KSFT_FAIL;
	350	goto exit;
	351	}
	352
	353	memset(&filter, 0, sizeof(filter));
	354	filter.mask \|= ALLOCINFO_FILTER_MASK_FUNCTION;
	355	strncpy(filter.fields.function, target_function, ALLOCINFO_STR_SIZE);
	356
	357	if (get_filtered_procfs_entries(procfs_entries, &filter)) {
	358	ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n");
	359	ret = KSFT_SKIP;
	360	goto exit;
	361	}
	362
	363	if (procfs_entries->count == 0) {
	364	ksft_print_msg("Function %s not found in procfs\n", target_function);
	365	ret = KSFT_SKIP;
	366	goto exit;
	367	}
	368
	369	target_size = procfs_entries->tag[0].counter.bytes;
	370
	371	memset(&filter, 0, sizeof(filter));
	372	filter.mask \|= ALLOCINFO_FILTER_MASK_MIN_SIZE \| ALLOCINFO_FILTER_MASK_MAX_SIZE;
	373	filter.min_size = target_size;
	374	filter.max_size = target_size;
	375
	376	while (1) {
	377	struct allocinfo_get_at get_at_params;
	378
	379	memset(&get_at_params, 0, sizeof(get_at_params));
	380	memcpy(&get_at_params.filter, &filter, sizeof(filter));
	381	get_at_params.pos = pos;
	382
	383	if (__allocinfo_get_at(fd, &get_at_params))
	384	break;
	385
	386	tags->count = 0;
	387	memcpy(&tags->tag[tags->count++], &get_at_params.data,
	388	sizeof(get_at_params.data));
	389
	390	while (tags->count < VEC_MAX_ENTRIES &&
	391	__allocinfo_get_next(fd, &tags->tag[tags->count]) == 0)
	392	tags->count++;
	393
	394	for (i = 0; i < tags->count; i++) {
	395	if (strcmp(tags->tag[i].tag.function, target_function) == 0) {
	396	found_tag = &tags->tag[i];
	397	break;
	398	}
	399	}
	400
	401	if (found_tag \|\| tags->count < VEC_MAX_ENTRIES)
	402	break;
	403
	404	pos += tags->count;
	405	}
	406
	407	if (__allocinfo_get_content_id(fd, &end_cont_id)) {
	408	ksft_print_msg("allocinfo_get_content_id failed\n");
	409	ret = KSFT_FAIL;
	410	goto exit;
	411	}
	412
	413	if (start_cont_id.id == end_cont_id.id)
	414	break;
	415
	416	ksft_print_msg("Module load detected during size verification, retrying...\n");
	417	} while (retry++ < max_retries);
	418
	419	if (start_cont_id.id == end_cont_id.id && !found_tag) {
	420	ksft_print_msg("Entry with function %s not found in IOCTL results\n",
	421	target_function);
	422	ret = KSFT_FAIL;
	423	} else if (start_cont_id.id != end_cont_id.id) {
	424	ksft_print_msg("Failed to match content_ids for procfs and IOCTL, skipping...\n");
	425	ret = KSFT_SKIP;
	426	} else if (found_tag && found_tag->counter.bytes != target_size) {
	427	ksft_print_msg("IOCTL entry size %llu does not match target size %llu\n",
	428	found_tag->counter.bytes, target_size);
	429	ret = KSFT_FAIL;
	430	}
	431
	432	exit:
	433	close(fd);
	434	freemem:
	435	free(tags);
	436	free(procfs_entries);
	437	return ret;
	438	}
	439
	440	static int test_lineno_filter(void)
	441	{
	442	struct allocinfo_tag_data_vec tags = malloc(sizeof(tags));
	443	struct allocinfo_tag_data_vec procfs_entries = malloc(sizeof(procfs_entries));
	444	struct allocinfo_filter filter;
	445	enum ioctl_ret ioctl_status;
	446	int ret = KSFT_PASS;
	447	__u64 target_lineno, i;
	448	struct allocinfo_tag_data *target_tag;
	449	bool found = false;
	450
	451	if (!tags \|\| !procfs_entries) {
	452	ksft_print_msg("Memory allocation failed.\n");
	453	ret = KSFT_FAIL;
	454	goto exit;
	455	}
	456
	457	memset(&filter, 0, sizeof(filter));
	458
	459	if (get_filtered_procfs_entries(procfs_entries, &filter)) {
	460	ksft_print_msg("Error retrieving entries from " ALLOCINFO_PROC "\n");
	461	ret = KSFT_SKIP;
	462	goto exit;
	463	}
	464	if (procfs_entries->count == 0) {
	465	ksft_print_msg("Could not retrieve procfs entries\n");
	466	ret = KSFT_SKIP;
	467	goto exit;
	468	}
	469	/*
	470	* We depend on the procfs results to determine the line number for the filter before
	471	* making the ioctl query. Hence, we cannot reuse run_filter_test here.
	472	*/
	473	target_tag = &procfs_entries->tag[0];
	474	target_lineno = target_tag->tag.lineno;
	475
	476	filter.mask \|= ALLOCINFO_FILTER_MASK_LINENO;
	477	filter.fields.lineno = target_lineno;
	478
	479	ioctl_status = get_filtered_ioctl_entries(tags, &filter, 0);
	480	if (ioctl_status == IOCTL_INVALID_DATA) {
	481	ksft_print_msg("Trouble retrieving valid IOCTL entries, skipping.\n");
	482	ret = KSFT_SKIP;
	483	goto exit;
	484	}
	485	if (ioctl_status == IOCTL_FAILURE) {
	486	ksft_print_msg("Error retrieving IOCTL entries.\n");
	487	ret = KSFT_FAIL;
	488	goto exit;
	489	}
	490
	491	for (i = 0; i < tags->count; i++) {
	492	if (tags->tag[i].tag.lineno != target_lineno) {
	493	ksft_print_msg("IOCTL entry %llu has incorrect lineno %llu.\n",
	494	i, tags->tag[i].tag.lineno);
	495	ret = KSFT_FAIL;
	496	goto exit;
	497	}
	498
	499	if (strncmp(tags->tag[i].tag.function, target_tag->tag.function,
	500	ALLOCINFO_STR_SIZE) == 0 &&
	501	strncmp(tags->tag[i].tag.filename, target_tag->tag.filename,
	502	ALLOCINFO_STR_SIZE) == 0)
	503	found = true;
	504	}
	505
	506	if (!found) {
	507	ksft_print_msg("Original procfs entry not found in IOCTL lineno filter results.\n");
	508	ret = KSFT_FAIL;
	509	}
	510
	511	exit:
	512	free(tags);
	513	free(procfs_entries);
	514	return ret;
	515	}
	516
	517	int main(int argc, char *argv[])
	518	{
	519	int ret;
	520
	521	ksft_set_plan(4);
	522
	523	ret = test_filename_filter();
	524	if (ret == KSFT_SKIP)
	525	ksft_test_result_skip("Skipping test_filename_filter\n");
	526	else
	527	ksft_test_result(ret == KSFT_PASS, "test_filename_filter\n");
	528
	529	ret = test_function_filter();
	530	if (ret == KSFT_SKIP)
	531	ksft_test_result_skip("Skipping test_function_filter\n");
	532	else
	533	ksft_test_result(ret == KSFT_PASS, "test_function_filter\n");
	534
	535	ret = test_size_filter();
	536	if (ret == KSFT_SKIP)
	537	ksft_test_result_skip("Skipping test_size_filter\n");
	538	else
	539	ksft_test_result(ret == KSFT_PASS, "test_size_filter\n");
	540
	541	ret = test_lineno_filter();
	542	if (ret == KSFT_SKIP)
	543	ksft_test_result_skip("Skipping test_lineno_filter\n");
	544	else
	545	ksft_test_result(ret == KSFT_PASS, "test_lineno_filter\n");
	546
	547	ksft_finished();
	548	}

tools/testing/selftests/cgroup/lib/include/cgroup_util.h+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#endif	7	#endif
8		8
9	#define MB(x) (x << 20)	9	#define MB(x) (x << 20)
		10	#define GB(x) ((unsigned long long)(x) << 30)
10		11
11	#define NSEC_PER_USEC 1000L	12	#define NSEC_PER_USEC 1000L
12	#define USEC_PER_SEC 1000000L	13	#define USEC_PER_SEC 1000000L

tools/testing/selftests/cgroup/test_zswap.c+11−2

 #define PATH_ZSWAP "/sys/module/zswap"
 #define PATH_ZSWAP_ENABLED "/sys/module/zswap/parameters/enabled"
+#define PATH_ZSWAP_STORED_PAGES "/sys/kernel/debug/zswap/stored_pages"
 static int read_int(const char *path, size_t *value)
 {
 static int get_zswap_stored_pages(size_t *value)
 {
-	return read_int("/sys/kernel/debug/zswap/stored_pages", value);
+	return read_int(PATH_ZSWAP_STORED_PAGES, value);
 }
 static long get_cg_wb_count(const char *cg)
 	/* Read sys info and compute test values accordingly */
 	if (sysinfo(&sys_info) != 0)
 		return KSFT_FAIL;
-	if (sys_info.totalram > 5000000000)
+	if (sys_info.totalram > GB(4)) {
+		ksft_print_msg(
+			"requires less than 4GB total ram, sys_info.totalram: %.1fGB\n",
+			(double)sys_info.totalram / GB(1));
 		return KSFT_SKIP;
+	}
+	if (access(PATH_ZSWAP_STORED_PAGES, R_OK)) {
+		ksft_print_msg("debugfs not mounted at /sys/kernel/debug\n");
+		return KSFT_SKIP;
+	}
 	values = mmap(0, sizeof(struct no_kmem_bypass_child_args), PROT_READ |
 			PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 	if (values == MAP_FAILED)

tools/testing/selftests/mm/.gitignore+9−67

@@ -1,68 +1,10 @@
1	# SPDX-License-Identifier: GPL-2.0-only	1	# SPDX-License-Identifier: GPL-2.0-only
2	cow	2	*
3	hugepage-mmap	3	!/**/
4	hugepage-mremap	4	!*.c
5	hugepage-shm	5	!*.h
6	hugepage-vmemmap	6	!*.sh
7	hugetlb-mmap	7	!.gitignore
8	hugetlb-mremap	8	!Makefile
9	hugetlb-shm	9	!config
10	hugetlb-vmemmap	10	!settings
11	hugetlb-madvise
12	hugetlb-read-hwpoison
13	hugetlb-soft-offline
14	khugepaged
15	map_hugetlb
16	map_populate
17	thuge-gen
18	compaction_test
19	memory-failure
20	migration
21	mlock2-tests
22	mrelease_test
23	mremap_dontunmap
24	mremap_test
25	on-fault-limit
26	transhuge-stress
27	pagemap_ioctl
28	pfnmap
29	process_madv
30	.tmp
31	protection_keys
32	protection_keys_32
33	protection_keys_64
34	madv_populate
35	uffd-stress
36	uffd-unit-tests
37	uffd-wp-mremap
38	mlock-intersect-test
39	mlock-random-test
40	virtual_address_range
41	gup_test
42	va_128TBswitch
43	map_fixed_noreplace
44	write_to_hugetlbfs
45	hmm-tests
46	memfd_secret
47	soft-dirty
48	split_huge_page_test
49	ksm_tests
50	local_config.h
51	local_config.mk
52	ksm_functional_tests
53	mdwe_test
54	gup_longterm
55	mkdirty
56	va_high_addr_switch
57	hugetlb_fault_after_madv
58	hugetlb_madv_vs_map
59	mseal_test
60	droppable
61	hugetlb_dio
62	pkey_sighandler_tests_32
63	pkey_sighandler_tests_64
64	guard-regions
65	merge
66	prctl_thp_disable
67	rmap
68	folio_split_race_test

tools/testing/selftests/mm/Makefile+4−3

 # script so kunit knows to run it, and add it to the list below.
 # If you do not YOUR TESTS WILL NOT RUN IN THE CI.
-LOCAL_HDRS += $(selfdir)/mm/local_config.h $(top_srcdir)/mm/gup_test.h
+LOCAL_HDRS += $(selfdir)/mm/local_config.h_gen $(top_srcdir)/mm/gup_test.h
 LOCAL_HDRS += $(selfdir)/mm/mseal_helpers.h
 include local_config.mk
 TEST_PROGS += ksft_hmm.sh
 TEST_PROGS += ksft_hugetlb.sh
 TEST_PROGS += ksft_hugevm.sh
+TEST_PROGS += ksft_kmemleak_confirm.sh
 TEST_PROGS += ksft_kmemleak_dedup.sh
 TEST_PROGS += ksft_ksm.sh
 TEST_PROGS += ksft_ksm_numa.sh
 $(OUTPUT)/rmap: LDLIBS += -lnuma
-local_config.mk local_config.h: check_config.sh
+local_config.mk local_config.h_gen: check_config.sh
 	$(call msg,CHK,config,$@)
 	$(Q)CC="$(CC)" CFLAGS="$(CFLAGS)" ./check_config.sh
-EXTRA_CLEAN += local_config.mk local_config.h
+EXTRA_CLEAN += local_config.mk local_config.h_gen
 ifeq ($(IOURING_EXTRA_LIBS),)
 all: warn_missing_liburing

tools/testing/selftests/mm/check_config.sh+1−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	# Probe for libraries and create header files to record the results. Both C	4	# Probe for libraries and create header files to record the results. Both C
5	# header files and Makefile include fragments are created.	5	# header files and Makefile include fragments are created.
6		6
7	OUTPUT_H_FILE=local_config.h	7	OUTPUT_H_FILE=local_config.h_gen
8	OUTPUT_MKFILE=local_config.mk	8	OUTPUT_MKFILE=local_config.mk
9		9
10	tmpname=$(mktemp)	10	tmpname=$(mktemp)

tools/testing/selftests/mm/compaction_test.c+21−17

@@ -29,30 +29,34 @@ struct map_list { · 28 unmodified lines
29		29
30	int read_memory_info(unsigned long memfree, unsigned long hugepagesize)	30	int read_memory_info(unsigned long memfree, unsigned long hugepagesize)
31	{	31	{
32	char buffer[256] = {0};	32	char buffer[256];
33	char cmd = "cat /proc/meminfo \| grep -i memfree \| grep -o '[0-9]'";	33	int found = 0;
34	FILE *cmdfile = popen(cmd, "r");	34	FILE *file;
		35	int ret = -1;
35		36
36	if (!(fgets(buffer, sizeof(buffer), cmdfile))) {	37	file = fopen("/proc/meminfo", "r");
37	ksft_print_msg("Failed to read meminfo: %s\n", strerror(errno));	38	if (!file) {
		39	ksft_print_msg("Failed to open /proc/meminfo: %s\n",
		40	strerror(errno));
38	return -1;	41	return -1;
39	}	42	}
40		43
41	pclose(cmdfile);	44	while (fgets(buffer, sizeof(buffer), file) && found != 2) {
42		45	if (sscanf(buffer, "MemFree: %lu kB", memfree) == 1 \|\|
43	*memfree = atoll(buffer);	46	sscanf(buffer, "Hugepagesize: %lu kB", hugepagesize) == 1)
44	cmd = "cat /proc/meminfo \| grep -i hugepagesize \| grep -o '[0-9]*'";	47	found++;
45	cmdfile = popen(cmd, "r");
46
47	if (!(fgets(buffer, sizeof(buffer), cmdfile))) {
48	ksft_print_msg("Failed to read meminfo: %s\n", strerror(errno));
49	return -1;
50	}	48	}
51		49
52	pclose(cmdfile);	50	if (ferror(file))
53	*hugepagesize = atoll(buffer);	51	ksft_print_msg("Failed to read /proc/meminfo: %s\n",
		52	strerror(errno));
		53	else if (found != 2)
		54	ksft_print_msg("Failed to parse /proc/meminfo\n");
		55	else
		56	ret = 0;
54		57
55	return 0;	58	fclose(file);
		59	return ret;
56	}	60	}
57		61
58	int prereq(void)	62	int prereq(void)

tools/testing/selftests/mm/cow.c+8−3

 #include <sys/wait.h>
 #include <linux/memfd.h>
-#include "local_config.h"
+#include "local_config.h_gen"
 #ifdef LOCAL_CONFIG_HAVE_LIBURING
 #include <liburing.h>
 #endif /* LOCAL_CONFIG_HAVE_LIBURING */
 	/* File consists of a single page filled with zeroes. */
 	if (fallocate(fd, 0, 0, pagesize)) {
-		ksft_perror("fallocate() failed");
+		if (errno == EOPNOTSUPP) {
-		log_test_result(KSFT_FAIL);
+			ksft_print_msg("fallocate() not supported by filesystem\n");
+			log_test_result(KSFT_SKIP);
+		} else {
+			ksft_perror("fallocate() failed");
+			log_test_result(KSFT_FAIL);
+		}
 		goto close;
 	}

tools/testing/selftests/mm/folio_split_race_test.c+1−1

@@ -182,7 +182,7 @@ static uint64_t run_iteration(void) · 181 unmodified lines
182	for (i = 0; i < TOTAL_PAGES; i++)	182	for (i = 0; i < TOTAL_PAGES; i++)
183	fill_page(mmap_base, i);	183	fill_page(mmap_base, i);
184		184
185	if (!check_huge_shmem(mmap_base, NR_PMD_PAGE, pmd_pagesize))	185	if (!check_huge_shmem(mmap_base, FILE_SIZE, NR_PMD_PAGE, pmd_pagesize))
186	ksft_exit_fail_msg("No shmem THP is allocated\n");	186	ksft_exit_fail_msg("No shmem THP is allocated\n");
187		187
188	if (pthread_barrier_init(&ctl.barrier, NULL, NUM_READER_THREADS + 1) != 0)	188	if (pthread_barrier_init(&ctl.barrier, NULL, NUM_READER_THREADS + 1) != 0)

tools/testing/selftests/mm/guard-regions.c+7−3

 {
 	const unsigned long page_size = self->page_size;
 	char *ptr;
-	int i;
+	int i, ret;
 	if (variant->backing == ANON_BACKED)
 		SKIP(return, "Truncation test specific to file-backed");
 	}
 	/* Now hole punch the guarded region. */
-	ASSERT_EQ(madvise(&ptr[3 * page_size], 4 * page_size,
+	ret = madvise(&ptr[3 * page_size], 4 * page_size, MADV_REMOVE);
-			  MADV_REMOVE), 0);
+	if (ret == -1 && errno == EOPNOTSUPP) {
+		ASSERT_EQ(munmap(ptr, 10 * page_size), 0);
+		SKIP(return, "MADV_REMOVE not supported by filesystem");
+	}
+	ASSERT_EQ(ret, 0);
 	/* Ensure guard regions remain. */
 	for (i = 0; i < 10; i++) {

tools/testing/selftests/mm/gup_longterm.c+1−1

@@ -21,7 +21,7 @@ · 20 unmodified lines
21	#include <linux/magic.h>	21	#include <linux/magic.h>
22	#include <linux/memfd.h>	22	#include <linux/memfd.h>
23		23
24	#include "local_config.h"	24	#include "local_config.h_gen"
25	#ifdef LOCAL_CONFIG_HAVE_LIBURING	25	#ifdef LOCAL_CONFIG_HAVE_LIBURING
26	#include <liburing.h>	26	#include <liburing.h>
27	#endif /* LOCAL_CONFIG_HAVE_LIBURING */	27	#endif /* LOCAL_CONFIG_HAVE_LIBURING */

tools/testing/selftests/mm/khugepaged.c+155−49

 #define BASE_ADDR ((void *)(1UL << 30))
 static unsigned long hpage_pmd_size;
+static int hpage_pmd_order;
 static unsigned long page_size;
 static int hpage_pmd_nr;
 static int anon_order;
+static int collapse_order;
 #define PID_SMAPS "/proc/self/smaps"
 #define TEST_FILE "collapse_test_file"
 	void *(*setup_area)(int nr_hpages);
 	void (*cleanup_area)(void *p, unsigned long size);
 	void (*fault)(void *p, unsigned long start, unsigned long end);
-	bool (*check_huge)(void *addr, int nr_hpages);
+	bool (*check_huge)(void *addr, size_t len, int nr_hpages, unsigned long hpage_size);
 	const char *name;
 };
 };
 static struct collapse_context *khugepaged_context;
+static struct collapse_context *mthp_khugepaged_context;
 static struct collapse_context *madvise_context;
 struct file_info {
 	char *str, *end;
 	finfo.dir = dir;
-	stat(finfo.dir, &path_stat);
+	if (stat(finfo.dir, &path_stat))
+		ksft_exit_fail_perror("stat()");
 	if (!S_ISDIR(path_stat.st_mode))
 		ksft_exit_fail_msg("%s: Not a directory (%s)\n", __func__, finfo.dir);
 	if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE,
 		     major(path_stat.st_dev), minor(path_stat.st_dev))
 	    >= sizeof(path))
 		ksft_exit_fail_msg("%s: Pathname is too long\n", __func__);
-	if (read_file(path, buf, sizeof(buf)) < 0)
+	if (!read_file(path, buf, sizeof(buf)))
-		ksft_exit_fail_perror("read_file(read_num)");
+		ksft_exit_fail_perror("read_file(uevent)");
 	if (strstr(buf, "DEVTYPE=disk")) {
 		/* Found it */
 		if (snprintf(finfo.dev_queue_read_ahead_path,
 	ksft_print_msg("Allocate huge page...");
 	if (madvise_collapse_retry(p, hpage_pmd_size))
 		ksft_exit_fail_perror("madvise(MADV_COLLAPSE)");
-	if (!ops->check_huge(p, 1))
+	if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
 		ksft_exit_fail_perror("madvise(MADV_COLLAPSE)");
 	if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE))
 		ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)");
 	fill_memory(p, start, end);
 }
-static bool anon_check_huge(void *addr, int nr_hpages)
+static bool anon_check_huge(void *addr, size_t len, int nr_hpages,
+		unsigned long hpage_size)
 {
-	return check_huge_anon(addr, nr_hpages, hpage_pmd_size);
+	return check_huge_anon(addr, len, nr_hpages, hpage_size);
 }
 static void *file_setup_area_common(int nr_hpages, enum file_setup_ops setup)
 		ksft_exit_fail_perror("madvise(MADV_POPULATE_WRITE)");
 }
-static bool file_check_huge(void *addr, int nr_hpages)
+static bool file_check_huge(void *addr, size_t len, int nr_hpages,
+		unsigned long hpage_size)
 {
 	switch (finfo.type) {
 	case VMA_FILE:
-		return check_huge_file(addr, nr_hpages, hpage_pmd_size);
+		return check_huge_file(addr, len, nr_hpages, hpage_size);
 	case VMA_SHMEM:
-		return check_huge_shmem(addr, nr_hpages, hpage_pmd_size);
+		return check_huge_shmem(addr, len, nr_hpages, hpage_size);
 	default:
 		exit(EXIT_FAILURE);
 		return false;
 	close(finfo.fd);
 }
-static bool shmem_check_huge(void *addr, int nr_hpages)
+static bool shmem_check_huge(void *addr, size_t len, int nr_hpages,
+		unsigned long hpage_size)
 {
-	return check_huge_shmem(addr, nr_hpages, hpage_pmd_size);
+	return check_huge_shmem(addr, len, nr_hpages, hpage_size);
 }
 static struct mem_ops __anon_ops = {
 	ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size);
 	if (((bool)ret) == expect)
 		fail("Fail: Bad return value");
-	else if (!ops->check_huge(p, expect ? nr_hpages : 0))
+	else if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, expect ? nr_hpages : 0, hpage_pmd_size))
 		fail("Fail: check_huge()");
 	else
 		success("OK");
 			     struct mem_ops *ops, bool expect)
 {
 	/* Sanity check */
-	if (!ops->check_huge(p, 0))
+	if (!ops->check_huge(p, nr_hpages * hpage_pmd_size, 0, hpage_pmd_size))
 		ksft_exit_fail_msg("Unexpected huge page\n");
 	__madvise_collapse(msg, p, nr_hpages, ops, expect);
 }
 #define TICK 500000
-static bool wait_for_scan(const char *msg, char *p, int nr_hpages,
+static bool wait_for_scan(const char *msg, char *p, size_t len,
-			  struct mem_ops *ops)
+		int nr_hpages, int collap_order, struct mem_ops *ops)
 {
+	unsigned long hpage_size = page_size << collap_order;
 	int full_scans;
 	int timeout = 6; /* 3 seconds */
 	/* Sanity check */
-	if (!ops->check_huge(p, 0))
+	if (!ops->check_huge(p, len, 0, hpage_size))
 		ksft_exit_fail_msg("Unexpected huge page\n");
-	madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE);
+	madvise(p, len, MADV_HUGEPAGE);
 	/* Wait until the second full_scan completed */
 	full_scans = thp_read_num("khugepaged/full_scans") + 2;
 	ksft_print_msg("%s...", msg);
 	while (timeout--) {
-		if (ops->check_huge(p, nr_hpages))
+		if (ops->check_huge(p, len, nr_hpages, hpage_size))
 			break;
 		if (thp_read_num("khugepaged/full_scans") >= full_scans)
 			break;
 static void khugepaged_collapse(const char *msg, char *p, int nr_hpages,
 				struct mem_ops *ops, bool expect)
 {
+	size_t len = nr_hpages * hpage_pmd_size;
 	/*
 	 * read&write file collapse fails since khugepaged does not flush
 	 * the target dirty folios
 	if (!is_tmpfs(ops) && ops == &__read_write_file_write_ops)
 		expect = false;
-	if (wait_for_scan(msg, p, nr_hpages, ops)) {
+	if (wait_for_scan(msg, p, len, nr_hpages, hpage_pmd_order, ops)) {
 		if (expect)
 			fail("Timeout");
 		else
 	if (ops != &__anon_ops)
 		ops->fault(p, 0, nr_hpages * hpage_pmd_size);
-	if (ops->check_huge(p, expect ? nr_hpages : 0))
+	if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_pmd_size))
+		success("OK");
+	else
+		fail("Fail");
+}
+static void mthp_khugepaged_collapse(const char *msg, char *p, int nr_hpages,
+				struct mem_ops *ops, bool expect)
+{
+	unsigned long hpage_size = page_size << collapse_order;
+	struct thp_settings settings = *thp_current_settings();
+	/* mTHP collpase only allocates PMD sized memory */
+	size_t len = hpage_pmd_size;
+	/* Set mTHP setting for mTHP collapse */
+	if (ops == &__anon_ops) {
+		settings.thp_enabled = THP_NEVER;
+		settings.hugepages[collapse_order].enabled = THP_MADVISE;
+	}
+	thp_push_settings(&settings);
+	if (wait_for_scan(msg, p, len, nr_hpages, collapse_order, ops)) {
+		if (expect)
+			fail("Timeout");
+		else
+			success("OK");
+		/* Restore THP settings for mTHP collapse. */
+		thp_pop_settings();
+		return;
+	}
+	/*
+	 * For file and shmem memory, khugepaged only retracts pte entries after
+	 * putting the new hugepage in the page cache. The hugepage must be
+	 * subsequently refaulted to install the pmd mapping for the mm.
+	 */
+	if (ops != &__anon_ops)
+		ops->fault(p, 0, nr_hpages * hpage_size);
+	if (ops->check_huge(p, len, expect ? nr_hpages : 0, hpage_size))
 		success("OK");
 	else
 		fail("Fail");
+	/* Restore THP settings for mTHP collapse. */
+	thp_pop_settings();
 }
 static struct collapse_context __khugepaged_context = {
 	.name = "khugepaged",
 };
+static struct collapse_context __mthp_khugepaged_context = {
+	.collapse = &mthp_khugepaged_collapse,
+	.enforce_pte_scan_limits = true,
+	.name = "mthp_khugepaged",
+};
 static struct collapse_context __madvise_context = {
 	.collapse = &madvise_collapse,
 	.enforce_pte_scan_limits = false,
 	p = alloc_mapping(1);
 	*p = 1;
 	ksft_print_msg("Allocate huge page on fault...");
-	if (check_huge_anon(p, 1, hpage_pmd_size))
+	if (check_huge_anon(p, hpage_pmd_size, 1, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	madvise(p, page_size, MADV_DONTNEED);
 	ksft_print_msg("Split huge PMD on MADV_DONTNEED...");
-	if (check_huge_anon(p, 0, hpage_pmd_size))
+	if (check_huge_anon(p, hpage_pmd_size, 0, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 static void collapse_full(struct collapse_context *c, struct mem_ops *ops)
 {
 	void *p;
-	int nr_hpages = 4;
+	int nr_pmds = 4, nr_hpages = 4;
 	unsigned long size = nr_hpages * hpage_pmd_size;
-	p = ops->setup_area(nr_hpages);
+	/* Only try 1 PMD sized range for mTHP collapse. */
+	if (c == &__mthp_khugepaged_context) {
+		nr_pmds = 1;
+		nr_hpages = 1 << (hpage_pmd_order - collapse_order);
+		size = hpage_pmd_size;
+	}
+	p = ops->setup_area(nr_pmds);
 	ops->fault(p, 0, size);
 	c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages,
 		    ops, true);
 static void collapse_empty(struct collapse_context *c, struct mem_ops *ops)
 {
+	int nr_hpages = 1;
+	void *p;
+	if (c == &__mthp_khugepaged_context)
+		nr_hpages = 1 << (hpage_pmd_order - collapse_order);
+	p = ops->setup_area(1);
+	c->collapse("Do not collapse empty PTE table", p, nr_hpages, ops, false);
+	ops->cleanup_area(p, hpage_pmd_size);
+	ksft_test_result_report(exit_status, "%s\n", __func__);
+}
+static void collapse_single_mthp(struct collapse_context *c, struct mem_ops *ops)
+{
+	unsigned long hpage_size = page_size << collapse_order;
 	void *p;
 	p = ops->setup_area(1);
-	c->collapse("Do not collapse empty PTE table", p, 1, ops, false);
+	/*
+	 * Only fault collapse_order sized ranges, and only check 1
+	 * collapse_order sized huge page.
+	 */
+	ops->fault(p, 0, hpage_size);
+	c->collapse("Collapse PTE table with half PTE entries present",
+		p, 1, ops, true);
 	ops->cleanup_area(p, hpage_pmd_size);
 	ksft_test_result_report(exit_status, "%s\n", __func__);
 }
 	madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
 	ksft_print_msg("Split huge page leaving single PTE mapping compound page...");
 	madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED);
-	if (ops->check_huge(p, 0))
+	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	ksft_print_msg("Split huge page leaving single PTE page table full of compound pages...");
 	madvise(p, page_size, MADV_NOHUGEPAGE);
 	madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
-	if (ops->check_huge(p, 0))
+	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	for (i = 0; i < hpage_pmd_nr; i++) {
 		madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE);
 		ops->fault(BASE_ADDR, 0, hpage_pmd_size);
-		if (!ops->check_huge(BASE_ADDR, 1))
+		if (!ops->check_huge(BASE_ADDR, hpage_pmd_size, 1, hpage_pmd_size))
 			ksft_exit_fail_msg("Failed to allocate huge page\n");
 		madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE);
 	ops->cleanup_area(BASE_ADDR, hpage_pmd_size);
 	ops->fault(p, 0, hpage_pmd_size);
-	if (!ops->check_huge(p, 1))
+	if (!ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	ksft_print_msg("Allocate small page...");
 	ops->fault(p, 0, page_size);
-	if (ops->check_huge(p, 0))
+	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	ksft_print_msg("Share small page over fork()...");
 	if (!fork()) {
 		/* Do not touch settings on child exit */
-		if (ops->check_huge(p, 0))
+		if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 			success("OK");
 		else
 			fail("Fail");
 	exit_status = WEXITSTATUS(wstatus);
 	ksft_print_msg("Check if parent still has small page...");
-	if (ops->check_huge(p, 0))
+	if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	ksft_print_msg("Share huge page over fork()...");
 	if (!fork()) {
 		/* Do not touch settings on child exit */
-		if (ops->check_huge(p, 1))
+		if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
 			success("OK");
 		else
 			fail("Fail");
 		ksft_print_msg("Split huge page PMD in child process...");
 		madvise(p, page_size, MADV_NOHUGEPAGE);
 		madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
-		if (ops->check_huge(p, 0))
+		if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 			success("OK");
 		else
 			fail("Fail");
 	exit_status = WEXITSTATUS(wstatus);
 	ksft_print_msg("Check if parent still has huge page...");
-	if (ops->check_huge(p, 1))
+	if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	ksft_print_msg("Share huge page over fork()...");
 	if (!fork()) {
 		/* Do not touch settings on child exit */
-		if (ops->check_huge(p, 1))
+		if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
 			success("OK");
 		else
 			fail("Fail");
 		ksft_print_msg("Trigger CoW on page %d of %d...",
 				hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr);
 		ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
-		if (ops->check_huge(p, 0))
+		if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 			success("OK");
 		else
 			fail("Fail");
 			       hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr);
 			ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) *
 				    page_size);
-			if (ops->check_huge(p, 0))
+			if (ops->check_huge(p, hpage_pmd_size, 0, hpage_pmd_size))
 				success("OK");
 			else
 				fail("Fail");
 	exit_status = WEXITSTATUS(wstatus);
 	ksft_print_msg("Check if parent still has huge page...");
-	if (ops->check_huge(p, 1))
+	if (ops->check_huge(p, hpage_pmd_size, 1, hpage_pmd_size))
 		success("OK");
 	else
 		fail("Fail");
 	ops->fault(p, 0, size);
 	/* Let khugepaged collapse and leave pmd cleared */
-	if (wait_for_scan("Collapse and leave PMD cleared", p, nr_hpages,
+	if (wait_for_scan("Collapse and leave PMD cleared", p, size, nr_hpages,
-			  ops)) {
+			  hpage_pmd_order, ops)) {
 		fail("Timeout");
 		return;
 	}
 {
 	fprintf(stderr, "\nUsage: ./khugepaged [OPTIONS] <test type> [dir]\n\n");
 	fprintf(stderr, "\t<test type>\t: <context>:<mem_type>\n");
-	fprintf(stderr, "\t<context>\t: [all|khugepaged|madvise]\n");
+	fprintf(stderr, "\t<context>\t: [all|khugepaged|mthp_khugepaged|madvise]\n");
 	fprintf(stderr, "\t<mem_type>\t: [all|anon|file|shmem]\n");
 	fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n");
 	fprintf(stderr, "\n\t\"file,all\" mem_type requires a file system\n");
 	fprintf(stderr,	"\twith PMD-sized large folio support\n");
 	fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n");
 	fprintf(stderr,	"\tmounted with huge=advise option for khugepaged tests to work\n");
+	fprintf(stderr, "\n\tmthp_khugepaged only supports anon mem_type now.\n");
 	fprintf(stderr,	"\n\tSupported Options:\n");
 	fprintf(stderr,	"\t\t-h: This help message.\n");
 	fprintf(stderr,	"\t\t-s: mTHP size, expressed as page order.\n");
 	fprintf(stderr,	"\t\t    Defaults to 0. Use this size for anon or shmem allocations.\n");
+	fprintf(stderr,	"\t\t-c: collapse order for mTHP collapse, expressed as page order.\n");
 	exit(1);
 }
 	char *buf;
 	const char *token;
-	while ((opt = getopt(argc, argv, "s:h")) != -1) {
+	while ((opt = getopt(argc, argv, "s:c:h")) != -1) {
 		switch (opt) {
 		case 's':
 			anon_order = atoi(optarg);
 			break;
+		case 'c':
+			collapse_order = atoi(optarg);
+			break;
 		case 'h':
 		default:
 			usage();
 		madvise_context =  &__madvise_context;
 	} else if (!strcmp(token, "khugepaged")) {
 		khugepaged_context =  &__khugepaged_context;
+	} else if (!strcmp(token, "mthp_khugepaged")) {
+		mthp_khugepaged_context =  &__mthp_khugepaged_context;
+		if (collapse_order <= 0 || collapse_order >= hpage_pmd_order)
+			usage();
 	} else if (!strcmp(token, "madvise")) {
 		madvise_context =  &__madvise_context;
 	} else {
 		read_write_file_write_ops =  &__read_write_file_write_ops;
 		anon_ops = &__anon_ops;
 		shmem_ops = &__shmem_ops;
+		if (mthp_khugepaged_context)
+			usage();
 	} else if (!strcmp(buf, "anon")) {
 		anon_ops = &__anon_ops;
 	} else if (!strcmp(buf, "file")) {
 		read_only_file_ops =  &__read_only_file_ops;
 		read_write_file_read_ops =  &__read_write_file_read_ops;
 		read_write_file_write_ops =  &__read_write_file_write_ops;
+		if (mthp_khugepaged_context)
+			usage();
 	} else if (!strcmp(buf, "shmem")) {
 		shmem_ops = &__shmem_ops;
+		if (mthp_khugepaged_context)
+			usage();
 	} else {
 		usage();
 	}
 int main(int argc, char **argv)
 {
-	int hpage_pmd_order;
 	struct thp_settings default_settings = {
 		.thp_enabled = THP_MADVISE,
 		.thp_defrag = THP_DEFRAG_ALWAYS,
 	if (!thp_is_enabled())
 		ksft_exit_skip("Transparent Hugepages not available\n");
-	parse_test_type(argc, argv);
-	setbuf(stdout, NULL);
 	page_size = getpagesize();
 	hpage_pmd_size = read_pmd_pagesize();
 	if (!hpage_pmd_size)
 	hpage_pmd_nr = hpage_pmd_size / page_size;
 	hpage_pmd_order = __builtin_ctz(hpage_pmd_nr);
+	parse_test_type(argc, argv);
+	setbuf(stdout, NULL);
 	default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1;
 	default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8;
 	default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2;
 	TEST(collapse_full, khugepaged_context, read_write_file_read_ops);
 	TEST(collapse_full, khugepaged_context, read_write_file_write_ops);
 	TEST(collapse_full, khugepaged_context, shmem_ops);
+	TEST(collapse_full, mthp_khugepaged_context, anon_ops);
 	TEST(collapse_full, madvise_context, anon_ops);
 	TEST(collapse_full, madvise_context, read_only_file_ops);
 	TEST(collapse_full, madvise_context, read_write_file_read_ops);
 	TEST(collapse_full, madvise_context, shmem_ops);
 	TEST(collapse_empty, khugepaged_context, anon_ops);
+	TEST(collapse_empty, mthp_khugepaged_context, anon_ops);
 	TEST(collapse_empty, madvise_context, anon_ops);
+	TEST(collapse_single_mthp, mthp_khugepaged_context, anon_ops);
 	TEST(collapse_single_pte_entry, khugepaged_context, anon_ops);
 	TEST(collapse_single_pte_entry, khugepaged_context, read_only_file_ops);
 	TEST(collapse_single_pte_entry, khugepaged_context, read_write_file_read_ops);

tools/testing/selftests/mm/ksft_kmemleak_confirm.sh+130−0

@@ -0,0 +1,130 @@
	1	#!/bin/bash
	2	# SPDX-License-Identifier: GPL-2.0
	3	#
	4	# Functional test for kmemleak's N-consecutive-scan leak confirmation
	5	# (the min_unref_scans module parameter).
	6	#
	7	# kmemleak only reports an object once it has stayed unreferenced for
	8	# min_unref_scans consecutive scans. A threshold of 1 reports on the first
	9	# scan (historical behaviour); higher values filter transient false
	10	# positives where a live object's only reference is briefly invisible to a
	11	# single scan (e.g. an RCU tree update in flight while the scan runs). The
	12	# test loads samples/kmemleak's helper module to create orphan allocations
	13	# and, counting only those orphans (matched by their [kmemleak_test]
	14	# backtrace so unrelated leaks already present on the system are ignored),
	15	# checks that:
	16	# - a freshly allocated object is greyed on its first scan (its checksum
	17	# settles then), so nothing can be reported before that priming scan;
	18	# each case below primes once first,
	19	# - at min_unref_scans=1 one scan after priming reports the orphans,
	20	# - raising the threshold to 2 needs two scans after priming: one is not
	21	# enough, the second reports,
	22	# - the parameter reads back what was written.
	23	#
	24	# The "one post-prime scan is not enough at min_unref_scans=2" check is the
	25	# core regression test: raising min_unref_scans must push the report
	26	# strictly later. Like ksft_kmemleak_dedup.sh, if the module yields no
	27	# detectable orphan at all in the running environment the test skips rather
	28	# than failing.
	29	#
	30	# Author: Breno Leitao <leitao@debian.org>
	31
	32	# KTAP output helpers (ktap_skip_all, ktap_exit_fail_msg, ktap_test_pass, ...).
	33	DIR="$(dirname "$(readlink -f "$0")")"
	34	# shellcheck source=../kselftest/ktap_helpers.sh
	35	source "${DIR}"/../kselftest/ktap_helpers.sh
	36
	37	KMEMLEAK=/sys/kernel/debug/kmemleak
	38	PARAM=/sys/module/kmemleak/parameters/min_unref_scans
	39	MODULE=kmemleak-test
	40	AGE=6 # seconds; must exceed kmemleak's 5s minimum object age
	41
	42	ktap_print_header
	43
	44	[ "$(id -u)" -eq 0 ] \|\| { ktap_skip_all "must run as root"; exit "$KSFT_SKIP"; }
	45	[ -r "$KMEMLEAK" ] \|\|
	46	{ ktap_skip_all "no kmemleak debugfs (CONFIG_DEBUG_KMEMLEAK)"; exit "$KSFT_SKIP"; }
	47	[ -w "$PARAM" ] \|\|
	48	{ ktap_skip_all "min_unref_scans module parameter not present"; exit "$KSFT_SKIP"; }
	49	modinfo "$MODULE" >/dev/null 2>&1 \|\|
	50	{ ktap_skip_all "$MODULE not built (CONFIG_SAMPLE_KMEMLEAK)"; exit "$KSFT_SKIP"; }
	51
	52	# kmemleak can be present but disabled at runtime (kmemleak=off boot arg,
	53	# or it self-disabled after an internal error); a "scan" then returns
	54	# EPERM. Probe once and skip if so.
	55	echo scan > "$KMEMLEAK" 2>/dev/null \|\|
	56	{ ktap_skip_all "kmemleak is disabled (check dmesg or kmemleak= boot arg)"; exit "$KSFT_SKIP"; }
	57
	58	prev=$(cat "$PARAM")
	59	# shellcheck disable=SC2317 # invoked indirectly via trap
	60	cleanup() {
	61	echo "$prev" > "$PARAM" 2>/dev/null # restore the parameter
	62	echo scan=on > "$KMEMLEAK" 2>/dev/null # re-enable auto scan
	63	rmmod "$MODULE" 2>/dev/null
	64	echo clear > "$KMEMLEAK" 2>/dev/null
	65	}
	66	trap cleanup EXIT
	67
	68	# Stop the automatic scan thread: only our manual scans should advance an
	69	# object's consecutive-unreferenced run. An auto scan landing between two
	70	# manual scans would change the result and make the test flaky.
	71	echo scan=off > "$KMEMLEAK" 2>/dev/null
	72
	73	# Create a fresh, aged set of orphan objects from the helper module's init
	74	# path (its kmalloc/vmalloc/percpu allocations are dropped right away).
	75	# Pre-existing reported leaks are greyed first ("clear") so only our
	76	# orphans are counted. The module is left loaded on purpose: once it is
	77	# unloaded its symbols are gone, so the orphan backtraces no longer resolve
	78	# to [kmemleak_test] and could not be matched below.
	79	gen_orphans() {
	80	rmmod "$MODULE" 2>/dev/null
	81	echo clear > "$KMEMLEAK"
	82	modprobe "$MODULE" \|\|
	83	{ ktap_skip_all "failed to load $MODULE"; exit "$KSFT_SKIP"; }
	84	sleep "$AGE"
	85	}
	86
	87	scan() { echo scan > "$KMEMLEAK"; }
	88
	89	# Number of helper-module orphans currently reported by kmemleak. Matching
	90	# the module's own backtrace ([kmemleak_test]) keeps the count immune to
	91	# unrelated leaks on the running system. kmemleak only lists an object here
	92	# once it has been reported, so this reflects the confirmation gating.
	93	count_orphans() {
	94	c=$(grep -c '\[kmemleak_test\]' "$KMEMLEAK" 2>/dev/null)
	95	echo "${c:-0}"
	96	}
	97
	98	# 0) the parameter reads back what was written.
	99	echo 3 > "$PARAM"
	100	[ "$(cat "$PARAM")" = "3" ] \|\| ktap_exit_fail_msg "min_unref_scans did not read back as 3"
	101
	102	# Priming scan: kmemleak greys a freshly allocated object on its first scan
	103	# (its checksum settles then), so nothing can be reported until a second
	104	# scan. Every case below runs this priming scan before counting.
	105	prime() { scan; }
	106
	107	# 1) min_unref_scans=1: one scan after priming reports the orphans. This
	108	# also establishes that the helper produces detectable orphans here.
	109	echo 1 > "$PARAM"
	110	gen_orphans
	111	prime
	112	scan
	113	first=$(count_orphans)
	114	[ "$first" -gt 0 ] \|\|
	115	{ ktap_skip_all "$MODULE produced no detectable orphans (cannot test min_unref_scans)"; exit "$KSFT_SKIP"; }
	116
	117	# 2) min_unref_scans=2: after priming, one scan is not enough (still
	118	# gated), the second reports. The gated-scan-zero check is the core
	119	# regression.
	120	echo 2 > "$PARAM"
	121	gen_orphans
	122	prime
	123	scan; s1=$(count_orphans)
	124	scan; s2=$(count_orphans)
	125	[ "$s1" -eq 0 ] \|\| ktap_exit_fail_msg "min_unref_scans=2: $s1 orphan(s) after 1 post-prime scan (must be 0)"
	126	[ "$s2" -gt 0 ] \|\| ktap_exit_fail_msg "min_unref_scans=2: no report after 2 post-prime scans (false negative)"
	127
	128	ktap_set_plan 1
	129	ktap_test_pass "min_unref_scans=1 reported $first orphan(s) one scan after priming; =2 held them one scan longer ($s1 after one scan, $s2 after two); param read-back ok"
	130	ktap_finished

tools/testing/selftests/mm/memory-failure.c+12−2

 	if (fd < 0)
 		SKIP(return, "failed to open test file.\n");
 	fs_type = get_fs_type(fd);
-	if (!fs_type || fs_type == TMPFS_MAGIC)
+	if (!fs_type || fs_type == TMPFS_MAGIC) {
+		close(fd);
 		SKIP(return, "unsupported filesystem :%x\n", fs_type);
+	}
 	addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE,
 		    MAP_SHARED, fd, 0);
 	if (fd < 0)
 		SKIP(return, "failed to open test file.\n");
 	fs_type = get_fs_type(fd);
-	if (!fs_type || fs_type == TMPFS_MAGIC)
+	/*
+	 * MADV_HARD poisoning of dirty page-cache data records an expected
+	 * -EIO in the file mapping. NFS reports this error on close(), so
+	 * skip this variant.
+	 */
+	if (!fs_type || fs_type == TMPFS_MAGIC ||
+	    (fs_type == NFS_SUPER_MAGIC && variant->type == MADV_HARD)) {
+		close(fd);
 		SKIP(return, "unsupported filesystem :%x\n", fs_type);
+	}
 	addr = mmap(0, self->page_size, PROT_READ | PROT_WRITE,
 		    MAP_SHARED, fd, 0);

tools/testing/selftests/mm/merge.c+57−0

@@ -1305,6 +1305,63 @@ TEST_F(merge, merge_vmas_with_mseal) · 1304 unmodified lines
1305	ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size);	1305	ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 2 * page_size);
1306	}	1306	}
1307		1307
		1308	TEST_F(merge, anon_and_page_offset_mismatch_memfd)
		1309	{
		1310	struct procmap_fd *procmap = &self->procmap;
		1311	unsigned int page_size = self->page_size;
		1312	char *carveout = self->carveout;
		1313	char ptr, ptr2;
		1314	int fd;
		1315
		1316	/* Create a 10 page memfd descriptor. */
		1317	fd = memfd_create("anon_page_offset_test", MFD_CLOEXEC);
		1318	ASSERT_NE(fd, -1);
		1319	ASSERT_EQ(ftruncate(fd, 10 * page_size), 0);
		1320
		1321	/* Map a region using the memfd at page offset 0. */
		1322	ptr = mmap(carveout, 5 * page_size, PROT_READ \| PROT_WRITE,
		1323	MAP_FIXED \| MAP_PRIVATE, fd, 0);
		1324	ASSERT_NE(ptr, MAP_FAILED);
		1325
		1326	/*
		1327	* Map another separately and trigger a CoW fault at page offset 5:
		1328	*
		1329	* \|-----------\| \|---------\|
		1330	* \| unfaulted \| \| faulted \|
		1331	* \|-----------\| \|---------\|
		1332	*/
		1333	ptr2 = mmap(&carveout[10 * page_size], 5 * page_size,
		1334	PROT_READ \| PROT_WRITE, MAP_FIXED \| MAP_PRIVATE,
		1335	fd, 5 * page_size);
		1336	ASSERT_NE(ptr2, MAP_FAILED);
		1337	ptr2[0] = 'x';
		1338
		1339	/*
		1340	* Now move it in place:
		1341	*
		1342	* \|----------\|
		1343	* \| \|
		1344	* v \|
		1345	* \|-----------\| \|---------\|
		1346	* \| unfaulted \| \| faulted \|
		1347	* \|-----------\| \|---------\|
		1348	*
		1349	* Because the anonymous page offset of the faulted region is now
		1350	* &carveout[10 * page_size], despite the two regions being mergeable
		1351	* due to file page offset, they are NOT mergeable due to anonymous
		1352	* page offset.
		1353	*/
		1354	ptr2 = sys_mremap(ptr2, 5 * page_size, 5 * page_size,
		1355	MREMAP_MAYMOVE \| MREMAP_FIXED,
		1356	&carveout[5 * page_size]);
		1357	ASSERT_NE(ptr2, MAP_FAILED);
		1358
		1359	/* Assert that they did not merge. */
		1360	ASSERT_TRUE(find_vma_procmap(procmap, ptr));
		1361	ASSERT_EQ(procmap->query.vma_start, (unsigned long)ptr);
		1362	ASSERT_EQ(procmap->query.vma_end, (unsigned long)ptr + 5 * page_size);
		1363	}
		1364
1308	TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev)	1365	TEST_F(merge_with_fork, mremap_faulted_to_unfaulted_prev)
1309	{	1366	{
1310	struct procmap_fd *procmap = &self->procmap;	1367	struct procmap_fd *procmap = &self->procmap;

tools/testing/selftests/mm/migration.c+20−17

 #include "kselftest_harness.h"
 #include "hugepage_settings.h"
-#include <strings.h>
+#include <string.h>
 #include <pthread.h>
 #include <numa.h>
 #include <numaif.h>
 #define TWOMEG		(2<<20)
 #define RUNTIME		(20)
-#define MAX_RETRIES	100
 #define ALIGN(x, a)	(((x) + (a - 1)) & (~((a) - 1)))
 HUGETLB_SETUP_DEFAULT_PAGES(1)
 	int ret, tmp;
 	int status = 0;
 	struct timespec ts1, ts2;
-	int failures = 0;
+	int success = 0;
 	if (clock_gettime(CLOCK_MONOTONIC, &ts1))
 		return -1;
 		if (clock_gettime(CLOCK_MONOTONIC, &ts2))
 			return -1;
-		if (ts2.tv_sec - ts1.tv_sec >= RUNTIME)
+		if (ts2.tv_sec - ts1.tv_sec >= RUNTIME) {
-			return 0;
+			/* Reaching both targets verifies a cross-node move. */
+			if (success >= 2)
+				return 0;
+			else
+				return -2;
+		}
 		ret = move_pages(0, 1, (void **) &ptr, &n2, &status,
 				MPOL_MF_MOVE_ALL);
-		if (ret) {
+		if (ret < 0) {
-			if (ret > 0) {
+			perror("Couldn't migrate pages");
-				/* Migration is best effort; try again */
+			return ret;
-				if (++failures < MAX_RETRIES)
+		}
-					continue;
+		/* Migration is best effort. Try again */
-				printf("Didn't migrate %d pages\n", ret);
+		if (ret > 0 || status < 0)
-			}
+			continue;
-			else
+		if (status != n2) {
-				perror("Couldn't migrate pages");
+			printf("Page is on node %d instead of target node %d\n",
+			       status, n2);
 			return -2;
 		}
-		failures = 0;
+		success++;
 		tmp = n2;
 		n2 = n1;
 		n1 = tmp;
 	}
-	return 0;
 }
 void *access_mem(void *ptr)

tools/testing/selftests/mm/mseal_test.c+1−2

 	if (!pkey_supported())
 		ksft_print_msg("PKEY not supported\n");
-	ksft_set_plan(88);
+	ksft_set_plan(87);
 	test_seal_addseal();
 	test_seal_unmapped_start();
 	test_seal_mprotect_partial_mprotect(false);
 	test_seal_mprotect_partial_mprotect(true);
-	test_seal_mprotect_two_vma_with_gap();
 	test_seal_mprotect_two_vma_with_gap();
 	test_seal_mprotect_merge(false);

tools/testing/selftests/mm/pagemap_ioctl.c+1−7

 	memset(mem, 1, size);
 	if (use_thp &&
 	    (madvise(mem, size, MADV_COLLAPSE) ||
-	     !check_huge_anon(mem, size / hpage_size, hpage_size))) {
+	     !check_huge_anon(mem, size, size / hpage_size, hpage_size))) {
 		ksft_test_result_skip("%s could not form a THP\n", name);
 		goto out;
 	}
 	int ret;
 	char *mem, *mem2;
 	struct page_region vec;
-	int pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
-	if (pagemap_fd < 0) {
-		fprintf(stderr, "open() failed\n");
-		exit(1);
-	}
 	/* 1. Map two pages */
 	mem = mmap(0, 2 * page_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);

tools/testing/selftests/mm/prctl_thp_disable.c+1−1

@@ -67,7 +67,7 @@ static int test_mmap_thp(enum thp_collapse_type madvise_buf, size_t pmdsize) · 66 unmodified lines
67	/* HACK: make sure we have a separate VMA that we can check reliably. */	67	/* HACK: make sure we have a separate VMA that we can check reliably. */
68	mprotect(mem, pmdsize, PROT_READ);	68	mprotect(mem, pmdsize, PROT_READ);
69		69
70	ret = check_huge_anon(mem, 1, pmdsize);	70	ret = check_huge_anon(mem, pmdsize, 1, pmdsize);
71	munmap(mmap_mem, mmap_size);	71	munmap(mmap_mem, mmap_size);
72	return ret;	72	return ret;
73	}	73	}

tools/testing/selftests/mm/run_vmtests.sh+2−0

@@ -410,6 +410,8 @@ CATEGORY="thp" run_test ./khugepaged all:shmem · 409 unmodified lines
410		410
411	CATEGORY="thp" run_test ./khugepaged -s 4 all:shmem	411	CATEGORY="thp" run_test ./khugepaged -s 4 all:shmem
412		412
		413	CATEGORY="thp" run_test ./khugepaged -c 4 mthp_khugepaged:anon
		414
413	# Try to create XFS if not provided	415	# Try to create XFS if not provided
414	if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then	416	if [ -z "${SPLIT_HUGE_PAGE_TEST_XFS_PATH}" ]; then
415	if test_selected "thp"; then	417	if test_selected "thp"; then

tools/testing/selftests/mm/soft-dirty.c+3−2

 	for (i = 0; i < hpage_len; i++)
 		map[i] = (char)i;
-	if (check_huge_anon(map, 1, hpage_len)) {
+	if (check_huge_anon(map, hpage_len, 1, hpage_len)) {
 		ksft_test_result_pass("Test %s huge page allocation\n", __func__);
 		clear_softdirty();
 			return;
 		}
 		unlink(fname);
-		ftruncate(test_fd, pagesize);
+		if (ftruncate(test_fd, pagesize) != 0)
+			ksft_exit_fail_msg("ftruncate failed\n");
 		map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
 			   MAP_SHARED, test_fd, 0);
 		if (map == MAP_FAILED)

tools/testing/selftests/mm/split_huge_page_test.c+8−131

@@ -104,129 +104,6 @@ fail: · 103 unmodified lines
104	return false;	104	return false;
105	}	105	}
106		106
107	static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd,
108	uint64_t *flags)
109	{
110	unsigned long pfn;
111
112	pfn = pagemap_get_pfn(pagemap_fd, vaddr);
113
114	/* non-present PFN */
115	if (pfn == -1UL)
116	return 1;
117
118	if (pageflags_get(pfn, kpageflags_fd, flags))
119	return -1;
120
121	return 0;
122	}
123
124	/*
125	* gather_after_split_folio_orders - scan through [vaddr_start, len) and record
126	* folio orders
127	*
128	* @vaddr_start: start vaddr
129	* @len: range length
130	* @pagemap_fd: file descriptor to /proc/<pid>/pagemap
131	* @kpageflags_fd: file descriptor to /proc/kpageflags
132	* @orders: output folio order array
133	* @nr_orders: folio order array size
134	*
135	* gather_after_split_folio_orders() scan through [vaddr_start, len) and check
136	* all folios within the range and record their orders. All order-0 pages will
137	* be recorded. Non-present vaddr is skipped.
138	*
139	* NOTE: the function is used to check folio orders after a split is performed,
140	* so it assumes [vaddr_start, len) fully maps to after-split folios within that
141	* range.
142	*
143	* Return: 0 - no error, -1 - unhandled cases
144	*/
145	static int gather_after_split_folio_orders(char *vaddr_start, size_t len,
146	int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders)
147	{
148	uint64_t page_flags = 0;
149	int cur_order = -1;
150	char *vaddr;
151
152	if (pagemap_fd == -1 \|\| kpageflags_fd == -1)
153	return -1;
154	if (!orders)
155	return -1;
156	if (nr_orders <= 0)
157	return -1;
158
159	for (vaddr = vaddr_start; vaddr < vaddr_start + len;) {
160	char *next_folio_vaddr;
161	int status;
162
163	status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd,
164	&page_flags);
165	if (status < 0)
166	return -1;
167
168	/* skip non present vaddr */
169	if (status == 1) {
170	vaddr += psize();
171	continue;
172	}
173
174	/* all order-0 pages with possible false postive (non folio) */
175	if (!(page_flags & (KPF_COMPOUND_HEAD \| KPF_COMPOUND_TAIL))) {
176	orders[0]++;
177	vaddr += psize();
178	continue;
179	}
180
181	/* skip non thp compound pages */
182	if (!(page_flags & KPF_THP)) {
183	vaddr += psize();
184	continue;
185	}
186
187	/* vpn points to part of a THP at this point */
188	if (page_flags & KPF_COMPOUND_HEAD)
189	cur_order = 1;
190	else {
191	vaddr += psize();
192	continue;
193	}
194
195	next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
196
197	if (next_folio_vaddr >= vaddr_start + len)
198	break;
199
200	while ((status = vaddr_pageflags_get(next_folio_vaddr,
201	pagemap_fd, kpageflags_fd,
202	&page_flags)) >= 0) {
203	/*
204	* non present vaddr, next compound head page, or
205	* order-0 page
206	*/
207	if (status == 1 \|\|
208	(page_flags & KPF_COMPOUND_HEAD) \|\|
209	!(page_flags & (KPF_COMPOUND_HEAD \| KPF_COMPOUND_TAIL))) {
210	if (cur_order < nr_orders) {
211	orders[cur_order]++;
212	cur_order = -1;
213	vaddr = next_folio_vaddr;
214	}
215	break;
216	}
217
218	cur_order++;
219	next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
220	}
221
222	if (status < 0)
223	return status;
224	}
225	if (cur_order > 0 && cur_order < nr_orders)
226	orders[cur_order]++;
227	return 0;
228	}
229
230	static int check_after_split_folio_orders(char *vaddr_start, size_t len,	107	static int check_after_split_folio_orders(char *vaddr_start, size_t len,
231	int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders)	108	int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders)
232	{	109	{
@@ -240,7 +117,7 @@ static int check_after_split_folio_orders(char *vaddr_start, size_t len, · 7 unmodified lines
240	ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders");	117	ksft_exit_fail_msg("Cannot allocate memory for vaddr_orders");
241		118
242	memset(vaddr_orders, 0, sizeof(int) * nr_orders);	119	memset(vaddr_orders, 0, sizeof(int) * nr_orders);
243	status = gather_after_split_folio_orders(vaddr_start, len, pagemap_fd,	120	status = gather_folio_orders(vaddr_start, len, pagemap_fd,
244	kpageflags_fd, vaddr_orders, nr_orders);	121	kpageflags_fd, vaddr_orders, nr_orders);
245	if (status)	122	if (status)
246	ksft_exit_fail_msg("gather folio info failed\n");	123	ksft_exit_fail_msg("gather folio info failed\n");
@@ -296,7 +173,7 @@ static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hp · 49 unmodified lines
296	unsigned long rss_anon_before, rss_anon_after;	173	unsigned long rss_anon_before, rss_anon_after;
297	size_t i;	174	size_t i;
298		175
299	if (!check_huge_anon(one_page, nr_hpages, pmd_pagesize))	176	if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, nr_hpages, pmd_pagesize))
300	ksft_exit_fail_msg("No THP is allocated\n");	177	ksft_exit_fail_msg("No THP is allocated\n");
301		178
302	rss_anon_before = rss_anon();	179	rss_anon_before = rss_anon();
@@ -311,7 +188,7 @@ static void verify_rss_anon_split_huge_page_all_zeroes(char *one_page, int nr_hp · 8 unmodified lines
311	if (one_page[i] != (char)0)	188	if (one_page[i] != (char)0)
312	ksft_exit_fail_msg("%ld byte corrupted\n", i);	189	ksft_exit_fail_msg("%ld byte corrupted\n", i);
313		190
314	if (!check_huge_anon(one_page, 0, pmd_pagesize))	191	if (!check_huge_anon(one_page, nr_hpages * pmd_pagesize, 0, pmd_pagesize))
315	ksft_exit_fail_msg("Still AnonHugePages not split\n");	192	ksft_exit_fail_msg("Still AnonHugePages not split\n");
316		193
317	rss_anon_after = rss_anon();	194	rss_anon_after = rss_anon();
@@ -347,7 +224,7 @@ static void split_pmd_thp_to_order(int order) · 29 unmodified lines
347	for (i = 0; i < len; i++)	224	for (i = 0; i < len; i++)
348	one_page[i] = (char)i;	225	one_page[i] = (char)i;
349		226
350	if (!check_huge_anon(one_page, 4, pmd_pagesize))	227	if (!check_huge_anon(one_page, 4 * pmd_pagesize, 4, pmd_pagesize))
351	ksft_exit_fail_msg("No THP is allocated\n");	228	ksft_exit_fail_msg("No THP is allocated\n");
352		229
353	/* split all THPs */	230	/* split all THPs */
@@ -366,7 +243,7 @@ static void split_pmd_thp_to_order(int order) · 12 unmodified lines
366	(pmd_order + 1)))	243	(pmd_order + 1)))
367	ksft_exit_fail_msg("Unexpected THP split\n");	244	ksft_exit_fail_msg("Unexpected THP split\n");
368		245
369	if (!check_huge_anon(one_page, 0, pmd_pagesize))	246	if (!check_huge_anon(one_page, 4 * pmd_pagesize, 0, pmd_pagesize))
370	ksft_exit_fail_msg("Still AnonHugePages not split\n");	247	ksft_exit_fail_msg("Still AnonHugePages not split\n");
371		248
372	ksft_test_result_pass("Split huge pages to order %d successful\n", order);	249	ksft_test_result_pass("Split huge pages to order %d successful\n", order);
@@ -393,7 +270,7 @@ static void split_pte_mapped_thp(void) · 20 unmodified lines
393	for (i = 0; i < thp_area_size; i++)	270	for (i = 0; i < thp_area_size; i++)
394	thp_area[i] = (char)i;	271	thp_area[i] = (char)i;
395		272
396	if (!check_huge_anon(thp_area, nr_thps, pmd_pagesize)) {	273	if (!check_huge_anon(thp_area, nr_thps * pmd_pagesize, nr_thps, pmd_pagesize)) {
397	ksft_test_result_skip("Not all THPs allocated\n");	274	ksft_test_result_skip("Not all THPs allocated\n");
398	goto out;	275	goto out;
399	}	276	}
@@ -657,7 +534,7 @@ static int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, · 257 unmodified lines
657		534
658	force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize);	535	force_read_pages(*addr, fd_size / pmd_pagesize, pmd_pagesize);
659		536
660	if (!check_huge_file(*addr, fd_size / pmd_pagesize, pmd_pagesize)) {	537	if (!check_huge_file(*addr, fd_size, fd_size / pmd_pagesize, pmd_pagesize)) {
661	ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n");	538	ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n");
662	munmap(*addr, fd_size);	539	munmap(*addr, fd_size);
663	close(*fd);	540	close(*fd);
@@ -735,7 +612,7 @@ static void split_thp_in_pagecache_to_order_at(size_t fd_size, · 71 unmodified lines
735	goto out;	612	goto out;
736	}	613	}
737		614
738	if (!check_huge_file(addr, 0, pmd_pagesize)) {	615	if (!check_huge_file(addr, fd_size, 0, pmd_pagesize)) {
739	ksft_print_msg("Still FilePmdMapped not split\n");	616	ksft_print_msg("Still FilePmdMapped not split\n");
740	err = EXIT_FAILURE;	617	err = EXIT_FAILURE;
741	goto out;	618	goto out;

tools/testing/selftests/mm/uffd-common.c+3−1

@@ -194,7 +194,9 @@ static void shmem_alias_mapping(uffd_global_test_opts_t gopts, __u64 start, · 193 unmodified lines
194		194
195	static void shmem_check_pmd_mapping(uffd_global_test_opts_t gopts, void p, int expect_nr_hpages)	195	static void shmem_check_pmd_mapping(uffd_global_test_opts_t gopts, void p, int expect_nr_hpages)
196	{	196	{
197	if (!check_huge_shmem(gopts->area_dst_alias, expect_nr_hpages,	197	size_t len = expect_nr_hpages * read_pmd_pagesize();
		198
		199	if (!check_huge_shmem(gopts->area_dst_alias, len, expect_nr_hpages,
198	read_pmd_pagesize()))	200	read_pmd_pagesize()))
199	err("Did not find expected %d number of hugepages",	201	err("Did not find expected %d number of hugepages",
200	expect_nr_hpages);	202	expect_nr_hpages);

tools/testing/selftests/mm/vm_util.c+197−10

@@ -15,6 +15,9 @@ · 14 unmodified lines
15	#define SMAP_FILE_PATH "/proc/self/smaps"	15	#define SMAP_FILE_PATH "/proc/self/smaps"
16	#define STATUS_FILE_PATH "/proc/self/status"	16	#define STATUS_FILE_PATH "/proc/self/status"
17	#define MAX_LINE_LENGTH 500	17	#define MAX_LINE_LENGTH 500
		18	#define PAGEMAP_PATH "/proc/self/pagemap"
		19	#define KPAGEFLAGS_PATH "/proc/kpageflags"
		20	#define MAX_NR_ORDERS 20
18		21
19	unsigned int __page_size;	22	unsigned int __page_size;
20	unsigned int __page_shift;	23	unsigned int __page_shift;
@@ -31,7 +34,7 @@ uint64_t pagemap_get_entry(int fd, char *start) · 10 unmodified lines
31	return entry;	34	return entry;
32	}	35	}
33		36
34	static uint64_t __pagemap_scan_get_categories(int fd, char start, struct page_region r)	37	static int __pagemap_scan_get_categories(int fd, char start, struct page_region r)
35	{	38	{
36	struct pm_scan_arg arg;	39	struct pm_scan_arg arg;
37		40
@@ -55,7 +58,7 @@ static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_r · 17 unmodified lines
55	static uint64_t pagemap_scan_get_categories(int fd, char *start)	58	static uint64_t pagemap_scan_get_categories(int fd, char *start)
56	{	59	{
57	struct page_region r;	60	struct page_region r;
58	long ret;	61	int ret;
59		62
60	ret = __pagemap_scan_get_categories(fd, start, &r);	63	ret = __pagemap_scan_get_categories(fd, start, &r);
61	if (ret < 0)	64	if (ret < 0)
@@ -194,6 +197,125 @@ err_out: · 132 unmodified lines
194	return rss_anon;	197	return rss_anon;
195	}	198	}
196		199
		200	static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd,
		201	uint64_t *flags)
		202	{
		203	unsigned long pfn;
		204
		205	pfn = pagemap_get_pfn(pagemap_fd, vaddr);
		206
		207	/* non-present PFN */
		208	if (pfn == -1UL)
		209	return 1;
		210
		211	if (pageflags_get(pfn, kpageflags_fd, flags))
		212	return -1;
		213
		214	return 0;
		215	}
		216
		217	/*
		218	* gather_folio_orders - scan through [vaddr_start, len) and record
		219	* folio orders
		220	*
		221	* @vaddr_start: start vaddr
		222	* @len: range length
		223	* @pagemap_fd: file descriptor to /proc/<pid>/pagemap
		224	* @kpageflags_fd: file descriptor to /proc/kpageflags
		225	* @orders: output folio order array
		226	* @nr_orders: folio order array size
		227	*
		228	* gather_folio_orders() scan through [vaddr_start, len) and check
		229	* all folios within the range and record their orders. All order-0 pages will
		230	* be recorded. Non-present vaddr is skipped.
		231	*
		232	* Return: 0 - no error, -1 - unhandled cases
		233	*/
		234	int gather_folio_orders(char *vaddr_start, size_t len,
		235	int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders)
		236	{
		237	uint64_t page_flags = 0;
		238	int cur_order = -1;
		239	char *vaddr;
		240
		241	if (pagemap_fd == -1 \|\| kpageflags_fd == -1)
		242	return -1;
		243	if (!orders)
		244	return -1;
		245	if (nr_orders <= 0)
		246	return -1;
		247
		248	for (vaddr = vaddr_start; vaddr < vaddr_start + len;) {
		249	char *next_folio_vaddr;
		250	int status;
		251
		252	status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd,
		253	&page_flags);
		254	if (status < 0)
		255	return -1;
		256
		257	/* skip non present vaddr */
		258	if (status == 1) {
		259	vaddr += psize();
		260	continue;
		261	}
		262
		263	/* all order-0 pages with possible false postive (non folio) */
		264	if (!(page_flags & (KPF_COMPOUND_HEAD \| KPF_COMPOUND_TAIL))) {
		265	orders[0]++;
		266	vaddr += psize();
		267	continue;
		268	}
		269
		270	/* skip non thp compound pages */
		271	if (!(page_flags & KPF_THP)) {
		272	vaddr += psize();
		273	continue;
		274	}
		275
		276	/* vpn points to part of a THP at this point */
		277	if (page_flags & KPF_COMPOUND_HEAD)
		278	cur_order = 1;
		279	else {
		280	vaddr += psize();
		281	continue;
		282	}
		283
		284	next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
		285
		286	if (next_folio_vaddr >= vaddr_start + len)
		287	break;
		288
		289	while ((status = vaddr_pageflags_get(next_folio_vaddr,
		290	pagemap_fd, kpageflags_fd,
		291	&page_flags)) >= 0) {
		292	/*
		293	* non present vaddr, next compound head page, or
		294	* order-0 page
		295	*/
		296	if (status == 1 \|\|
		297	(page_flags & KPF_COMPOUND_HEAD) \|\|
		298	!(page_flags & (KPF_COMPOUND_HEAD \| KPF_COMPOUND_TAIL))) {
		299	if (cur_order < nr_orders) {
		300	orders[cur_order]++;
		301	cur_order = -1;
		302	vaddr = next_folio_vaddr;
		303	}
		304	break;
		305	}
		306
		307	cur_order++;
		308	next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
		309	}
		310
		311	if (status < 0)
		312	return status;
		313	}
		314	if (cur_order > 0 && cur_order < nr_orders)
		315	orders[cur_order]++;
		316	return 0;
		317	}
		318
197	char __get_smap_entry(void addr, const char pattern, char buf, size_t len)	319	char __get_smap_entry(void addr, const char pattern, char buf, size_t len)
198	{	320	{
199	int ret;	321	int ret;
@@ -229,7 +351,7 @@ err_out: · 29 unmodified lines
229	return entry;	351	return entry;
230	}	352	}
231		353
232	bool __check_huge(void addr, char pattern, int nr_hpages,	354	static bool __check_pmd_huge(void addr, char pattern, int nr_hpages,
233	uint64_t hpage_size)	355	uint64_t hpage_size)
234	{	356	{
235	char buffer[MAX_LINE_LENGTH];	357	char buffer[MAX_LINE_LENGTH];
@@ -247,19 +369,84 @@ err_out: · 11 unmodified lines
247	return thp == (nr_hpages * (hpage_size >> 10));	369	return thp == (nr_hpages * (hpage_size >> 10));
248	}	370	}
249		371
250	bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)	372	static bool check_large_folios(void *addr, size_t len, int nr_hpages,
		373	uint64_t hpage_size)
251	{	374	{
252	return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);	375	int order = 0, pagesize = getpagesize();
		376	unsigned int nr_pages = hpage_size / pagesize;
		377	int orders[MAX_NR_ORDERS], status;
		378	int pagemap_fd, kpageflags_fd;
		379	bool ret = false;
		380
		381	if (!nr_pages)
		382	ksft_exit_fail_msg("invalid hugepage size\n");
		383
		384	order = 31 - __builtin_clz(nr_pages);
		385	if (!order \|\| order >= MAX_NR_ORDERS)
		386	ksft_exit_fail_msg("invalid order\n");
		387
		388	memset(orders, 0, sizeof(int) * MAX_NR_ORDERS);
		389	pagemap_fd = open(PAGEMAP_PATH, O_RDONLY);
		390	if (pagemap_fd == -1)
		391	ksft_exit_fail_msg("read pagemap fail\n");
		392
		393	kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY);
		394	if (kpageflags_fd == -1) {
		395	close(pagemap_fd);
		396	ksft_exit_fail_msg("read kpageflags fail\n");
		397	}
		398
		399	status = gather_folio_orders(addr, len, pagemap_fd,
		400	kpageflags_fd, orders, MAX_NR_ORDERS);
		401	if (status)
		402	goto out;
		403
		404	if (orders[order] == nr_hpages)
		405	ret = true;
		406
		407	out:
		408	close(pagemap_fd);
		409	close(kpageflags_fd);
		410	return ret;
253	}	411	}
254		412
255	bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)	413	bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
256	{	414	{
257	return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);	415	uint64_t pmd_pagesize = read_pmd_pagesize();
		416
		417	if (!pmd_pagesize)
		418	ksft_exit_fail_msg("reading PMD pagesize failed\n");
		419
		420	if (hpage_size == pmd_pagesize)
		421	return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
		422
		423	return check_large_folios(addr, len, nr_hpages, hpage_size);
258	}	424	}
259		425
260	bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)	426	bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
261	{	427	{
262	return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);	428	uint64_t pmd_pagesize = read_pmd_pagesize();
		429
		430	if (!pmd_pagesize)
		431	ksft_exit_fail_msg("reading PMD pagesize failed\n");
		432
		433	if (hpage_size == pmd_pagesize)
		434	return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
		435
		436	return check_large_folios(addr, len, nr_hpages, hpage_size);
		437	}
		438
		439	bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
		440	{
		441	uint64_t pmd_pagesize = read_pmd_pagesize();
		442
		443	if (!pmd_pagesize)
		444	ksft_exit_fail_msg("reading PMD pagesize failed\n");
		445
		446	if (hpage_size == pmd_pagesize)
		447	return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
		448
		449	return check_large_folios(addr, len, nr_hpages, hpage_size);
263	}	450	}
264		451
265	int64_t allocate_transhuge(void *ptr, int pagemap_fd)	452	int64_t allocate_transhuge(void *ptr, int pagemap_fd)
@@ -755,7 +942,7 @@ unsigned long read_num(const char *path) · 489 unmodified lines
755	{	942	{
756	char buf[21];	943	char buf[21];
757		944
758	if (read_file(path, buf, sizeof(buf)) < 0)	945	if (!read_file(path, buf, sizeof(buf)))
759	ksft_exit_fail_perror("read_file()");	946	ksft_exit_fail_perror("read_file()");
760		947
761	return strtoul(buf, NULL, 10);	948	return strtoul(buf, NULL, 10);

tools/testing/selftests/mm/vm_util.h+5−3

 bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len);
 uint64_t read_pmd_pagesize(void);
 unsigned long rss_anon(void);
-bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size);
+bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size);
-bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size);
+bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size);
-bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size);
+bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size);
 int64_t allocate_transhuge(void *ptr, int pagemap_fd);
 int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags);
+int gather_folio_orders(char *vaddr_start, size_t len,
+		int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders);
 int uffd_register(int uffd, void *addr, uint64_t len,
 		  bool miss, bool wp, bool minor);

tools/testing/selftests/proc/proc-maps-race.c+2−1

@@ -490,7 +490,8 @@ static bool query_addr_at(int maps_fd, void *addr, · 489 unmodified lines
490		490
491	static inline bool split_vma(FIXTURE_DATA(proc_maps_race) *self)	491	static inline bool split_vma(FIXTURE_DATA(proc_maps_race) *self)
492	{	492	{
493	return mmap(self->mod_info->addr, self->page_size, self->mod_info->prot \| PROT_EXEC,	493	/* PROT_NONE differs from both readable neighbors. */
		494	return mmap(self->mod_info->addr, self->page_size, PROT_NONE,
494	MAP_PRIVATE \| MAP_ANONYMOUS \| MAP_FIXED, -1, 0) != MAP_FAILED;	495	MAP_PRIVATE \| MAP_ANONYMOUS \| MAP_FIXED, -1, 0) != MAP_FAILED;
495	}	496	}
496		497

tools/testing/vma/include/dup.h+62−2

@@ -243,7 +243,7 @@ enum { · 242 unmodified lines
243	#define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE)	243	#define VM_NOHUGEPAGE INIT_VM_FLAG(NOHUGEPAGE)
244	#define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE)	244	#define VM_MERGEABLE INIT_VM_FLAG(MERGEABLE)
245	#define VM_STACK INIT_VM_FLAG(STACK)	245	#define VM_STACK INIT_VM_FLAG(STACK)
246	#ifdef CONFIG_STACK_GROWS_UP	246	#ifdef CONFIG_STACK_GROWSUP
247	#define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY)	247	#define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY)
248	#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT)	248	#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT)
249	#else	249	#else
@@ -577,6 +577,7 @@ struct vm_area_struct { · 327 unmodified lines
577	*/	577	*/
578	unsigned int vm_lock_seq;	578	unsigned int vm_lock_seq;
579	#endif	579	#endif
		580	unsigned int __vm_anon_pgoff_lo;
580		581
581	/*	582	/*
582	* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma	583	* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
@@ -612,6 +613,9 @@ struct vm_area_struct { · 29 unmodified lines
612	#ifdef CONFIG_PER_VMA_LOCK	613	#ifdef CONFIG_PER_VMA_LOCK
613	/* Unstable RCU readers are allowed to read this. */	614	/* Unstable RCU readers are allowed to read this. */
614	refcount_t vm_refcnt;	615	refcount_t vm_refcnt;
		616	#endif
		617	#ifdef CONFIG_64BIT
		618	unsigned int __vm_anon_pgoff_hi;
615	#endif	619	#endif
616	/*	620	/*
617	* For areas with an address space and backing store,	621	* For areas with an address space and backing store,
@@ -1158,6 +1162,17 @@ static inline bool vma_is_shared_maywrite(struct vm_area_struct *vma) · 540 unmodified lines
1158	return is_shared_maywrite(&vma->flags);	1162	return is_shared_maywrite(&vma->flags);
1159	}	1163	}
1160		1164
		1165	static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags)
		1166	{
		1167	return vma_flags_test(flags, VMA_MAYWRITE_BIT) &&
		1168	!vma_flags_test(flags, VMA_SHARED_BIT);
		1169	}
		1170
		1171	static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma)
		1172	{
		1173	return vma_flags_is_cow_mapping(&vma->flags);
		1174	}
		1175
1161	static inline struct vm_area_struct vma_next(struct vma_iterator vmi)	1176	static inline struct vm_area_struct vma_next(struct vma_iterator vmi)
1162	{	1177	{
1163	/*	1178	/*
@@ -1320,6 +1335,28 @@ static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) · 156 unmodified lines
1320	return vma_start_pgoff(vma) + vma_pages(vma);	1335	return vma_start_pgoff(vma) + vma_pages(vma);
1321	}	1336	}
1322		1337
		1338	static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma)
		1339	{
		1340	pgoff_t pgoff = 0;
		1341
		1342	#ifdef CONFIG_64BIT
		1343	pgoff += vma->__vm_anon_pgoff_hi;
		1344	pgoff <<= 32;
		1345	#endif
		1346	pgoff += vma->__vm_anon_pgoff_lo;
		1347	return pgoff;
		1348	}
		1349
		1350	static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma)
		1351	{
		1352	return vma_start_anon_pgoff(vma) + vma_pages(vma);
		1353	}
		1354
		1355	static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma)
		1356	{
		1357	return vma_end_anon_pgoff(vma) - 1;
		1358	}
		1359
1323	static inline int vfs_mmap_prepare(struct file file, struct vm_area_desc desc)	1360	static inline int vfs_mmap_prepare(struct file file, struct vm_area_desc desc)
1324	{	1361	{
1325	return file->f_op->mmap_prepare(desc);	1362	return file->f_op->mmap_prepare(desc);
@@ -1391,7 +1428,7 @@ static inline void vma_iter_set(struct vma_iterator *vmi, unsigned long addr) · 65 unmodified lines
1391	mas_set(&vmi->mas, addr);	1428	mas_set(&vmi->mas, addr);
1392	}	1429	}
1393		1430
1394	static inline bool vma_is_anonymous(struct vm_area_struct *vma)	1431	static inline bool vma_is_anonymous(const struct vm_area_struct *vma)
1395	{	1432	{
1396	return !vma->vm_ops;	1433	return !vma->vm_ops;
1397	}	1434	}
@@ -1584,3 +1621,26 @@ static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) · 186 unmodified lines
1584	{	1621	{
1585	return vma_flags_to_page_prot(vma->flags);	1622	return vma_flags_to_page_prot(vma->flags);
1586	}	1623	}
		1624
		1625	static inline pgoff_t __linear_anon_page_index(const struct vm_area_struct *vma,
		1626	const unsigned long address)
		1627	{
		1628	pgoff_t pgoff;
		1629
		1630	pgoff = linear_page_delta(vma, address);
		1631	pgoff += vma_start_anon_pgoff(vma);
		1632	return pgoff;
		1633	}
		1634
		1635	static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma,
		1636	const unsigned long address)
		1637	{
		1638	const pgoff_t pgoff = __linear_anon_page_index(vma, address);
		1639
		1640	VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma));
		1641	/* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */
		1642	if (vma_is_anonymous(vma) && !vma->vm_file)
		1643	VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address));
		1644
		1645	return pgoff;
		1646	}

tools/testing/vma/shared.c+2−1

@@ -23,7 +23,8 @@ struct vm_area_struct alloc_vma(struct mm_struct mm, · 22 unmodified lines
23		23
24	vma->vm_start = start;	24	vma->vm_start = start;
25	vma->vm_end = end;	25	vma->vm_end = end;
26	vma->vm_pgoff = pgoff;	26	vma_set_pgoff(vma, pgoff);
		27	vma_set_anon_pgoff(vma, start >> PAGE_SHIFT);
27	vma->flags = vma_flags;	28	vma->flags = vma_flags;
28	vma_assert_detached(vma);	29	vma_assert_detached(vma);
29		30

tools/testing/vma/tests/merge.c+39−10

 	vmg->start = start;
 	vmg->end = end;
 	vmg->pgoff = pgoff;
+	vmg->anon_pgoff = start >> PAGE_SHIFT;
 	vmg->vma_flags = vma_flags;
 	vmg->just_expand = false;
 		.end = 0x2000,
 		.vma_flags = vma_flags,
 		.pgoff = 1,
+		.anon_pgoff = 1,
 	};
 	ASSERT_FALSE(attach_vma(&mm, vma_left));
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x3000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	ASSERT_FLAGS_SAME_MASK(&vma->flags, vma_flags);
 	detach_free_vma(vma);
 	ASSERT_EQ(vma->vm_start, 0x1000);
 	ASSERT_EQ(vma->vm_end, 0x2000);
 	ASSERT_EQ(vma_start_pgoff(vma), 1);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 1);
 	/*
 	 * Now walk through the three split VMAs and make sure they are as
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x1000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	detach_free_vma(vma);
 	vma_iter_clear(&vmi);
 	ASSERT_EQ(vma->vm_start, 0x1000);
 	ASSERT_EQ(vma->vm_end, 0x2000);
 	ASSERT_EQ(vma_start_pgoff(vma), 1);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 1);
 	detach_free_vma(vma);
 	vma_iter_clear(&vmi);
 	ASSERT_EQ(vma->vm_start, 0x2000);
 	ASSERT_EQ(vma->vm_end, 0x3000);
 	ASSERT_EQ(vma_start_pgoff(vma), 2);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 2);
 	detach_free_vma(vma);
 	mtree_destroy(&mm.mm_mt);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x3000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	detach_free_vma(vma);
 	mtree_destroy(&mm.mm_mt);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x1000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	detach_free_vma(vma);
 	mtree_destroy(&mm.mm_mt);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x5000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 3);
 	ASSERT_EQ(vma->vm_start, 0x6000);
 	ASSERT_EQ(vma->vm_end, 0x9000);
 	ASSERT_EQ(vma_start_pgoff(vma), 6);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 6);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 3);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x9000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
 	ASSERT_EQ(vma->vm_start, 0xa000);
 	ASSERT_EQ(vma->vm_end, 0xc000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0xa);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0xa);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0xc000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 1);
 		ASSERT_EQ(vma->vm_start, 0);
 		ASSERT_EQ(vma->vm_end, 0xc000);
 		ASSERT_EQ(vma_start_pgoff(vma), 0);
+		ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 		ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
 		detach_free_vma(vma);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_prev->vm_start, 0);
 	ASSERT_EQ(vma_prev->vm_end, 0x5000);
-	ASSERT_EQ(vma_prev->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma_prev), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0);
 	ASSERT_EQ(cleanup_mm(&mm, &vmi), 2);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_prev->vm_start, 0);
 	ASSERT_EQ(vma_prev->vm_end, 0x5000);
-	ASSERT_EQ(vma_prev->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma_prev), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0);
 	ASSERT_EQ(cleanup_mm(&mm, &vmi), 2);
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x5000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	ASSERT_EQ(vma->vm_ops, &vm_ops);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_next->vm_start, 0x3000);
 	ASSERT_EQ(vma_next->vm_end, 0x9000);
-	ASSERT_EQ(vma_next->vm_pgoff, 3);
+	ASSERT_EQ(vma_start_pgoff(vma_next), 3);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_next), 3);
 	ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma);
 	ASSERT_EQ(vma->vm_start, 0x2000);
 	ASSERT_EQ(vma->vm_end, 0x3000);
 	ASSERT_EQ(vma_start_pgoff(vma), 2);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 2);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_TRUE(vma_write_started(vma_next));
 	ASSERT_EQ(mm.map_count, 2);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_next->vm_start, 0x2000);
 	ASSERT_EQ(vma_next->vm_end, 0x9000);
-	ASSERT_EQ(vma_next->vm_pgoff, 2);
+	ASSERT_EQ(vma_start_pgoff(vma_next), 2);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_next), 2);
 	ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma_next));
 	ASSERT_EQ(mm.map_count, 1);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_prev->vm_start, 0);
 	ASSERT_EQ(vma_prev->vm_end, 0x6000);
-	ASSERT_EQ(vma_prev->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma_prev), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0);
 	ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
 	ASSERT_EQ(vma->vm_start, 0x6000);
 	ASSERT_EQ(vma->vm_end, 0x7000);
 	ASSERT_EQ(vma_start_pgoff(vma), 6);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 6);
 	ASSERT_TRUE(vma_write_started(vma_prev));
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 2);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_prev->vm_start, 0);
 	ASSERT_EQ(vma_prev->vm_end, 0x7000);
-	ASSERT_EQ(vma_prev->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma_prev), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0);
 	ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma_prev));
 	ASSERT_EQ(mm.map_count, 1);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_prev->vm_start, 0);
 	ASSERT_EQ(vma_prev->vm_end, 0x9000);
-	ASSERT_EQ(vma_prev->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma_prev), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0);
 	ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
 	ASSERT_TRUE(vma_write_started(vma_prev));
 	ASSERT_EQ(mm.map_count, 1);
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_prev->vm_start, 0);
 	ASSERT_EQ(vma_prev->vm_end, 0x7000);
-	ASSERT_EQ(vma_prev->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma_prev), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0);
 	ASSERT_TRUE(vma_write_started(vma_prev));
 	ASSERT_FALSE(vma_write_started(vma_next));
 	ASSERT_EQ(vmg.state, VMA_MERGE_SUCCESS);
 	ASSERT_EQ(vma_prev->vm_start, 0);
 	ASSERT_EQ(vma_prev->vm_end, 0x7000);
-	ASSERT_EQ(vma_prev->vm_pgoff, 0);
+	ASSERT_EQ(vma_start_pgoff(vma_prev), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma_prev), 0);
 	ASSERT_TRUE(vma_write_started(vma_prev));
 	ASSERT_FALSE(vma_write_started(vma_next));
 	ASSERT_EQ(vma->vm_start, 0);
 	ASSERT_EQ(vma->vm_end, 0x4000);
 	ASSERT_EQ(vma_start_pgoff(vma), 0);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 0);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(mm.map_count, 1);
 	struct mm_struct mm = {};
 	VMA_ITERATOR(vmi, &mm, 0);
 	struct vm_area_struct *vma_prev, *vma;
-	VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5);
+	VMG_STATE(vmg, &mm, &vmi, 0x5000, 0x9000, vma_flags, 5, 5);
 	/*
 	 * Place a VMA prior to the one we're expanding so we assert that we do
 	ASSERT_EQ(vma->vm_start, 0x3000);
 	ASSERT_EQ(vma->vm_end, 0x9000);
 	ASSERT_EQ(vma_start_pgoff(vma), 3);
+	ASSERT_EQ(vma_start_anon_pgoff(vma), 3);
 	ASSERT_TRUE(vma_write_started(vma));
 	ASSERT_EQ(vma_iter_addr(&vmi), 0x3000);
 	vma_assert_attached(vma);

tools/testing/vma/tests/vma.c+47−3

@@ -33,12 +33,56 @@ static bool test_copy_vma(void) · 32 unmodified lines
33	struct mm_struct mm = {};	33	struct mm_struct mm = {};
34	bool need_locks = false;	34	bool need_locks = false;
35	VMA_ITERATOR(vmi, &mm, 0);	35	VMA_ITERATOR(vmi, &mm, 0);
36	struct vm_area_struct vma, vma_new, *vma_next;	36	struct vm_area_struct vma, vma_prev, vma_new, vma_next, *vma_orig;
		37
		38	/* Move forwards, adjacent to old self - self-merge. */
		39
		40	vma = alloc_and_link_vma(&mm, 0x1000, 0x2000, 1, vma_flags);
		41	vma_set_anonymous(vma);
		42	vma_orig = vma;
		43	vma_new = copy_vma(&vma, 0x2000, 0x1000, 1, 1, &need_locks);
		44	ASSERT_EQ(vma_new, vma_orig);
		45	ASSERT_EQ(vma, vma_orig);
		46	ASSERT_EQ(vma_new->vm_start, 0x1000);
		47	ASSERT_EQ(vma_new->vm_end, 0x3000);
		48
		49	cleanup_mm(&mm, &vmi);
		50
		51	/* Move backwards, adjacent to old self - self-merge. */
		52
		53	vma = alloc_and_link_vma(&mm, 0x2000, 0x3000, 2, vma_flags);
		54	vma_set_anonymous(vma);
		55	vma_orig = vma;
		56	vma_new = copy_vma(&vma, 0x1000, 0x1000, 2, 2, &need_locks);
		57	ASSERT_EQ(vma_new, vma_orig);
		58	ASSERT_EQ(vma, vma_orig);
		59	ASSERT_EQ(vma_new->vm_start, 0x1000);
		60	ASSERT_EQ(vma_new->vm_end, 0x3000);
		61
		62	cleanup_mm(&mm, &vmi);
		63
		64	/*
		65	* Move backwards between prior VMA and old self - self-merge and vma
		66	* updated to a new VMA.
		67	*/
		68
		69	vma_prev = alloc_and_link_vma(&mm, 0x1000, 0x2000, 1, vma_flags);
		70	vma_set_anonymous(vma_prev);
		71	vma = alloc_and_link_vma(&mm, 0x3000, 0x4000, 3, vma_flags);
		72	vma_set_anonymous(vma);
		73	vma_orig = vma;
		74	vma_new = copy_vma(&vma, 0x2000, 0x1000, 3, 3, &need_locks);
		75	ASSERT_NE(vma_new, vma_orig);
		76	ASSERT_EQ(vma_new, vma);
		77	ASSERT_EQ(vma_new->vm_start, 0x1000);
		78	ASSERT_EQ(vma_new->vm_end, 0x4000);
		79
		80	cleanup_mm(&mm, &vmi);
37		81
38	/* Move backwards and do not merge. */	82	/* Move backwards and do not merge. */
39		83
40	vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags);	84	vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags);
41	vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks);	85	vma_new = copy_vma(&vma, 0, 0x2000, 0, 3, &need_locks);
42	ASSERT_NE(vma_new, vma);	86	ASSERT_NE(vma_new, vma);
43	ASSERT_EQ(vma_new->vm_start, 0);	87	ASSERT_EQ(vma_new->vm_start, 0);
44	ASSERT_EQ(vma_new->vm_end, 0x2000);	88	ASSERT_EQ(vma_new->vm_end, 0x2000);
@@ -51,7 +95,7 @@ static bool test_copy_vma(void) · 6 unmodified lines
51		95
52	vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags);	96	vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags);
53	vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags);	97	vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags);
54	vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks);	98	vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, 4, &need_locks);
55	vma_assert_attached(vma_new);	99	vma_assert_attached(vma_new);
56		100
57	ASSERT_EQ(vma_new, vma_next);	101	ASSERT_EQ(vma_new, vma_next);

tools/testing/vma/vma_internal.h+1−0

@@ -53,6 +53,7 @@ typedef __bitwise unsigned int vm_fault_t; · 52 unmodified lines
53		53
54	#define VM_WARN_ON(_expr) (WARN_ON(_expr))	54	#define VM_WARN_ON(_expr) (WARN_ON(_expr))
55	#define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr))	55	#define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr))
		56	#define VM_WARN_ON_ONCE_VMA(_expr, _vma) (WARN_ON_ONCE(_expr))
56	#define VM_WARN_ON_VMG(_expr, _vmg) (WARN_ON(_expr))	57	#define VM_WARN_ON_VMG(_expr, _vmg) (WARN_ON(_expr))
57	#define VM_BUG_ON(_expr) (BUG_ON(_expr))	58	#define VM_BUG_ON(_expr) (BUG_ON(_expr))
58	#define VM_BUG_ON_VMA(_expr, _vma) (BUG_ON(_expr))	59	#define VM_BUG_ON_VMA(_expr, _vma) (BUG_ON(_expr))