History

Merge branch 'akpm' (patches from Andrew)

Merge more updates from Andrew Morton: "147 patches, based on 7d2a07b769330c34b4deabeed939325c77a7ec2f. Subsystems affected by this patch series: mm (memory-hotplug, rmap, ioremap, highmem, cleanups, secretmem, kfence, damon, and vmscan), alpha, percpu, procfs, misc, core-kernel, MAINTAINERS, lib, checkpatch, epoll, init, nilfs2, coredump, fork, pids, criu, kconfig, selftests, ipc, and scripts" * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (94 commits) scripts: check_extable: fix typo in user error message mm/workingset: correct kernel-doc notations ipc: replace costly bailout check in sysvipc_find_ipc() selftests/memfd: remove unused variable Kconfig.debug: drop selecting non-existing HARDLOCKUP_DETECTOR_ARCH configs: remove the obsolete CONFIG_INPUT_POLLDEV prctl: allow to setup brk for et_dyn executables pid: cleanup the stale comment mentioning pidmap_init(). kernel/fork.c: unexport get_{mm,task}_exe_file coredump: fix memleak in dump_vma_snapshot() fs/coredump.c: log if a core dump is aborted due to changed file permissions nilfs2: use refcount_dec_and_lock() to fix potential UAF nilfs2: fix memory leak in nilfs_sysfs_delete_snapshot_group nilfs2: fix memory leak in nilfs_sysfs_create_snapshot_group nilfs2: fix memory leak in nilfs_sysfs_delete_##name##_group nilfs2: fix memory leak in nilfs_sysfs_create_##name##_group nilfs2: fix NULL pointer in nilfs_##name##_attr_release nilfs2: fix memory leak in nilfs_sysfs_create_device_group trap: cleanup trap_init() init: move usermodehelper_enable() to populate_rootfs() ...

Linus Torvalds committed 5 years ago

Commit 2d338202 parents cc09ee8 b285437149 files · +5341 −930Browse files

Unified · Split

149 files
  1. Documentation/admin-guide/mm/damon/index.rst +15 −0
  2. Documentation/admin-guide/mm/damon/start.rst +114 −0
  3. Documentation/admin-guide/mm/damon/usage.rst +112 −0
  4. Documentation/admin-guide/mm/index.rst +1 −0
  5. Documentation/admin-guide/mm/memory-hotplug.rst +455 −345
  6. Documentation/dev-tools/kfence.rst +53 −45
  7. Documentation/kbuild/llvm.rst +3 −2
  8. Documentation/vm/damon/api.rst +20 −0
  9. Documentation/vm/damon/design.rst +166 −0
  10. Documentation/vm/damon/faq.rst +51 −0
  11. Documentation/vm/damon/index.rst +30 −0
  12. Documentation/vm/index.rst +1 −0
  13. MAINTAINERS +13 −2
  14. arch/Kconfig +1 −1
  15. arch/alpha/include/asm/agp.h +2 −2
  16. arch/alpha/kernel/pci-sysfs.c +8 −4
  17. arch/arc/kernel/traps.c +0 −5
  18. arch/arm/configs/dove_defconfig +0 −1
  19. arch/arm/configs/pxa_defconfig +0 −1
  20. arch/arm/kernel/traps.c +0 −5
  21. arch/arm64/mm/mmu.c +1 −2
  22. arch/h8300/kernel/traps.c +0 −4
  23. arch/hexagon/kernel/traps.c +0 −4
  24. arch/ia64/mm/init.c +1 −2
  25. arch/mips/configs/lemote2f_defconfig +0 −1
  26. arch/mips/configs/pic32mzda_defconfig +0 −1
  27. arch/mips/configs/rt305x_defconfig +0 −1
  28. arch/mips/configs/xway_defconfig +0 −1
  29. arch/nds32/kernel/traps.c +0 −5
  30. arch/nios2/kernel/traps.c +0 −5
  31. arch/openrisc/kernel/traps.c +0 −5
  32. arch/parisc/configs/generic-32bit_defconfig +0 −1
  33. arch/parisc/kernel/traps.c +0 −4
  34. arch/powerpc/kernel/traps.c +0 −5
  35. arch/powerpc/mm/mem.c +1 −2
  36. arch/powerpc/platforms/pseries/hotplug-memory.c +4 −5
  37. arch/riscv/Kconfig +1 −1
  38. arch/riscv/kernel/traps.c +0 −5
  39. arch/s390/mm/init.c +1 −2
  40. arch/sh/mm/init.c +1 −2
  41. arch/um/kernel/trap.c +0 −4
  42. arch/x86/configs/i386_defconfig +0 −1
  43. arch/x86/configs/x86_64_defconfig +0 −1
  44. arch/x86/mm/init_32.c +1 −2
  45. arch/x86/mm/init_64.c +1 −2
  46. drivers/acpi/acpi_memhotplug.c +31 −15
  47. drivers/base/memory.c +204 −21
  48. drivers/base/node.c +0 −2
  49. drivers/dax/kmem.c +33 −10
  50. drivers/devfreq/devfreq.c +1 −1
  51. drivers/hwmon/mr75203.c +1 −1
  52. drivers/iio/common/hid-sensors/hid-sensor-attributes.c +1 −2
  53. drivers/iio/light/as73211.c +1 −2
  54. drivers/media/i2c/ov02a10.c +1 −1
  55. drivers/mtd/nand/raw/intel-nand-controller.c +1 −1
  56. drivers/phy/st/phy-stm32-usbphyc.c +1 −1
  57. drivers/thermal/devfreq_cooling.c +1 −1
  58. drivers/virtio/virtio_mem.c +21 −5
  59. fs/coredump.c +12 −3
  60. fs/eventpoll.c +10 −8
  61. fs/nilfs2/sysfs.c +11 −15
  62. fs/nilfs2/the_nilfs.c +4 −5
  63. fs/proc/array.c +4 −14
  64. fs/proc/base.c +4 −1
  65. include/asm-generic/early_ioremap.h +0 −6
  66. include/linux/damon.h +268 −0
  67. include/linux/highmem-internal.h +22 −5
  68. include/linux/memory.h +54 −1
  69. include/linux/memory_hotplug.h +22 −12
  70. include/linux/mmzone.h +7 −12
  71. include/linux/once.h +1 −1
  72. include/linux/page-flags.h +5 −3
  73. include/linux/page_ext.h +1 −1
  74. include/linux/page_idle.h +3 −3
  75. include/linux/pagemap.h +3 −4
  76. include/linux/sched/user.h +2 −1
  77. include/linux/threads.h +1 −1
  78. include/linux/units.h +7 −3
  79. include/linux/vmalloc.h +0 −3
  80. include/trace/events/damon.h +43 −0
  81. include/trace/events/mmflags.h +1 −1
  82. include/trace/events/page_ref.h +2 −2
  83. init/initramfs.c +2 −0
  84. init/main.c +2 −1
  85. init/noinitramfs.c +2 −0
  86. ipc/util.c +4 −12
  87. kernel/acct.c +1 −1
  88. kernel/fork.c +0 −2
  89. kernel/profile.c +11 −10
  90. kernel/sys.c +0 −7
  91. kernel/user.c +25 −0
  92. lib/Kconfig.debug +4 −5
  93. lib/dump_stack.c +2 −1
  94. lib/iov_iter.c +6 −2
  95. lib/math/Kconfig +1 −1
  96. lib/math/rational.c +3 −0
  97. lib/test_printf.c +1 −1
  98. lib/test_sort.c +19 −21
  99. lib/vsprintf.c +1 −1
  100. mm/Kconfig +11 −4
  101. mm/Makefile +3 −1
  102. mm/compaction.c +7 −13
  103. mm/damon/Kconfig +68 −0
  104. mm/damon/Makefile +5 −0
  105. mm/damon/core-test.h +253 −0
  106. mm/damon/core.c +720 −0
  107. mm/damon/dbgfs-test.h +126 −0
  108. mm/damon/dbgfs.c +623 −0
  109. mm/damon/vaddr-test.h +329 −0
  110. mm/damon/vaddr.c +672 −0
  111. mm/early_ioremap.c +0 −5
  112. mm/highmem.c +1 −1
  113. mm/ioremap.c +0 −25
  114. mm/kfence/core.c +3 −0
  115. mm/kfence/kfence.h +2 −0
  116. mm/kfence/kfence_test.c +3 −0
  117. mm/kfence/report.c +13 −6
  118. mm/kmemleak.c +1 −1
  119. mm/memory_hotplug.c +341 −33
  120. mm/memremap.c +1 −4
  121. mm/page_alloc.c +5 −22
  122. mm/page_ext.c +11 −1
  123. mm/page_idle.c +0 −10
  124. mm/page_isolation.c +1 −6
  125. mm/page_owner.c +1 −13
  126. mm/percpu.c +0 −1
  127. mm/rmap.c +4 −2
  128. mm/secretmem.c +5 −4
  129. mm/vmalloc.c +17 −5
  130. mm/workingset.c +1 −1
  131. scripts/check_extable.sh +1 −1
  132. scripts/checkpatch.pl +57 −36
  133. tools/include/linux/bitmap.h +2 −2
  134. tools/perf/bench/find-bit-bench.c +1 −1
  135. tools/perf/builtin-c2c.c +3 −3
  136. tools/perf/builtin-record.c +1 −1
  137. tools/perf/tests/bitmap.c +1 −1
  138. tools/perf/tests/mem2node.c +1 −1
  139. tools/perf/util/affinity.c +2 −2
  140. tools/perf/util/header.c +2 −2
  141. tools/perf/util/metricgroup.c +1 −1
  142. tools/perf/util/mmap.c +2 −2
  143. tools/testing/selftests/damon/Makefile +7 −0
  144. tools/testing/selftests/damon/_chk_dependency.sh +28 −0
  145. tools/testing/selftests/damon/debugfs_attrs.sh +75 −0
  146. tools/testing/selftests/kvm/dirty_log_perf_test.c +1 −1
  147. tools/testing/selftests/kvm/dirty_log_test.c +2 −2
  148. tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c +1 −1
  149. tools/testing/selftests/memfd/memfd_test.c +1 −1
Documentation/admin-guide/mm/damon/index.rst+15−0
@@ -0,0 +1,15 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+========================
4+Monitoring Data Accesses
5+========================
6+
7+:doc:`DAMON </vm/damon/index>` allows light-weight data access monitoring.
8+Using DAMON, users can analyze the memory access patterns of their systems and
9+optimize those.
10+
11+.. toctree::
12+ :maxdepth: 2
13+
14+ start
15+ usage
Documentation/admin-guide/mm/damon/start.rst+114−0
@@ -0,0 +1,114 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+===============
4+Getting Started
5+===============
6+
7+This document briefly describes how you can use DAMON by demonstrating its
8+default user space tool. Please note that this document describes only a part
9+of its features for brevity. Please refer to :doc:`usage` for more details.
10+
11+
12+TL; DR
13+======
14+
15+Follow the commands below to monitor and visualize the memory access pattern of
16+your workload. ::
17+
18+ # # build the kernel with CONFIG_DAMON_*=y, install it, and reboot
19+ # mount -t debugfs none /sys/kernel/debug/
20+ # git clone https://github.com/awslabs/damo
21+ # ./damo/damo record $(pidof <your workload>)
22+ # ./damo/damo report heat --plot_ascii
23+
24+The final command draws the access heatmap of ``<your workload>``. The heatmap
25+shows which memory region (x-axis) is accessed when (y-axis) and how frequently
26+(number; the higher the more accesses have been observed). ::
27+
28+ 111111111111111111111111111111111111111111111111111111110000
29+ 111121111111111111111111111111211111111111111111111111110000
30+ 000000000000000000000000000000000000000000000000001555552000
31+ 000000000000000000000000000000000000000000000222223555552000
32+ 000000000000000000000000000000000000000011111677775000000000
33+ 000000000000000000000000000000000000000488888000000000000000
34+ 000000000000000000000000000000000177888400000000000000000000
35+ 000000000000000000000000000046666522222100000000000000000000
36+ 000000000000000000000014444344444300000000000000000000000000
37+ 000000000000000002222245555510000000000000000000000000000000
38+ # access_frequency: 0 1 2 3 4 5 6 7 8 9
39+ # x-axis: space (140286319947776-140286426374096: 101.496 MiB)
40+ # y-axis: time (605442256436361-605479951866441: 37.695430s)
41+ # resolution: 60x10 (1.692 MiB and 3.770s for each character)
42+
43+
44+Prerequisites
45+=============
46+
47+Kernel
48+------
49+
50+You should first ensure your system is running on a kernel built with
51+``CONFIG_DAMON_*=y``.
52+
53+
54+User Space Tool
55+---------------
56+
57+For the demonstration, we will use the default user space tool for DAMON,
58+called DAMON Operator (DAMO). It is available at
59+https://github.com/awslabs/damo. The examples below assume that ``damo`` is on
60+your ``$PATH``. It's not mandatory, though.
61+
62+Because DAMO is using the debugfs interface (refer to :doc:`usage` for the
63+detail) of DAMON, you should ensure debugfs is mounted. Mount it manually as
64+below::
65+
66+ # mount -t debugfs none /sys/kernel/debug/
67+
68+or append the following line to your ``/etc/fstab`` file so that your system
69+can automatically mount debugfs upon booting::
70+
71+ debugfs /sys/kernel/debug debugfs defaults 0 0
72+
73+
74+Recording Data Access Patterns
75+==============================
76+
77+The commands below record the memory access patterns of a program and save the
78+monitoring results to a file. ::
79+
80+ $ git clone https://github.com/sjp38/masim
81+ $ cd masim; make; ./masim ./configs/zigzag.cfg &
82+ $ sudo damo record -o damon.data $(pidof masim)
83+
84+The first two lines of the commands download an artificial memory access
85+generator program and run it in the background. The generator will repeatedly
86+access two 100 MiB sized memory regions one by one. You can substitute this
87+with your real workload. The last line asks ``damo`` to record the access
88+pattern in the ``damon.data`` file.
89+
90+
91+Visualizing Recorded Patterns
92+=============================
93+
94+The following three commands visualize the recorded access patterns and save
95+the results as separate image files. ::
96+
97+ $ damo report heats --heatmap access_pattern_heatmap.png
98+ $ damo report wss --range 0 101 1 --plot wss_dist.png
99+ $ damo report wss --range 0 101 1 --sortby time --plot wss_chron_change.png
100+
101+- ``access_pattern_heatmap.png`` will visualize the data access pattern in a
102+ heatmap, showing which memory region (y-axis) got accessed when (x-axis)
103+ and how frequently (color).
104+- ``wss_dist.png`` will show the distribution of the working set size.
105+- ``wss_chron_change.png`` will show how the working set size has
106+ chronologically changed.
107+
108+You can view the visualizations of this example workload at [1]_.
109+Visualizations of other realistic workloads are available at [2]_ [3]_ [4]_.
110+
111+.. [1] https://damonitor.github.io/doc/html/v17/admin-guide/mm/damon/start.html#visualizing-recorded-patterns
112+.. [2] https://damonitor.github.io/test/result/visual/latest/rec.heatmap.1.png.html
113+.. [3] https://damonitor.github.io/test/result/visual/latest/rec.wss_sz.png.html
114+.. [4] https://damonitor.github.io/test/result/visual/latest/rec.wss_time.png.html
Documentation/admin-guide/mm/damon/usage.rst+112−0
@@ -0,0 +1,112 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+===============
4+Detailed Usages
5+===============
6+
7+DAMON provides below three interfaces for different users.
8+
9+- *DAMON user space tool.*
10+ This is for privileged people such as system administrators who want a
11+ just-working human-friendly interface. Using this, users can use the DAMON’s
12+ major features in a human-friendly way. It may not be highly tuned for
13+ special cases, though. It supports only virtual address spaces monitoring.
14+- *debugfs interface.*
15+ This is for privileged user space programmers who want more optimized use of
16+ DAMON. Using this, users can use DAMON’s major features by reading
17+ from and writing to special debugfs files. Therefore, you can write and use
18+ your personalized DAMON debugfs wrapper programs that reads/writes the
19+ debugfs files instead of you. The DAMON user space tool is also a reference
20+ implementation of such programs. It supports only virtual address spaces
21+ monitoring.
22+- *Kernel Space Programming Interface.*
23+ This is for kernel space programmers. Using this, users can utilize every
24+ feature of DAMON most flexibly and efficiently by writing kernel space
25+ DAMON application programs for you. You can even extend DAMON for various
26+ address spaces.
27+
28+Nevertheless, you could write your own user space tool using the debugfs
29+interface. A reference implementation is available at
30+https://github.com/awslabs/damo. If you are a kernel programmer, you could
31+refer to :doc:`/vm/damon/api` for the kernel space programming interface. For
32+the reason, this document describes only the debugfs interface
33+
34+debugfs Interface
35+=================
36+
37+DAMON exports three files, ``attrs``, ``target_ids``, and ``monitor_on`` under
38+its debugfs directory, ``<debugfs>/damon/``.
39+
40+
41+Attributes
42+----------
43+
44+Users can get and set the ``sampling interval``, ``aggregation interval``,
45+``regions update interval``, and min/max number of monitoring target regions by
46+reading from and writing to the ``attrs`` file. To know about the monitoring
47+attributes in detail, please refer to the :doc:`/vm/damon/design`. For
48+example, below commands set those values to 5 ms, 100 ms, 1,000 ms, 10 and
49+1000, and then check it again::
50+
51+ # cd <debugfs>/damon
52+ # echo 5000 100000 1000000 10 1000 > attrs
53+ # cat attrs
54+ 5000 100000 1000000 10 1000
55+
56+
57+Target IDs
58+----------
59+
60+Some types of address spaces supports multiple monitoring target. For example,
61+the virtual memory address spaces monitoring can have multiple processes as the
62+monitoring targets. Users can set the targets by writing relevant id values of
63+the targets to, and get the ids of the current targets by reading from the
64+``target_ids`` file. In case of the virtual address spaces monitoring, the
65+values should be pids of the monitoring target processes. For example, below
66+commands set processes having pids 42 and 4242 as the monitoring targets and
67+check it again::
68+
69+ # cd <debugfs>/damon
70+ # echo 42 4242 > target_ids
71+ # cat target_ids
72+ 42 4242
73+
74+Note that setting the target ids doesn't start the monitoring.
75+
76+
77+Turning On/Off
78+--------------
79+
80+Setting the files as described above doesn't incur effect unless you explicitly
81+start the monitoring. You can start, stop, and check the current status of the
82+monitoring by writing to and reading from the ``monitor_on`` file. Writing
83+``on`` to the file starts the monitoring of the targets with the attributes.
84+Writing ``off`` to the file stops those. DAMON also stops if every target
85+process is terminated. Below example commands turn on, off, and check the
86+status of DAMON::
87+
88+ # cd <debugfs>/damon
89+ # echo on > monitor_on
90+ # echo off > monitor_on
91+ # cat monitor_on
92+ off
93+
94+Please note that you cannot write to the above-mentioned debugfs files while
95+the monitoring is turned on. If you write to the files while DAMON is running,
96+an error code such as ``-EBUSY`` will be returned.
97+
98+
99+Tracepoint for Monitoring Results
100+=================================
101+
102+DAMON provides the monitoring results via a tracepoint,
103+``damon:damon_aggregated``. While the monitoring is turned on, you could
104+record the tracepoint events and show results using tracepoint supporting tools
105+like ``perf``. For example::
106+
107+ # echo on > monitor_on
108+ # perf record -e damon:damon_aggregated &
109+ # sleep 5
110+ # kill 9 $(pidof perf)
111+ # echo off > monitor_on
112+ # perf script
Documentation/admin-guide/mm/index.rst+1−0
@@ -27,6 +27,7 @@ the Linux memory management. · 26 unmodified lines
2727
2828 concepts
2929 cma_debugfs
30+ damon/index
3031 hugetlbpage
3132 idle_page_tracking
3233 ksm
Documentation/admin-guide/mm/memory-hotplug.rst+455−345
@@ -1,466 +1,576 @@
11 .. _admin_guide_memory_hotplug:
22
3−==============
4−Memory Hotplug
5−==============
3+==================
4+Memory Hot(Un)Plug
5+==================
66
7−:Created: Jul 28 2007
8−:Updated: Add some details about locking internals: Aug 20 2018
9−
10−This document is about memory hotplug including how-to-use and current status.
11−Because Memory Hotplug is still under development, contents of this text will
12−be changed often.
7+This document describes generic Linux support for memory hot(un)plug with
8+a focus on System RAM, including ZONE_MOVABLE support.
139
1410 .. contents:: :local:
1511
16−.. note::
12+Introduction
13+============
1714
18− (1) x86_64's has special implementation for memory hotplug.
19− This text does not describe it.
20− (2) This text assumes that sysfs is mounted at ``/sys``.
15+Memory hot(un)plug allows for increasing and decreasing the size of physical
16+memory available to a machine at runtime. In the simplest case, it consists of
17+physically plugging or unplugging a DIMM at runtime, coordinated with the
18+operating system.
2119
20+Memory hot(un)plug is used for various purposes:
2221
23−Introduction
24−============
22+- The physical memory available to a machine can be adjusted at runtime, up- or
23+ downgrading the memory capacity. This dynamic memory resizing, sometimes
24+ referred to as "capacity on demand", is frequently used with virtual machines
25+ and logical partitions.
26+
27+- Replacing hardware, such as DIMMs or whole NUMA nodes, without downtime. One
28+ example is replacing failing memory modules.
2529
26−Purpose of memory hotplug
27−-------------------------
30+- Reducing energy consumption either by physically unplugging memory modules or
31+ by logically unplugging (parts of) memory modules from Linux.
2832
29−Memory Hotplug allows users to increase/decrease the amount of memory.
30−Generally, there are two purposes.
33+Further, the basic memory hot(un)plug infrastructure in Linux is nowadays also
34+used to expose persistent memory, other performance-differentiated memory and
35+reserved memory regions as ordinary system RAM to Linux.
3136
32−(A) For changing the amount of memory.
33− This is to allow a feature like capacity on demand.
34−(B) For installing/removing DIMMs or NUMA-nodes physically.
35− This is to exchange DIMMs/NUMA-nodes, reduce power consumption, etc.
37+Linux only supports memory hot(un)plug on selected 64 bit architectures, such as
38+x86_64, arm64, ppc64, s390x and ia64.
3639
37−(A) is required by highly virtualized environments and (B) is required by
38−hardware which supports memory power management.
40+Memory Hot(Un)Plug Granularity
41+------------------------------
3942
40−Linux memory hotplug is designed for both purpose.
43+Memory hot(un)plug in Linux uses the SPARSEMEM memory model, which divides the
44+physical memory address space into chunks of the same size: memory sections. The
45+size of a memory section is architecture dependent. For example, x86_64 uses
46+128 MiB and ppc64 uses 16 MiB.
4147
42−Phases of memory hotplug
48+Memory sections are combined into chunks referred to as "memory blocks". The
49+size of a memory block is architecture dependent and corresponds to the smallest
50+granularity that can be hot(un)plugged. The default size of a memory block is
51+the same as memory section size, unless an architecture specifies otherwise.
52+
53+All memory blocks have the same size.
54+
55+Phases of Memory Hotplug
4356 ------------------------
4457
45−There are 2 phases in Memory Hotplug:
58+Memory hotplug consists of two phases:
4659
47− 1) Physical Memory Hotplug phase
48− 2) Logical Memory Hotplug phase.
60+(1) Adding the memory to Linux
61+(2) Onlining memory blocks
4962
50−The First phase is to communicate hardware/firmware and make/erase
51−environment for hotplugged memory. Basically, this phase is necessary
52−for the purpose (B), but this is good phase for communication between
53−highly virtualized environments too.
63+In the first phase, metadata, such as the memory map ("memmap") and page tables
64+for the direct mapping, is allocated and initialized, and memory blocks are
65+created; the latter also creates sysfs files for managing newly created memory
66+blocks.
5467
55−When memory is hotplugged, the kernel recognizes new memory, makes new memory
56−management tables, and makes sysfs files for new memory's operation.
68+In the second phase, added memory is exposed to the page allocator. After this
69+phase, the memory is visible in memory statistics, such as free and total
70+memory, of the system.
5771
58−If firmware supports notification of connection of new memory to OS,
59−this phase is triggered automatically. ACPI can notify this event. If not,
60−"probe" operation by system administration is used instead.
61−(see :ref:`memory_hotplug_physical_mem`).
72+Phases of Memory Hotunplug
73+--------------------------
6274
63−Logical Memory Hotplug phase is to change memory state into
64−available/unavailable for users. Amount of memory from user's view is
65−changed by this phase. The kernel makes all memory in it as free pages
66−when a memory range is available.
75+Memory hotunplug consists of two phases:
6776
68−In this document, this phase is described as online/offline.
77+(1) Offlining memory blocks
78+(2) Removing the memory from Linux
6979
70−Logical Memory Hotplug phase is triggered by write of sysfs file by system
71−administrator. For the hot-add case, it must be executed after Physical Hotplug
72−phase by hand.
73−(However, if you writes udev's hotplug scripts for memory hotplug, these
74−phases can be execute in seamless way.)
80+In the fist phase, memory is "hidden" from the page allocator again, for
81+example, by migrating busy memory to other memory locations and removing all
82+relevant free pages from the page allocator After this phase, the memory is no
83+longer visible in memory statistics of the system.
7584
76−Unit of Memory online/offline operation
77−---------------------------------------
85+In the second phase, the memory blocks are removed and metadata is freed.
7886
79−Memory hotplug uses SPARSEMEM memory model which allows memory to be divided
80−into chunks of the same size. These chunks are called "sections". The size of
81−a memory section is architecture dependent. For example, power uses 16MiB, ia64
82−uses 1GiB.
87+Memory Hotplug Notifications
88+============================
8389
84−Memory sections are combined into chunks referred to as "memory blocks". The
85−size of a memory block is architecture dependent and represents the logical
86−unit upon which memory online/offline operations are to be performed. The
87−default size of a memory block is the same as memory section size unless an
88−architecture specifies otherwise. (see :ref:`memory_hotplug_sysfs_files`.)
90+There are various ways how Linux is notified about memory hotplug events such
91+that it can start adding hotplugged memory. This description is limited to
92+systems that support ACPI; mechanisms specific to other firmware interfaces or
93+virtual machines are not described.
8994
90−To determine the size (in bytes) of a memory block please read this file::
95+ACPI Notifications
96+------------------
9197
92− /sys/devices/system/memory/block_size_bytes
98+Platforms that support ACPI, such as x86_64, can support memory hotplug
99+notifications via ACPI.
93100
94−Kernel Configuration
95−====================
101+In general, a firmware supporting memory hotplug defines a memory class object
102+HID "PNP0C80". When notified about hotplug of a new memory device, the ACPI
103+driver will hotplug the memory to Linux.
96104
97−To use memory hotplug feature, kernel must be compiled with following
98−config options.
105+If the firmware supports hotplug of NUMA nodes, it defines an object _HID
106+"ACPI0004", "PNP0A05", or "PNP0A06". When notified about an hotplug event, all
107+assigned memory devices are added to Linux by the ACPI driver.
99108
100−- For all memory hotplug:
101− - Memory model -> Sparse Memory (``CONFIG_SPARSEMEM``)
102− - Allow for memory hot-add (``CONFIG_MEMORY_HOTPLUG``)
109+Similarly, Linux can be notified about requests to hotunplug a memory device or
110+a NUMA node via ACPI. The ACPI driver will try offlining all relevant memory
111+blocks, and, if successful, hotunplug the memory from Linux.
103112
104−- To enable memory removal, the following are also necessary:
105− - Allow for memory hot remove (``CONFIG_MEMORY_HOTREMOVE``)
106− - Page Migration (``CONFIG_MIGRATION``)
113+Manual Probing
114+--------------
107115
108−- For ACPI memory hotplug, the following are also necessary:
109− - Memory hotplug (under ACPI Support menu) (``CONFIG_ACPI_HOTPLUG_MEMORY``)
110− - This option can be kernel module.
116+On some architectures, the firmware may not be able to notify the operating
117+system about a memory hotplug event. Instead, the memory has to be manually
118+probed from user space.
111119
112−- As a related configuration, if your box has a feature of NUMA-node hotplug
113− via ACPI, then this option is necessary too.
120+The probe interface is located at::
114121
115− - ACPI0004,PNP0A05 and PNP0A06 Container Driver (under ACPI Support menu)
116− (``CONFIG_ACPI_CONTAINER``).
122+ /sys/devices/system/memory/probe
117123
118− This option can be kernel module too.
124+Only complete memory blocks can be probed. Individual memory blocks are probed
125+by providing the physical start address of the memory block::
119126
127+ % echo addr > /sys/devices/system/memory/probe
120128
121−.. _memory_hotplug_sysfs_files:
129+Which results in a memory block for the range [addr, addr + memory_block_size)
130+being created.
122131
123−sysfs files for memory hotplug
124−==============================
132+.. note::
125133
126−All memory blocks have their device information in sysfs. Each memory block
127−is described under ``/sys/devices/system/memory`` as::
134+ Using the probe interface is discouraged as it is easy to crash the kernel,
135+ because Linux cannot validate user input; this interface might be removed in
136+ the future.
128137
129− /sys/devices/system/memory/memoryXXX
138+Onlining and Offlining Memory Blocks
139+====================================
130140
131−where XXX is the memory block id.
141+After a memory block has been created, Linux has to be instructed to actually
142+make use of that memory: the memory block has to be "online".
132143
133−For the memory block covered by the sysfs directory. It is expected that all
134−memory sections in this range are present and no memory holes exist in the
135−range. Currently there is no way to determine if there is a memory hole, but
136−the existence of one should not affect the hotplug capabilities of the memory
137−block.
144+Before a memory block can be removed, Linux has to stop using any memory part of
145+the memory block: the memory block has to be "offlined".
138146
139−For example, assume 1GiB memory block size. A device for a memory starting at
140−0x100000000 is ``/sys/device/system/memory/memory4``::
147+The Linux kernel can be configured to automatically online added memory blocks
148+and drivers automatically trigger offlining of memory blocks when trying
149+hotunplug of memory. Memory blocks can only be removed once offlining succeeded
150+and drivers may trigger offlining of memory blocks when attempting hotunplug of
151+memory.
141152
142− (0x100000000 / 1Gib = 4)
153+Onlining Memory Blocks Manually
154+-------------------------------
143155
144−This device covers address range [0x100000000 ... 0x140000000)
156+If auto-onlining of memory blocks isn't enabled, user-space has to manually
157+trigger onlining of memory blocks. Often, udev rules are used to automate this
158+task in user space.
145159
146−Under each memory block, you can see 5 files:
160+Onlining of a memory block can be triggered via::
147161
148−- ``/sys/devices/system/memory/memoryXXX/phys_index``
149−- ``/sys/devices/system/memory/memoryXXX/phys_device``
150−- ``/sys/devices/system/memory/memoryXXX/state``
151−- ``/sys/devices/system/memory/memoryXXX/removable``
152−- ``/sys/devices/system/memory/memoryXXX/valid_zones``
162+ % echo online > /sys/devices/system/memory/memoryXXX/state
153163
154−=================== ============================================================
155−``phys_index`` read-only and contains memory block id, same as XXX.
156−``state`` read-write
164+Or alternatively::
157165
158− - at read: contains online/offline state of memory.
159− - at write: user can specify "online_kernel",
166+ % echo 1 > /sys/devices/system/memory/memoryXXX/online
160167
161− "online_movable", "online", "offline" command
162− which will be performed on all sections in the block.
163−``phys_device`` read-only: legacy interface only ever used on s390x to
164− expose the covered storage increment.
165−``removable`` read-only: legacy interface that indicated whether a memory
166− block was likely to be offlineable or not. Newer kernel
167− versions return "1" if and only if the kernel supports
168− memory offlining.
169−``valid_zones`` read-only: designed to show by which zone memory provided by
170− a memory block is managed, and to show by which zone memory
171− provided by an offline memory block could be managed when
172− onlining.
173−
174− The first column shows it`s default zone.
175−
176− "memory6/valid_zones: Normal Movable" shows this memoryblock
177− can be onlined to ZONE_NORMAL by default and to ZONE_MOVABLE
178− by online_movable.
179−
180− "memory7/valid_zones: Movable Normal" shows this memoryblock
181− can be onlined to ZONE_MOVABLE by default and to ZONE_NORMAL
182− by online_kernel.
183−=================== ============================================================
168+The kernel will select the target zone automatically, usually defaulting to
169+``ZONE_NORMAL`` unless ``movablecore=1`` has been specified on the kernel
170+command line or if the memory block would intersect the ZONE_MOVABLE already.
184171
185−.. note::
172+One can explicitly request to associate an offline memory block with
173+ZONE_MOVABLE by::
186174
187− These directories/files appear after physical memory hotplug phase.
175+ % echo online_movable > /sys/devices/system/memory/memoryXXX/state
188176
189−If CONFIG_NUMA is enabled the memoryXXX/ directories can also be accessed
190−via symbolic links located in the ``/sys/devices/system/node/node*`` directories.
177+Or one can explicitly request a kernel zone (usually ZONE_NORMAL) by::
191178
192−For example::
179+ % echo online_kernel > /sys/devices/system/memory/memoryXXX/state
193180
194− /sys/devices/system/node/node0/memory9 -> ../../memory/memory9
181+In any case, if onlining succeeds, the state of the memory block is changed to
182+be "online". If it fails, the state of the memory block will remain unchanged
183+and the above commands will fail.
195184
196−A backlink will also be created::
185+Onlining Memory Blocks Automatically
186+------------------------------------
197187
198− /sys/devices/system/memory/memory9/node0 -> ../../node/node0
188+The kernel can be configured to try auto-onlining of newly added memory blocks.
189+If this feature is disabled, the memory blocks will stay offline until
190+explicitly onlined from user space.
199191
200−.. _memory_hotplug_physical_mem:
192+The configured auto-online behavior can be observed via::
201193
202−Physical memory hot-add phase
203−=============================
194+ % cat /sys/devices/system/memory/auto_online_blocks
204195
205−Hardware(Firmware) Support
206−--------------------------
196+Auto-onlining can be enabled by writing ``online``, ``online_kernel`` or
197+``online_movable`` to that file, like::
207198
208−On x86_64/ia64 platform, memory hotplug by ACPI is supported.
199+ % echo online > /sys/devices/system/memory/auto_online_blocks
209200
210−In general, the firmware (ACPI) which supports memory hotplug defines
211−memory class object of _HID "PNP0C80". When a notify is asserted to PNP0C80,
212−Linux's ACPI handler does hot-add memory to the system and calls a hotplug udev
213−script. This will be done automatically.
201+Modifying the auto-online behavior will only affect all subsequently added
202+memory blocks only.
214203
215−But scripts for memory hotplug are not contained in generic udev package(now).
216−You may have to write it by yourself or online/offline memory by hand.
217−Please see :ref:`memory_hotplug_how_to_online_memory` and
218−:ref:`memory_hotplug_how_to_offline_memory`.
204+.. note::
219205
220−If firmware supports NUMA-node hotplug, and defines an object _HID "ACPI0004",
221−"PNP0A05", or "PNP0A06", notification is asserted to it, and ACPI handler
222−calls hotplug code for all of objects which are defined in it.
223−If memory device is found, memory hotplug code will be called.
206+ In corner cases, auto-onlining can fail. The kernel won't retry. Note that
207+ auto-onlining is not expected to fail in default configurations.
224208
225−Notify memory hot-add event by hand
226−-----------------------------------
209+.. note::
227210
228−On some architectures, the firmware may not notify the kernel of a memory
229−hotplug event. Therefore, the memory "probe" interface is supported to
230−explicitly notify the kernel. This interface depends on
231−CONFIG_ARCH_MEMORY_PROBE and can be configured on powerpc, sh, and x86
232−if hotplug is supported, although for x86 this should be handled by ACPI
233−notification.
211+ DLPAR on ppc64 ignores the ``offline`` setting and will still online added
212+ memory blocks; if onlining fails, memory blocks are removed again.
234213
235−Probe interface is located at::
214+Offlining Memory Blocks
215+-----------------------
236216
237− /sys/devices/system/memory/probe
217+In the current implementation, Linux's memory offlining will try migrating all
218+movable pages off the affected memory block. As most kernel allocations, such as
219+page tables, are unmovable, page migration can fail and, therefore, inhibit
220+memory offlining from succeeding.
238221
239−You can tell the physical address of new memory to the kernel by::
222+Having the memory provided by memory block managed by ZONE_MOVABLE significantly
223+increases memory offlining reliability; still, memory offlining can fail in
224+some corner cases.
240225
241− % echo start_address_of_new_memory > /sys/devices/system/memory/probe
226+Further, memory offlining might retry for a long time (or even forever), until
227+aborted by the user.
242228
243−Then, [start_address_of_new_memory, start_address_of_new_memory +
244−memory_block_size] memory range is hot-added. In this case, hotplug script is
245−not called (in current implementation). You'll have to online memory by
246−yourself. Please see :ref:`memory_hotplug_how_to_online_memory`.
229+Offlining of a memory block can be triggered via::
247230
248−Logical Memory hot-add phase
249−============================
231+ % echo offline > /sys/devices/system/memory/memoryXXX/state
250232
251−State of memory
252−---------------
233+Or alternatively::
253234
254−To see (online/offline) state of a memory block, read 'state' file::
235+ % echo 0 > /sys/devices/system/memory/memoryXXX/online
255236
256− % cat /sys/device/system/memory/memoryXXX/state
237+If offlining succeeds, the state of the memory block is changed to be "offline".
238+If it fails, the state of the memory block will remain unchanged and the above
239+commands will fail, for example, via::
257240
241+ bash: echo: write error: Device or resource busy
258242
259−- If the memory block is online, you'll read "online".
260−- If the memory block is offline, you'll read "offline".
243+or via::
261244
245+ bash: echo: write error: Invalid argument
262246
263−.. _memory_hotplug_how_to_online_memory:
247+Observing the State of Memory Blocks
248+------------------------------------
264249
265−How to online memory
266−--------------------
250+The state (online/offline/going-offline) of a memory block can be observed
251+either via::
267252
268−When the memory is hot-added, the kernel decides whether or not to "online"
269−it according to the policy which can be read from "auto_online_blocks" file::
253+ % cat /sys/device/system/memory/memoryXXX/state
270254
271− % cat /sys/devices/system/memory/auto_online_blocks
255+Or alternatively (1/0) via::
272256
273−The default depends on the CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE kernel config
274−option. If it is disabled the default is "offline" which means the newly added
275−memory is not in a ready-to-use state and you have to "online" the newly added
276−memory blocks manually. Automatic onlining can be requested by writing "online"
277−to "auto_online_blocks" file::
257+ % cat /sys/device/system/memory/memoryXXX/online
278258
279− % echo online > /sys/devices/system/memory/auto_online_blocks
259+For an online memory block, the managing zone can be observed via::
280260
281−This sets a global policy and impacts all memory blocks that will subsequently
282−be hotplugged. Currently offline blocks keep their state. It is possible, under
283−certain circumstances, that some memory blocks will be added but will fail to
284−online. User space tools can check their "state" files
285−(``/sys/devices/system/memory/memoryXXX/state``) and try to online them manually.
261+ % cat /sys/device/system/memory/memoryXXX/valid_zones
286262
287−If the automatic onlining wasn't requested, failed, or some memory block was
288−offlined it is possible to change the individual block's state by writing to the
289−"state" file::
263+Configuring Memory Hot(Un)Plug
264+==============================
290265
291− % echo online > /sys/devices/system/memory/memoryXXX/state
266+There are various ways how system administrators can configure memory
267+hot(un)plug and interact with memory blocks, especially, to online them.
292268
293−This onlining will not change the ZONE type of the target memory block,
294−If the memory block doesn't belong to any zone an appropriate kernel zone
295−(usually ZONE_NORMAL) will be used unless movable_node kernel command line
296−option is specified when ZONE_MOVABLE will be used.
269+Memory Hot(Un)Plug Configuration via Sysfs
270+------------------------------------------
297271
298−You can explicitly request to associate it with ZONE_MOVABLE by::
272+Some memory hot(un)plug properties can be configured or inspected via sysfs in::
299273
300− % echo online_movable > /sys/devices/system/memory/memoryXXX/state
274+ /sys/devices/system/memory/
301275
302−.. note:: current limit: this memory block must be adjacent to ZONE_MOVABLE
276+The following files are currently defined:
303277
304−Or you can explicitly request a kernel zone (usually ZONE_NORMAL) by::
278+====================== =========================================================
279+``auto_online_blocks`` read-write: set or get the default state of new memory
280+ blocks; configure auto-onlining.
305281
306− % echo online_kernel > /sys/devices/system/memory/memoryXXX/state
282+ The default value depends on the
283+ CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE kernel configuration
284+ option.
307285
308−.. note:: current limit: this memory block must be adjacent to ZONE_NORMAL
286+ See the ``state`` property of memory blocks for details.
287+``block_size_bytes`` read-only: the size in bytes of a memory block.
288+``probe`` write-only: add (probe) selected memory blocks manually
289+ from user space by supplying the physical start address.
309290
310−An explicit zone onlining can fail (e.g. when the range is already within
311−and existing and incompatible zone already).
291+ Availability depends on the CONFIG_ARCH_MEMORY_PROBE
292+ kernel configuration option.
293+``uevent`` read-write: generic udev file for device subsystems.
294+====================== =========================================================
312295
313−After this, memory block XXX's state will be 'online' and the amount of
314−available memory will be increased.
296+.. note::
315297
316−This may be changed in future.
298+ When the CONFIG_MEMORY_FAILURE kernel configuration option is enabled, two
299+ additional files ``hard_offline_page`` and ``soft_offline_page`` are available
300+ to trigger hwpoisoning of pages, for example, for testing purposes. Note that
301+ this functionality is not really related to memory hot(un)plug or actual
302+ offlining of memory blocks.
317303
318−Logical memory remove
319−=====================
304+Memory Block Configuration via Sysfs
305+------------------------------------
320306
321−Memory offline and ZONE_MOVABLE
322−-------------------------------
307+Each memory block is represented as a memory block device that can be
308+onlined or offlined. All memory blocks have their device information located in
309+sysfs. Each present memory block is listed under
310+``/sys/devices/system/memory`` as::
323311
324−Memory offlining is more complicated than memory online. Because memory offline
325−has to make the whole memory block be unused, memory offline can fail if
326−the memory block includes memory which cannot be freed.
312+ /sys/devices/system/memory/memoryXXX
327313
328−In general, memory offline can use 2 techniques.
314+where XXX is the memory block id; the number of digits is variable.
329315
330−(1) reclaim and free all memory in the memory block.
331−(2) migrate all pages in the memory block.
316+A present memory block indicates that some memory in the range is present;
317+however, a memory block might span memory holes. A memory block spanning memory
318+holes cannot be offlined.
332319
333−In the current implementation, Linux's memory offline uses method (2), freeing
334−all pages in the memory block by page migration. But not all pages are
335−migratable. Under current Linux, migratable pages are anonymous pages and
336−page caches. For offlining a memory block by migration, the kernel has to
337−guarantee that the memory block contains only migratable pages.
320+For example, assume 1 GiB memory block size. A device for a memory starting at
321+0x100000000 is ``/sys/device/system/memory/memory4``::
338322
339−Now, a boot option for making a memory block which consists of migratable pages
340−is supported. By specifying "kernelcore=" or "movablecore=" boot option, you can
341−create ZONE_MOVABLE...a zone which is just used for movable pages.
342−(See also Documentation/admin-guide/kernel-parameters.rst)
323+ (0x100000000 / 1Gib = 4)
343324
344−Assume the system has "TOTAL" amount of memory at boot time, this boot option
345−creates ZONE_MOVABLE as following.
325+This device covers address range [0x100000000 ... 0x140000000)
346326
347−1) When kernelcore=YYYY boot option is used,
348− Size of memory not for movable pages (not for offline) is YYYY.
349− Size of memory for movable pages (for offline) is TOTAL-YYYY.
327+The following files are currently defined:
350328
351−2) When movablecore=ZZZZ boot option is used,
352− Size of memory not for movable pages (not for offline) is TOTAL - ZZZZ.
353− Size of memory for movable pages (for offline) is ZZZZ.
329+=================== ============================================================
330+``online`` read-write: simplified interface to trigger onlining /
331+ offlining and to observe the state of a memory block.
332+ When onlining, the zone is selected automatically.
333+``phys_device`` read-only: legacy interface only ever used on s390x to
334+ expose the covered storage increment.
335+``phys_index`` read-only: the memory block id (XXX).
336+``removable`` read-only: legacy interface that indicated whether a memory
337+ block was likely to be offlineable or not. Nowadays, the
338+ kernel return ``1`` if and only if it supports memory
339+ offlining.
340+``state`` read-write: advanced interface to trigger onlining /
341+ offlining and to observe the state of a memory block.
342+
343+ When writing, ``online``, ``offline``, ``online_kernel`` and
344+ ``online_movable`` are supported.
345+
346+ ``online_movable`` specifies onlining to ZONE_MOVABLE.
347+ ``online_kernel`` specifies onlining to the default kernel
348+ zone for the memory block, such as ZONE_NORMAL.
349+ ``online`` let's the kernel select the zone automatically.
350+
351+ When reading, ``online``, ``offline`` and ``going-offline``
352+ may be returned.
353+``uevent`` read-write: generic uevent file for devices.
354+``valid_zones`` read-only: when a block is online, shows the zone it
355+ belongs to; when a block is offline, shows what zone will
356+ manage it when the block will be onlined.
357+
358+ For online memory blocks, ``DMA``, ``DMA32``, ``Normal``,
359+ ``Movable`` and ``none`` may be returned. ``none`` indicates
360+ that memory provided by a memory block is managed by
361+ multiple zones or spans multiple nodes; such memory blocks
362+ cannot be offlined. ``Movable`` indicates ZONE_MOVABLE.
363+ Other values indicate a kernel zone.
364+
365+ For offline memory blocks, the first column shows the
366+ zone the kernel would select when onlining the memory block
367+ right now without further specifying a zone.
368+
369+ Availability depends on the CONFIG_MEMORY_HOTREMOVE
370+ kernel configuration option.
371+=================== ============================================================
354372
355373 .. note::
356374
357− Unfortunately, there is no information to show which memory block belongs
358− to ZONE_MOVABLE. This is TBD.
375+ If the CONFIG_NUMA kernel configuration option is enabled, the memoryXXX/
376+ directories can also be accessed via symbolic links located in the
377+ ``/sys/devices/system/node/node*`` directories.
378+
379+ For example::
380+
381+ /sys/devices/system/node/node0/memory9 -> ../../memory/memory9
382+
383+ A backlink will also be created::
384+
385+ /sys/devices/system/memory/memory9/node0 -> ../../node/node0
386+
387+Command Line Parameters
388+-----------------------
389+
390+Some command line parameters affect memory hot(un)plug handling. The following
391+command line parameters are relevant:
392+
393+======================== =======================================================
394+``memhp_default_state`` configure auto-onlining by essentially setting
395+ ``/sys/devices/system/memory/auto_online_blocks``.
396+``movablecore`` configure automatic zone selection of the kernel. When
397+ set, the kernel will default to ZONE_MOVABLE, unless
398+ other zones can be kept contiguous.
399+======================== =======================================================
400+
401+Module Parameters
402+------------------
359403
360− Memory offlining can fail when dissolving a free huge page on ZONE_MOVABLE
361− and the feature of freeing unused vmemmap pages associated with each hugetlb
362− page is enabled.
404+Instead of additional command line parameters or sysfs files, the
405+``memory_hotplug`` subsystem now provides a dedicated namespace for module
406+parameters. Module parameters can be set via the command line by predicating
407+them with ``memory_hotplug.`` such as::
408+
409+ memory_hotplug.memmap_on_memory=1
410+
411+and they can be observed (and some even modified at runtime) via::
412+
413+ /sys/modules/memory_hotplug/parameters/
414+
415+The following module parameters are currently defined:
416+
417+======================== =======================================================
418+``memmap_on_memory`` read-write: Allocate memory for the memmap from the
419+ added memory block itself. Even if enabled, actual
420+ support depends on various other system properties and
421+ should only be regarded as a hint whether the behavior
422+ would be desired.
423+
424+ While allocating the memmap from the memory block
425+ itself makes memory hotplug less likely to fail and
426+ keeps the memmap on the same NUMA node in any case, it
427+ can fragment physical memory in a way that huge pages
428+ in bigger granularity cannot be formed on hotplugged
429+ memory.
430+======================== =======================================================
431+
432+ZONE_MOVABLE
433+============
434+
435+ZONE_MOVABLE is an important mechanism for more reliable memory offlining.
436+Further, having system RAM managed by ZONE_MOVABLE instead of one of the
437+kernel zones can increase the number of possible transparent huge pages and
438+dynamically allocated huge pages.
439+
440+Most kernel allocations are unmovable. Important examples include the memory
441+map (usually 1/64ths of memory), page tables, and kmalloc(). Such allocations
442+can only be served from the kernel zones.
443+
444+Most user space pages, such as anonymous memory, and page cache pages are
445+movable. Such allocations can be served from ZONE_MOVABLE and the kernel zones.
446+
447+Only movable allocations are served from ZONE_MOVABLE, resulting in unmovable
448+allocations being limited to the kernel zones. Without ZONE_MOVABLE, there is
449+absolutely no guarantee whether a memory block can be offlined successfully.
450+
451+Zone Imbalances
452+---------------
363453
364− This can happen when we have plenty of ZONE_MOVABLE memory, but not enough
365− kernel memory to allocate vmemmmap pages. We may even be able to migrate
366− huge page contents, but will not be able to dissolve the source huge page.
367− This will prevent an offline operation and is unfortunate as memory offlining
368− is expected to succeed on movable zones. Users that depend on memory hotplug
369− to succeed for movable zones should carefully consider whether the memory
370− savings gained from this feature are worth the risk of possibly not being
371− able to offline memory in certain situations.
454+Having too much system RAM managed by ZONE_MOVABLE is called a zone imbalance,
455+which can harm the system or degrade performance. As one example, the kernel
456+might crash because it runs out of free memory for unmovable allocations,
457+although there is still plenty of free memory left in ZONE_MOVABLE.
458+
459+Usually, MOVABLE:KERNEL ratios of up to 3:1 or even 4:1 are fine. Ratios of 63:1
460+are definitely impossible due to the overhead for the memory map.
461+
462+Actual safe zone ratios depend on the workload. Extreme cases, like excessive
463+long-term pinning of pages, might not be able to deal with ZONE_MOVABLE at all.
372464
373465 .. note::
374− Techniques that rely on long-term pinnings of memory (especially, RDMA and
375− vfio) are fundamentally problematic with ZONE_MOVABLE and, therefore, memory
376− hot remove. Pinned pages cannot reside on ZONE_MOVABLE, to guarantee that
377− memory can still get hot removed - be aware that pinning can fail even if
378− there is plenty of free memory in ZONE_MOVABLE. In addition, using
379− ZONE_MOVABLE might make page pinning more expensive, because pages have to be
380− migrated off that zone first.
381466
382−.. _memory_hotplug_how_to_offline_memory:
467+ CMA memory part of a kernel zone essentially behaves like memory in
468+ ZONE_MOVABLE and similar considerations apply, especially when combining
469+ CMA with ZONE_MOVABLE.
383470
384−How to offline memory
385−---------------------
471+ZONE_MOVABLE Sizing Considerations
472+----------------------------------
386473
387−You can offline a memory block by using the same sysfs interface that was used
388−in memory onlining::
474+We usually expect that a large portion of available system RAM will actually
475+be consumed by user space, either directly or indirectly via the page cache. In
476+the normal case, ZONE_MOVABLE can be used when allocating such pages just fine.
389477
390− % echo offline > /sys/devices/system/memory/memoryXXX/state
478+With that in mind, it makes sense that we can have a big portion of system RAM
479+managed by ZONE_MOVABLE. However, there are some things to consider when using
480+ZONE_MOVABLE, especially when fine-tuning zone ratios:
481+
482+- Having a lot of offline memory blocks. Even offline memory blocks consume
483+ memory for metadata and page tables in the direct map; having a lot of offline
484+ memory blocks is not a typical case, though.
485+
486+- Memory ballooning without balloon compaction is incompatible with
487+ ZONE_MOVABLE. Only some implementations, such as virtio-balloon and
488+ pseries CMM, fully support balloon compaction.
489+
490+ Further, the CONFIG_BALLOON_COMPACTION kernel configuration option might be
491+ disabled. In that case, balloon inflation will only perform unmovable
492+ allocations and silently create a zone imbalance, usually triggered by
493+ inflation requests from the hypervisor.
494+
495+- Gigantic pages are unmovable, resulting in user space consuming a
496+ lot of unmovable memory.
497+
498+- Huge pages are unmovable when an architectures does not support huge
499+ page migration, resulting in a similar issue as with gigantic pages.
500+
501+- Page tables are unmovable. Excessive swapping, mapping extremely large
502+ files or ZONE_DEVICE memory can be problematic, although only really relevant
503+ in corner cases. When we manage a lot of user space memory that has been
504+ swapped out or is served from a file/persistent memory/... we still need a lot
505+ of page tables to manage that memory once user space accessed that memory.
506+
507+- In certain DAX configurations the memory map for the device memory will be
508+ allocated from the kernel zones.
509+
510+- KASAN can have a significant memory overhead, for example, consuming 1/8th of
511+ the total system memory size as (unmovable) tracking metadata.
512+
513+- Long-term pinning of pages. Techniques that rely on long-term pinnings
514+ (especially, RDMA and vfio/mdev) are fundamentally problematic with
515+ ZONE_MOVABLE, and therefore, memory offlining. Pinned pages cannot reside
516+ on ZONE_MOVABLE as that would turn these pages unmovable. Therefore, they
517+ have to be migrated off that zone while pinning. Pinning a page can fail
518+ even if there is plenty of free memory in ZONE_MOVABLE.
519+
520+ In addition, using ZONE_MOVABLE might make page pinning more expensive,
521+ because of the page migration overhead.
522+
523+By default, all the memory configured at boot time is managed by the kernel
524+zones and ZONE_MOVABLE is not used.
525+
526+To enable ZONE_MOVABLE to include the memory present at boot and to control the
527+ratio between movable and kernel zones there are two command line options:
528+``kernelcore=`` and ``movablecore=``. See
529+Documentation/admin-guide/kernel-parameters.rst for their description.
530+
531+Memory Offlining and ZONE_MOVABLE
532+---------------------------------
533+
534+Even with ZONE_MOVABLE, there are some corner cases where offlining a memory
535+block might fail:
536+
537+- Memory blocks with memory holes; this applies to memory blocks present during
538+ boot and can apply to memory blocks hotplugged via the XEN balloon and the
539+ Hyper-V balloon.
540+
541+- Mixed NUMA nodes and mixed zones within a single memory block prevent memory
542+ offlining; this applies to memory blocks present during boot only.
543+
544+- Special memory blocks prevented by the system from getting offlined. Examples
545+ include any memory available during boot on arm64 or memory blocks spanning
546+ the crashkernel area on s390x; this usually applies to memory blocks present
547+ during boot only.
548+
549+- Memory blocks overlapping with CMA areas cannot be offlined, this applies to
550+ memory blocks present during boot only.
551+
552+- Concurrent activity that operates on the same physical memory area, such as
553+ allocating gigantic pages, can result in temporary offlining failures.
554+
555+- Out of memory when dissolving huge pages, especially when freeing unused
556+ vmemmap pages associated with each hugetlb page is enabled.
557+
558+ Offlining code may be able to migrate huge page contents, but may not be able
559+ to dissolve the source huge page because it fails allocating (unmovable) pages
560+ for the vmemmap, because the system might not have free memory in the kernel
561+ zones left.
562+
563+ Users that depend on memory offlining to succeed for movable zones should
564+ carefully consider whether the memory savings gained from this feature are
565+ worth the risk of possibly not being able to offline memory in certain
566+ situations.
567+
568+Further, when running into out of memory situations while migrating pages, or
569+when still encountering permanently unmovable pages within ZONE_MOVABLE
570+(-> BUG), memory offlining will keep retrying until it eventually succeeds.
571+
572+When offlining is triggered from user space, the offlining context can be
573+terminated by sending a fatal signal. A timeout based offlining can easily be
574+implemented via::
391575
392−If offline succeeds, the state of the memory block is changed to be "offline".
393−If it fails, some error core (like -EBUSY) will be returned by the kernel.
394−Even if a memory block does not belong to ZONE_MOVABLE, you can try to offline
395−it. If it doesn't contain 'unmovable' memory, you'll get success.
396−
397−A memory block under ZONE_MOVABLE is considered to be able to be offlined
398−easily. But under some busy state, it may return -EBUSY. Even if a memory
399−block cannot be offlined due to -EBUSY, you can retry offlining it and may be
400−able to offline it (or not). (For example, a page is referred to by some kernel
401−internal call and released soon.)
402−
403−Consideration:
404− Memory hotplug's design direction is to make the possibility of memory
405− offlining higher and to guarantee unplugging memory under any situation. But
406− it needs more work. Returning -EBUSY under some situation may be good because
407− the user can decide to retry more or not by himself. Currently, memory
408− offlining code does some amount of retry with 120 seconds timeout.
409−
410−Physical memory remove
411−======================
412−
413−Need more implementation yet....
414− - Notification completion of remove works by OS to firmware.
415− - Guard from remove if not yet.
416−
417−
418−Locking Internals
419−=================
420−
421−When adding/removing memory that uses memory block devices (i.e. ordinary RAM),
422−the device_hotplug_lock should be held to:
423−
424−- synchronize against online/offline requests (e.g. via sysfs). This way, memory
425− block devices can only be accessed (.online/.state attributes) by user
426− space once memory has been fully added. And when removing memory, we
427− know nobody is in critical sections.
428−- synchronize against CPU hotplug and similar (e.g. relevant for ACPI and PPC)
429−
430−Especially, there is a possible lock inversion that is avoided using
431−device_hotplug_lock when adding memory and user space tries to online that
432−memory faster than expected:
433−
434−- device_online() will first take the device_lock(), followed by
435− mem_hotplug_lock
436−- add_memory_resource() will first take the mem_hotplug_lock, followed by
437− the device_lock() (while creating the devices, during bus_add_device()).
438−
439−As the device is visible to user space before taking the device_lock(), this
440−can result in a lock inversion.
441−
442−onlining/offlining of memory should be done via device_online()/
443−device_offline() - to make sure it is properly synchronized to actions
444−via sysfs. Holding device_hotplug_lock is advised (to e.g. protect online_type)
445−
446−When adding/removing/onlining/offlining memory or adding/removing
447−heterogeneous/device memory, we should always hold the mem_hotplug_lock in
448−write mode to serialise memory hotplug (e.g. access to global/zone
449−variables).
450−
451−In addition, mem_hotplug_lock (in contrast to device_hotplug_lock) in read
452−mode allows for a quite efficient get_online_mems/put_online_mems
453−implementation, so code accessing memory can protect from that memory
454−vanishing.
455−
456−
457−Future Work
458−===========
459−
460− - allowing memory hot-add to ZONE_MOVABLE. maybe we need some switch like
461− sysctl or new control file.
462− - showing memory block and physical device relationship.
463− - test and make it better memory offlining.
464− - support HugeTLB page migration and offlining.
465− - memmap removing at memory offline.
466− - physical remove memory.
576+ % timeout $TIMEOUT offline_block | failure_handling
Documentation/dev-tools/kfence.rst+53−45
@@ -65,25 +65,27 @@ Error reports · 64 unmodified lines
6565 A typical out-of-bounds access looks like this::
6666
6767 ==================================================================
68− BUG: KFENCE: out-of-bounds read in test_out_of_bounds_read+0xa3/0x22b
68+ BUG: KFENCE: out-of-bounds read in test_out_of_bounds_read+0xa6/0x234
6969
70− Out-of-bounds read at 0xffffffffb672efff (1B left of kfence-#17):
71− test_out_of_bounds_read+0xa3/0x22b
72− kunit_try_run_case+0x51/0x85
70+ Out-of-bounds read at 0xffff8c3f2e291fff (1B left of kfence-#72):
71+ test_out_of_bounds_read+0xa6/0x234
72+ kunit_try_run_case+0x61/0xa0
7373 kunit_generic_run_threadfn_adapter+0x16/0x30
74− kthread+0x137/0x160
74+ kthread+0x176/0x1b0
7575 ret_from_fork+0x22/0x30
7676
77− kfence-#17 [0xffffffffb672f000-0xffffffffb672f01f, size=32, cache=kmalloc-32] allocated by task 507:
78− test_alloc+0xf3/0x25b
79− test_out_of_bounds_read+0x98/0x22b
80− kunit_try_run_case+0x51/0x85
77+ kfence-#72: 0xffff8c3f2e292000-0xffff8c3f2e29201f, size=32, cache=kmalloc-32
78+
79+ allocated by task 484 on cpu 0 at 32.919330s:
80+ test_alloc+0xfe/0x738
81+ test_out_of_bounds_read+0x9b/0x234
82+ kunit_try_run_case+0x61/0xa0
8183 kunit_generic_run_threadfn_adapter+0x16/0x30
82− kthread+0x137/0x160
84+ kthread+0x176/0x1b0
8385 ret_from_fork+0x22/0x30
8486
85− CPU: 4 PID: 107 Comm: kunit_try_catch Not tainted 5.8.0-rc6+ #7
86− Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014
87+ CPU: 0 PID: 484 Comm: kunit_try_catch Not tainted 5.13.0-rc3+ #7
88+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014
8789 ==================================================================
8890
8991 The header of the report provides a short summary of the function involved in
@@ -96,30 +98,32 @@ Use-after-free accesses are reported as:: · 6 unmodified lines
9698 ==================================================================
9799 BUG: KFENCE: use-after-free read in test_use_after_free_read+0xb3/0x143
98100
99− Use-after-free read at 0xffffffffb673dfe0 (in kfence-#24):
101+ Use-after-free read at 0xffff8c3f2e2a0000 (in kfence-#79):
100102 test_use_after_free_read+0xb3/0x143
101− kunit_try_run_case+0x51/0x85
103+ kunit_try_run_case+0x61/0xa0
102104 kunit_generic_run_threadfn_adapter+0x16/0x30
103− kthread+0x137/0x160
105+ kthread+0x176/0x1b0
104106 ret_from_fork+0x22/0x30
105107
106− kfence-#24 [0xffffffffb673dfe0-0xffffffffb673dfff, size=32, cache=kmalloc-32] allocated by task 507:
107− test_alloc+0xf3/0x25b
108+ kfence-#79: 0xffff8c3f2e2a0000-0xffff8c3f2e2a001f, size=32, cache=kmalloc-32
109+
110+ allocated by task 488 on cpu 2 at 33.871326s:
111+ test_alloc+0xfe/0x738
108112 test_use_after_free_read+0x76/0x143
109− kunit_try_run_case+0x51/0x85
113+ kunit_try_run_case+0x61/0xa0
110114 kunit_generic_run_threadfn_adapter+0x16/0x30
111− kthread+0x137/0x160
115+ kthread+0x176/0x1b0
112116 ret_from_fork+0x22/0x30
113117
114− freed by task 507:
118+ freed by task 488 on cpu 2 at 33.871358s:
115119 test_use_after_free_read+0xa8/0x143
116− kunit_try_run_case+0x51/0x85
120+ kunit_try_run_case+0x61/0xa0
117121 kunit_generic_run_threadfn_adapter+0x16/0x30
118− kthread+0x137/0x160
122+ kthread+0x176/0x1b0
119123 ret_from_fork+0x22/0x30
120124
121− CPU: 4 PID: 109 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7
122− Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014
125+ CPU: 2 PID: 488 Comm: kunit_try_catch Tainted: G B 5.13.0-rc3+ #7
126+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014
123127 ==================================================================
124128
125129 KFENCE also reports on invalid frees, such as double-frees::
@@ -127,30 +131,32 @@ KFENCE also reports on invalid frees, such as double-frees:: · 1 unmodified line
127131 ==================================================================
128132 BUG: KFENCE: invalid free in test_double_free+0xdc/0x171
129133
130− Invalid free of 0xffffffffb6741000:
134+ Invalid free of 0xffff8c3f2e2a4000 (in kfence-#81):
131135 test_double_free+0xdc/0x171
132− kunit_try_run_case+0x51/0x85
136+ kunit_try_run_case+0x61/0xa0
133137 kunit_generic_run_threadfn_adapter+0x16/0x30
134− kthread+0x137/0x160
138+ kthread+0x176/0x1b0
135139 ret_from_fork+0x22/0x30
136140
137− kfence-#26 [0xffffffffb6741000-0xffffffffb674101f, size=32, cache=kmalloc-32] allocated by task 507:
138− test_alloc+0xf3/0x25b
141+ kfence-#81: 0xffff8c3f2e2a4000-0xffff8c3f2e2a401f, size=32, cache=kmalloc-32
142+
143+ allocated by task 490 on cpu 1 at 34.175321s:
144+ test_alloc+0xfe/0x738
139145 test_double_free+0x76/0x171
140− kunit_try_run_case+0x51/0x85
146+ kunit_try_run_case+0x61/0xa0
141147 kunit_generic_run_threadfn_adapter+0x16/0x30
142− kthread+0x137/0x160
148+ kthread+0x176/0x1b0
143149 ret_from_fork+0x22/0x30
144150
145− freed by task 507:
151+ freed by task 490 on cpu 1 at 34.175348s:
146152 test_double_free+0xa8/0x171
147− kunit_try_run_case+0x51/0x85
153+ kunit_try_run_case+0x61/0xa0
148154 kunit_generic_run_threadfn_adapter+0x16/0x30
149− kthread+0x137/0x160
155+ kthread+0x176/0x1b0
150156 ret_from_fork+0x22/0x30
151157
152− CPU: 4 PID: 111 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7
153− Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014
158+ CPU: 1 PID: 490 Comm: kunit_try_catch Tainted: G B 5.13.0-rc3+ #7
159+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014
154160 ==================================================================
155161
156162 KFENCE also uses pattern-based redzones on the other side of an object's guard
@@ -160,23 +166,25 @@ These are reported on frees:: · 3 unmodified lines
160166 ==================================================================
161167 BUG: KFENCE: memory corruption in test_kmalloc_aligned_oob_write+0xef/0x184
162168
163− Corrupted memory at 0xffffffffb6797ff9 [ 0xac . . . . . . ] (in kfence-#69):
169+ Corrupted memory at 0xffff8c3f2e33aff9 [ 0xac . . . . . . ] (in kfence-#156):
164170 test_kmalloc_aligned_oob_write+0xef/0x184
165− kunit_try_run_case+0x51/0x85
171+ kunit_try_run_case+0x61/0xa0
166172 kunit_generic_run_threadfn_adapter+0x16/0x30
167− kthread+0x137/0x160
173+ kthread+0x176/0x1b0
168174 ret_from_fork+0x22/0x30
169175
170− kfence-#69 [0xffffffffb6797fb0-0xffffffffb6797ff8, size=73, cache=kmalloc-96] allocated by task 507:
171− test_alloc+0xf3/0x25b
176+ kfence-#156: 0xffff8c3f2e33afb0-0xffff8c3f2e33aff8, size=73, cache=kmalloc-96
177+
178+ allocated by task 502 on cpu 7 at 42.159302s:
179+ test_alloc+0xfe/0x738
172180 test_kmalloc_aligned_oob_write+0x57/0x184
173− kunit_try_run_case+0x51/0x85
181+ kunit_try_run_case+0x61/0xa0
174182 kunit_generic_run_threadfn_adapter+0x16/0x30
175− kthread+0x137/0x160
183+ kthread+0x176/0x1b0
176184 ret_from_fork+0x22/0x30
177185
178− CPU: 4 PID: 120 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7
179− Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014
186+ CPU: 7 PID: 502 Comm: kunit_try_catch Tainted: G B 5.13.0-rc3+ #7
187+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014
180188 ==================================================================
181189
182190 For such errors, the address where the corruption occurred as well as the
Documentation/kbuild/llvm.rst+3−2
@@ -130,9 +130,10 @@ Getting Help · 129 unmodified lines
130130 ------------
131131
132132 - `Website <https://clangbuiltlinux.github.io/>`_
133−- `Mailing List <https://groups.google.com/forum/#!forum/clang-built-linux>`_: <clang-built-linux@googlegroups.com>
133+- `Mailing List <https://lore.kernel.org/llvm/>`_: <llvm@lists.linux.dev>
134+- `Old Mailing List Archives <https://groups.google.com/g/clang-built-linux>`_
134135 - `Issue Tracker <https://github.com/ClangBuiltLinux/linux/issues>`_
135−- IRC: #clangbuiltlinux on chat.freenode.net
136+- IRC: #clangbuiltlinux on irc.libera.chat
136137 - `Telegram <https://t.me/ClangBuiltLinux>`_: @ClangBuiltLinux
137138 - `Wiki <https://github.com/ClangBuiltLinux/linux/wiki>`_
138139 - `Beginner Bugs <https://github.com/ClangBuiltLinux/linux/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22>`_
Documentation/vm/damon/api.rst+20−0
@@ -0,0 +1,20 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+=============
4+API Reference
5+=============
6+
7+Kernel space programs can use every feature of DAMON using below APIs. All you
8+need to do is including ``damon.h``, which is located in ``include/linux/`` of
9+the source tree.
10+
11+Structures
12+==========
13+
14+.. kernel-doc:: include/linux/damon.h
15+
16+
17+Functions
18+=========
19+
20+.. kernel-doc:: mm/damon/core.c
Documentation/vm/damon/design.rst+166−0
@@ -0,0 +1,166 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+======
4+Design
5+======
6+
7+Configurable Layers
8+===================
9+
10+DAMON provides data access monitoring functionality while making the accuracy
11+and the overhead controllable. The fundamental access monitorings require
12+primitives that dependent on and optimized for the target address space. On
13+the other hand, the accuracy and overhead tradeoff mechanism, which is the core
14+of DAMON, is in the pure logic space. DAMON separates the two parts in
15+different layers and defines its interface to allow various low level
16+primitives implementations configurable with the core logic.
17+
18+Due to this separated design and the configurable interface, users can extend
19+DAMON for any address space by configuring the core logics with appropriate low
20+level primitive implementations. If appropriate one is not provided, users can
21+implement the primitives on their own.
22+
23+For example, physical memory, virtual memory, swap space, those for specific
24+processes, NUMA nodes, files, and backing memory devices would be supportable.
25+Also, if some architectures or devices support special optimized access check
26+primitives, those will be easily configurable.
27+
28+
29+Reference Implementations of Address Space Specific Primitives
30+==============================================================
31+
32+The low level primitives for the fundamental access monitoring are defined in
33+two parts:
34+
35+1. Identification of the monitoring target address range for the address space.
36+2. Access check of specific address range in the target space.
37+
38+DAMON currently provides the implementation of the primitives for only the
39+virtual address spaces. Below two subsections describe how it works.
40+
41+
42+VMA-based Target Address Range Construction
43+-------------------------------------------
44+
45+Only small parts in the super-huge virtual address space of the processes are
46+mapped to the physical memory and accessed. Thus, tracking the unmapped
47+address regions is just wasteful. However, because DAMON can deal with some
48+level of noise using the adaptive regions adjustment mechanism, tracking every
49+mapping is not strictly required but could even incur a high overhead in some
50+cases. That said, too huge unmapped areas inside the monitoring target should
51+be removed to not take the time for the adaptive mechanism.
52+
53+For the reason, this implementation converts the complex mappings to three
54+distinct regions that cover every mapped area of the address space. The two
55+gaps between the three regions are the two biggest unmapped areas in the given
56+address space. The two biggest unmapped areas would be the gap between the
57+heap and the uppermost mmap()-ed region, and the gap between the lowermost
58+mmap()-ed region and the stack in most of the cases. Because these gaps are
59+exceptionally huge in usual address spaces, excluding these will be sufficient
60+to make a reasonable trade-off. Below shows this in detail::
61+
62+ <heap>
63+ <BIG UNMAPPED REGION 1>
64+ <uppermost mmap()-ed region>
65+ (small mmap()-ed regions and munmap()-ed regions)
66+ <lowermost mmap()-ed region>
67+ <BIG UNMAPPED REGION 2>
68+ <stack>
69+
70+
71+PTE Accessed-bit Based Access Check
72+-----------------------------------
73+
74+The implementation for the virtual address space uses PTE Accessed-bit for
75+basic access checks. It finds the relevant PTE Accessed bit from the address
76+by walking the page table for the target task of the address. In this way, the
77+implementation finds and clears the bit for next sampling target address and
78+checks whether the bit set again after one sampling period. This could disturb
79+other kernel subsystems using the Accessed bits, namely Idle page tracking and
80+the reclaim logic. To avoid such disturbances, DAMON makes it mutually
81+exclusive with Idle page tracking and uses ``PG_idle`` and ``PG_young`` page
82+flags to solve the conflict with the reclaim logic, as Idle page tracking does.
83+
84+
85+Address Space Independent Core Mechanisms
86+=========================================
87+
88+Below four sections describe each of the DAMON core mechanisms and the five
89+monitoring attributes, ``sampling interval``, ``aggregation interval``,
90+``regions update interval``, ``minimum number of regions``, and ``maximum
91+number of regions``.
92+
93+
94+Access Frequency Monitoring
95+---------------------------
96+
97+The output of DAMON says what pages are how frequently accessed for a given
98+duration. The resolution of the access frequency is controlled by setting
99+``sampling interval`` and ``aggregation interval``. In detail, DAMON checks
100+access to each page per ``sampling interval`` and aggregates the results. In
101+other words, counts the number of the accesses to each page. After each
102+``aggregation interval`` passes, DAMON calls callback functions that previously
103+registered by users so that users can read the aggregated results and then
104+clears the results. This can be described in below simple pseudo-code::
105+
106+ while monitoring_on:
107+ for page in monitoring_target:
108+ if accessed(page):
109+ nr_accesses[page] += 1
110+ if time() % aggregation_interval == 0:
111+ for callback in user_registered_callbacks:
112+ callback(monitoring_target, nr_accesses)
113+ for page in monitoring_target:
114+ nr_accesses[page] = 0
115+ sleep(sampling interval)
116+
117+The monitoring overhead of this mechanism will arbitrarily increase as the
118+size of the target workload grows.
119+
120+
121+Region Based Sampling
122+---------------------
123+
124+To avoid the unbounded increase of the overhead, DAMON groups adjacent pages
125+that assumed to have the same access frequencies into a region. As long as the
126+assumption (pages in a region have the same access frequencies) is kept, only
127+one page in the region is required to be checked. Thus, for each ``sampling
128+interval``, DAMON randomly picks one page in each region, waits for one
129+``sampling interval``, checks whether the page is accessed meanwhile, and
130+increases the access frequency of the region if so. Therefore, the monitoring
131+overhead is controllable by setting the number of regions. DAMON allows users
132+to set the minimum and the maximum number of regions for the trade-off.
133+
134+This scheme, however, cannot preserve the quality of the output if the
135+assumption is not guaranteed.
136+
137+
138+Adaptive Regions Adjustment
139+---------------------------
140+
141+Even somehow the initial monitoring target regions are well constructed to
142+fulfill the assumption (pages in same region have similar access frequencies),
143+the data access pattern can be dynamically changed. This will result in low
144+monitoring quality. To keep the assumption as much as possible, DAMON
145+adaptively merges and splits each region based on their access frequency.
146+
147+For each ``aggregation interval``, it compares the access frequencies of
148+adjacent regions and merges those if the frequency difference is small. Then,
149+after it reports and clears the aggregated access frequency of each region, it
150+splits each region into two or three regions if the total number of regions
151+will not exceed the user-specified maximum number of regions after the split.
152+
153+In this way, DAMON provides its best-effort quality and minimal overhead while
154+keeping the bounds users set for their trade-off.
155+
156+
157+Dynamic Target Space Updates Handling
158+-------------------------------------
159+
160+The monitoring target address range could dynamically changed. For example,
161+virtual memory could be dynamically mapped and unmapped. Physical memory could
162+be hot-plugged.
163+
164+As the changes could be quite frequent in some cases, DAMON checks the dynamic
165+memory mapping changes and applies it to the abstracted target area only for
166+each of a user-specified time interval (``regions update interval``).
Documentation/vm/damon/faq.rst+51−0
@@ -0,0 +1,51 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+==========================
4+Frequently Asked Questions
5+==========================
6+
7+Why a new subsystem, instead of extending perf or other user space tools?
8+=========================================================================
9+
10+First, because it needs to be lightweight as much as possible so that it can be
11+used online, any unnecessary overhead such as kernel - user space context
12+switching cost should be avoided. Second, DAMON aims to be used by other
13+programs including the kernel. Therefore, having a dependency on specific
14+tools like perf is not desirable. These are the two biggest reasons why DAMON
15+is implemented in the kernel space.
16+
17+
18+Can 'idle pages tracking' or 'perf mem' substitute DAMON?
19+=========================================================
20+
21+Idle page tracking is a low level primitive for access check of the physical
22+address space. 'perf mem' is similar, though it can use sampling to minimize
23+the overhead. On the other hand, DAMON is a higher-level framework for the
24+monitoring of various address spaces. It is focused on memory management
25+optimization and provides sophisticated accuracy/overhead handling mechanisms.
26+Therefore, 'idle pages tracking' and 'perf mem' could provide a subset of
27+DAMON's output, but cannot substitute DAMON.
28+
29+
30+Does DAMON support virtual memory only?
31+=======================================
32+
33+No. The core of the DAMON is address space independent. The address space
34+specific low level primitive parts including monitoring target regions
35+constructions and actual access checks can be implemented and configured on the
36+DAMON core by the users. In this way, DAMON users can monitor any address
37+space with any access check technique.
38+
39+Nonetheless, DAMON provides vma tracking and PTE Accessed bit check based
40+implementations of the address space dependent functions for the virtual memory
41+by default, for a reference and convenient use. In near future, we will
42+provide those for physical memory address space.
43+
44+
45+Can I simply monitor page granularity?
46+======================================
47+
48+Yes. You can do so by setting the ``min_nr_regions`` attribute higher than the
49+working set size divided by the page size. Because the monitoring target
50+regions size is forced to be ``>=page size``, the region split will make no
51+effect.
Documentation/vm/damon/index.rst+30−0
@@ -0,0 +1,30 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+==========================
4+DAMON: Data Access MONitor
5+==========================
6+
7+DAMON is a data access monitoring framework subsystem for the Linux kernel.
8+The core mechanisms of DAMON (refer to :doc:`design` for the detail) make it
9+
10+ - *accurate* (the monitoring output is useful enough for DRAM level memory
11+ management; It might not appropriate for CPU Cache levels, though),
12+ - *light-weight* (the monitoring overhead is low enough to be applied online),
13+ and
14+ - *scalable* (the upper-bound of the overhead is in constant range regardless
15+ of the size of target workloads).
16+
17+Using this framework, therefore, the kernel's memory management mechanisms can
18+make advanced decisions. Experimental memory management optimization works
19+that incurring high data accesses monitoring overhead could implemented again.
20+In user space, meanwhile, users who have some special workloads can write
21+personalized applications for better understanding and optimizations of their
22+workloads and systems.
23+
24+.. toctree::
25+ :maxdepth: 2
26+
27+ faq
28+ design
29+ api
30+ plans
Documentation/vm/index.rst+1−0
@@ -32,6 +32,7 @@ descriptions of data structures and algorithms. · 31 unmodified lines
3232 arch_pgtable_helpers
3333 balance
3434 cleancache
35+ damon/index
3536 free_page_reporting
3637 frontswap
3738 highmem
MAINTAINERS+13−2
@@ -4526,7 +4526,7 @@ F: .clang-format · 4525 unmodified lines
45264526 CLANG/LLVM BUILD SUPPORT
45274527 M: Nathan Chancellor <nathan@kernel.org>
45284528 M: Nick Desaulniers <ndesaulniers@google.com>
4529−L: clang-built-linux@googlegroups.com
4529+L: llvm@lists.linux.dev
45304530 S: Supported
45314531 W: https://clangbuiltlinux.github.io/
45324532 B: https://github.com/ClangBuiltLinux/linux/issues
@@ -4542,7 +4542,7 @@ M: Sami Tolvanen <samitolvanen@google.com> · 9 unmodified lines
45424542 M: Kees Cook <keescook@chromium.org>
45434543 R: Nathan Chancellor <nathan@kernel.org>
45444544 R: Nick Desaulniers <ndesaulniers@google.com>
4545−L: clang-built-linux@googlegroups.com
4545+L: llvm@lists.linux.dev
45464546 S: Supported
45474547 B: https://github.com/ClangBuiltLinux/linux/issues
45484548 T: git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/clang/features
@@ -5149,6 +5149,17 @@ F: net/ax25/ax25_out.c · 600 unmodified lines
51495149 F: net/ax25/ax25_timer.c
51505150 F: net/ax25/sysctl_net_ax25.c
51515151
5152+DATA ACCESS MONITOR
5153+M: SeongJae Park <sjpark@amazon.de>
5154+L: linux-mm@kvack.org
5155+S: Maintained
5156+F: Documentation/admin-guide/mm/damon/
5157+F: Documentation/vm/damon/
5158+F: include/linux/damon.h
5159+F: include/trace/events/damon.h
5160+F: mm/damon/
5161+F: tools/testing/selftests/damon/
5162+
51525163 DAVICOM FAST ETHERNET (DMFE) NETWORK DRIVER
51535164 L: netdev@vger.kernel.org
51545165 S: Orphan
arch/Kconfig+1−1
@@ -889,7 +889,7 @@ config HAVE_SOFTIRQ_ON_OWN_STACK · 888 unmodified lines
889889 bool
890890 help
891891 Architecture provides a function to run __do_softirq() on a
892− seperate stack.
892+ separate stack.
893893
894894 config PGTABLE_LEVELS
895895 int
arch/alpha/include/asm/agp.h+2−2
@@ -6,8 +6,8 @@ · 5 unmodified lines
66
77 /* dummy for now */
88
9−#define map_page_into_agp(page)
10−#define unmap_page_from_agp(page)
9+#define map_page_into_agp(page) do { } while (0)
10+#define unmap_page_from_agp(page) do { } while (0)
1111 #define flush_agp_cache() mb()
1212
1313 /* GATT allocation. Returns/accepts GATT kernel virtual address. */
arch/alpha/kernel/pci-sysfs.c+8−4
@@ -60,6 +60,8 @@ static int __pci_mmap_fits(struct pci_dev *pdev, int num, · 59 unmodified lines
6060 * @sparse: address space type
6161 *
6262 * Use the bus mapping routines to map a PCI resource into userspace.
63+ *
64+ * Return: %0 on success, negative error code otherwise
6365 */
6466 static int pci_mmap_resource(struct kobject *kobj,
6567 struct bin_attribute *attr,
@@ -106,7 +108,7 @@ static int pci_mmap_resource_dense(struct file *filp, struct kobject *kobj, · 40 unmodified lines
106108
107109 /**
108110 * pci_remove_resource_files - cleanup resource files
109− * @dev: dev to cleanup
111+ * @pdev: pci_dev to cleanup
110112 *
111113 * If we created resource files for @dev, remove them from sysfs and
112114 * free their resources.
@@ -221,10 +223,12 @@ static int pci_create_attr(struct pci_dev *pdev, int num) · 108 unmodified lines
221223 }
222224
223225 /**
224− * pci_create_resource_files - create resource files in sysfs for @dev
225− * @dev: dev in question
226+ * pci_create_resource_files - create resource files in sysfs for @pdev
227+ * @pdev: pci_dev in question
226228 *
227229 * Walk the resources in @dev creating files for each resource available.
230+ *
231+ * Return: %0 on success, or negative error code
228232 */
229233 int pci_create_resource_files(struct pci_dev *pdev)
230234 {
@@ -296,7 +300,7 @@ int pci_mmap_legacy_page_range(struct pci_bus *bus, struct vm_area_struct *vma, · 65 unmodified lines
296300
297301 /**
298302 * pci_adjust_legacy_attr - adjustment of legacy file attributes
299− * @b: bus to create files under
303+ * @bus: bus to create files under
300304 * @mmap_type: I/O port or memory
301305 *
302306 * Adjust file name and size for sparse mappings.
arch/arc/kernel/traps.c+0−5
@@ -20,11 +20,6 @@ · 19 unmodified lines
2020 #include <asm/unaligned.h>
2121 #include <asm/kprobes.h>
2222
23−void __init trap_init(void)
24−{
25− return;
26−}
27−
2823 void die(const char *str, struct pt_regs *regs, unsigned long address)
2924 {
3025 show_kernel_fault_diag(str, regs, address);
arch/arm/configs/dove_defconfig+0−1
@@ -56,7 +56,6 @@ CONFIG_ATA=y · 55 unmodified lines
5656 CONFIG_SATA_MV=y
5757 CONFIG_NETDEVICES=y
5858 CONFIG_MV643XX_ETH=y
59−CONFIG_INPUT_POLLDEV=y
6059 # CONFIG_INPUT_MOUSEDEV is not set
6160 CONFIG_INPUT_EVDEV=y
6261 # CONFIG_KEYBOARD_ATKBD is not set
arch/arm/configs/pxa_defconfig+0−1
@@ -284,7 +284,6 @@ CONFIG_RT2800USB=m · 283 unmodified lines
284284 CONFIG_MWIFIEX=m
285285 CONFIG_MWIFIEX_SDIO=m
286286 CONFIG_INPUT_FF_MEMLESS=m
287−CONFIG_INPUT_POLLDEV=y
288287 CONFIG_INPUT_MATRIXKMAP=y
289288 CONFIG_INPUT_MOUSEDEV=m
290289 CONFIG_INPUT_MOUSEDEV_SCREEN_X=640
arch/arm/kernel/traps.c+0−5
@@ -781,11 +781,6 @@ void abort(void) · 780 unmodified lines
781781 panic("Oops failed to kill thread");
782782 }
783783
784−void __init trap_init(void)
785−{
786− return;
787−}
788−
789784 #ifdef CONFIG_KUSER_HELPERS
790785 static void __init kuser_init(void *vectors)
791786 {
arch/arm64/mm/mmu.c+1−2
@@ -1502,8 +1502,7 @@ int arch_add_memory(int nid, u64 start, u64 size, · 1501 unmodified lines
15021502 return ret;
15031503 }
15041504
1505−void arch_remove_memory(int nid, u64 start, u64 size,
1506− struct vmem_altmap *altmap)
1505+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
15071506 {
15081507 unsigned long start_pfn = start >> PAGE_SHIFT;
15091508 unsigned long nr_pages = size >> PAGE_SHIFT;
arch/h8300/kernel/traps.c+0−4
@@ -39,10 +39,6 @@ void __init base_trap_init(void) · 38 unmodified lines
3939 {
4040 }
4141
42−void __init trap_init(void)
43−{
44−}
45−
4642 asmlinkage void set_esp0(unsigned long ssp)
4743 {
4844 current->thread.esp0 = ssp;
arch/hexagon/kernel/traps.c+0−4
@@ -28,10 +28,6 @@ · 27 unmodified lines
2828 #define TRAP_SYSCALL 1
2929 #define TRAP_DEBUG 0xdb
3030
31−void __init trap_init(void)
32−{
33−}
34−
3531 #ifdef CONFIG_GENERIC_BUG
3632 /* Maybe should resemble arch/sh/kernel/traps.c ?? */
3733 int is_valid_bugaddr(unsigned long addr)
arch/ia64/mm/init.c+1−2
@@ -484,8 +484,7 @@ int arch_add_memory(int nid, u64 start, u64 size, · 483 unmodified lines
484484 return ret;
485485 }
486486
487−void arch_remove_memory(int nid, u64 start, u64 size,
488− struct vmem_altmap *altmap)
487+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
489488 {
490489 unsigned long start_pfn = start >> PAGE_SHIFT;
491490 unsigned long nr_pages = size >> PAGE_SHIFT;
arch/mips/configs/lemote2f_defconfig+0−1
@@ -116,7 +116,6 @@ CONFIG_8139TOO=y · 115 unmodified lines
116116 CONFIG_R8169=y
117117 CONFIG_USB_USBNET=m
118118 CONFIG_USB_NET_CDC_EEM=m
119−CONFIG_INPUT_POLLDEV=m
120119 CONFIG_INPUT_EVDEV=y
121120 # CONFIG_MOUSE_PS2_ALPS is not set
122121 # CONFIG_MOUSE_PS2_LOGIPS2PP is not set
arch/mips/configs/pic32mzda_defconfig+0−1
@@ -34,7 +34,6 @@ CONFIG_SCSI_CONSTANTS=y · 33 unmodified lines
3434 CONFIG_SCSI_SCAN_ASYNC=y
3535 # CONFIG_SCSI_LOWLEVEL is not set
3636 CONFIG_INPUT_LEDS=m
37−CONFIG_INPUT_POLLDEV=y
3837 CONFIG_INPUT_MOUSEDEV=m
3938 CONFIG_INPUT_EVDEV=y
4039 CONFIG_INPUT_EVBUG=m
arch/mips/configs/rt305x_defconfig+0−1
@@ -90,7 +90,6 @@ CONFIG_PPPOE=m · 89 unmodified lines
9090 CONFIG_PPP_ASYNC=m
9191 CONFIG_ISDN=y
9292 CONFIG_INPUT=m
93−CONFIG_INPUT_POLLDEV=m
9493 # CONFIG_KEYBOARD_ATKBD is not set
9594 # CONFIG_INPUT_MOUSE is not set
9695 CONFIG_INPUT_MISC=y
arch/mips/configs/xway_defconfig+0−1
@@ -96,7 +96,6 @@ CONFIG_PPPOE=m · 95 unmodified lines
9696 CONFIG_PPP_ASYNC=m
9797 CONFIG_ISDN=y
9898 CONFIG_INPUT=m
99−CONFIG_INPUT_POLLDEV=m
10099 # CONFIG_KEYBOARD_ATKBD is not set
101100 # CONFIG_INPUT_MOUSE is not set
102101 CONFIG_INPUT_MISC=y
arch/nds32/kernel/traps.c+0−5
@@ -183,11 +183,6 @@ void __pgd_error(const char *file, int line, unsigned long val) · 182 unmodified lines
183183 }
184184
185185 extern char *exception_vector, *exception_vector_end;
186−void __init trap_init(void)
187−{
188− return;
189−}
190−
191186 void __init early_trap_init(void)
192187 {
193188 unsigned long ivb = 0;
arch/nios2/kernel/traps.c+0−5
@@ -105,11 +105,6 @@ void show_stack(struct task_struct *task, unsigned long *stack, · 104 unmodified lines
105105 printk("%s\n", loglvl);
106106 }
107107
108−void __init trap_init(void)
109−{
110− /* Nothing to do here */
111−}
112−
113108 /* Breakpoint handler */
114109 asmlinkage void breakpoint_c(struct pt_regs *fp)
115110 {
arch/openrisc/kernel/traps.c+0−5
@@ -231,11 +231,6 @@ void unhandled_exception(struct pt_regs *regs, int ea, int vector) · 230 unmodified lines
231231 die("Oops", regs, 9);
232232 }
233233
234−void __init trap_init(void)
235−{
236− /* Nothing needs to be done */
237−}
238−
239234 asmlinkage void do_trap(struct pt_regs *regs, unsigned long address)
240235 {
241236 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc);
arch/parisc/configs/generic-32bit_defconfig+0−1
@@ -111,7 +111,6 @@ CONFIG_PPP_BSDCOMP=m · 110 unmodified lines
111111 CONFIG_PPP_DEFLATE=m
112112 CONFIG_PPPOE=m
113113 # CONFIG_WLAN is not set
114−CONFIG_INPUT_POLLDEV=y
115114 CONFIG_KEYBOARD_HIL_OLD=m
116115 CONFIG_KEYBOARD_HIL=m
117116 CONFIG_MOUSE_SERIAL=y
arch/parisc/kernel/traps.c+0−4
@@ -859,7 +859,3 @@ void __init early_trap_init(void) · 858 unmodified lines
859859
860860 initialize_ivt(&fault_vector_20);
861861 }
862−
863−void __init trap_init(void)
864−{
865−}
arch/powerpc/kernel/traps.c+0−5
@@ -2219,11 +2219,6 @@ DEFINE_INTERRUPT_HANDLER(kernel_bad_stack) · 2218 unmodified lines
22192219 die("Bad kernel stack pointer", regs, SIGABRT);
22202220 }
22212221
2222−void __init trap_init(void)
2223−{
2224−}
2225−
2226−
22272222 #ifdef CONFIG_PPC_EMULATED_STATS
22282223
22292224 #define WARN_EMULATED_SETUP(type) .type = { .name = #type }
arch/powerpc/mm/mem.c+1−2
@@ -119,8 +119,7 @@ int __ref arch_add_memory(int nid, u64 start, u64 size, · 118 unmodified lines
119119 return rc;
120120 }
121121
122−void __ref arch_remove_memory(int nid, u64 start, u64 size,
123− struct vmem_altmap *altmap)
122+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
124123 {
125124 unsigned long start_pfn = start >> PAGE_SHIFT;
126125 unsigned long nr_pages = size >> PAGE_SHIFT;
arch/powerpc/platforms/pseries/hotplug-memory.c+4−5
@@ -286,7 +286,7 @@ static int pseries_remove_memblock(unsigned long base, unsigned long memblock_si · 285 unmodified lines
286286 {
287287 unsigned long block_sz, start_pfn;
288288 int sections_per_block;
289− int i, nid;
289+ int i;
290290
291291 start_pfn = base >> PAGE_SHIFT;
292292
@@ -297,10 +297,9 @@ static int pseries_remove_memblock(unsigned long base, unsigned long memblock_si · 4 unmodified lines
297297
298298 block_sz = pseries_memory_block_size();
299299 sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
300− nid = memory_add_physaddr_to_nid(base);
301300
302301 for (i = 0; i < sections_per_block; i++) {
303− __remove_memory(nid, base, MIN_MEMORY_BLOCK_SIZE);
302+ __remove_memory(base, MIN_MEMORY_BLOCK_SIZE);
304303 base += MIN_MEMORY_BLOCK_SIZE;
305304 }
306305
@@ -387,7 +386,7 @@ static int dlpar_remove_lmb(struct drmem_lmb *lmb) · 80 unmodified lines
387386
388387 block_sz = pseries_memory_block_size();
389388
390− __remove_memory(mem_block->nid, lmb->base_addr, block_sz);
389+ __remove_memory(lmb->base_addr, block_sz);
391390 put_device(&mem_block->dev);
392391
393392 /* Update memory regions for memory remove */
@@ -660,7 +659,7 @@ static int dlpar_add_lmb(struct drmem_lmb *lmb) · 266 unmodified lines
660659
661660 rc = dlpar_online_lmb(lmb);
662661 if (rc) {
663− __remove_memory(nid, lmb->base_addr, block_sz);
662+ __remove_memory(lmb->base_addr, block_sz);
664663 invalidate_lmb_associativity_index(lmb);
665664 } else {
666665 lmb->flags |= DRCONF_MEM_ASSIGNED;
arch/riscv/Kconfig+1−1
@@ -51,7 +51,7 @@ config RISCV · 50 unmodified lines
5151 select GENERIC_EARLY_IOREMAP
5252 select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO
5353 select GENERIC_IDLE_POLL_SETUP
54− select GENERIC_IOREMAP
54+ select GENERIC_IOREMAP if MMU
5555 select GENERIC_IRQ_MULTI_HANDLER
5656 select GENERIC_IRQ_SHOW
5757 select GENERIC_IRQ_SHOW_LEVEL
arch/riscv/kernel/traps.c+0−5
@@ -199,11 +199,6 @@ int is_valid_bugaddr(unsigned long pc) · 198 unmodified lines
199199 }
200200 #endif /* CONFIG_GENERIC_BUG */
201201
202−/* stvec & scratch is already set from head.S */
203−void __init trap_init(void)
204−{
205−}
206−
207202 #ifdef CONFIG_VMAP_STACK
208203 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
209204 overflow_stack)__aligned(16);
arch/s390/mm/init.c+1−2
@@ -307,8 +307,7 @@ int arch_add_memory(int nid, u64 start, u64 size, · 306 unmodified lines
307307 return rc;
308308 }
309309
310−void arch_remove_memory(int nid, u64 start, u64 size,
311− struct vmem_altmap *altmap)
310+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
312311 {
313312 unsigned long start_pfn = start >> PAGE_SHIFT;
314313 unsigned long nr_pages = size >> PAGE_SHIFT;
arch/sh/mm/init.c+1−2
@@ -414,8 +414,7 @@ int arch_add_memory(int nid, u64 start, u64 size, · 413 unmodified lines
414414 return ret;
415415 }
416416
417−void arch_remove_memory(int nid, u64 start, u64 size,
418− struct vmem_altmap *altmap)
417+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
419418 {
420419 unsigned long start_pfn = PFN_DOWN(start);
421420 unsigned long nr_pages = size >> PAGE_SHIFT;
arch/um/kernel/trap.c+0−4
@@ -311,7 +311,3 @@ void winch(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs) · 310 unmodified lines
311311 {
312312 do_IRQ(WINCH_IRQ, regs);
313313 }
314−
315−void trap_init(void)
316−{
317−}
arch/x86/configs/i386_defconfig+0−1
@@ -156,7 +156,6 @@ CONFIG_FORCEDETH=y · 155 unmodified lines
156156 CONFIG_8139TOO=y
157157 # CONFIG_8139TOO_PIO is not set
158158 CONFIG_R8169=y
159−CONFIG_INPUT_POLLDEV=y
160159 CONFIG_INPUT_EVDEV=y
161160 CONFIG_INPUT_JOYSTICK=y
162161 CONFIG_INPUT_TABLET=y
arch/x86/configs/x86_64_defconfig+0−1
@@ -148,7 +148,6 @@ CONFIG_SKY2=y · 147 unmodified lines
148148 CONFIG_FORCEDETH=y
149149 CONFIG_8139TOO=y
150150 CONFIG_R8169=y
151−CONFIG_INPUT_POLLDEV=y
152151 CONFIG_INPUT_EVDEV=y
153152 CONFIG_INPUT_JOYSTICK=y
154153 CONFIG_INPUT_TABLET=y
arch/x86/mm/init_32.c+1−2
@@ -801,8 +801,7 @@ int arch_add_memory(int nid, u64 start, u64 size, · 800 unmodified lines
801801 return __add_pages(nid, start_pfn, nr_pages, params);
802802 }
803803
804−void arch_remove_memory(int nid, u64 start, u64 size,
805− struct vmem_altmap *altmap)
804+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
806805 {
807806 unsigned long start_pfn = start >> PAGE_SHIFT;
808807 unsigned long nr_pages = size >> PAGE_SHIFT;
arch/x86/mm/init_64.c+1−2
@@ -1255,8 +1255,7 @@ kernel_physical_mapping_remove(unsigned long start, unsigned long end) · 1254 unmodified lines
12551255 remove_pagetable(start, end, true, NULL);
12561256 }
12571257
1258−void __ref arch_remove_memory(int nid, u64 start, u64 size,
1259− struct vmem_altmap *altmap)
1258+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
12601259 {
12611260 unsigned long start_pfn = start >> PAGE_SHIFT;
12621261 unsigned long nr_pages = size >> PAGE_SHIFT;
drivers/acpi/acpi_memhotplug.c+31−15
@@ -54,6 +54,7 @@ struct acpi_memory_info { · 53 unmodified lines
5454 struct acpi_memory_device {
5555 struct acpi_device *device;
5656 struct list_head res_list;
57+ int mgid;
5758 };
5859
5960 static acpi_status
@@ -169,12 +170,33 @@ static void acpi_unbind_memory_blocks(struct acpi_memory_info *info) · 109 unmodified lines
169170 static int acpi_memory_enable_device(struct acpi_memory_device *mem_device)
170171 {
171172 acpi_handle handle = mem_device->device->handle;
173+ mhp_t mhp_flags = MHP_NID_IS_MGID;
172174 int result, num_enabled = 0;
173175 struct acpi_memory_info *info;
174− mhp_t mhp_flags = MHP_NONE;
175− int node;
176+ u64 total_length = 0;
177+ int node, mgid;
176178
177179 node = acpi_get_node(handle);
180+
181+ list_for_each_entry(info, &mem_device->res_list, list) {
182+ if (!info->length)
183+ continue;
184+ /* We want a single node for the whole memory group */
185+ if (node < 0)
186+ node = memory_add_physaddr_to_nid(info->start_addr);
187+ total_length += info->length;
188+ }
189+
190+ if (!total_length) {
191+ dev_err(&mem_device->device->dev, "device is empty\n");
192+ return -EINVAL;
193+ }
194+
195+ mgid = memory_group_register_static(node, PFN_UP(total_length));
196+ if (mgid < 0)
197+ return mgid;
198+ mem_device->mgid = mgid;
199+
178200 /*
179201 * Tell the VM there is more memory here...
180202 * Note: Assume that this function returns zero on success
@@ -182,22 +204,16 @@ static int acpi_memory_enable_device(struct acpi_memory_device *mem_device) · 1 unmodified line
182204 * (i.e. memory-hot-remove function)
183205 */
184206 list_for_each_entry(info, &mem_device->res_list, list) {
185− if (info->enabled) { /* just sanity check...*/
186− num_enabled++;
187− continue;
188− }
189207 /*
190208 * If the memory block size is zero, please ignore it.
191209 * Don't try to do the following memory hotplug flowchart.
192210 */
193211 if (!info->length)
194212 continue;
195− if (node < 0)
196− node = memory_add_physaddr_to_nid(info->start_addr);
197213
198214 if (mhp_supports_memmap_on_memory(info->length))
199215 mhp_flags |= MHP_MEMMAP_ON_MEMORY;
200− result = __add_memory(node, info->start_addr, info->length,
216+ result = __add_memory(mgid, info->start_addr, info->length,
201217 mhp_flags);
202218
203219 /*
@@ -239,19 +255,14 @@ static int acpi_memory_enable_device(struct acpi_memory_device *mem_device) · 35 unmodified lines
239255
240256 static void acpi_memory_remove_memory(struct acpi_memory_device *mem_device)
241257 {
242− acpi_handle handle = mem_device->device->handle;
243258 struct acpi_memory_info *info, *n;
244− int nid = acpi_get_node(handle);
245259
246260 list_for_each_entry_safe(info, n, &mem_device->res_list, list) {
247261 if (!info->enabled)
248262 continue;
249263
250− if (nid == NUMA_NO_NODE)
251− nid = memory_add_physaddr_to_nid(info->start_addr);
252−
253264 acpi_unbind_memory_blocks(info);
254− __remove_memory(nid, info->start_addr, info->length);
265+ __remove_memory(info->start_addr, info->length);
255266 list_del(&info->list);
256267 kfree(info);
257268 }
@@ -262,6 +273,10 @@ static void acpi_memory_device_free(struct acpi_memory_device *mem_device) · 4 unmodified lines
262273 if (!mem_device)
263274 return;
264275
276+ /* In case we succeeded adding *some* memory, unregistering fails. */
277+ if (mem_device->mgid >= 0)
278+ memory_group_unregister(mem_device->mgid);
279+
265280 acpi_memory_free_device_resources(mem_device);
266281 mem_device->device->driver_data = NULL;
267282 kfree(mem_device);
@@ -282,6 +297,7 @@ static int acpi_memory_device_add(struct acpi_device *device, · 14 unmodified lines
282297
283298 INIT_LIST_HEAD(&mem_device->res_list);
284299 mem_device->device = device;
300+ mem_device->mgid = -1;
285301 sprintf(acpi_device_name(device), "%s", ACPI_MEMORY_DEVICE_NAME);
286302 sprintf(acpi_device_class(device), "%s", ACPI_MEMORY_DEVICE_CLASS);
287303 device->driver_data = mem_device;
drivers/base/memory.c+204−21
@@ -82,6 +82,12 @@ static struct bus_type memory_subsys = { · 81 unmodified lines
8282 */
8383 static DEFINE_XARRAY(memory_blocks);
8484
85+/*
86+ * Memory groups, indexed by memory group id (mgid).
87+ */
88+static DEFINE_XARRAY_FLAGS(memory_groups, XA_FLAGS_ALLOC);
89+#define MEMORY_GROUP_MARK_DYNAMIC XA_MARK_1
90+
8591 static BLOCKING_NOTIFIER_HEAD(memory_chain);
8692
8793 int register_memory_notifier(struct notifier_block *nb)
@@ -177,7 +183,8 @@ static int memory_block_online(struct memory_block *mem) · 89 unmodified lines
177183 struct zone *zone;
178184 int ret;
179185
180− zone = zone_for_pfn_range(mem->online_type, mem->nid, start_pfn, nr_pages);
186+ zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group,
187+ start_pfn, nr_pages);
181188
182189 /*
183190 * Although vmemmap pages have a different lifecycle than the pages
@@ -193,7 +200,7 @@ static int memory_block_online(struct memory_block *mem) · 9 unmodified lines
193200 }
194201
195202 ret = online_pages(start_pfn + nr_vmemmap_pages,
196− nr_pages - nr_vmemmap_pages, zone);
203+ nr_pages - nr_vmemmap_pages, zone, mem->group);
197204 if (ret) {
198205 if (nr_vmemmap_pages)
199206 mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages);
@@ -205,7 +212,8 @@ static int memory_block_online(struct memory_block *mem) · 5 unmodified lines
205212 * now already properly populated.
206213 */
207214 if (nr_vmemmap_pages)
208− adjust_present_page_count(zone, nr_vmemmap_pages);
215+ adjust_present_page_count(pfn_to_page(start_pfn), mem->group,
216+ nr_vmemmap_pages);
209217
210218 return ret;
211219 }
@@ -215,24 +223,23 @@ static int memory_block_offline(struct memory_block *mem) · 3 unmodified lines
215223 unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);
216224 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
217225 unsigned long nr_vmemmap_pages = mem->nr_vmemmap_pages;
218− struct zone *zone;
219226 int ret;
220227
221228 /*
222229 * Unaccount before offlining, such that unpopulated zone and kthreads
223230 * can properly be torn down in offline_pages().
224231 */
225− if (nr_vmemmap_pages) {
226− zone = page_zone(pfn_to_page(start_pfn));
227− adjust_present_page_count(zone, -nr_vmemmap_pages);
228− }
232+ if (nr_vmemmap_pages)
233+ adjust_present_page_count(pfn_to_page(start_pfn), mem->group,
234+ -nr_vmemmap_pages);
229235
230236 ret = offline_pages(start_pfn + nr_vmemmap_pages,
231− nr_pages - nr_vmemmap_pages);
237+ nr_pages - nr_vmemmap_pages, mem->group);
232238 if (ret) {
233239 /* offline_pages() failed. Account back. */
234240 if (nr_vmemmap_pages)
235− adjust_present_page_count(zone, nr_vmemmap_pages);
241+ adjust_present_page_count(pfn_to_page(start_pfn),
242+ mem->group, nr_vmemmap_pages);
236243 return ret;
237244 }
238245
@@ -374,12 +381,13 @@ static ssize_t phys_device_show(struct device *dev, · 135 unmodified lines
374381
375382 #ifdef CONFIG_MEMORY_HOTREMOVE
376383 static int print_allowed_zone(char *buf, int len, int nid,
384+ struct memory_group *group,
377385 unsigned long start_pfn, unsigned long nr_pages,
378386 int online_type, struct zone *default_zone)
379387 {
380388 struct zone *zone;
381389
382− zone = zone_for_pfn_range(online_type, nid, start_pfn, nr_pages);
390+ zone = zone_for_pfn_range(online_type, nid, group, start_pfn, nr_pages);
383391 if (zone == default_zone)
384392 return 0;
385393
@@ -392,9 +400,10 @@ static ssize_t valid_zones_show(struct device *dev, · 6 unmodified lines
392400 struct memory_block *mem = to_memory_block(dev);
393401 unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);
394402 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
403+ struct memory_group *group = mem->group;
395404 struct zone *default_zone;
405+ int nid = mem->nid;
396406 int len = 0;
397− int nid;
398407
399408 /*
400409 * Check the existing zone. Make sure that we do that only on the
@@ -413,14 +422,13 @@ static ssize_t valid_zones_show(struct device *dev, · 12 unmodified lines
413422 goto out;
414423 }
415424
416− nid = mem->nid;
417− default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, start_pfn,
418− nr_pages);
425+ default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, group,
426+ start_pfn, nr_pages);
419427
420428 len += sysfs_emit_at(buf, len, "%s", default_zone->name);
421− len += print_allowed_zone(buf, len, nid, start_pfn, nr_pages,
429+ len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages,
422430 MMOP_ONLINE_KERNEL, default_zone);
423− len += print_allowed_zone(buf, len, nid, start_pfn, nr_pages,
431+ len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages,
424432 MMOP_ONLINE_MOVABLE, default_zone);
425433 out:
426434 len += sysfs_emit_at(buf, len, "\n");
@@ -634,7 +642,8 @@ int register_memory(struct memory_block *memory) · 207 unmodified lines
634642 }
635643
636644 static int init_memory_block(unsigned long block_id, unsigned long state,
637− unsigned long nr_vmemmap_pages)
645+ unsigned long nr_vmemmap_pages,
646+ struct memory_group *group)
638647 {
639648 struct memory_block *mem;
640649 int ret = 0;
@@ -652,6 +661,12 @@ static int init_memory_block(unsigned long block_id, unsigned long state, · 11 unmodified lines
652661 mem->state = state;
653662 mem->nid = NUMA_NO_NODE;
654663 mem->nr_vmemmap_pages = nr_vmemmap_pages;
664+ INIT_LIST_HEAD(&mem->group_next);
665+
666+ if (group) {
667+ mem->group = group;
668+ list_add(&mem->group_next, &group->memory_blocks);
669+ }
655670
656671 ret = register_memory(mem);
657672
@@ -671,7 +686,7 @@ static int add_memory_block(unsigned long base_section_nr) · 13 unmodified lines
671686 if (section_count == 0)
672687 return 0;
673688 return init_memory_block(memory_block_id(base_section_nr),
674− MEM_ONLINE, 0);
689+ MEM_ONLINE, 0, NULL);
675690 }
676691
677692 static void unregister_memory(struct memory_block *memory)
@@ -681,6 +696,11 @@ static void unregister_memory(struct memory_block *memory) · 3 unmodified lines
681696
682697 WARN_ON(xa_erase(&memory_blocks, memory->dev.id) == NULL);
683698
699+ if (memory->group) {
700+ list_del(&memory->group_next);
701+ memory->group = NULL;
702+ }
703+
684704 /* drop the ref. we got via find_memory_block() */
685705 put_device(&memory->dev);
686706 device_unregister(&memory->dev);
@@ -694,7 +714,8 @@ static void unregister_memory(struct memory_block *memory) · 7 unmodified lines
694714 * Called under device_hotplug_lock.
695715 */
696716 int create_memory_block_devices(unsigned long start, unsigned long size,
697− unsigned long vmemmap_pages)
717+ unsigned long vmemmap_pages,
718+ struct memory_group *group)
698719 {
699720 const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
700721 unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
@@ -707,7 +728,8 @@ int create_memory_block_devices(unsigned long start, unsigned long size, · 6 unmodified lines
707728 return -EINVAL;
708729
709730 for (block_id = start_block_id; block_id != end_block_id; block_id++) {
710− ret = init_memory_block(block_id, MEM_OFFLINE, vmemmap_pages);
731+ ret = init_memory_block(block_id, MEM_OFFLINE, vmemmap_pages,
732+ group);
711733 if (ret)
712734 break;
713735 }
@@ -891,3 +913,164 @@ int for_each_memory_block(void *arg, walk_memory_blocks_func_t func) · 177 unmodified lines
891913 return bus_for_each_dev(&memory_subsys, NULL, &cb_data,
892914 for_each_memory_block_cb);
893915 }
916+
917+/*
918+ * This is an internal helper to unify allocation and initialization of
919+ * memory groups. Note that the passed memory group will be copied to a
920+ * dynamically allocated memory group. After this call, the passed
921+ * memory group should no longer be used.
922+ */
923+static int memory_group_register(struct memory_group group)
924+{
925+ struct memory_group *new_group;
926+ uint32_t mgid;
927+ int ret;
928+
929+ if (!node_possible(group.nid))
930+ return -EINVAL;
931+
932+ new_group = kzalloc(sizeof(group), GFP_KERNEL);
933+ if (!new_group)
934+ return -ENOMEM;
935+ *new_group = group;
936+ INIT_LIST_HEAD(&new_group->memory_blocks);
937+
938+ ret = xa_alloc(&memory_groups, &mgid, new_group, xa_limit_31b,
939+ GFP_KERNEL);
940+ if (ret) {
941+ kfree(new_group);
942+ return ret;
943+ } else if (group.is_dynamic) {
944+ xa_set_mark(&memory_groups, mgid, MEMORY_GROUP_MARK_DYNAMIC);
945+ }
946+ return mgid;
947+}
948+
949+/**
950+ * memory_group_register_static() - Register a static memory group.
951+ * @nid: The node id.
952+ * @max_pages: The maximum number of pages we'll have in this static memory
953+ * group.
954+ *
955+ * Register a new static memory group and return the memory group id.
956+ * All memory in the group belongs to a single unit, such as a DIMM. All
957+ * memory belonging to a static memory group is added in one go to be removed
958+ * in one go -- it's static.
959+ *
960+ * Returns an error if out of memory, if the node id is invalid, if no new
961+ * memory groups can be registered, or if max_pages is invalid (0). Otherwise,
962+ * returns the new memory group id.
963+ */
964+int memory_group_register_static(int nid, unsigned long max_pages)
965+{
966+ struct memory_group group = {
967+ .nid = nid,
968+ .s = {
969+ .max_pages = max_pages,
970+ },
971+ };
972+
973+ if (!max_pages)
974+ return -EINVAL;
975+ return memory_group_register(group);
976+}
977+EXPORT_SYMBOL_GPL(memory_group_register_static);
978+
979+/**
980+ * memory_group_register_dynamic() - Register a dynamic memory group.
981+ * @nid: The node id.
982+ * @unit_pages: Unit in pages in which is memory added/removed in this dynamic
983+ * memory group.
984+ *
985+ * Register a new dynamic memory group and return the memory group id.
986+ * Memory within a dynamic memory group is added/removed dynamically
987+ * in unit_pages.
988+ *
989+ * Returns an error if out of memory, if the node id is invalid, if no new
990+ * memory groups can be registered, or if unit_pages is invalid (0, not a
991+ * power of two, smaller than a single memory block). Otherwise, returns the
992+ * new memory group id.
993+ */
994+int memory_group_register_dynamic(int nid, unsigned long unit_pages)
995+{
996+ struct memory_group group = {
997+ .nid = nid,
998+ .is_dynamic = true,
999+ .d = {
1000+ .unit_pages = unit_pages,
1001+ },
1002+ };
1003+
1004+ if (!unit_pages || !is_power_of_2(unit_pages) ||
1005+ unit_pages < PHYS_PFN(memory_block_size_bytes()))
1006+ return -EINVAL;
1007+ return memory_group_register(group);
1008+}
1009+EXPORT_SYMBOL_GPL(memory_group_register_dynamic);
1010+
1011+/**
1012+ * memory_group_unregister() - Unregister a memory group.
1013+ * @mgid: the memory group id
1014+ *
1015+ * Unregister a memory group. If any memory block still belongs to this
1016+ * memory group, unregistering will fail.
1017+ *
1018+ * Returns -EINVAL if the memory group id is invalid, returns -EBUSY if some
1019+ * memory blocks still belong to this memory group and returns 0 if
1020+ * unregistering succeeded.
1021+ */
1022+int memory_group_unregister(int mgid)
1023+{
1024+ struct memory_group *group;
1025+
1026+ if (mgid < 0)
1027+ return -EINVAL;
1028+
1029+ group = xa_load(&memory_groups, mgid);
1030+ if (!group)
1031+ return -EINVAL;
1032+ if (!list_empty(&group->memory_blocks))
1033+ return -EBUSY;
1034+ xa_erase(&memory_groups, mgid);
1035+ kfree(group);
1036+ return 0;
1037+}
1038+EXPORT_SYMBOL_GPL(memory_group_unregister);
1039+
1040+/*
1041+ * This is an internal helper only to be used in core memory hotplug code to
1042+ * lookup a memory group. We don't care about locking, as we don't expect a
1043+ * memory group to get unregistered while adding memory to it -- because
1044+ * the group and the memory is managed by the same driver.
1045+ */
1046+struct memory_group *memory_group_find_by_id(int mgid)
1047+{
1048+ return xa_load(&memory_groups, mgid);
1049+}
1050+
1051+/*
1052+ * This is an internal helper only to be used in core memory hotplug code to
1053+ * walk all dynamic memory groups excluding a given memory group, either
1054+ * belonging to a specific node, or belonging to any node.
1055+ */
1056+int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func,
1057+ struct memory_group *excluded, void *arg)
1058+{
1059+ struct memory_group *group;
1060+ unsigned long index;
1061+ int ret = 0;
1062+
1063+ xa_for_each_marked(&memory_groups, index, group,
1064+ MEMORY_GROUP_MARK_DYNAMIC) {
1065+ if (group == excluded)
1066+ continue;
1067+#ifdef CONFIG_NUMA
1068+ if (nid != NUMA_NO_NODE && group->nid != nid)
1069+ continue;
1070+#endif /* CONFIG_NUMA */
1071+ ret = func(group, arg);
1072+ if (ret)
1073+ break;
1074+ }
1075+ return ret;
1076+}
drivers/base/node.c+0−2
@@ -785,8 +785,6 @@ int unregister_cpu_under_node(unsigned int cpu, unsigned int nid) · 784 unmodified lines
785785 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
786786 static int __ref get_nid_for_pfn(unsigned long pfn)
787787 {
788− if (!pfn_valid_within(pfn))
789− return -1;
790788 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
791789 if (system_state < SYSTEM_RUNNING)
792790 return early_pfn_to_nid(pfn);
drivers/dax/kmem.c+33−10
@@ -37,15 +37,16 @@ static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) · 36 unmodified lines
3737
3838 struct dax_kmem_data {
3939 const char *res_name;
40+ int mgid;
4041 struct resource *res[];
4142 };
4243
4344 static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
4445 {
4546 struct device *dev = &dev_dax->dev;
47+ unsigned long total_len = 0;
4648 struct dax_kmem_data *data;
47− int rc = -ENOMEM;
48− int i, mapped = 0;
49+ int i, rc, mapped = 0;
4950 int numa_node;
5051
5152 /*
@@ -61,24 +62,44 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) · 9 unmodified lines
6162 return -EINVAL;
6263 }
6364
65+ for (i = 0; i < dev_dax->nr_range; i++) {
66+ struct range range;
67+
68+ rc = dax_kmem_range(dev_dax, i, &range);
69+ if (rc) {
70+ dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n",
71+ i, range.start, range.end);
72+ continue;
73+ }
74+ total_len += range_len(&range);
75+ }
76+
77+ if (!total_len) {
78+ dev_warn(dev, "rejecting DAX region without any memory after alignment\n");
79+ return -EINVAL;
80+ }
81+
6482 data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL);
6583 if (!data)
6684 return -ENOMEM;
6785
86+ rc = -ENOMEM;
6887 data->res_name = kstrdup(dev_name(dev), GFP_KERNEL);
6988 if (!data->res_name)
7089 goto err_res_name;
7190
91+ rc = memory_group_register_static(numa_node, total_len);
92+ if (rc < 0)
93+ goto err_reg_mgid;
94+ data->mgid = rc;
95+
7296 for (i = 0; i < dev_dax->nr_range; i++) {
7397 struct resource *res;
7498 struct range range;
7599
76100 rc = dax_kmem_range(dev_dax, i, &range);
77− if (rc) {
78− dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n",
79− i, range.start, range.end);
101+ if (rc)
80102 continue;
81− }
82103
83104 /* Region is permanently reserved if hotremove fails. */
84105 res = request_mem_region(range.start, range_len(&range), data->res_name);
@@ -108,8 +129,8 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) · 23 unmodified lines
108129 * Ensure that future kexec'd kernels will not treat
109130 * this as RAM automatically.
110131 */
111− rc = add_memory_driver_managed(numa_node, range.start,
112− range_len(&range), kmem_name, MHP_NONE);
132+ rc = add_memory_driver_managed(data->mgid, range.start,
133+ range_len(&range), kmem_name, MHP_NID_IS_MGID);
113134
114135 if (rc) {
115136 dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n",
@@ -129,6 +150,8 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) · 13 unmodified lines
129150 return 0;
130151
131152 err_request_mem:
153+ memory_group_unregister(data->mgid);
154+err_reg_mgid:
132155 kfree(data->res_name);
133156 err_res_name:
134157 kfree(data);
@@ -156,8 +179,7 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) · 21 unmodified lines
156179 if (rc)
157180 continue;
158181
159− rc = remove_memory(dev_dax->target_node, range.start,
160− range_len(&range));
182+ rc = remove_memory(range.start, range_len(&range));
161183 if (rc == 0) {
162184 release_resource(data->res[i]);
163185 kfree(data->res[i]);
@@ -172,6 +194,7 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) · 8 unmodified lines
172194 }
173195
174196 if (success >= dev_dax->nr_range) {
197+ memory_group_unregister(data->mgid);
175198 kfree(data->res_name);
176199 kfree(data);
177200 dev_set_drvdata(dev, NULL);
drivers/devfreq/devfreq.c+1−1
@@ -27,6 +27,7 @@ · 26 unmodified lines
2727 #include <linux/hrtimer.h>
2828 #include <linux/of.h>
2929 #include <linux/pm_qos.h>
30+#include <linux/units.h>
3031 #include "governor.h"
3132
3233 #define CREATE_TRACE_POINTS
@@ -34,7 +35,6 @@ · 1 unmodified line
3435
3536 #define IS_SUPPORTED_FLAG(f, name) ((f & DEVFREQ_GOV_FLAG_##name) ? true : false)
3637 #define IS_SUPPORTED_ATTR(f, name) ((f & DEVFREQ_GOV_ATTR_##name) ? true : false)
37−#define HZ_PER_KHZ 1000
3838
3939 static struct class *devfreq_class;
4040 static struct dentry *devfreq_debugfs;
drivers/hwmon/mr75203.c+1−1
@@ -17,6 +17,7 @@ · 16 unmodified lines
1717 #include <linux/property.h>
1818 #include <linux/regmap.h>
1919 #include <linux/reset.h>
20+#include <linux/units.h>
2021
2122 /* PVT Common register */
2223 #define PVT_IP_CONFIG 0x04
@@ -37,7 +38,6 @@ · 14 unmodified lines
3738 #define CLK_SYNTH_EN BIT(24)
3839 #define CLK_SYS_CYCLES_MAX 514
3940 #define CLK_SYS_CYCLES_MIN 2
40−#define HZ_PER_MHZ 1000000L
4141
4242 #define SDIF_DISABLE 0x04
4343
drivers/iio/common/hid-sensors/hid-sensor-attributes.c+1−2
@@ -6,12 +6,11 @@ · 5 unmodified lines
66 #include <linux/module.h>
77 #include <linux/kernel.h>
88 #include <linux/time.h>
9+#include <linux/units.h>
910
1011 #include <linux/hid-sensor-hub.h>
1112 #include <linux/iio/iio.h>
1213
13−#define HZ_PER_MHZ 1000000L
14−
1514 static struct {
1615 u32 usage_id;
1716 int unit; /* 0 for default others from HID sensor spec */
drivers/iio/light/as73211.c+1−2
@@ -24,8 +24,7 @@ · 23 unmodified lines
2424 #include <linux/module.h>
2525 #include <linux/mutex.h>
2626 #include <linux/pm.h>
27−
28−#define HZ_PER_KHZ 1000
27+#include <linux/units.h>
2928
3029 #define AS73211_DRV_NAME "as73211"
3130
drivers/media/i2c/ov02a10.c+1−1
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include <linux/module.h>
1010 #include <linux/pm_runtime.h>
1111 #include <linux/regulator/consumer.h>
12+#include <linux/units.h>
1213 #include <media/media-entity.h>
1314 #include <media/v4l2-async.h>
1415 #include <media/v4l2-ctrls.h>
@@ -64,7 +65,6 @@ · 49 unmodified lines
6465 /* Test pattern control */
6566 #define OV02A10_REG_TEST_PATTERN 0xb6
6667
67−#define HZ_PER_MHZ 1000000L
6868 #define OV02A10_LINK_FREQ_390MHZ (390 * HZ_PER_MHZ)
6969 #define OV02A10_ECLK_FREQ (24 * HZ_PER_MHZ)
7070
drivers/mtd/nand/raw/intel-nand-controller.c+1−1
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 #include <linux/sched.h>
2121 #include <linux/slab.h>
2222 #include <linux/types.h>
23+#include <linux/units.h>
2324 #include <asm/unaligned.h>
2425
2526 #define EBU_CLC 0x000
@@ -102,7 +103,6 @@ · 76 unmodified lines
102103
103104 #define MAX_CS 2
104105
105−#define HZ_PER_MHZ 1000000L
106106 #define USEC_PER_SEC 1000000L
107107
108108 struct ebu_nand_cs {
drivers/phy/st/phy-stm32-usbphyc.c+1−1
@@ -15,6 +15,7 @@ · 14 unmodified lines
1515 #include <linux/of_platform.h>
1616 #include <linux/phy/phy.h>
1717 #include <linux/reset.h>
18+#include <linux/units.h>
1819
1920 #define STM32_USBPHYC_PLL 0x0
2021 #define STM32_USBPHYC_MISC 0x8
@@ -47,7 +48,6 @@ · 26 unmodified lines
4748 #define PLL_FVCO_MHZ 2880
4849 #define PLL_INFF_MIN_RATE_HZ 19200000
4950 #define PLL_INFF_MAX_RATE_HZ 38400000
50−#define HZ_PER_MHZ 1000000L
5151
5252 struct pll_params {
5353 u8 ndiv;
drivers/thermal/devfreq_cooling.c+1−1
@@ -18,10 +18,10 @@ · 17 unmodified lines
1818 #include <linux/pm_opp.h>
1919 #include <linux/pm_qos.h>
2020 #include <linux/thermal.h>
21+#include <linux/units.h>
2122
2223 #include <trace/events/thermal.h>
2324
24−#define HZ_PER_KHZ 1000
2525 #define SCALE_ERROR_MITIGATION 100
2626
2727 /**
drivers/virtio/virtio_mem.c+21−5
@@ -143,6 +143,8 @@ struct virtio_mem { · 142 unmodified lines
143143 * add_memory_driver_managed().
144144 */
145145 const char *resource_name;
146+ /* Memory group identification. */
147+ int mgid;
146148
147149 /*
148150 * We don't want to add too much memory if it's not getting onlined,
@@ -626,8 +628,8 @@ static int virtio_mem_add_memory(struct virtio_mem *vm, uint64_t addr, · 477 unmodified lines
626628 addr + size - 1);
627629 /* Memory might get onlined immediately. */
628630 atomic64_add(size, &vm->offline_size);
629− rc = add_memory_driver_managed(vm->nid, addr, size, vm->resource_name,
630− MHP_MERGE_RESOURCE);
631+ rc = add_memory_driver_managed(vm->mgid, addr, size, vm->resource_name,
632+ MHP_MERGE_RESOURCE | MHP_NID_IS_MGID);
631633 if (rc) {
632634 atomic64_sub(size, &vm->offline_size);
633635 dev_warn(&vm->vdev->dev, "adding memory failed: %d\n", rc);
@@ -677,7 +679,7 @@ static int virtio_mem_remove_memory(struct virtio_mem *vm, uint64_t addr, · 43 unmodified lines
677679
678680 dev_dbg(&vm->vdev->dev, "removing memory: 0x%llx - 0x%llx\n", addr,
679681 addr + size - 1);
680− rc = remove_memory(vm->nid, addr, size);
682+ rc = remove_memory(addr, size);
681683 if (!rc) {
682684 atomic64_sub(size, &vm->offline_size);
683685 /*
@@ -720,7 +722,7 @@ static int virtio_mem_offline_and_remove_memory(struct virtio_mem *vm, · 36 unmodified lines
720722 "offlining and removing memory: 0x%llx - 0x%llx\n", addr,
721723 addr + size - 1);
722724
723− rc = offline_and_remove_memory(vm->nid, addr, size);
725+ rc = offline_and_remove_memory(addr, size);
724726 if (!rc) {
725727 atomic64_sub(size, &vm->offline_size);
726728 /*
@@ -2569,6 +2571,7 @@ static bool virtio_mem_has_memory_added(struct virtio_mem *vm) · 1842 unmodified lines
25692571 static int virtio_mem_probe(struct virtio_device *vdev)
25702572 {
25712573 struct virtio_mem *vm;
2574+ uint64_t unit_pages;
25722575 int rc;
25732576
25742577 BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
@@ -2603,6 +2606,16 @@ static int virtio_mem_probe(struct virtio_device *vdev) · 28 unmodified lines
26032606 if (rc)
26042607 goto out_del_vq;
26052608
2609+ /* use a single dynamic memory group to cover the whole memory device */
2610+ if (vm->in_sbm)
2611+ unit_pages = PHYS_PFN(memory_block_size_bytes());
2612+ else
2613+ unit_pages = PHYS_PFN(vm->bbm.bb_size);
2614+ rc = memory_group_register_dynamic(vm->nid, unit_pages);
2615+ if (rc < 0)
2616+ goto out_del_resource;
2617+ vm->mgid = rc;
2618+
26062619 /*
26072620 * If we still have memory plugged, we have to unplug all memory first.
26082621 * Registering our parent resource makes sure that this memory isn't
@@ -2617,7 +2630,7 @@ static int virtio_mem_probe(struct virtio_device *vdev) · 8 unmodified lines
26172630 vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
26182631 rc = register_memory_notifier(&vm->memory_notifier);
26192632 if (rc)
2620− goto out_del_resource;
2633+ goto out_unreg_group;
26212634 rc = register_virtio_mem_device(vm);
26222635 if (rc)
26232636 goto out_unreg_mem;
@@ -2631,6 +2644,8 @@ static int virtio_mem_probe(struct virtio_device *vdev) · 7 unmodified lines
26312644 return 0;
26322645 out_unreg_mem:
26332646 unregister_memory_notifier(&vm->memory_notifier);
2647+out_unreg_group:
2648+ memory_group_unregister(vm->mgid);
26342649 out_del_resource:
26352650 virtio_mem_delete_resource(vm);
26362651 out_del_vq:
@@ -2695,6 +2710,7 @@ static void virtio_mem_remove(struct virtio_device *vdev) · 58 unmodified lines
26952710 } else {
26962711 virtio_mem_delete_resource(vm);
26972712 kfree_const(vm->resource_name);
2713+ memory_group_unregister(vm->mgid);
26982714 }
26992715
27002716 /* remove all tracking data - no locking needed */
fs/coredump.c+12−3
@@ -782,10 +782,17 @@ void do_coredump(const kernel_siginfo_t *siginfo) · 781 unmodified lines
782782 * filesystem.
783783 */
784784 mnt_userns = file_mnt_user_ns(cprm.file);
785− if (!uid_eq(i_uid_into_mnt(mnt_userns, inode), current_fsuid()))
785+ if (!uid_eq(i_uid_into_mnt(mnt_userns, inode),
786+ current_fsuid())) {
787+ pr_info_ratelimited("Core dump to %s aborted: cannot preserve file owner\n",
788+ cn.corename);
786789 goto close_fail;
787− if ((inode->i_mode & 0677) != 0600)
790+ }
791+ if ((inode->i_mode & 0677) != 0600) {
792+ pr_info_ratelimited("Core dump to %s aborted: cannot preserve file permissions\n",
793+ cn.corename);
788794 goto close_fail;
795+ }
789796 if (!(cprm.file->f_mode & FMODE_CAN_WRITE))
790797 goto close_fail;
791798 if (do_truncate(mnt_userns, cprm.file->f_path.dentry,
@@ -1127,8 +1134,10 @@ int dump_vma_snapshot(struct coredump_params *cprm, int *vma_count, · 335 unmodified lines
11271134
11281135 mmap_write_unlock(mm);
11291136
1130− if (WARN_ON(i != *vma_count))
1137+ if (WARN_ON(i != *vma_count)) {
1138+ kvfree(*vma_meta);
11311139 return -EFAULT;
1140+ }
11321141
11331142 *vma_data_size_ptr = vma_data_size;
11341143 return 0;
fs/eventpoll.c+10−8
@@ -723,7 +723,7 @@ static int ep_remove(struct eventpoll *ep, struct epitem *epi) · 722 unmodified lines
723723 */
724724 call_rcu(&epi->rcu, epi_rcu_free);
725725
726− atomic_long_dec(&ep->user->epoll_watches);
726+ percpu_counter_dec(&ep->user->epoll_watches);
727727
728728 return 0;
729729 }
@@ -1439,7 +1439,6 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, · 709 unmodified lines
14391439 {
14401440 int error, pwake = 0;
14411441 __poll_t revents;
1442− long user_watches;
14431442 struct epitem *epi;
14441443 struct ep_pqueue epq;
14451444 struct eventpoll *tep = NULL;
@@ -1449,11 +1448,15 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, · 3 unmodified lines
14491448
14501449 lockdep_assert_irqs_enabled();
14511450
1452− user_watches = atomic_long_read(&ep->user->epoll_watches);
1453− if (unlikely(user_watches >= max_user_watches))
1451+ if (unlikely(percpu_counter_compare(&ep->user->epoll_watches,
1452+ max_user_watches) >= 0))
14541453 return -ENOSPC;
1455− if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL)))
1454+ percpu_counter_inc(&ep->user->epoll_watches);
1455+
1456+ if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) {
1457+ percpu_counter_dec(&ep->user->epoll_watches);
14561458 return -ENOMEM;
1459+ }
14571460
14581461 /* Item initialization follow here ... */
14591462 INIT_LIST_HEAD(&epi->rdllink);
@@ -1466,17 +1469,16 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, · 6 unmodified lines
14661469 mutex_lock_nested(&tep->mtx, 1);
14671470 /* Add the current item to the list of active epoll hook for this file */
14681471 if (unlikely(attach_epitem(tfile, epi) < 0)) {
1469− kmem_cache_free(epi_cache, epi);
14701472 if (tep)
14711473 mutex_unlock(&tep->mtx);
1474+ kmem_cache_free(epi_cache, epi);
1475+ percpu_counter_dec(&ep->user->epoll_watches);
14721476 return -ENOMEM;
14731477 }
14741478
14751479 if (full_check && !tep)
14761480 list_file(tfile);
14771481
1478− atomic_long_inc(&ep->user->epoll_watches);
1479−
14801482 /*
14811483 * Add the current item to the RB tree. All RB tree operations are
14821484 * protected by "mtx", and ep_insert() is called with "mtx" held.
fs/nilfs2/sysfs.c+11−15
@@ -51,11 +51,9 @@ static const struct sysfs_ops nilfs_##name##_attr_ops = { \ · 50 unmodified lines
5151 #define NILFS_DEV_INT_GROUP_TYPE(name, parent_name) \
5252 static void nilfs_##name##_attr_release(struct kobject *kobj) \
5353 { \
54− struct nilfs_sysfs_##parent_name##_subgroups *subgroups; \
55− struct the_nilfs *nilfs = container_of(kobj->parent, \
56− struct the_nilfs, \
57− ns_##parent_name##_kobj); \
58− subgroups = nilfs->ns_##parent_name##_subgroups; \
54+ struct nilfs_sysfs_##parent_name##_subgroups *subgroups = container_of(kobj, \
55+ struct nilfs_sysfs_##parent_name##_subgroups, \
56+ sg_##name##_kobj); \
5957 complete(&subgroups->sg_##name##_kobj_unregister); \
6058 } \
6159 static struct kobj_type nilfs_##name##_ktype = { \
@@ -81,12 +79,12 @@ static int nilfs_sysfs_create_##name##_group(struct the_nilfs *nilfs) \ · 19 unmodified lines
8179 err = kobject_init_and_add(kobj, &nilfs_##name##_ktype, parent, \
8280 #name); \
8381 if (err) \
84− return err; \
85− return 0; \
82+ kobject_put(kobj); \
83+ return err; \
8684 } \
8785 static void nilfs_sysfs_delete_##name##_group(struct the_nilfs *nilfs) \
8886 { \
89− kobject_del(&nilfs->ns_##parent_name##_subgroups->sg_##name##_kobj); \
87+ kobject_put(&nilfs->ns_##parent_name##_subgroups->sg_##name##_kobj); \
9088 }
9189
9290 /************************************************************************
@@ -197,14 +195,14 @@ int nilfs_sysfs_create_snapshot_group(struct nilfs_root *root) · 104 unmodified lines
197195 }
198196
199197 if (err)
200− return err;
198+ kobject_put(&root->snapshot_kobj);
201199
202− return 0;
200+ return err;
203201 }
204202
205203 void nilfs_sysfs_delete_snapshot_group(struct nilfs_root *root)
206204 {
207− kobject_del(&root->snapshot_kobj);
205+ kobject_put(&root->snapshot_kobj);
208206 }
209207
210208 /************************************************************************
@@ -986,7 +984,7 @@ int nilfs_sysfs_create_device_group(struct super_block *sb) · 775 unmodified lines
986984 err = kobject_init_and_add(&nilfs->ns_dev_kobj, &nilfs_dev_ktype, NULL,
987985 "%s", sb->s_id);
988986 if (err)
989− goto free_dev_subgroups;
987+ goto cleanup_dev_kobject;
990988
991989 err = nilfs_sysfs_create_mounted_snapshots_group(nilfs);
992990 if (err)
@@ -1023,9 +1021,7 @@ delete_mounted_snapshots_group: · 30 unmodified lines
10231021 nilfs_sysfs_delete_mounted_snapshots_group(nilfs);
10241022
10251023 cleanup_dev_kobject:
1026− kobject_del(&nilfs->ns_dev_kobj);
1027−
1028−free_dev_subgroups:
1024+ kobject_put(&nilfs->ns_dev_kobj);
10291025 kfree(nilfs->ns_dev_subgroups);
10301026
10311027 failed_create_device_group:
fs/nilfs2/the_nilfs.c+4−5
@@ -792,14 +792,13 @@ nilfs_find_or_create_root(struct the_nilfs *nilfs, __u64 cno) · 791 unmodified lines
792792
793793 void nilfs_put_root(struct nilfs_root *root)
794794 {
795− if (refcount_dec_and_test(&root->count)) {
796− struct the_nilfs *nilfs = root->nilfs;
795+ struct the_nilfs *nilfs = root->nilfs;
797796
798− nilfs_sysfs_delete_snapshot_group(root);
799−
800− spin_lock(&nilfs->ns_cptree_lock);
797+ if (refcount_dec_and_lock(&root->count, &nilfs->ns_cptree_lock)) {
801798 rb_erase(&root->rb_node, &nilfs->ns_cptree);
802799 spin_unlock(&nilfs->ns_cptree_lock);
800+
801+ nilfs_sysfs_delete_snapshot_group(root);
803802 iput(root->ifile);
804803
805804 kfree(root);
fs/proc/array.c+4−14
@@ -98,27 +98,17 @@ · 97 unmodified lines
9898
9999 void proc_task_name(struct seq_file *m, struct task_struct *p, bool escape)
100100 {
101− char *buf;
102− size_t size;
103101 char tcomm[64];
104− int ret;
105102
106103 if (p->flags & PF_WQ_WORKER)
107104 wq_worker_comm(tcomm, sizeof(tcomm), p);
108105 else
109106 __get_task_comm(tcomm, sizeof(tcomm), p);
110107
111− size = seq_get_buf(m, &buf);
112− if (escape) {
113− ret = string_escape_str(tcomm, buf, size,
114− ESCAPE_SPACE | ESCAPE_SPECIAL, "\n\\");
115− if (ret >= size)
116− ret = -1;
117− } else {
118− ret = strscpy(buf, tcomm, size);
119− }
120−
121− seq_commit(m, ret);
108+ if (escape)
109+ seq_escape_str(m, tcomm, ESCAPE_SPACE | ESCAPE_SPECIAL, "\n\\");
110+ else
111+ seq_printf(m, "%.64s", tcomm);
122112 }
123113
124114 /*
fs/proc/base.c+4−1
@@ -95,6 +95,7 @@ · 94 unmodified lines
9595 #include <linux/posix-timers.h>
9696 #include <linux/time_namespace.h>
9797 #include <linux/resctrl.h>
98+#include <linux/cn_proc.h>
9899 #include <trace/events/oom.h>
99100 #include "internal.h"
100101 #include "fd.h"
@@ -1674,8 +1675,10 @@ static ssize_t comm_write(struct file *file, const char __user *buf, · 1573 unmodified lines
16741675 if (!p)
16751676 return -ESRCH;
16761677
1677− if (same_thread_group(current, p))
1678+ if (same_thread_group(current, p)) {
16781679 set_task_comm(p, buffer);
1680+ proc_comm_connector(p);
1681+ }
16791682 else
16801683 count = -EINVAL;
16811684
include/asm-generic/early_ioremap.h+0−6
@@ -19,12 +19,6 @@ extern void *early_memremap_prot(resource_size_t phys_addr, · 18 unmodified lines
1919 extern void early_iounmap(void __iomem *addr, unsigned long size);
2020 extern void early_memunmap(void *addr, unsigned long size);
2121
22−/*
23− * Weak function called by early_ioremap_reset(). It does nothing, but
24− * architectures may provide their own version to do any needed cleanups.
25− */
26−extern void early_ioremap_shutdown(void);
27−
2822 #if defined(CONFIG_GENERIC_EARLY_IOREMAP) && defined(CONFIG_MMU)
2923 /* Arch-specific initialization */
3024 extern void early_ioremap_init(void);
include/linux/damon.h+268−0
@@ -0,0 +1,268 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * DAMON api
4+ *
5+ * Author: SeongJae Park <sjpark@amazon.de>
6+ */
7+
8+#ifndef _DAMON_H_
9+#define _DAMON_H_
10+
11+#include <linux/mutex.h>
12+#include <linux/time64.h>
13+#include <linux/types.h>
14+
15+/* Minimal region size. Every damon_region is aligned by this. */
16+#define DAMON_MIN_REGION PAGE_SIZE
17+
18+/**
19+ * struct damon_addr_range - Represents an address region of [@start, @end).
20+ * @start: Start address of the region (inclusive).
21+ * @end: End address of the region (exclusive).
22+ */
23+struct damon_addr_range {
24+ unsigned long start;
25+ unsigned long end;
26+};
27+
28+/**
29+ * struct damon_region - Represents a monitoring target region.
30+ * @ar: The address range of the region.
31+ * @sampling_addr: Address of the sample for the next access check.
32+ * @nr_accesses: Access frequency of this region.
33+ * @list: List head for siblings.
34+ */
35+struct damon_region {
36+ struct damon_addr_range ar;
37+ unsigned long sampling_addr;
38+ unsigned int nr_accesses;
39+ struct list_head list;
40+};
41+
42+/**
43+ * struct damon_target - Represents a monitoring target.
44+ * @id: Unique identifier for this target.
45+ * @nr_regions: Number of monitoring target regions of this target.
46+ * @regions_list: Head of the monitoring target regions of this target.
47+ * @list: List head for siblings.
48+ *
49+ * Each monitoring context could have multiple targets. For example, a context
50+ * for virtual memory address spaces could have multiple target processes. The
51+ * @id of each target should be unique among the targets of the context. For
52+ * example, in the virtual address monitoring context, it could be a pidfd or
53+ * an address of an mm_struct.
54+ */
55+struct damon_target {
56+ unsigned long id;
57+ unsigned int nr_regions;
58+ struct list_head regions_list;
59+ struct list_head list;
60+};
61+
62+struct damon_ctx;
63+
64+/**
65+ * struct damon_primitive Monitoring primitives for given use cases.
66+ *
67+ * @init: Initialize primitive-internal data structures.
68+ * @update: Update primitive-internal data structures.
69+ * @prepare_access_checks: Prepare next access check of target regions.
70+ * @check_accesses: Check the accesses to target regions.
71+ * @reset_aggregated: Reset aggregated accesses monitoring results.
72+ * @target_valid: Determine if the target is valid.
73+ * @cleanup: Clean up the context.
74+ *
75+ * DAMON can be extended for various address spaces and usages. For this,
76+ * users should register the low level primitives for their target address
77+ * space and usecase via the &damon_ctx.primitive. Then, the monitoring thread
78+ * (&damon_ctx.kdamond) calls @init and @prepare_access_checks before starting
79+ * the monitoring, @update after each &damon_ctx.primitive_update_interval, and
80+ * @check_accesses, @target_valid and @prepare_access_checks after each
81+ * &damon_ctx.sample_interval. Finally, @reset_aggregated is called after each
82+ * &damon_ctx.aggr_interval.
83+ *
84+ * @init should initialize primitive-internal data structures. For example,
85+ * this could be used to construct proper monitoring target regions and link
86+ * those to @damon_ctx.adaptive_targets.
87+ * @update should update the primitive-internal data structures. For example,
88+ * this could be used to update monitoring target regions for current status.
89+ * @prepare_access_checks should manipulate the monitoring regions to be
90+ * prepared for the next access check.
91+ * @check_accesses should check the accesses to each region that made after the
92+ * last preparation and update the number of observed accesses of each region.
93+ * It should also return max number of observed accesses that made as a result
94+ * of its update. The value will be used for regions adjustment threshold.
95+ * @reset_aggregated should reset the access monitoring results that aggregated
96+ * by @check_accesses.
97+ * @target_valid should check whether the target is still valid for the
98+ * monitoring.
99+ * @cleanup is called from @kdamond just before its termination.
100+ */
101+struct damon_primitive {
102+ void (*init)(struct damon_ctx *context);
103+ void (*update)(struct damon_ctx *context);
104+ void (*prepare_access_checks)(struct damon_ctx *context);
105+ unsigned int (*check_accesses)(struct damon_ctx *context);
106+ void (*reset_aggregated)(struct damon_ctx *context);
107+ bool (*target_valid)(void *target);
108+ void (*cleanup)(struct damon_ctx *context);
109+};
110+
111+/*
112+ * struct damon_callback Monitoring events notification callbacks.
113+ *
114+ * @before_start: Called before starting the monitoring.
115+ * @after_sampling: Called after each sampling.
116+ * @after_aggregation: Called after each aggregation.
117+ * @before_terminate: Called before terminating the monitoring.
118+ * @private: User private data.
119+ *
120+ * The monitoring thread (&damon_ctx.kdamond) calls @before_start and
121+ * @before_terminate just before starting and finishing the monitoring,
122+ * respectively. Therefore, those are good places for installing and cleaning
123+ * @private.
124+ *
125+ * The monitoring thread calls @after_sampling and @after_aggregation for each
126+ * of the sampling intervals and aggregation intervals, respectively.
127+ * Therefore, users can safely access the monitoring results without additional
128+ * protection. For the reason, users are recommended to use these callback for
129+ * the accesses to the results.
130+ *
131+ * If any callback returns non-zero, monitoring stops.
132+ */
133+struct damon_callback {
134+ void *private;
135+
136+ int (*before_start)(struct damon_ctx *context);
137+ int (*after_sampling)(struct damon_ctx *context);
138+ int (*after_aggregation)(struct damon_ctx *context);
139+ int (*before_terminate)(struct damon_ctx *context);
140+};
141+
142+/**
143+ * struct damon_ctx - Represents a context for each monitoring. This is the
144+ * main interface that allows users to set the attributes and get the results
145+ * of the monitoring.
146+ *
147+ * @sample_interval: The time between access samplings.
148+ * @aggr_interval: The time between monitor results aggregations.
149+ * @primitive_update_interval: The time between monitoring primitive updates.
150+ *
151+ * For each @sample_interval, DAMON checks whether each region is accessed or
152+ * not. It aggregates and keeps the access information (number of accesses to
153+ * each region) for @aggr_interval time. DAMON also checks whether the target
154+ * memory regions need update (e.g., by ``mmap()`` calls from the application,
155+ * in case of virtual memory monitoring) and applies the changes for each
156+ * @primitive_update_interval. All time intervals are in micro-seconds.
157+ * Please refer to &struct damon_primitive and &struct damon_callback for more
158+ * detail.
159+ *
160+ * @kdamond: Kernel thread who does the monitoring.
161+ * @kdamond_stop: Notifies whether kdamond should stop.
162+ * @kdamond_lock: Mutex for the synchronizations with @kdamond.
163+ *
164+ * For each monitoring context, one kernel thread for the monitoring is
165+ * created. The pointer to the thread is stored in @kdamond.
166+ *
167+ * Once started, the monitoring thread runs until explicitly required to be
168+ * terminated or every monitoring target is invalid. The validity of the
169+ * targets is checked via the &damon_primitive.target_valid of @primitive. The
170+ * termination can also be explicitly requested by writing non-zero to
171+ * @kdamond_stop. The thread sets @kdamond to NULL when it terminates.
172+ * Therefore, users can know whether the monitoring is ongoing or terminated by
173+ * reading @kdamond. Reads and writes to @kdamond and @kdamond_stop from
174+ * outside of the monitoring thread must be protected by @kdamond_lock.
175+ *
176+ * Note that the monitoring thread protects only @kdamond and @kdamond_stop via
177+ * @kdamond_lock. Accesses to other fields must be protected by themselves.
178+ *
179+ * @primitive: Set of monitoring primitives for given use cases.
180+ * @callback: Set of callbacks for monitoring events notifications.
181+ *
182+ * @min_nr_regions: The minimum number of adaptive monitoring regions.
183+ * @max_nr_regions: The maximum number of adaptive monitoring regions.
184+ * @adaptive_targets: Head of monitoring targets (&damon_target) list.
185+ */
186+struct damon_ctx {
187+ unsigned long sample_interval;
188+ unsigned long aggr_interval;
189+ unsigned long primitive_update_interval;
190+
191+/* private: internal use only */
192+ struct timespec64 last_aggregation;
193+ struct timespec64 last_primitive_update;
194+
195+/* public: */
196+ struct task_struct *kdamond;
197+ bool kdamond_stop;
198+ struct mutex kdamond_lock;
199+
200+ struct damon_primitive primitive;
201+ struct damon_callback callback;
202+
203+ unsigned long min_nr_regions;
204+ unsigned long max_nr_regions;
205+ struct list_head adaptive_targets;
206+};
207+
208+#define damon_next_region(r) \
209+ (container_of(r->list.next, struct damon_region, list))
210+
211+#define damon_prev_region(r) \
212+ (container_of(r->list.prev, struct damon_region, list))
213+
214+#define damon_for_each_region(r, t) \
215+ list_for_each_entry(r, &t->regions_list, list)
216+
217+#define damon_for_each_region_safe(r, next, t) \
218+ list_for_each_entry_safe(r, next, &t->regions_list, list)
219+
220+#define damon_for_each_target(t, ctx) \
221+ list_for_each_entry(t, &(ctx)->adaptive_targets, list)
222+
223+#define damon_for_each_target_safe(t, next, ctx) \
224+ list_for_each_entry_safe(t, next, &(ctx)->adaptive_targets, list)
225+
226+#ifdef CONFIG_DAMON
227+
228+struct damon_region *damon_new_region(unsigned long start, unsigned long end);
229+inline void damon_insert_region(struct damon_region *r,
230+ struct damon_region *prev, struct damon_region *next,
231+ struct damon_target *t);
232+void damon_add_region(struct damon_region *r, struct damon_target *t);
233+void damon_destroy_region(struct damon_region *r, struct damon_target *t);
234+
235+struct damon_target *damon_new_target(unsigned long id);
236+void damon_add_target(struct damon_ctx *ctx, struct damon_target *t);
237+void damon_free_target(struct damon_target *t);
238+void damon_destroy_target(struct damon_target *t);
239+unsigned int damon_nr_regions(struct damon_target *t);
240+
241+struct damon_ctx *damon_new_ctx(void);
242+void damon_destroy_ctx(struct damon_ctx *ctx);
243+int damon_set_targets(struct damon_ctx *ctx,
244+ unsigned long *ids, ssize_t nr_ids);
245+int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int,
246+ unsigned long aggr_int, unsigned long primitive_upd_int,
247+ unsigned long min_nr_reg, unsigned long max_nr_reg);
248+int damon_nr_running_ctxs(void);
249+
250+int damon_start(struct damon_ctx **ctxs, int nr_ctxs);
251+int damon_stop(struct damon_ctx **ctxs, int nr_ctxs);
252+
253+#endif /* CONFIG_DAMON */
254+
255+#ifdef CONFIG_DAMON_VADDR
256+
257+/* Monitoring primitives for virtual memory address spaces */
258+void damon_va_init(struct damon_ctx *ctx);
259+void damon_va_update(struct damon_ctx *ctx);
260+void damon_va_prepare_access_checks(struct damon_ctx *ctx);
261+unsigned int damon_va_check_accesses(struct damon_ctx *ctx);
262+bool damon_va_target_valid(void *t);
263+void damon_va_cleanup(struct damon_ctx *ctx);
264+void damon_va_set_primitives(struct damon_ctx *ctx);
265+
266+#endif /* CONFIG_DAMON_VADDR */
267+
268+#endif /* _DAMON_H */
include/linux/highmem-internal.h+22−5
@@ -90,7 +90,11 @@ static inline void __kunmap_local(void *vaddr) · 89 unmodified lines
9090
9191 static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot)
9292 {
93− preempt_disable();
93+ if (IS_ENABLED(CONFIG_PREEMPT_RT))
94+ migrate_disable();
95+ else
96+ preempt_disable();
97+
9498 pagefault_disable();
9599 return __kmap_local_page_prot(page, prot);
96100 }
@@ -102,7 +106,11 @@ static inline void *kmap_atomic(struct page *page) · 5 unmodified lines
102106
103107 static inline void *kmap_atomic_pfn(unsigned long pfn)
104108 {
105− preempt_disable();
109+ if (IS_ENABLED(CONFIG_PREEMPT_RT))
110+ migrate_disable();
111+ else
112+ preempt_disable();
113+
106114 pagefault_disable();
107115 return __kmap_local_pfn_prot(pfn, kmap_prot);
108116 }
@@ -111,7 +119,10 @@ static inline void __kunmap_atomic(void *addr) · 2 unmodified lines
111119 {
112120 kunmap_local_indexed(addr);
113121 pagefault_enable();
114− preempt_enable();
122+ if (IS_ENABLED(CONFIG_PREEMPT_RT))
123+ migrate_enable();
124+ else
125+ preempt_enable();
115126 }
116127
117128 unsigned int __nr_free_highpages(void);
@@ -179,7 +190,10 @@ static inline void __kunmap_local(void *addr) · 61 unmodified lines
179190
180191 static inline void *kmap_atomic(struct page *page)
181192 {
182− preempt_disable();
193+ if (IS_ENABLED(CONFIG_PREEMPT_RT))
194+ migrate_disable();
195+ else
196+ preempt_disable();
183197 pagefault_disable();
184198 return page_address(page);
185199 }
@@ -200,7 +214,10 @@ static inline void __kunmap_atomic(void *addr) · 14 unmodified lines
200214 kunmap_flush_on_unmap(addr);
201215 #endif
202216 pagefault_enable();
203− preempt_enable();
217+ if (IS_ENABLED(CONFIG_PREEMPT_RT))
218+ migrate_enable();
219+ else
220+ preempt_enable();
204221 }
205222
206223 static inline unsigned int nr_free_highpages(void) { return 0; }
include/linux/memory.h+54−1
@@ -23,6 +23,48 @@ · 22 unmodified lines
2323
2424 #define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS)
2525
26+/**
27+ * struct memory_group - a logical group of memory blocks
28+ * @nid: The node id for all memory blocks inside the memory group.
29+ * @blocks: List of all memory blocks belonging to this memory group.
30+ * @present_kernel_pages: Present (online) memory outside ZONE_MOVABLE of this
31+ * memory group.
32+ * @present_movable_pages: Present (online) memory in ZONE_MOVABLE of this
33+ * memory group.
34+ * @is_dynamic: The memory group type: static vs. dynamic
35+ * @s.max_pages: Valid with &memory_group.is_dynamic == false. The maximum
36+ * number of pages we'll have in this static memory group.
37+ * @d.unit_pages: Valid with &memory_group.is_dynamic == true. Unit in pages
38+ * in which memory is added/removed in this dynamic memory group.
39+ * This granularity defines the alignment of a unit in physical
40+ * address space; it has to be at least as big as a single
41+ * memory block.
42+ *
43+ * A memory group logically groups memory blocks; each memory block
44+ * belongs to at most one memory group. A memory group corresponds to
45+ * a memory device, such as a DIMM or a NUMA node, which spans multiple
46+ * memory blocks and might even span multiple non-contiguous physical memory
47+ * ranges.
48+ *
49+ * Modification of members after registration is serialized by memory
50+ * hot(un)plug code.
51+ */
52+struct memory_group {
53+ int nid;
54+ struct list_head memory_blocks;
55+ unsigned long present_kernel_pages;
56+ unsigned long present_movable_pages;
57+ bool is_dynamic;
58+ union {
59+ struct {
60+ unsigned long max_pages;
61+ } s;
62+ struct {
63+ unsigned long unit_pages;
64+ } d;
65+ };
66+};
67+
2668 struct memory_block {
2769 unsigned long start_section_nr;
2870 unsigned long state; /* serialized by the dev->lock */
@@ -34,6 +76,8 @@ struct memory_block { · 5 unmodified lines
3476 * lay at the beginning of the memory block.
3577 */
3678 unsigned long nr_vmemmap_pages;
79+ struct memory_group *group; /* group (if any) for this block */
80+ struct list_head group_next; /* next block inside memory group */
3781 };
3882
3983 int arch_get_memory_phys_device(unsigned long start_pfn);
@@ -86,7 +130,8 @@ static inline int memory_notify(unsigned long val, void *v) · 46 unmodified lines
86130 extern int register_memory_notifier(struct notifier_block *nb);
87131 extern void unregister_memory_notifier(struct notifier_block *nb);
88132 int create_memory_block_devices(unsigned long start, unsigned long size,
89− unsigned long vmemmap_pages);
133+ unsigned long vmemmap_pages,
134+ struct memory_group *group);
90135 void remove_memory_block_devices(unsigned long start, unsigned long size);
91136 extern void memory_dev_init(void);
92137 extern int memory_notify(unsigned long val, void *v);
@@ -96,6 +141,14 @@ extern int walk_memory_blocks(unsigned long start, unsigned long size, · 3 unmodified lines
96141 void *arg, walk_memory_blocks_func_t func);
97142 extern int for_each_memory_block(void *arg, walk_memory_blocks_func_t func);
98143 #define CONFIG_MEM_BLOCK_SIZE (PAGES_PER_SECTION<<PAGE_SHIFT)
144+
145+extern int memory_group_register_static(int nid, unsigned long max_pages);
146+extern int memory_group_register_dynamic(int nid, unsigned long unit_pages);
147+extern int memory_group_unregister(int mgid);
148+struct memory_group *memory_group_find_by_id(int mgid);
149+typedef int (*walk_memory_groups_func_t)(struct memory_group *, void *);
150+int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func,
151+ struct memory_group *excluded, void *arg);
99152 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
100153
101154 #ifdef CONFIG_MEMORY_HOTPLUG
include/linux/memory_hotplug.h+22−12
@@ -12,6 +12,7 @@ struct zone; · 11 unmodified lines
1212 struct pglist_data;
1313 struct mem_section;
1414 struct memory_block;
15+struct memory_group;
1516 struct resource;
1617 struct vmem_altmap;
1718
@@ -50,6 +51,11 @@ typedef int __bitwise mhp_t; · 32 unmodified lines
5051 * Only selected architectures support it with SPARSE_VMEMMAP.
5152 */
5253 #define MHP_MEMMAP_ON_MEMORY ((__force mhp_t)BIT(1))
54+/*
55+ * The nid field specifies a memory group id (mgid) instead. The memory group
56+ * implies the node id (nid).
57+ */
58+#define MHP_NID_IS_MGID ((__force mhp_t)BIT(2))
5359
5460 /*
5561 * Extended parameters for memory hotplug:
@@ -95,13 +101,15 @@ static inline void zone_seqlock_init(struct zone *zone) · 39 unmodified lines
95101 extern int zone_grow_free_lists(struct zone *zone, unsigned long new_nr_pages);
96102 extern int zone_grow_waitqueues(struct zone *zone, unsigned long nr_pages);
97103 extern int add_one_highpage(struct page *page, int pfn, int bad_ppro);
98−extern void adjust_present_page_count(struct zone *zone, long nr_pages);
104+extern void adjust_present_page_count(struct page *page,
105+ struct memory_group *group,
106+ long nr_pages);
99107 /* VM interface that may be used by firmware interface */
100108 extern int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages,
101109 struct zone *zone);
102110 extern void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages);
103111 extern int online_pages(unsigned long pfn, unsigned long nr_pages,
104− struct zone *zone);
112+ struct zone *zone, struct memory_group *group);
105113 extern struct zone *test_pages_in_a_zone(unsigned long start_pfn,
106114 unsigned long end_pfn);
107115 extern void __offline_isolated_pages(unsigned long start_pfn,
@@ -130,8 +138,7 @@ static inline bool movable_node_is_enabled(void) · 22 unmodified lines
130138 return movable_node_enabled;
131139 }
132140
133−extern void arch_remove_memory(int nid, u64 start, u64 size,
134− struct vmem_altmap *altmap);
141+extern void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap);
135142 extern void __remove_pages(unsigned long start_pfn, unsigned long nr_pages,
136143 struct vmem_altmap *altmap);
137144
@@ -292,25 +299,27 @@ static inline void pgdat_resize_init(struct pglist_data *pgdat) {} · 154 unmodified lines
292299 #ifdef CONFIG_MEMORY_HOTREMOVE
293300
294301 extern void try_offline_node(int nid);
295−extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages);
296−extern int remove_memory(int nid, u64 start, u64 size);
297−extern void __remove_memory(int nid, u64 start, u64 size);
298−extern int offline_and_remove_memory(int nid, u64 start, u64 size);
302+extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
303+ struct memory_group *group);
304+extern int remove_memory(u64 start, u64 size);
305+extern void __remove_memory(u64 start, u64 size);
306+extern int offline_and_remove_memory(u64 start, u64 size);
299307
300308 #else
301309 static inline void try_offline_node(int nid) {}
302310
303−static inline int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
311+static inline int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
312+ struct memory_group *group)
304313 {
305314 return -EINVAL;
306315 }
307316
308−static inline int remove_memory(int nid, u64 start, u64 size)
317+static inline int remove_memory(u64 start, u64 size)
309318 {
310319 return -EBUSY;
311320 }
312321
313−static inline void __remove_memory(int nid, u64 start, u64 size) {}
322+static inline void __remove_memory(u64 start, u64 size) {}
314323 #endif /* CONFIG_MEMORY_HOTREMOVE */
315324
316325 extern void set_zone_contiguous(struct zone *zone);
@@ -339,7 +348,8 @@ extern void sparse_remove_section(struct mem_section *ms, · 22 unmodified lines
339348 unsigned long map_offset, struct vmem_altmap *altmap);
340349 extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map,
341350 unsigned long pnum);
342−extern struct zone *zone_for_pfn_range(int online_type, int nid, unsigned start_pfn,
351+extern struct zone *zone_for_pfn_range(int online_type, int nid,
352+ struct memory_group *group, unsigned long start_pfn,
343353 unsigned long nr_pages);
344354 extern int arch_create_linear_mapping(int nid, u64 start, u64 size,
345355 struct mhp_params *params);
include/linux/mmzone.h+7−12
@@ -540,6 +540,10 @@ struct zone { · 539 unmodified lines
540540 * is calculated as:
541541 * present_pages = spanned_pages - absent_pages(pages in holes);
542542 *
543+ * present_early_pages is present pages existing within the zone
544+ * located on memory available since early boot, excluding hotplugged
545+ * memory.
546+ *
543547 * managed_pages is present pages managed by the buddy system, which
544548 * is calculated as (reserved_pages includes pages allocated by the
545549 * bootmem allocator):
@@ -572,6 +576,9 @@ struct zone { · 26 unmodified lines
572576 atomic_long_t managed_pages;
573577 unsigned long spanned_pages;
574578 unsigned long present_pages;
579+#if defined(CONFIG_MEMORY_HOTPLUG)
580+ unsigned long present_early_pages;
581+#endif
575582 #ifdef CONFIG_CMA
576583 unsigned long cma_pages;
577584 #endif
@@ -1525,18 +1532,6 @@ void sparse_init(void); · 947 unmodified lines
15251532 #define subsection_map_init(_pfn, _nr_pages) do {} while (0)
15261533 #endif /* CONFIG_SPARSEMEM */
15271534
1528−/*
1529− * If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we
1530− * need to check pfn validity within that MAX_ORDER_NR_PAGES block.
1531− * pfn_valid_within() should be used in this case; we optimise this away
1532− * when we have no holes within a MAX_ORDER_NR_PAGES block.
1533− */
1534−#ifdef CONFIG_HOLES_IN_ZONE
1535−#define pfn_valid_within(pfn) pfn_valid(pfn)
1536−#else
1537−#define pfn_valid_within(pfn) (1)
1538−#endif
1539−
15401535 #endif /* !__GENERATING_BOUNDS.H */
15411536 #endif /* !__ASSEMBLY__ */
15421537 #endif /* _LINUX_MMZONE_H */
include/linux/once.h+1−1
@@ -16,7 +16,7 @@ void __do_once_done(bool *done, struct static_key_true *once_key, · 15 unmodified lines
1616 * out the condition into a nop. DO_ONCE() guarantees type safety of
1717 * arguments!
1818 *
19− * Not that the following is not equivalent ...
19+ * Note that the following is not equivalent ...
2020 *
2121 * DO_ONCE(func, arg);
2222 * DO_ONCE(func, arg);
include/linux/page-flags.h+5−3
@@ -131,7 +131,7 @@ enum pageflags { · 130 unmodified lines
131131 #ifdef CONFIG_MEMORY_FAILURE
132132 PG_hwpoison, /* hardware poisoned page. Don't touch */
133133 #endif
134−#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
134+#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
135135 PG_young,
136136 PG_idle,
137137 #endif
@@ -178,6 +178,8 @@ enum pageflags { · 40 unmodified lines
178178 PG_reported = PG_uptodate,
179179 };
180180
181+#define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1)
182+
181183 #ifndef __GENERATING_BOUNDS_H
182184
183185 static inline unsigned long _compound_head(const struct page *page)
@@ -439,7 +441,7 @@ PAGEFLAG_FALSE(HWPoison) · 255 unmodified lines
439441 #define __PG_HWPOISON 0
440442 #endif
441443
442−#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
444+#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
443445 TESTPAGEFLAG(Young, young, PF_ANY)
444446 SETPAGEFLAG(Young, young, PF_ANY)
445447 TESTCLEARFLAG(Young, young, PF_ANY)
@@ -831,7 +833,7 @@ static inline void ClearPageSlabPfmemalloc(struct page *page) · 385 unmodified lines
831833 * alloc-free cycle to prevent from reusing the page.
832834 */
833835 #define PAGE_FLAGS_CHECK_AT_PREP \
834− (((1UL << NR_PAGEFLAGS) - 1) & ~__PG_HWPOISON)
836+ (PAGEFLAGS_MASK & ~__PG_HWPOISON)
835837
836838 #define PAGE_FLAGS_PRIVATE \
837839 (1UL << PG_private | 1UL << PG_private_2)
include/linux/page_ext.h+1−1
@@ -19,7 +19,7 @@ struct page_ext_operations { · 18 unmodified lines
1919 enum page_ext_flags {
2020 PAGE_EXT_OWNER,
2121 PAGE_EXT_OWNER_ALLOCATED,
22−#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
22+#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
2323 PAGE_EXT_YOUNG,
2424 PAGE_EXT_IDLE,
2525 #endif
include/linux/page_idle.h+3−3
@@ -6,7 +6,7 @@ · 5 unmodified lines
66 #include <linux/page-flags.h>
77 #include <linux/page_ext.h>
88
9−#ifdef CONFIG_IDLE_PAGE_TRACKING
9+#ifdef CONFIG_PAGE_IDLE_FLAG
1010
1111 #ifdef CONFIG_64BIT
1212 static inline bool page_is_young(struct page *page)
@@ -106,7 +106,7 @@ static inline void clear_page_idle(struct page *page) · 93 unmodified lines
106106 }
107107 #endif /* CONFIG_64BIT */
108108
109−#else /* !CONFIG_IDLE_PAGE_TRACKING */
109+#else /* !CONFIG_PAGE_IDLE_FLAG */
110110
111111 static inline bool page_is_young(struct page *page)
112112 {
@@ -135,6 +135,6 @@ static inline void clear_page_idle(struct page *page) · 22 unmodified lines
135135 {
136136 }
137137
138−#endif /* CONFIG_IDLE_PAGE_TRACKING */
138+#endif /* CONFIG_PAGE_IDLE_FLAG */
139139
140140 #endif /* _LINUX_MM_PAGE_IDLE_H */
include/linux/pagemap.h+3−4
@@ -521,18 +521,17 @@ static inline struct page *read_mapping_page(struct address_space *mapping, · 520 unmodified lines
521521 */
522522 static inline pgoff_t page_to_index(struct page *page)
523523 {
524− pgoff_t pgoff;
524+ struct page *head;
525525
526526 if (likely(!PageTransTail(page)))
527527 return page->index;
528528
529+ head = compound_head(page);
529530 /*
530531 * We don't initialize ->index for tail pages: calculate based on
531532 * head page
532533 */
533− pgoff = compound_head(page)->index;
534− pgoff += page - compound_head(page);
535− return pgoff;
534+ return head->index + page - head;
536535 }
537536
538537 extern pgoff_t hugetlb_basepage_index(struct page *page);
include/linux/sched/user.h+2−1
@@ -4,6 +4,7 @@ · 3 unmodified lines
44
55 #include <linux/uidgid.h>
66 #include <linux/atomic.h>
7+#include <linux/percpu_counter.h>
78 #include <linux/refcount.h>
89 #include <linux/ratelimit.h>
910
@@ -13,7 +14,7 @@ · 3 unmodified lines
1314 struct user_struct {
1415 refcount_t __count; /* reference count */
1516 #ifdef CONFIG_EPOLL
16− atomic_long_t epoll_watches; /* The number of file descriptors currently watched */
17+ struct percpu_counter epoll_watches; /* The number of file descriptors currently watched */
1718 #endif
1819 unsigned long unix_inflight; /* How many files in flight in unix sockets */
1920 atomic_long_t pipe_bufs; /* how many pages are allocated in pipe buffers */
include/linux/threads.h+1−1
@@ -38,7 +38,7 @@ · 37 unmodified lines
3838 * Define a minimum number of pids per cpu. Heuristically based
3939 * on original pid max of 32k for 32 cpus. Also, increase the
4040 * minimum settable value for pid_max on the running system based
41− * on similar defaults. See kernel/pid.c:pidmap_init() for details.
41+ * on similar defaults. See kernel/pid.c:pid_idr_init() for details.
4242 */
4343 #define PIDS_PER_CPU_DEFAULT 1024
4444 #define PIDS_PER_CPU_MIN 8
include/linux/units.h+7−3
@@ -20,9 +20,13 @@ · 19 unmodified lines
2020 #define PICO 1000000000000ULL
2121 #define FEMTO 1000000000000000ULL
2222
23−#define MILLIWATT_PER_WATT 1000L
24−#define MICROWATT_PER_MILLIWATT 1000L
25−#define MICROWATT_PER_WATT 1000000L
23+#define HZ_PER_KHZ 1000UL
24+#define KHZ_PER_MHZ 1000UL
25+#define HZ_PER_MHZ 1000000UL
26+
27+#define MILLIWATT_PER_WATT 1000UL
28+#define MICROWATT_PER_MILLIWATT 1000UL
29+#define MICROWATT_PER_WATT 1000000UL
2630
2731 #define ABSOLUTE_ZERO_MILLICELSIUS -273150
2832
include/linux/vmalloc.h+0−3
@@ -225,9 +225,6 @@ static inline bool is_vm_area_hugepages(const void *addr) · 224 unmodified lines
225225 }
226226
227227 #ifdef CONFIG_MMU
228−int vmap_range(unsigned long addr, unsigned long end,
229− phys_addr_t phys_addr, pgprot_t prot,
230− unsigned int max_page_shift);
231228 void vunmap_range(unsigned long addr, unsigned long end);
232229 static inline void set_vm_flush_reset_perms(void *addr)
233230 {
include/trace/events/damon.h+43−0
@@ -0,0 +1,43 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#undef TRACE_SYSTEM
3+#define TRACE_SYSTEM damon
4+
5+#if !defined(_TRACE_DAMON_H) || defined(TRACE_HEADER_MULTI_READ)
6+#define _TRACE_DAMON_H
7+
8+#include <linux/damon.h>
9+#include <linux/types.h>
10+#include <linux/tracepoint.h>
11+
12+TRACE_EVENT(damon_aggregated,
13+
14+ TP_PROTO(struct damon_target *t, struct damon_region *r,
15+ unsigned int nr_regions),
16+
17+ TP_ARGS(t, r, nr_regions),
18+
19+ TP_STRUCT__entry(
20+ __field(unsigned long, target_id)
21+ __field(unsigned int, nr_regions)
22+ __field(unsigned long, start)
23+ __field(unsigned long, end)
24+ __field(unsigned int, nr_accesses)
25+ ),
26+
27+ TP_fast_assign(
28+ __entry->target_id = t->id;
29+ __entry->nr_regions = nr_regions;
30+ __entry->start = r->ar.start;
31+ __entry->end = r->ar.end;
32+ __entry->nr_accesses = r->nr_accesses;
33+ ),
34+
35+ TP_printk("target_id=%lu nr_regions=%u %lu-%lu: %u",
36+ __entry->target_id, __entry->nr_regions,
37+ __entry->start, __entry->end, __entry->nr_accesses)
38+);
39+
40+#endif /* _TRACE_DAMON_H */
41+
42+/* This part must be outside protection */
43+#include <trace/define_trace.h>
include/trace/events/mmflags.h+1−1
@@ -75,7 +75,7 @@ · 74 unmodified lines
7575 #define IF_HAVE_PG_HWPOISON(flag,string)
7676 #endif
7777
78−#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
78+#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
7979 #define IF_HAVE_PG_IDLE(flag,string) ,{1UL << flag, string}
8080 #else
8181 #define IF_HAVE_PG_IDLE(flag,string)
include/trace/events/page_ref.h+2−2
@@ -38,7 +38,7 @@ DECLARE_EVENT_CLASS(page_ref_mod_template, · 37 unmodified lines
3838
3939 TP_printk("pfn=0x%lx flags=%s count=%d mapcount=%d mapping=%p mt=%d val=%d",
4040 __entry->pfn,
41− show_page_flags(__entry->flags & ((1UL << NR_PAGEFLAGS) - 1)),
41+ show_page_flags(__entry->flags & PAGEFLAGS_MASK),
4242 __entry->count,
4343 __entry->mapcount, __entry->mapping, __entry->mt,
4444 __entry->val)
@@ -88,7 +88,7 @@ DECLARE_EVENT_CLASS(page_ref_mod_and_test_template, · 43 unmodified lines
8888
8989 TP_printk("pfn=0x%lx flags=%s count=%d mapcount=%d mapping=%p mt=%d val=%d ret=%d",
9090 __entry->pfn,
91− show_page_flags(__entry->flags & ((1UL << NR_PAGEFLAGS) - 1)),
91+ show_page_flags(__entry->flags & PAGEFLAGS_MASK),
9292 __entry->count,
9393 __entry->mapcount, __entry->mapping, __entry->mt,
9494 __entry->val, __entry->ret)
init/initramfs.c+2−0
@@ -15,6 +15,7 @@ · 14 unmodified lines
1515 #include <linux/mm.h>
1616 #include <linux/namei.h>
1717 #include <linux/init_syscalls.h>
18+#include <linux/umh.h>
1819
1920 static ssize_t __init xwrite(struct file *file, const char *p, size_t count,
2021 loff_t *pos)
@@ -727,6 +728,7 @@ static int __init populate_rootfs(void) · 706 unmodified lines
727728 {
728729 initramfs_cookie = async_schedule_domain(do_populate_rootfs, NULL,
729730 &initramfs_domain);
731+ usermodehelper_enable();
730732 if (!initramfs_async)
731733 wait_for_initramfs();
732734 return 0;
init/main.c+2−1
@@ -777,6 +777,8 @@ void __init __weak poking_init(void) { } · 776 unmodified lines
777777
778778 void __init __weak pgtable_cache_init(void) { }
779779
780+void __init __weak trap_init(void) { }
781+
780782 bool initcall_debug;
781783 core_param(initcall_debug, initcall_debug, bool, 0644);
782784
@@ -1392,7 +1394,6 @@ static void __init do_basic_setup(void) · 609 unmodified lines
13921394 driver_init();
13931395 init_irq_proc();
13941396 do_ctors();
1395− usermodehelper_enable();
13961397 do_initcalls();
13971398 }
13981399
init/noinitramfs.c+2−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #include <linux/kdev_t.h>
1111 #include <linux/syscalls.h>
1212 #include <linux/init_syscalls.h>
13+#include <linux/umh.h>
1314
1415 /*
1516 * Create a simple rootfs that is similar to the default initramfs
@@ -18,6 +19,7 @@ static int __init default_rootfs(void) · 2 unmodified lines
1819 {
1920 int err;
2021
22+ usermodehelper_enable();
2123 err = init_mkdir("/dev", 0755);
2224 if (err < 0)
2325 goto out;
ipc/util.c+4−12
@@ -788,21 +788,13 @@ struct pid_namespace *ipc_seq_pid_ns(struct seq_file *s) · 787 unmodified lines
788788 static struct kern_ipc_perm *sysvipc_find_ipc(struct ipc_ids *ids, loff_t pos,
789789 loff_t *new_pos)
790790 {
791− struct kern_ipc_perm *ipc;
792− int total, id;
793−
794− total = 0;
795− for (id = 0; id < pos && total < ids->in_use; id++) {
796− ipc = idr_find(&ids->ipcs_idr, id);
797− if (ipc != NULL)
798− total++;
799− }
791+ struct kern_ipc_perm *ipc = NULL;
792+ int max_idx = ipc_get_maxidx(ids);
800793
801− ipc = NULL;
802− if (total >= ids->in_use)
794+ if (max_idx == -1 || pos > max_idx)
803795 goto out;
804796
805− for (; pos < ipc_mni; pos++) {
797+ for (; pos <= max_idx; pos++) {
806798 ipc = idr_find(&ids->ipcs_idr, pos);
807799 if (ipc != NULL) {
808800 rcu_read_lock();
kernel/acct.c+1−1
@@ -478,7 +478,7 @@ static void do_acct_process(struct bsd_acct_struct *acct) · 477 unmodified lines
478478 /*
479479 * Accounting records are not subject to resource limits.
480480 */
481− flim = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
481+ flim = rlimit(RLIMIT_FSIZE);
482482 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
483483 /* Perform file operations on behalf of whoever enabled accounting */
484484 orig_cred = override_creds(file->f_cred);
kernel/fork.c+0−2
@@ -1262,7 +1262,6 @@ struct file *get_mm_exe_file(struct mm_struct *mm) · 1261 unmodified lines
12621262 rcu_read_unlock();
12631263 return exe_file;
12641264 }
1265−EXPORT_SYMBOL(get_mm_exe_file);
12661265
12671266 /**
12681267 * get_task_exe_file - acquire a reference to the task's executable file
@@ -1285,7 +1284,6 @@ struct file *get_task_exe_file(struct task_struct *task) · 16 unmodified lines
12851284 task_unlock(task);
12861285 return exe_file;
12871286 }
1288−EXPORT_SYMBOL(get_task_exe_file);
12891287
12901288 /**
12911289 * get_task_mm - acquire a reference to the task's mm
kernel/profile.c+11−10
@@ -41,7 +41,8 @@ struct profile_hit { · 40 unmodified lines
4141 #define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
4242
4343 static atomic_t *prof_buffer;
44−static unsigned long prof_len, prof_shift;
44+static unsigned long prof_len;
45+static unsigned short int prof_shift;
4546
4647 int prof_on __read_mostly;
4748 EXPORT_SYMBOL_GPL(prof_on);
@@ -67,8 +68,8 @@ int profile_setup(char *str) · 19 unmodified lines
6768 if (str[strlen(sleepstr)] == ',')
6869 str += strlen(sleepstr) + 1;
6970 if (get_option(&str, &par))
70− prof_shift = par;
71− pr_info("kernel sleep profiling enabled (shift: %ld)\n",
71+ prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
72+ pr_info("kernel sleep profiling enabled (shift: %u)\n",
7273 prof_shift);
7374 #else
7475 pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
@@ -78,21 +79,21 @@ int profile_setup(char *str) · 3 unmodified lines
7879 if (str[strlen(schedstr)] == ',')
7980 str += strlen(schedstr) + 1;
8081 if (get_option(&str, &par))
81− prof_shift = par;
82− pr_info("kernel schedule profiling enabled (shift: %ld)\n",
82+ prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
83+ pr_info("kernel schedule profiling enabled (shift: %u)\n",
8384 prof_shift);
8485 } else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
8586 prof_on = KVM_PROFILING;
8687 if (str[strlen(kvmstr)] == ',')
8788 str += strlen(kvmstr) + 1;
8889 if (get_option(&str, &par))
89− prof_shift = par;
90− pr_info("kernel KVM profiling enabled (shift: %ld)\n",
90+ prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
91+ pr_info("kernel KVM profiling enabled (shift: %u)\n",
9192 prof_shift);
9293 } else if (get_option(&str, &par)) {
93− prof_shift = par;
94+ prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
9495 prof_on = CPU_PROFILING;
95− pr_info("kernel profiling enabled (shift: %ld)\n",
96+ pr_info("kernel profiling enabled (shift: %u)\n",
9697 prof_shift);
9798 }
9899 return 1;
@@ -468,7 +469,7 @@ read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos) · 369 unmodified lines
468469 unsigned long p = *ppos;
469470 ssize_t read;
470471 char *pnt;
471− unsigned int sample_step = 1 << prof_shift;
472+ unsigned long sample_step = 1UL << prof_shift;
472473
473474 profile_flip_buffers();
474475 if (p >= (prof_len+1)*sizeof(unsigned int))
kernel/sys.c+0−7
@@ -1929,13 +1929,6 @@ static int validate_prctl_map_addr(struct prctl_mm_map *prctl_map) · 1928 unmodified lines
19291929
19301930 error = -EINVAL;
19311931
1932− /*
1933− * @brk should be after @end_data in traditional maps.
1934− */
1935− if (prctl_map->start_brk <= prctl_map->end_data ||
1936− prctl_map->brk <= prctl_map->end_data)
1937− goto out;
1938−
19391932 /*
19401933 * Neither we should allow to override limits if they set.
19411934 */
kernel/user.c+25−0
@@ -129,6 +129,22 @@ static struct user_struct *uid_hash_find(kuid_t uid, struct hlist_head *hashent) · 128 unmodified lines
129129 return NULL;
130130 }
131131
132+static int user_epoll_alloc(struct user_struct *up)
133+{
134+#ifdef CONFIG_EPOLL
135+ return percpu_counter_init(&up->epoll_watches, 0, GFP_KERNEL);
136+#else
137+ return 0;
138+#endif
139+}
140+
141+static void user_epoll_free(struct user_struct *up)
142+{
143+#ifdef CONFIG_EPOLL
144+ percpu_counter_destroy(&up->epoll_watches);
145+#endif
146+}
147+
132148 /* IRQs are disabled and uidhash_lock is held upon function entry.
133149 * IRQ state (as stored in flags) is restored and uidhash_lock released
134150 * upon function exit.
@@ -138,6 +154,7 @@ static void free_user(struct user_struct *up, unsigned long flags) · 3 unmodified lines
138154 {
139155 uid_hash_remove(up);
140156 spin_unlock_irqrestore(&uidhash_lock, flags);
157+ user_epoll_free(up);
141158 kmem_cache_free(uid_cachep, up);
142159 }
143160
@@ -185,6 +202,10 @@ struct user_struct *alloc_uid(kuid_t uid) · 41 unmodified lines
185202
186203 new->uid = uid;
187204 refcount_set(&new->__count, 1);
205+ if (user_epoll_alloc(new)) {
206+ kmem_cache_free(uid_cachep, new);
207+ return NULL;
208+ }
188209 ratelimit_state_init(&new->ratelimit, HZ, 100);
189210 ratelimit_set_flags(&new->ratelimit, RATELIMIT_MSG_ON_RELEASE);
190211
@@ -195,6 +216,7 @@ struct user_struct *alloc_uid(kuid_t uid) · 4 unmodified lines
195216 spin_lock_irq(&uidhash_lock);
196217 up = uid_hash_find(uid, hashent);
197218 if (up) {
219+ user_epoll_free(new);
198220 kmem_cache_free(uid_cachep, new);
199221 } else {
200222 uid_hash_insert(new, hashent);
@@ -216,6 +238,9 @@ static int __init uid_cache_init(void) · 15 unmodified lines
216238 for(n = 0; n < UIDHASH_SZ; ++n)
217239 INIT_HLIST_HEAD(uidhash_table + n);
218240
241+ if (user_epoll_alloc(&root_user))
242+ panic("root_user epoll percpu counter alloc failed");
243+
219244 /* Insert the root user immediately (init already runs as root) */
220245 spin_lock_irq(&uidhash_lock);
221246 uid_hash_insert(&root_user, uidhashentry(GLOBAL_ROOT_UID));
lib/Kconfig.debug+4−5
@@ -1064,7 +1064,6 @@ config HARDLOCKUP_DETECTOR · 1063 unmodified lines
10641064 depends on HAVE_HARDLOCKUP_DETECTOR_PERF || HAVE_HARDLOCKUP_DETECTOR_ARCH
10651065 select LOCKUP_DETECTOR
10661066 select HARDLOCKUP_DETECTOR_PERF if HAVE_HARDLOCKUP_DETECTOR_PERF
1067− select HARDLOCKUP_DETECTOR_ARCH if HAVE_HARDLOCKUP_DETECTOR_ARCH
10681067 help
10691068 Say Y here to enable the kernel to act as a watchdog to detect
10701069 hard lockups.
@@ -2061,8 +2060,9 @@ config TEST_MIN_HEAP · 990 unmodified lines
20612060 If unsure, say N.
20622061
20632062 config TEST_SORT
2064− tristate "Array-based sort test"
2065− depends on DEBUG_KERNEL || m
2063+ tristate "Array-based sort test" if !KUNIT_ALL_TESTS
2064+ depends on KUNIT
2065+ default KUNIT_ALL_TESTS
20662066 help
20672067 This option enables the self-test function of 'sort()' at boot,
20682068 or at module load time.
@@ -2443,8 +2443,7 @@ config SLUB_KUNIT_TEST · 374 unmodified lines
24432443
24442444 config RATIONAL_KUNIT_TEST
24452445 tristate "KUnit test for rational.c" if !KUNIT_ALL_TESTS
2446− depends on KUNIT
2447− select RATIONAL
2446+ depends on KUNIT && RATIONAL
24482447 default KUNIT_ALL_TESTS
24492448 help
24502449 This builds the rational math unit test.
lib/dump_stack.c+2−1
@@ -89,7 +89,8 @@ static void __dump_stack(const char *log_lvl) · 88 unmodified lines
8989 }
9090
9191 /**
92− * dump_stack - dump the current task information and its stack trace
92+ * dump_stack_lvl - dump the current task information and its stack trace
93+ * @log_lvl: log level
9394 *
9495 * Architectures can override this implementation by implementing its own.
9596 */
lib/iov_iter.c+6−2
@@ -672,7 +672,7 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, · 671 unmodified lines
672672 * _copy_mc_to_iter - copy to iter with source memory error exception handling
673673 * @addr: source kernel address
674674 * @bytes: total transfer length
675− * @iter: destination iterator
675+ * @i: destination iterator
676676 *
677677 * The pmem driver deploys this for the dax operation
678678 * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
@@ -690,6 +690,8 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, · 11 unmodified lines
690690 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
691691 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
692692 * a short copy.
693+ *
694+ * Return: number of bytes copied (may be %0)
693695 */
694696 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
695697 {
@@ -744,7 +746,7 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); · 48 unmodified lines
744746 * _copy_from_iter_flushcache - write destination through cpu cache
745747 * @addr: destination kernel address
746748 * @bytes: total transfer length
747− * @iter: source iterator
749+ * @i: source iterator
748750 *
749751 * The pmem driver arranges for filesystem-dax to use this facility via
750752 * dax_copy_from_iter() for ensuring that writes to persistent memory
@@ -753,6 +755,8 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); · 2 unmodified lines
753755 * all iterator types. The _copy_from_iter_nocache() only attempts to
754756 * bypass the cache for the ITER_IOVEC case, and on some archs may use
755757 * instructions that strand dirty-data in the cache.
758+ *
759+ * Return: number of bytes copied (may be %0)
756760 */
757761 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
758762 {
lib/math/Kconfig+1−1
@@ -14,4 +14,4 @@ config PRIME_NUMBERS · 13 unmodified lines
1414 If unsure, say N.
1515
1616 config RATIONAL
17− bool
17+ tristate
lib/math/rational.c+3−0
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #include <linux/export.h>
1414 #include <linux/minmax.h>
1515 #include <linux/limits.h>
16+#include <linux/module.h>
1617
1718 /*
1819 * calculate best rational approximation for a given fraction
@@ -106,3 +107,5 @@ void rational_best_approximation( · 87 unmodified lines
106107 }
107108
108109 EXPORT_SYMBOL(rational_best_approximation);
110+
111+MODULE_LICENSE("GPL v2");
lib/test_printf.c+1−1
@@ -614,7 +614,7 @@ page_flags_test(int section, int node, int zone, int last_cpupid, · 613 unmodified lines
614614 bool append = false;
615615 int i;
616616
617− flags &= BIT(NR_PAGEFLAGS) - 1;
617+ flags &= PAGEFLAGS_MASK;
618618 if (flags) {
619619 page_flags |= flags;
620620 snprintf(cmp_buf + size, BUF_SIZE - size, "%s", name);
lib/test_sort.c+19−21
@@ -1,4 +1,7 @@
11 // SPDX-License-Identifier: GPL-2.0-only
2+
3+#include <kunit/test.h>
4+
25 #include <linux/sort.h>
36 #include <linux/slab.h>
47 #include <linux/module.h>
@@ -7,18 +10,17 @@ · 2 unmodified lines
710
811 #define TEST_LEN 1000
912
10−static int __init cmpint(const void *a, const void *b)
13+static int cmpint(const void *a, const void *b)
1114 {
1215 return *(int *)a - *(int *)b;
1316 }
1417
15−static int __init test_sort_init(void)
18+static void test_sort(struct kunit *test)
1619 {
17− int *a, i, r = 1, err = -ENOMEM;
20+ int *a, i, r = 1;
1821
19− a = kmalloc_array(TEST_LEN, sizeof(*a), GFP_KERNEL);
20− if (!a)
21− return err;
22+ a = kunit_kmalloc_array(test, TEST_LEN, sizeof(*a), GFP_KERNEL);
23+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, a);
2224
2325 for (i = 0; i < TEST_LEN; i++) {
2426 r = (r * 725861) % 6599;
@@ -27,24 +29,20 @@ static int __init test_sort_init(void) · 2 unmodified lines
2729
2830 sort(a, TEST_LEN, sizeof(*a), cmpint, NULL);
2931
30− err = -EINVAL;
3132 for (i = 0; i < TEST_LEN-1; i++)
32− if (a[i] > a[i+1]) {
33− pr_err("test has failed\n");
34− goto exit;
35− }
36− err = 0;
37− pr_info("test passed\n");
38−exit:
39− kfree(a);
40− return err;
33+ KUNIT_ASSERT_LE(test, a[i], a[i + 1]);
4134 }
4235
43−static void __exit test_sort_exit(void)
44−{
45−}
36+static struct kunit_case sort_test_cases[] = {
37+ KUNIT_CASE(test_sort),
38+ {}
39+};
40+
41+static struct kunit_suite sort_test_suite = {
42+ .name = "lib_sort",
43+ .test_cases = sort_test_cases,
44+};
4645
47−module_init(test_sort_init);
48−module_exit(test_sort_exit);
46+kunit_test_suites(&sort_test_suite);
4947
5048 MODULE_LICENSE("GPL");
lib/vsprintf.c+1−1
@@ -2019,7 +2019,7 @@ static const struct page_flags_fields pff[] = { · 2018 unmodified lines
20192019 static
20202020 char *format_page_flags(char *buf, char *end, unsigned long flags)
20212021 {
2022− unsigned long main_flags = flags & (BIT(NR_PAGEFLAGS) - 1);
2022+ unsigned long main_flags = flags & PAGEFLAGS_MASK;
20232023 bool append = false;
20242024 int i;
20252025
mm/Kconfig+11−4
@@ -96,9 +96,6 @@ config HAVE_FAST_GUP · 95 unmodified lines
9696 depends on MMU
9797 bool
9898
99−config HOLES_IN_ZONE
100− bool
101−
10299 # Don't discard allocated memory used to track "memory" and "reserved" memblocks
103100 # after early boot, so it can still be used to test for validity of memory.
104101 # Also, memblocks are updated with memory hot(un)plug.
@@ -742,10 +739,18 @@ config DEFERRED_STRUCT_PAGE_INIT · 637 unmodified lines
742739 lifetime of the system until these kthreads finish the
743740 initialisation.
744741
742+config PAGE_IDLE_FLAG
743+ bool
744+ select PAGE_EXTENSION if !64BIT
745+ help
746+ This adds PG_idle and PG_young flags to 'struct page'. PTE Accessed
747+ bit writers can set the state of the bit in the flags so that PTE
748+ Accessed bit readers may avoid disturbance.
749+
745750 config IDLE_PAGE_TRACKING
746751 bool "Enable idle page tracking"
747752 depends on SYSFS && MMU
748− select PAGE_EXTENSION if !64BIT
753+ select PAGE_IDLE_FLAG
749754 help
750755 This feature allows to estimate the amount of user pages that have
751756 not been touched during a given period of time. This information can
@@ -889,4 +894,6 @@ config IO_MAPPING · 137 unmodified lines
889894 config SECRETMEM
890895 def_bool ARCH_HAS_SET_DIRECT_MAP && !EMBEDDED
891896
897+source "mm/damon/Kconfig"
898+
892899 endmenu
mm/Makefile+3−1
@@ -38,7 +38,7 @@ mmu-y := nommu.o · 37 unmodified lines
3838 mmu-$(CONFIG_MMU) := highmem.o memory.o mincore.o \
3939 mlock.o mmap.o mmu_gather.o mprotect.o mremap.o \
4040 msync.o page_vma_mapped.o pagewalk.o \
41− pgtable-generic.o rmap.o vmalloc.o ioremap.o
41+ pgtable-generic.o rmap.o vmalloc.o
4242
4343
4444 ifdef CONFIG_CROSS_MEMORY_ATTACH
@@ -118,6 +118,7 @@ obj-$(CONFIG_CMA_SYSFS) += cma_sysfs.o · 73 unmodified lines
118118 obj-$(CONFIG_USERFAULTFD) += userfaultfd.o
119119 obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o
120120 obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o
121+obj-$(CONFIG_DAMON) += damon/
121122 obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o
122123 obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o
123124 obj-$(CONFIG_ZONE_DEVICE) += memremap.o
@@ -128,3 +129,4 @@ obj-$(CONFIG_PTDUMP_CORE) += ptdump.o · 4 unmodified lines
128129 obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
129130 obj-$(CONFIG_IO_MAPPING) += io-mapping.o
130131 obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o
132+obj-$(CONFIG_GENERIC_IOREMAP) += ioremap.o
mm/compaction.c+7−13
@@ -306,16 +306,14 @@ __reset_isolation_pfn(struct zone *zone, unsigned long pfn, bool check_source, · 305 unmodified lines
306306 * is necessary for the block to be a migration source/target.
307307 */
308308 do {
309− if (pfn_valid_within(pfn)) {
310− if (check_source && PageLRU(page)) {
311− clear_pageblock_skip(page);
312− return true;
313− }
309+ if (check_source && PageLRU(page)) {
310+ clear_pageblock_skip(page);
311+ return true;
312+ }
314313
315− if (check_target && PageBuddy(page)) {
316− clear_pageblock_skip(page);
317− return true;
318− }
314+ if (check_target && PageBuddy(page)) {
315+ clear_pageblock_skip(page);
316+ return true;
319317 }
320318
321319 page += (1 << PAGE_ALLOC_COSTLY_ORDER);
@@ -585,8 +583,6 @@ static unsigned long isolate_freepages_block(struct compact_control *cc, · 263 unmodified lines
585583 break;
586584
587585 nr_scanned++;
588− if (!pfn_valid_within(blockpfn))
589− goto isolate_fail;
590586
591587 /*
592588 * For compound pages such as THP and hugetlbfs, we can save
@@ -885,8 +881,6 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, · 292 unmodified lines
885881 cond_resched();
886882 }
887883
888− if (!pfn_valid_within(low_pfn))
889− goto isolate_fail;
890884 nr_scanned++;
891885
892886 page = pfn_to_page(low_pfn);
mm/damon/Kconfig+68−0
@@ -0,0 +1,68 @@
1+# SPDX-License-Identifier: GPL-2.0-only
2+
3+menu "Data Access Monitoring"
4+
5+config DAMON
6+ bool "DAMON: Data Access Monitoring Framework"
7+ help
8+ This builds a framework that allows kernel subsystems to monitor
9+ access frequency of each memory region. The information can be useful
10+ for performance-centric DRAM level memory management.
11+
12+ See https://damonitor.github.io/doc/html/latest-damon/index.html for
13+ more information.
14+
15+config DAMON_KUNIT_TEST
16+ bool "Test for damon" if !KUNIT_ALL_TESTS
17+ depends on DAMON && KUNIT=y
18+ default KUNIT_ALL_TESTS
19+ help
20+ This builds the DAMON Kunit test suite.
21+
22+ For more information on KUnit and unit tests in general, please refer
23+ to the KUnit documentation.
24+
25+ If unsure, say N.
26+
27+config DAMON_VADDR
28+ bool "Data access monitoring primitives for virtual address spaces"
29+ depends on DAMON && MMU
30+ select PAGE_IDLE_FLAG
31+ help
32+ This builds the default data access monitoring primitives for DAMON
33+ that works for virtual address spaces.
34+
35+config DAMON_VADDR_KUNIT_TEST
36+ bool "Test for DAMON primitives" if !KUNIT_ALL_TESTS
37+ depends on DAMON_VADDR && KUNIT=y
38+ default KUNIT_ALL_TESTS
39+ help
40+ This builds the DAMON virtual addresses primitives Kunit test suite.
41+
42+ For more information on KUnit and unit tests in general, please refer
43+ to the KUnit documentation.
44+
45+ If unsure, say N.
46+
47+config DAMON_DBGFS
48+ bool "DAMON debugfs interface"
49+ depends on DAMON_VADDR && DEBUG_FS
50+ help
51+ This builds the debugfs interface for DAMON. The user space admins
52+ can use the interface for arbitrary data access monitoring.
53+
54+ If unsure, say N.
55+
56+config DAMON_DBGFS_KUNIT_TEST
57+ bool "Test for damon debugfs interface" if !KUNIT_ALL_TESTS
58+ depends on DAMON_DBGFS && KUNIT=y
59+ default KUNIT_ALL_TESTS
60+ help
61+ This builds the DAMON debugfs interface Kunit test suite.
62+
63+ For more information on KUnit and unit tests in general, please refer
64+ to the KUnit documentation.
65+
66+ If unsure, say N.
67+
68+endmenu
mm/damon/Makefile+5−0
@@ -0,0 +1,5 @@
1+# SPDX-License-Identifier: GPL-2.0
2+
3+obj-$(CONFIG_DAMON) := core.o
4+obj-$(CONFIG_DAMON_VADDR) += vaddr.o
5+obj-$(CONFIG_DAMON_DBGFS) += dbgfs.o
mm/damon/core-test.h+253−0
@@ -0,0 +1,253 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Data Access Monitor Unit Tests
4+ *
5+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved.
6+ *
7+ * Author: SeongJae Park <sjpark@amazon.de>
8+ */
9+
10+#ifdef CONFIG_DAMON_KUNIT_TEST
11+
12+#ifndef _DAMON_CORE_TEST_H
13+#define _DAMON_CORE_TEST_H
14+
15+#include <kunit/test.h>
16+
17+static void damon_test_regions(struct kunit *test)
18+{
19+ struct damon_region *r;
20+ struct damon_target *t;
21+
22+ r = damon_new_region(1, 2);
23+ KUNIT_EXPECT_EQ(test, 1ul, r->ar.start);
24+ KUNIT_EXPECT_EQ(test, 2ul, r->ar.end);
25+ KUNIT_EXPECT_EQ(test, 0u, r->nr_accesses);
26+
27+ t = damon_new_target(42);
28+ KUNIT_EXPECT_EQ(test, 0u, damon_nr_regions(t));
29+
30+ damon_add_region(r, t);
31+ KUNIT_EXPECT_EQ(test, 1u, damon_nr_regions(t));
32+
33+ damon_del_region(r, t);
34+ KUNIT_EXPECT_EQ(test, 0u, damon_nr_regions(t));
35+
36+ damon_free_target(t);
37+}
38+
39+static unsigned int nr_damon_targets(struct damon_ctx *ctx)
40+{
41+ struct damon_target *t;
42+ unsigned int nr_targets = 0;
43+
44+ damon_for_each_target(t, ctx)
45+ nr_targets++;
46+
47+ return nr_targets;
48+}
49+
50+static void damon_test_target(struct kunit *test)
51+{
52+ struct damon_ctx *c = damon_new_ctx();
53+ struct damon_target *t;
54+
55+ t = damon_new_target(42);
56+ KUNIT_EXPECT_EQ(test, 42ul, t->id);
57+ KUNIT_EXPECT_EQ(test, 0u, nr_damon_targets(c));
58+
59+ damon_add_target(c, t);
60+ KUNIT_EXPECT_EQ(test, 1u, nr_damon_targets(c));
61+
62+ damon_destroy_target(t);
63+ KUNIT_EXPECT_EQ(test, 0u, nr_damon_targets(c));
64+
65+ damon_destroy_ctx(c);
66+}
67+
68+/*
69+ * Test kdamond_reset_aggregated()
70+ *
71+ * DAMON checks access to each region and aggregates this information as the
72+ * access frequency of each region. In detail, it increases '->nr_accesses' of
73+ * regions that an access has confirmed. 'kdamond_reset_aggregated()' flushes
74+ * the aggregated information ('->nr_accesses' of each regions) to the result
75+ * buffer. As a result of the flushing, the '->nr_accesses' of regions are
76+ * initialized to zero.
77+ */
78+static void damon_test_aggregate(struct kunit *test)
79+{
80+ struct damon_ctx *ctx = damon_new_ctx();
81+ unsigned long target_ids[] = {1, 2, 3};
82+ unsigned long saddr[][3] = {{10, 20, 30}, {5, 42, 49}, {13, 33, 55} };
83+ unsigned long eaddr[][3] = {{15, 27, 40}, {31, 45, 55}, {23, 44, 66} };
84+ unsigned long accesses[][3] = {{42, 95, 84}, {10, 20, 30}, {0, 1, 2} };
85+ struct damon_target *t;
86+ struct damon_region *r;
87+ int it, ir;
88+
89+ damon_set_targets(ctx, target_ids, 3);
90+
91+ it = 0;
92+ damon_for_each_target(t, ctx) {
93+ for (ir = 0; ir < 3; ir++) {
94+ r = damon_new_region(saddr[it][ir], eaddr[it][ir]);
95+ r->nr_accesses = accesses[it][ir];
96+ damon_add_region(r, t);
97+ }
98+ it++;
99+ }
100+ kdamond_reset_aggregated(ctx);
101+ it = 0;
102+ damon_for_each_target(t, ctx) {
103+ ir = 0;
104+ /* '->nr_accesses' should be zeroed */
105+ damon_for_each_region(r, t) {
106+ KUNIT_EXPECT_EQ(test, 0u, r->nr_accesses);
107+ ir++;
108+ }
109+ /* regions should be preserved */
110+ KUNIT_EXPECT_EQ(test, 3, ir);
111+ it++;
112+ }
113+ /* targets also should be preserved */
114+ KUNIT_EXPECT_EQ(test, 3, it);
115+
116+ damon_destroy_ctx(ctx);
117+}
118+
119+static void damon_test_split_at(struct kunit *test)
120+{
121+ struct damon_ctx *c = damon_new_ctx();
122+ struct damon_target *t;
123+ struct damon_region *r;
124+
125+ t = damon_new_target(42);
126+ r = damon_new_region(0, 100);
127+ damon_add_region(r, t);
128+ damon_split_region_at(c, t, r, 25);
129+ KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
130+ KUNIT_EXPECT_EQ(test, r->ar.end, 25ul);
131+
132+ r = damon_next_region(r);
133+ KUNIT_EXPECT_EQ(test, r->ar.start, 25ul);
134+ KUNIT_EXPECT_EQ(test, r->ar.end, 100ul);
135+
136+ damon_free_target(t);
137+ damon_destroy_ctx(c);
138+}
139+
140+static void damon_test_merge_two(struct kunit *test)
141+{
142+ struct damon_target *t;
143+ struct damon_region *r, *r2, *r3;
144+ int i;
145+
146+ t = damon_new_target(42);
147+ r = damon_new_region(0, 100);
148+ r->nr_accesses = 10;
149+ damon_add_region(r, t);
150+ r2 = damon_new_region(100, 300);
151+ r2->nr_accesses = 20;
152+ damon_add_region(r2, t);
153+
154+ damon_merge_two_regions(t, r, r2);
155+ KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
156+ KUNIT_EXPECT_EQ(test, r->ar.end, 300ul);
157+ KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u);
158+
159+ i = 0;
160+ damon_for_each_region(r3, t) {
161+ KUNIT_EXPECT_PTR_EQ(test, r, r3);
162+ i++;
163+ }
164+ KUNIT_EXPECT_EQ(test, i, 1);
165+
166+ damon_free_target(t);
167+}
168+
169+static struct damon_region *__nth_region_of(struct damon_target *t, int idx)
170+{
171+ struct damon_region *r;
172+ unsigned int i = 0;
173+
174+ damon_for_each_region(r, t) {
175+ if (i++ == idx)
176+ return r;
177+ }
178+
179+ return NULL;
180+}
181+
182+static void damon_test_merge_regions_of(struct kunit *test)
183+{
184+ struct damon_target *t;
185+ struct damon_region *r;
186+ unsigned long sa[] = {0, 100, 114, 122, 130, 156, 170, 184};
187+ unsigned long ea[] = {100, 112, 122, 130, 156, 170, 184, 230};
188+ unsigned int nrs[] = {0, 0, 10, 10, 20, 30, 1, 2};
189+
190+ unsigned long saddrs[] = {0, 114, 130, 156, 170};
191+ unsigned long eaddrs[] = {112, 130, 156, 170, 230};
192+ int i;
193+
194+ t = damon_new_target(42);
195+ for (i = 0; i < ARRAY_SIZE(sa); i++) {
196+ r = damon_new_region(sa[i], ea[i]);
197+ r->nr_accesses = nrs[i];
198+ damon_add_region(r, t);
199+ }
200+
201+ damon_merge_regions_of(t, 9, 9999);
202+ /* 0-112, 114-130, 130-156, 156-170 */
203+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u);
204+ for (i = 0; i < 5; i++) {
205+ r = __nth_region_of(t, i);
206+ KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]);
207+ KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]);
208+ }
209+ damon_free_target(t);
210+}
211+
212+static void damon_test_split_regions_of(struct kunit *test)
213+{
214+ struct damon_ctx *c = damon_new_ctx();
215+ struct damon_target *t;
216+ struct damon_region *r;
217+
218+ t = damon_new_target(42);
219+ r = damon_new_region(0, 22);
220+ damon_add_region(r, t);
221+ damon_split_regions_of(c, t, 2);
222+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2u);
223+ damon_free_target(t);
224+
225+ t = damon_new_target(42);
226+ r = damon_new_region(0, 220);
227+ damon_add_region(r, t);
228+ damon_split_regions_of(c, t, 4);
229+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 4u);
230+ damon_free_target(t);
231+ damon_destroy_ctx(c);
232+}
233+
234+static struct kunit_case damon_test_cases[] = {
235+ KUNIT_CASE(damon_test_target),
236+ KUNIT_CASE(damon_test_regions),
237+ KUNIT_CASE(damon_test_aggregate),
238+ KUNIT_CASE(damon_test_split_at),
239+ KUNIT_CASE(damon_test_merge_two),
240+ KUNIT_CASE(damon_test_merge_regions_of),
241+ KUNIT_CASE(damon_test_split_regions_of),
242+ {},
243+};
244+
245+static struct kunit_suite damon_test_suite = {
246+ .name = "damon",
247+ .test_cases = damon_test_cases,
248+};
249+kunit_test_suite(damon_test_suite);
250+
251+#endif /* _DAMON_CORE_TEST_H */
252+
253+#endif /* CONFIG_DAMON_KUNIT_TEST */
mm/damon/core.c+720−0
@@ -0,0 +1,720 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * Data Access Monitor
4+ *
5+ * Author: SeongJae Park <sjpark@amazon.de>
6+ */
7+
8+#define pr_fmt(fmt) "damon: " fmt
9+
10+#include <linux/damon.h>
11+#include <linux/delay.h>
12+#include <linux/kthread.h>
13+#include <linux/random.h>
14+#include <linux/slab.h>
15+
16+#define CREATE_TRACE_POINTS
17+#include <trace/events/damon.h>
18+
19+#ifdef CONFIG_DAMON_KUNIT_TEST
20+#undef DAMON_MIN_REGION
21+#define DAMON_MIN_REGION 1
22+#endif
23+
24+/* Get a random number in [l, r) */
25+#define damon_rand(l, r) (l + prandom_u32_max(r - l))
26+
27+static DEFINE_MUTEX(damon_lock);
28+static int nr_running_ctxs;
29+
30+/*
31+ * Construct a damon_region struct
32+ *
33+ * Returns the pointer to the new struct if success, or NULL otherwise
34+ */
35+struct damon_region *damon_new_region(unsigned long start, unsigned long end)
36+{
37+ struct damon_region *region;
38+
39+ region = kmalloc(sizeof(*region), GFP_KERNEL);
40+ if (!region)
41+ return NULL;
42+
43+ region->ar.start = start;
44+ region->ar.end = end;
45+ region->nr_accesses = 0;
46+ INIT_LIST_HEAD(&region->list);
47+
48+ return region;
49+}
50+
51+/*
52+ * Add a region between two other regions
53+ */
54+inline void damon_insert_region(struct damon_region *r,
55+ struct damon_region *prev, struct damon_region *next,
56+ struct damon_target *t)
57+{
58+ __list_add(&r->list, &prev->list, &next->list);
59+ t->nr_regions++;
60+}
61+
62+void damon_add_region(struct damon_region *r, struct damon_target *t)
63+{
64+ list_add_tail(&r->list, &t->regions_list);
65+ t->nr_regions++;
66+}
67+
68+static void damon_del_region(struct damon_region *r, struct damon_target *t)
69+{
70+ list_del(&r->list);
71+ t->nr_regions--;
72+}
73+
74+static void damon_free_region(struct damon_region *r)
75+{
76+ kfree(r);
77+}
78+
79+void damon_destroy_region(struct damon_region *r, struct damon_target *t)
80+{
81+ damon_del_region(r, t);
82+ damon_free_region(r);
83+}
84+
85+/*
86+ * Construct a damon_target struct
87+ *
88+ * Returns the pointer to the new struct if success, or NULL otherwise
89+ */
90+struct damon_target *damon_new_target(unsigned long id)
91+{
92+ struct damon_target *t;
93+
94+ t = kmalloc(sizeof(*t), GFP_KERNEL);
95+ if (!t)
96+ return NULL;
97+
98+ t->id = id;
99+ t->nr_regions = 0;
100+ INIT_LIST_HEAD(&t->regions_list);
101+
102+ return t;
103+}
104+
105+void damon_add_target(struct damon_ctx *ctx, struct damon_target *t)
106+{
107+ list_add_tail(&t->list, &ctx->adaptive_targets);
108+}
109+
110+static void damon_del_target(struct damon_target *t)
111+{
112+ list_del(&t->list);
113+}
114+
115+void damon_free_target(struct damon_target *t)
116+{
117+ struct damon_region *r, *next;
118+
119+ damon_for_each_region_safe(r, next, t)
120+ damon_free_region(r);
121+ kfree(t);
122+}
123+
124+void damon_destroy_target(struct damon_target *t)
125+{
126+ damon_del_target(t);
127+ damon_free_target(t);
128+}
129+
130+unsigned int damon_nr_regions(struct damon_target *t)
131+{
132+ return t->nr_regions;
133+}
134+
135+struct damon_ctx *damon_new_ctx(void)
136+{
137+ struct damon_ctx *ctx;
138+
139+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
140+ if (!ctx)
141+ return NULL;
142+
143+ ctx->sample_interval = 5 * 1000;
144+ ctx->aggr_interval = 100 * 1000;
145+ ctx->primitive_update_interval = 60 * 1000 * 1000;
146+
147+ ktime_get_coarse_ts64(&ctx->last_aggregation);
148+ ctx->last_primitive_update = ctx->last_aggregation;
149+
150+ mutex_init(&ctx->kdamond_lock);
151+
152+ ctx->min_nr_regions = 10;
153+ ctx->max_nr_regions = 1000;
154+
155+ INIT_LIST_HEAD(&ctx->adaptive_targets);
156+
157+ return ctx;
158+}
159+
160+static void damon_destroy_targets(struct damon_ctx *ctx)
161+{
162+ struct damon_target *t, *next_t;
163+
164+ if (ctx->primitive.cleanup) {
165+ ctx->primitive.cleanup(ctx);
166+ return;
167+ }
168+
169+ damon_for_each_target_safe(t, next_t, ctx)
170+ damon_destroy_target(t);
171+}
172+
173+void damon_destroy_ctx(struct damon_ctx *ctx)
174+{
175+ damon_destroy_targets(ctx);
176+ kfree(ctx);
177+}
178+
179+/**
180+ * damon_set_targets() - Set monitoring targets.
181+ * @ctx: monitoring context
182+ * @ids: array of target ids
183+ * @nr_ids: number of entries in @ids
184+ *
185+ * This function should not be called while the kdamond is running.
186+ *
187+ * Return: 0 on success, negative error code otherwise.
188+ */
189+int damon_set_targets(struct damon_ctx *ctx,
190+ unsigned long *ids, ssize_t nr_ids)
191+{
192+ ssize_t i;
193+ struct damon_target *t, *next;
194+
195+ damon_destroy_targets(ctx);
196+
197+ for (i = 0; i < nr_ids; i++) {
198+ t = damon_new_target(ids[i]);
199+ if (!t) {
200+ pr_err("Failed to alloc damon_target\n");
201+ /* The caller should do cleanup of the ids itself */
202+ damon_for_each_target_safe(t, next, ctx)
203+ damon_destroy_target(t);
204+ return -ENOMEM;
205+ }
206+ damon_add_target(ctx, t);
207+ }
208+
209+ return 0;
210+}
211+
212+/**
213+ * damon_set_attrs() - Set attributes for the monitoring.
214+ * @ctx: monitoring context
215+ * @sample_int: time interval between samplings
216+ * @aggr_int: time interval between aggregations
217+ * @primitive_upd_int: time interval between monitoring primitive updates
218+ * @min_nr_reg: minimal number of regions
219+ * @max_nr_reg: maximum number of regions
220+ *
221+ * This function should not be called while the kdamond is running.
222+ * Every time interval is in micro-seconds.
223+ *
224+ * Return: 0 on success, negative error code otherwise.
225+ */
226+int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int,
227+ unsigned long aggr_int, unsigned long primitive_upd_int,
228+ unsigned long min_nr_reg, unsigned long max_nr_reg)
229+{
230+ if (min_nr_reg < 3) {
231+ pr_err("min_nr_regions (%lu) must be at least 3\n",
232+ min_nr_reg);
233+ return -EINVAL;
234+ }
235+ if (min_nr_reg > max_nr_reg) {
236+ pr_err("invalid nr_regions. min (%lu) > max (%lu)\n",
237+ min_nr_reg, max_nr_reg);
238+ return -EINVAL;
239+ }
240+
241+ ctx->sample_interval = sample_int;
242+ ctx->aggr_interval = aggr_int;
243+ ctx->primitive_update_interval = primitive_upd_int;
244+ ctx->min_nr_regions = min_nr_reg;
245+ ctx->max_nr_regions = max_nr_reg;
246+
247+ return 0;
248+}
249+
250+/**
251+ * damon_nr_running_ctxs() - Return number of currently running contexts.
252+ */
253+int damon_nr_running_ctxs(void)
254+{
255+ int nr_ctxs;
256+
257+ mutex_lock(&damon_lock);
258+ nr_ctxs = nr_running_ctxs;
259+ mutex_unlock(&damon_lock);
260+
261+ return nr_ctxs;
262+}
263+
264+/* Returns the size upper limit for each monitoring region */
265+static unsigned long damon_region_sz_limit(struct damon_ctx *ctx)
266+{
267+ struct damon_target *t;
268+ struct damon_region *r;
269+ unsigned long sz = 0;
270+
271+ damon_for_each_target(t, ctx) {
272+ damon_for_each_region(r, t)
273+ sz += r->ar.end - r->ar.start;
274+ }
275+
276+ if (ctx->min_nr_regions)
277+ sz /= ctx->min_nr_regions;
278+ if (sz < DAMON_MIN_REGION)
279+ sz = DAMON_MIN_REGION;
280+
281+ return sz;
282+}
283+
284+static bool damon_kdamond_running(struct damon_ctx *ctx)
285+{
286+ bool running;
287+
288+ mutex_lock(&ctx->kdamond_lock);
289+ running = ctx->kdamond != NULL;
290+ mutex_unlock(&ctx->kdamond_lock);
291+
292+ return running;
293+}
294+
295+static int kdamond_fn(void *data);
296+
297+/*
298+ * __damon_start() - Starts monitoring with given context.
299+ * @ctx: monitoring context
300+ *
301+ * This function should be called while damon_lock is hold.
302+ *
303+ * Return: 0 on success, negative error code otherwise.
304+ */
305+static int __damon_start(struct damon_ctx *ctx)
306+{
307+ int err = -EBUSY;
308+
309+ mutex_lock(&ctx->kdamond_lock);
310+ if (!ctx->kdamond) {
311+ err = 0;
312+ ctx->kdamond_stop = false;
313+ ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d",
314+ nr_running_ctxs);
315+ if (IS_ERR(ctx->kdamond)) {
316+ err = PTR_ERR(ctx->kdamond);
317+ ctx->kdamond = 0;
318+ }
319+ }
320+ mutex_unlock(&ctx->kdamond_lock);
321+
322+ return err;
323+}
324+
325+/**
326+ * damon_start() - Starts the monitorings for a given group of contexts.
327+ * @ctxs: an array of the pointers for contexts to start monitoring
328+ * @nr_ctxs: size of @ctxs
329+ *
330+ * This function starts a group of monitoring threads for a group of monitoring
331+ * contexts. One thread per each context is created and run in parallel. The
332+ * caller should handle synchronization between the threads by itself. If a
333+ * group of threads that created by other 'damon_start()' call is currently
334+ * running, this function does nothing but returns -EBUSY.
335+ *
336+ * Return: 0 on success, negative error code otherwise.
337+ */
338+int damon_start(struct damon_ctx **ctxs, int nr_ctxs)
339+{
340+ int i;
341+ int err = 0;
342+
343+ mutex_lock(&damon_lock);
344+ if (nr_running_ctxs) {
345+ mutex_unlock(&damon_lock);
346+ return -EBUSY;
347+ }
348+
349+ for (i = 0; i < nr_ctxs; i++) {
350+ err = __damon_start(ctxs[i]);
351+ if (err)
352+ break;
353+ nr_running_ctxs++;
354+ }
355+ mutex_unlock(&damon_lock);
356+
357+ return err;
358+}
359+
360+/*
361+ * __damon_stop() - Stops monitoring of given context.
362+ * @ctx: monitoring context
363+ *
364+ * Return: 0 on success, negative error code otherwise.
365+ */
366+static int __damon_stop(struct damon_ctx *ctx)
367+{
368+ mutex_lock(&ctx->kdamond_lock);
369+ if (ctx->kdamond) {
370+ ctx->kdamond_stop = true;
371+ mutex_unlock(&ctx->kdamond_lock);
372+ while (damon_kdamond_running(ctx))
373+ usleep_range(ctx->sample_interval,
374+ ctx->sample_interval * 2);
375+ return 0;
376+ }
377+ mutex_unlock(&ctx->kdamond_lock);
378+
379+ return -EPERM;
380+}
381+
382+/**
383+ * damon_stop() - Stops the monitorings for a given group of contexts.
384+ * @ctxs: an array of the pointers for contexts to stop monitoring
385+ * @nr_ctxs: size of @ctxs
386+ *
387+ * Return: 0 on success, negative error code otherwise.
388+ */
389+int damon_stop(struct damon_ctx **ctxs, int nr_ctxs)
390+{
391+ int i, err = 0;
392+
393+ for (i = 0; i < nr_ctxs; i++) {
394+ /* nr_running_ctxs is decremented in kdamond_fn */
395+ err = __damon_stop(ctxs[i]);
396+ if (err)
397+ return err;
398+ }
399+
400+ return err;
401+}
402+
403+/*
404+ * damon_check_reset_time_interval() - Check if a time interval is elapsed.
405+ * @baseline: the time to check whether the interval has elapsed since
406+ * @interval: the time interval (microseconds)
407+ *
408+ * See whether the given time interval has passed since the given baseline
409+ * time. If so, it also updates the baseline to current time for next check.
410+ *
411+ * Return: true if the time interval has passed, or false otherwise.
412+ */
413+static bool damon_check_reset_time_interval(struct timespec64 *baseline,
414+ unsigned long interval)
415+{
416+ struct timespec64 now;
417+
418+ ktime_get_coarse_ts64(&now);
419+ if ((timespec64_to_ns(&now) - timespec64_to_ns(baseline)) <
420+ interval * 1000)
421+ return false;
422+ *baseline = now;
423+ return true;
424+}
425+
426+/*
427+ * Check whether it is time to flush the aggregated information
428+ */
429+static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx)
430+{
431+ return damon_check_reset_time_interval(&ctx->last_aggregation,
432+ ctx->aggr_interval);
433+}
434+
435+/*
436+ * Reset the aggregated monitoring results ('nr_accesses' of each region).
437+ */
438+static void kdamond_reset_aggregated(struct damon_ctx *c)
439+{
440+ struct damon_target *t;
441+
442+ damon_for_each_target(t, c) {
443+ struct damon_region *r;
444+
445+ damon_for_each_region(r, t) {
446+ trace_damon_aggregated(t, r, damon_nr_regions(t));
447+ r->nr_accesses = 0;
448+ }
449+ }
450+}
451+
452+#define sz_damon_region(r) (r->ar.end - r->ar.start)
453+
454+/*
455+ * Merge two adjacent regions into one region
456+ */
457+static void damon_merge_two_regions(struct damon_target *t,
458+ struct damon_region *l, struct damon_region *r)
459+{
460+ unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r);
461+
462+ l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) /
463+ (sz_l + sz_r);
464+ l->ar.end = r->ar.end;
465+ damon_destroy_region(r, t);
466+}
467+
468+#define diff_of(a, b) (a > b ? a - b : b - a)
469+
470+/*
471+ * Merge adjacent regions having similar access frequencies
472+ *
473+ * t target affected by this merge operation
474+ * thres '->nr_accesses' diff threshold for the merge
475+ * sz_limit size upper limit of each region
476+ */
477+static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
478+ unsigned long sz_limit)
479+{
480+ struct damon_region *r, *prev = NULL, *next;
481+
482+ damon_for_each_region_safe(r, next, t) {
483+ if (prev && prev->ar.end == r->ar.start &&
484+ diff_of(prev->nr_accesses, r->nr_accesses) <= thres &&
485+ sz_damon_region(prev) + sz_damon_region(r) <= sz_limit)
486+ damon_merge_two_regions(t, prev, r);
487+ else
488+ prev = r;
489+ }
490+}
491+
492+/*
493+ * Merge adjacent regions having similar access frequencies
494+ *
495+ * threshold '->nr_accesses' diff threshold for the merge
496+ * sz_limit size upper limit of each region
497+ *
498+ * This function merges monitoring target regions which are adjacent and their
499+ * access frequencies are similar. This is for minimizing the monitoring
500+ * overhead under the dynamically changeable access pattern. If a merge was
501+ * unnecessarily made, later 'kdamond_split_regions()' will revert it.
502+ */
503+static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
504+ unsigned long sz_limit)
505+{
506+ struct damon_target *t;
507+
508+ damon_for_each_target(t, c)
509+ damon_merge_regions_of(t, threshold, sz_limit);
510+}
511+
512+/*
513+ * Split a region in two
514+ *
515+ * r the region to be split
516+ * sz_r size of the first sub-region that will be made
517+ */
518+static void damon_split_region_at(struct damon_ctx *ctx,
519+ struct damon_target *t, struct damon_region *r,
520+ unsigned long sz_r)
521+{
522+ struct damon_region *new;
523+
524+ new = damon_new_region(r->ar.start + sz_r, r->ar.end);
525+ if (!new)
526+ return;
527+
528+ r->ar.end = new->ar.start;
529+
530+ damon_insert_region(new, r, damon_next_region(r), t);
531+}
532+
533+/* Split every region in the given target into 'nr_subs' regions */
534+static void damon_split_regions_of(struct damon_ctx *ctx,
535+ struct damon_target *t, int nr_subs)
536+{
537+ struct damon_region *r, *next;
538+ unsigned long sz_region, sz_sub = 0;
539+ int i;
540+
541+ damon_for_each_region_safe(r, next, t) {
542+ sz_region = r->ar.end - r->ar.start;
543+
544+ for (i = 0; i < nr_subs - 1 &&
545+ sz_region > 2 * DAMON_MIN_REGION; i++) {
546+ /*
547+ * Randomly select size of left sub-region to be at
548+ * least 10 percent and at most 90% of original region
549+ */
550+ sz_sub = ALIGN_DOWN(damon_rand(1, 10) *
551+ sz_region / 10, DAMON_MIN_REGION);
552+ /* Do not allow blank region */
553+ if (sz_sub == 0 || sz_sub >= sz_region)
554+ continue;
555+
556+ damon_split_region_at(ctx, t, r, sz_sub);
557+ sz_region = sz_sub;
558+ }
559+ }
560+}
561+
562+/*
563+ * Split every target region into randomly-sized small regions
564+ *
565+ * This function splits every target region into random-sized small regions if
566+ * current total number of the regions is equal or smaller than half of the
567+ * user-specified maximum number of regions. This is for maximizing the
568+ * monitoring accuracy under the dynamically changeable access patterns. If a
569+ * split was unnecessarily made, later 'kdamond_merge_regions()' will revert
570+ * it.
571+ */
572+static void kdamond_split_regions(struct damon_ctx *ctx)
573+{
574+ struct damon_target *t;
575+ unsigned int nr_regions = 0;
576+ static unsigned int last_nr_regions;
577+ int nr_subregions = 2;
578+
579+ damon_for_each_target(t, ctx)
580+ nr_regions += damon_nr_regions(t);
581+
582+ if (nr_regions > ctx->max_nr_regions / 2)
583+ return;
584+
585+ /* Maybe the middle of the region has different access frequency */
586+ if (last_nr_regions == nr_regions &&
587+ nr_regions < ctx->max_nr_regions / 3)
588+ nr_subregions = 3;
589+
590+ damon_for_each_target(t, ctx)
591+ damon_split_regions_of(ctx, t, nr_subregions);
592+
593+ last_nr_regions = nr_regions;
594+}
595+
596+/*
597+ * Check whether it is time to check and apply the target monitoring regions
598+ *
599+ * Returns true if it is.
600+ */
601+static bool kdamond_need_update_primitive(struct damon_ctx *ctx)
602+{
603+ return damon_check_reset_time_interval(&ctx->last_primitive_update,
604+ ctx->primitive_update_interval);
605+}
606+
607+/*
608+ * Check whether current monitoring should be stopped
609+ *
610+ * The monitoring is stopped when either the user requested to stop, or all
611+ * monitoring targets are invalid.
612+ *
613+ * Returns true if need to stop current monitoring.
614+ */
615+static bool kdamond_need_stop(struct damon_ctx *ctx)
616+{
617+ struct damon_target *t;
618+ bool stop;
619+
620+ mutex_lock(&ctx->kdamond_lock);
621+ stop = ctx->kdamond_stop;
622+ mutex_unlock(&ctx->kdamond_lock);
623+ if (stop)
624+ return true;
625+
626+ if (!ctx->primitive.target_valid)
627+ return false;
628+
629+ damon_for_each_target(t, ctx) {
630+ if (ctx->primitive.target_valid(t))
631+ return false;
632+ }
633+
634+ return true;
635+}
636+
637+static void set_kdamond_stop(struct damon_ctx *ctx)
638+{
639+ mutex_lock(&ctx->kdamond_lock);
640+ ctx->kdamond_stop = true;
641+ mutex_unlock(&ctx->kdamond_lock);
642+}
643+
644+/*
645+ * The monitoring daemon that runs as a kernel thread
646+ */
647+static int kdamond_fn(void *data)
648+{
649+ struct damon_ctx *ctx = (struct damon_ctx *)data;
650+ struct damon_target *t;
651+ struct damon_region *r, *next;
652+ unsigned int max_nr_accesses = 0;
653+ unsigned long sz_limit = 0;
654+
655+ mutex_lock(&ctx->kdamond_lock);
656+ pr_info("kdamond (%d) starts\n", ctx->kdamond->pid);
657+ mutex_unlock(&ctx->kdamond_lock);
658+
659+ if (ctx->primitive.init)
660+ ctx->primitive.init(ctx);
661+ if (ctx->callback.before_start && ctx->callback.before_start(ctx))
662+ set_kdamond_stop(ctx);
663+
664+ sz_limit = damon_region_sz_limit(ctx);
665+
666+ while (!kdamond_need_stop(ctx)) {
667+ if (ctx->primitive.prepare_access_checks)
668+ ctx->primitive.prepare_access_checks(ctx);
669+ if (ctx->callback.after_sampling &&
670+ ctx->callback.after_sampling(ctx))
671+ set_kdamond_stop(ctx);
672+
673+ usleep_range(ctx->sample_interval, ctx->sample_interval + 1);
674+
675+ if (ctx->primitive.check_accesses)
676+ max_nr_accesses = ctx->primitive.check_accesses(ctx);
677+
678+ if (kdamond_aggregate_interval_passed(ctx)) {
679+ kdamond_merge_regions(ctx,
680+ max_nr_accesses / 10,
681+ sz_limit);
682+ if (ctx->callback.after_aggregation &&
683+ ctx->callback.after_aggregation(ctx))
684+ set_kdamond_stop(ctx);
685+ kdamond_reset_aggregated(ctx);
686+ kdamond_split_regions(ctx);
687+ if (ctx->primitive.reset_aggregated)
688+ ctx->primitive.reset_aggregated(ctx);
689+ }
690+
691+ if (kdamond_need_update_primitive(ctx)) {
692+ if (ctx->primitive.update)
693+ ctx->primitive.update(ctx);
694+ sz_limit = damon_region_sz_limit(ctx);
695+ }
696+ }
697+ damon_for_each_target(t, ctx) {
698+ damon_for_each_region_safe(r, next, t)
699+ damon_destroy_region(r, t);
700+ }
701+
702+ if (ctx->callback.before_terminate &&
703+ ctx->callback.before_terminate(ctx))
704+ set_kdamond_stop(ctx);
705+ if (ctx->primitive.cleanup)
706+ ctx->primitive.cleanup(ctx);
707+
708+ pr_debug("kdamond (%d) finishes\n", ctx->kdamond->pid);
709+ mutex_lock(&ctx->kdamond_lock);
710+ ctx->kdamond = NULL;
711+ mutex_unlock(&ctx->kdamond_lock);
712+
713+ mutex_lock(&damon_lock);
714+ nr_running_ctxs--;
715+ mutex_unlock(&damon_lock);
716+
717+ do_exit(0);
718+}
719+
720+#include "core-test.h"
mm/damon/dbgfs-test.h+126−0
@@ -0,0 +1,126 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * DAMON Debugfs Interface Unit Tests
4+ *
5+ * Author: SeongJae Park <sjpark@amazon.de>
6+ */
7+
8+#ifdef CONFIG_DAMON_DBGFS_KUNIT_TEST
9+
10+#ifndef _DAMON_DBGFS_TEST_H
11+#define _DAMON_DBGFS_TEST_H
12+
13+#include <kunit/test.h>
14+
15+static void damon_dbgfs_test_str_to_target_ids(struct kunit *test)
16+{
17+ char *question;
18+ unsigned long *answers;
19+ unsigned long expected[] = {12, 35, 46};
20+ ssize_t nr_integers = 0, i;
21+
22+ question = "123";
23+ answers = str_to_target_ids(question, strnlen(question, 128),
24+ &nr_integers);
25+ KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers);
26+ KUNIT_EXPECT_EQ(test, 123ul, answers[0]);
27+ kfree(answers);
28+
29+ question = "123abc";
30+ answers = str_to_target_ids(question, strnlen(question, 128),
31+ &nr_integers);
32+ KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers);
33+ KUNIT_EXPECT_EQ(test, 123ul, answers[0]);
34+ kfree(answers);
35+
36+ question = "a123";
37+ answers = str_to_target_ids(question, strnlen(question, 128),
38+ &nr_integers);
39+ KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers);
40+ kfree(answers);
41+
42+ question = "12 35";
43+ answers = str_to_target_ids(question, strnlen(question, 128),
44+ &nr_integers);
45+ KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers);
46+ for (i = 0; i < nr_integers; i++)
47+ KUNIT_EXPECT_EQ(test, expected[i], answers[i]);
48+ kfree(answers);
49+
50+ question = "12 35 46";
51+ answers = str_to_target_ids(question, strnlen(question, 128),
52+ &nr_integers);
53+ KUNIT_EXPECT_EQ(test, (ssize_t)3, nr_integers);
54+ for (i = 0; i < nr_integers; i++)
55+ KUNIT_EXPECT_EQ(test, expected[i], answers[i]);
56+ kfree(answers);
57+
58+ question = "12 35 abc 46";
59+ answers = str_to_target_ids(question, strnlen(question, 128),
60+ &nr_integers);
61+ KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers);
62+ for (i = 0; i < 2; i++)
63+ KUNIT_EXPECT_EQ(test, expected[i], answers[i]);
64+ kfree(answers);
65+
66+ question = "";
67+ answers = str_to_target_ids(question, strnlen(question, 128),
68+ &nr_integers);
69+ KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers);
70+ kfree(answers);
71+
72+ question = "\n";
73+ answers = str_to_target_ids(question, strnlen(question, 128),
74+ &nr_integers);
75+ KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers);
76+ kfree(answers);
77+}
78+
79+static void damon_dbgfs_test_set_targets(struct kunit *test)
80+{
81+ struct damon_ctx *ctx = dbgfs_new_ctx();
82+ unsigned long ids[] = {1, 2, 3};
83+ char buf[64];
84+
85+ /* Make DAMON consider target id as plain number */
86+ ctx->primitive.target_valid = NULL;
87+ ctx->primitive.cleanup = NULL;
88+
89+ damon_set_targets(ctx, ids, 3);
90+ sprint_target_ids(ctx, buf, 64);
91+ KUNIT_EXPECT_STREQ(test, (char *)buf, "1 2 3\n");
92+
93+ damon_set_targets(ctx, NULL, 0);
94+ sprint_target_ids(ctx, buf, 64);
95+ KUNIT_EXPECT_STREQ(test, (char *)buf, "\n");
96+
97+ damon_set_targets(ctx, (unsigned long []){1, 2}, 2);
98+ sprint_target_ids(ctx, buf, 64);
99+ KUNIT_EXPECT_STREQ(test, (char *)buf, "1 2\n");
100+
101+ damon_set_targets(ctx, (unsigned long []){2}, 1);
102+ sprint_target_ids(ctx, buf, 64);
103+ KUNIT_EXPECT_STREQ(test, (char *)buf, "2\n");
104+
105+ damon_set_targets(ctx, NULL, 0);
106+ sprint_target_ids(ctx, buf, 64);
107+ KUNIT_EXPECT_STREQ(test, (char *)buf, "\n");
108+
109+ dbgfs_destroy_ctx(ctx);
110+}
111+
112+static struct kunit_case damon_test_cases[] = {
113+ KUNIT_CASE(damon_dbgfs_test_str_to_target_ids),
114+ KUNIT_CASE(damon_dbgfs_test_set_targets),
115+ {},
116+};
117+
118+static struct kunit_suite damon_test_suite = {
119+ .name = "damon-dbgfs",
120+ .test_cases = damon_test_cases,
121+};
122+kunit_test_suite(damon_test_suite);
123+
124+#endif /* _DAMON_TEST_H */
125+
126+#endif /* CONFIG_DAMON_KUNIT_TEST */
mm/damon/dbgfs.c+623−0
@@ -0,0 +1,623 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * DAMON Debugfs Interface
4+ *
5+ * Author: SeongJae Park <sjpark@amazon.de>
6+ */
7+
8+#define pr_fmt(fmt) "damon-dbgfs: " fmt
9+
10+#include <linux/damon.h>
11+#include <linux/debugfs.h>
12+#include <linux/file.h>
13+#include <linux/mm.h>
14+#include <linux/module.h>
15+#include <linux/page_idle.h>
16+#include <linux/slab.h>
17+
18+static struct damon_ctx **dbgfs_ctxs;
19+static int dbgfs_nr_ctxs;
20+static struct dentry **dbgfs_dirs;
21+static DEFINE_MUTEX(damon_dbgfs_lock);
22+
23+/*
24+ * Returns non-empty string on success, negative error code otherwise.
25+ */
26+static char *user_input_str(const char __user *buf, size_t count, loff_t *ppos)
27+{
28+ char *kbuf;
29+ ssize_t ret;
30+
31+ /* We do not accept continuous write */
32+ if (*ppos)
33+ return ERR_PTR(-EINVAL);
34+
35+ kbuf = kmalloc(count + 1, GFP_KERNEL);
36+ if (!kbuf)
37+ return ERR_PTR(-ENOMEM);
38+
39+ ret = simple_write_to_buffer(kbuf, count + 1, ppos, buf, count);
40+ if (ret != count) {
41+ kfree(kbuf);
42+ return ERR_PTR(-EIO);
43+ }
44+ kbuf[ret] = '\0';
45+
46+ return kbuf;
47+}
48+
49+static ssize_t dbgfs_attrs_read(struct file *file,
50+ char __user *buf, size_t count, loff_t *ppos)
51+{
52+ struct damon_ctx *ctx = file->private_data;
53+ char kbuf[128];
54+ int ret;
55+
56+ mutex_lock(&ctx->kdamond_lock);
57+ ret = scnprintf(kbuf, ARRAY_SIZE(kbuf), "%lu %lu %lu %lu %lu\n",
58+ ctx->sample_interval, ctx->aggr_interval,
59+ ctx->primitive_update_interval, ctx->min_nr_regions,
60+ ctx->max_nr_regions);
61+ mutex_unlock(&ctx->kdamond_lock);
62+
63+ return simple_read_from_buffer(buf, count, ppos, kbuf, ret);
64+}
65+
66+static ssize_t dbgfs_attrs_write(struct file *file,
67+ const char __user *buf, size_t count, loff_t *ppos)
68+{
69+ struct damon_ctx *ctx = file->private_data;
70+ unsigned long s, a, r, minr, maxr;
71+ char *kbuf;
72+ ssize_t ret = count;
73+ int err;
74+
75+ kbuf = user_input_str(buf, count, ppos);
76+ if (IS_ERR(kbuf))
77+ return PTR_ERR(kbuf);
78+
79+ if (sscanf(kbuf, "%lu %lu %lu %lu %lu",
80+ &s, &a, &r, &minr, &maxr) != 5) {
81+ ret = -EINVAL;
82+ goto out;
83+ }
84+
85+ mutex_lock(&ctx->kdamond_lock);
86+ if (ctx->kdamond) {
87+ ret = -EBUSY;
88+ goto unlock_out;
89+ }
90+
91+ err = damon_set_attrs(ctx, s, a, r, minr, maxr);
92+ if (err)
93+ ret = err;
94+unlock_out:
95+ mutex_unlock(&ctx->kdamond_lock);
96+out:
97+ kfree(kbuf);
98+ return ret;
99+}
100+
101+static inline bool targetid_is_pid(const struct damon_ctx *ctx)
102+{
103+ return ctx->primitive.target_valid == damon_va_target_valid;
104+}
105+
106+static ssize_t sprint_target_ids(struct damon_ctx *ctx, char *buf, ssize_t len)
107+{
108+ struct damon_target *t;
109+ unsigned long id;
110+ int written = 0;
111+ int rc;
112+
113+ damon_for_each_target(t, ctx) {
114+ id = t->id;
115+ if (targetid_is_pid(ctx))
116+ /* Show pid numbers to debugfs users */
117+ id = (unsigned long)pid_vnr((struct pid *)id);
118+
119+ rc = scnprintf(&buf[written], len - written, "%lu ", id);
120+ if (!rc)
121+ return -ENOMEM;
122+ written += rc;
123+ }
124+ if (written)
125+ written -= 1;
126+ written += scnprintf(&buf[written], len - written, "\n");
127+ return written;
128+}
129+
130+static ssize_t dbgfs_target_ids_read(struct file *file,
131+ char __user *buf, size_t count, loff_t *ppos)
132+{
133+ struct damon_ctx *ctx = file->private_data;
134+ ssize_t len;
135+ char ids_buf[320];
136+
137+ mutex_lock(&ctx->kdamond_lock);
138+ len = sprint_target_ids(ctx, ids_buf, 320);
139+ mutex_unlock(&ctx->kdamond_lock);
140+ if (len < 0)
141+ return len;
142+
143+ return simple_read_from_buffer(buf, count, ppos, ids_buf, len);
144+}
145+
146+/*
147+ * Converts a string into an array of unsigned long integers
148+ *
149+ * Returns an array of unsigned long integers if the conversion success, or
150+ * NULL otherwise.
151+ */
152+static unsigned long *str_to_target_ids(const char *str, ssize_t len,
153+ ssize_t *nr_ids)
154+{
155+ unsigned long *ids;
156+ const int max_nr_ids = 32;
157+ unsigned long id;
158+ int pos = 0, parsed, ret;
159+
160+ *nr_ids = 0;
161+ ids = kmalloc_array(max_nr_ids, sizeof(id), GFP_KERNEL);
162+ if (!ids)
163+ return NULL;
164+ while (*nr_ids < max_nr_ids && pos < len) {
165+ ret = sscanf(&str[pos], "%lu%n", &id, &parsed);
166+ pos += parsed;
167+ if (ret != 1)
168+ break;
169+ ids[*nr_ids] = id;
170+ *nr_ids += 1;
171+ }
172+
173+ return ids;
174+}
175+
176+static void dbgfs_put_pids(unsigned long *ids, int nr_ids)
177+{
178+ int i;
179+
180+ for (i = 0; i < nr_ids; i++)
181+ put_pid((struct pid *)ids[i]);
182+}
183+
184+static ssize_t dbgfs_target_ids_write(struct file *file,
185+ const char __user *buf, size_t count, loff_t *ppos)
186+{
187+ struct damon_ctx *ctx = file->private_data;
188+ char *kbuf, *nrs;
189+ unsigned long *targets;
190+ ssize_t nr_targets;
191+ ssize_t ret = count;
192+ int i;
193+ int err;
194+
195+ kbuf = user_input_str(buf, count, ppos);
196+ if (IS_ERR(kbuf))
197+ return PTR_ERR(kbuf);
198+
199+ nrs = kbuf;
200+
201+ targets = str_to_target_ids(nrs, ret, &nr_targets);
202+ if (!targets) {
203+ ret = -ENOMEM;
204+ goto out;
205+ }
206+
207+ if (targetid_is_pid(ctx)) {
208+ for (i = 0; i < nr_targets; i++) {
209+ targets[i] = (unsigned long)find_get_pid(
210+ (int)targets[i]);
211+ if (!targets[i]) {
212+ dbgfs_put_pids(targets, i);
213+ ret = -EINVAL;
214+ goto free_targets_out;
215+ }
216+ }
217+ }
218+
219+ mutex_lock(&ctx->kdamond_lock);
220+ if (ctx->kdamond) {
221+ if (targetid_is_pid(ctx))
222+ dbgfs_put_pids(targets, nr_targets);
223+ ret = -EBUSY;
224+ goto unlock_out;
225+ }
226+
227+ err = damon_set_targets(ctx, targets, nr_targets);
228+ if (err) {
229+ if (targetid_is_pid(ctx))
230+ dbgfs_put_pids(targets, nr_targets);
231+ ret = err;
232+ }
233+
234+unlock_out:
235+ mutex_unlock(&ctx->kdamond_lock);
236+free_targets_out:
237+ kfree(targets);
238+out:
239+ kfree(kbuf);
240+ return ret;
241+}
242+
243+static ssize_t dbgfs_kdamond_pid_read(struct file *file,
244+ char __user *buf, size_t count, loff_t *ppos)
245+{
246+ struct damon_ctx *ctx = file->private_data;
247+ char *kbuf;
248+ ssize_t len;
249+
250+ kbuf = kmalloc(count, GFP_KERNEL);
251+ if (!kbuf)
252+ return -ENOMEM;
253+
254+ mutex_lock(&ctx->kdamond_lock);
255+ if (ctx->kdamond)
256+ len = scnprintf(kbuf, count, "%d\n", ctx->kdamond->pid);
257+ else
258+ len = scnprintf(kbuf, count, "none\n");
259+ mutex_unlock(&ctx->kdamond_lock);
260+ if (!len)
261+ goto out;
262+ len = simple_read_from_buffer(buf, count, ppos, kbuf, len);
263+
264+out:
265+ kfree(kbuf);
266+ return len;
267+}
268+
269+static int damon_dbgfs_open(struct inode *inode, struct file *file)
270+{
271+ file->private_data = inode->i_private;
272+
273+ return nonseekable_open(inode, file);
274+}
275+
276+static const struct file_operations attrs_fops = {
277+ .open = damon_dbgfs_open,
278+ .read = dbgfs_attrs_read,
279+ .write = dbgfs_attrs_write,
280+};
281+
282+static const struct file_operations target_ids_fops = {
283+ .open = damon_dbgfs_open,
284+ .read = dbgfs_target_ids_read,
285+ .write = dbgfs_target_ids_write,
286+};
287+
288+static const struct file_operations kdamond_pid_fops = {
289+ .open = damon_dbgfs_open,
290+ .read = dbgfs_kdamond_pid_read,
291+};
292+
293+static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx)
294+{
295+ const char * const file_names[] = {"attrs", "target_ids",
296+ "kdamond_pid"};
297+ const struct file_operations *fops[] = {&attrs_fops, &target_ids_fops,
298+ &kdamond_pid_fops};
299+ int i;
300+
301+ for (i = 0; i < ARRAY_SIZE(file_names); i++)
302+ debugfs_create_file(file_names[i], 0600, dir, ctx, fops[i]);
303+}
304+
305+static int dbgfs_before_terminate(struct damon_ctx *ctx)
306+{
307+ struct damon_target *t, *next;
308+
309+ if (!targetid_is_pid(ctx))
310+ return 0;
311+
312+ damon_for_each_target_safe(t, next, ctx) {
313+ put_pid((struct pid *)t->id);
314+ damon_destroy_target(t);
315+ }
316+ return 0;
317+}
318+
319+static struct damon_ctx *dbgfs_new_ctx(void)
320+{
321+ struct damon_ctx *ctx;
322+
323+ ctx = damon_new_ctx();
324+ if (!ctx)
325+ return NULL;
326+
327+ damon_va_set_primitives(ctx);
328+ ctx->callback.before_terminate = dbgfs_before_terminate;
329+ return ctx;
330+}
331+
332+static void dbgfs_destroy_ctx(struct damon_ctx *ctx)
333+{
334+ damon_destroy_ctx(ctx);
335+}
336+
337+/*
338+ * Make a context of @name and create a debugfs directory for it.
339+ *
340+ * This function should be called while holding damon_dbgfs_lock.
341+ *
342+ * Returns 0 on success, negative error code otherwise.
343+ */
344+static int dbgfs_mk_context(char *name)
345+{
346+ struct dentry *root, **new_dirs, *new_dir;
347+ struct damon_ctx **new_ctxs, *new_ctx;
348+
349+ if (damon_nr_running_ctxs())
350+ return -EBUSY;
351+
352+ new_ctxs = krealloc(dbgfs_ctxs, sizeof(*dbgfs_ctxs) *
353+ (dbgfs_nr_ctxs + 1), GFP_KERNEL);
354+ if (!new_ctxs)
355+ return -ENOMEM;
356+ dbgfs_ctxs = new_ctxs;
357+
358+ new_dirs = krealloc(dbgfs_dirs, sizeof(*dbgfs_dirs) *
359+ (dbgfs_nr_ctxs + 1), GFP_KERNEL);
360+ if (!new_dirs)
361+ return -ENOMEM;
362+ dbgfs_dirs = new_dirs;
363+
364+ root = dbgfs_dirs[0];
365+ if (!root)
366+ return -ENOENT;
367+
368+ new_dir = debugfs_create_dir(name, root);
369+ dbgfs_dirs[dbgfs_nr_ctxs] = new_dir;
370+
371+ new_ctx = dbgfs_new_ctx();
372+ if (!new_ctx) {
373+ debugfs_remove(new_dir);
374+ dbgfs_dirs[dbgfs_nr_ctxs] = NULL;
375+ return -ENOMEM;
376+ }
377+
378+ dbgfs_ctxs[dbgfs_nr_ctxs] = new_ctx;
379+ dbgfs_fill_ctx_dir(dbgfs_dirs[dbgfs_nr_ctxs],
380+ dbgfs_ctxs[dbgfs_nr_ctxs]);
381+ dbgfs_nr_ctxs++;
382+
383+ return 0;
384+}
385+
386+static ssize_t dbgfs_mk_context_write(struct file *file,
387+ const char __user *buf, size_t count, loff_t *ppos)
388+{
389+ char *kbuf;
390+ char *ctx_name;
391+ ssize_t ret = count;
392+ int err;
393+
394+ kbuf = user_input_str(buf, count, ppos);
395+ if (IS_ERR(kbuf))
396+ return PTR_ERR(kbuf);
397+ ctx_name = kmalloc(count + 1, GFP_KERNEL);
398+ if (!ctx_name) {
399+ kfree(kbuf);
400+ return -ENOMEM;
401+ }
402+
403+ /* Trim white space */
404+ if (sscanf(kbuf, "%s", ctx_name) != 1) {
405+ ret = -EINVAL;
406+ goto out;
407+ }
408+
409+ mutex_lock(&damon_dbgfs_lock);
410+ err = dbgfs_mk_context(ctx_name);
411+ if (err)
412+ ret = err;
413+ mutex_unlock(&damon_dbgfs_lock);
414+
415+out:
416+ kfree(kbuf);
417+ kfree(ctx_name);
418+ return ret;
419+}
420+
421+/*
422+ * Remove a context of @name and its debugfs directory.
423+ *
424+ * This function should be called while holding damon_dbgfs_lock.
425+ *
426+ * Return 0 on success, negative error code otherwise.
427+ */
428+static int dbgfs_rm_context(char *name)
429+{
430+ struct dentry *root, *dir, **new_dirs;
431+ struct damon_ctx **new_ctxs;
432+ int i, j;
433+
434+ if (damon_nr_running_ctxs())
435+ return -EBUSY;
436+
437+ root = dbgfs_dirs[0];
438+ if (!root)
439+ return -ENOENT;
440+
441+ dir = debugfs_lookup(name, root);
442+ if (!dir)
443+ return -ENOENT;
444+
445+ new_dirs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_dirs),
446+ GFP_KERNEL);
447+ if (!new_dirs)
448+ return -ENOMEM;
449+
450+ new_ctxs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_ctxs),
451+ GFP_KERNEL);
452+ if (!new_ctxs) {
453+ kfree(new_dirs);
454+ return -ENOMEM;
455+ }
456+
457+ for (i = 0, j = 0; i < dbgfs_nr_ctxs; i++) {
458+ if (dbgfs_dirs[i] == dir) {
459+ debugfs_remove(dbgfs_dirs[i]);
460+ dbgfs_destroy_ctx(dbgfs_ctxs[i]);
461+ continue;
462+ }
463+ new_dirs[j] = dbgfs_dirs[i];
464+ new_ctxs[j++] = dbgfs_ctxs[i];
465+ }
466+
467+ kfree(dbgfs_dirs);
468+ kfree(dbgfs_ctxs);
469+
470+ dbgfs_dirs = new_dirs;
471+ dbgfs_ctxs = new_ctxs;
472+ dbgfs_nr_ctxs--;
473+
474+ return 0;
475+}
476+
477+static ssize_t dbgfs_rm_context_write(struct file *file,
478+ const char __user *buf, size_t count, loff_t *ppos)
479+{
480+ char *kbuf;
481+ ssize_t ret = count;
482+ int err;
483+ char *ctx_name;
484+
485+ kbuf = user_input_str(buf, count, ppos);
486+ if (IS_ERR(kbuf))
487+ return PTR_ERR(kbuf);
488+ ctx_name = kmalloc(count + 1, GFP_KERNEL);
489+ if (!ctx_name) {
490+ kfree(kbuf);
491+ return -ENOMEM;
492+ }
493+
494+ /* Trim white space */
495+ if (sscanf(kbuf, "%s", ctx_name) != 1) {
496+ ret = -EINVAL;
497+ goto out;
498+ }
499+
500+ mutex_lock(&damon_dbgfs_lock);
501+ err = dbgfs_rm_context(ctx_name);
502+ if (err)
503+ ret = err;
504+ mutex_unlock(&damon_dbgfs_lock);
505+
506+out:
507+ kfree(kbuf);
508+ kfree(ctx_name);
509+ return ret;
510+}
511+
512+static ssize_t dbgfs_monitor_on_read(struct file *file,
513+ char __user *buf, size_t count, loff_t *ppos)
514+{
515+ char monitor_on_buf[5];
516+ bool monitor_on = damon_nr_running_ctxs() != 0;
517+ int len;
518+
519+ len = scnprintf(monitor_on_buf, 5, monitor_on ? "on\n" : "off\n");
520+
521+ return simple_read_from_buffer(buf, count, ppos, monitor_on_buf, len);
522+}
523+
524+static ssize_t dbgfs_monitor_on_write(struct file *file,
525+ const char __user *buf, size_t count, loff_t *ppos)
526+{
527+ ssize_t ret = count;
528+ char *kbuf;
529+ int err;
530+
531+ kbuf = user_input_str(buf, count, ppos);
532+ if (IS_ERR(kbuf))
533+ return PTR_ERR(kbuf);
534+
535+ /* Remove white space */
536+ if (sscanf(kbuf, "%s", kbuf) != 1) {
537+ kfree(kbuf);
538+ return -EINVAL;
539+ }
540+
541+ if (!strncmp(kbuf, "on", count))
542+ err = damon_start(dbgfs_ctxs, dbgfs_nr_ctxs);
543+ else if (!strncmp(kbuf, "off", count))
544+ err = damon_stop(dbgfs_ctxs, dbgfs_nr_ctxs);
545+ else
546+ err = -EINVAL;
547+
548+ if (err)
549+ ret = err;
550+ kfree(kbuf);
551+ return ret;
552+}
553+
554+static const struct file_operations mk_contexts_fops = {
555+ .write = dbgfs_mk_context_write,
556+};
557+
558+static const struct file_operations rm_contexts_fops = {
559+ .write = dbgfs_rm_context_write,
560+};
561+
562+static const struct file_operations monitor_on_fops = {
563+ .read = dbgfs_monitor_on_read,
564+ .write = dbgfs_monitor_on_write,
565+};
566+
567+static int __init __damon_dbgfs_init(void)
568+{
569+ struct dentry *dbgfs_root;
570+ const char * const file_names[] = {"mk_contexts", "rm_contexts",
571+ "monitor_on"};
572+ const struct file_operations *fops[] = {&mk_contexts_fops,
573+ &rm_contexts_fops, &monitor_on_fops};
574+ int i;
575+
576+ dbgfs_root = debugfs_create_dir("damon", NULL);
577+
578+ for (i = 0; i < ARRAY_SIZE(file_names); i++)
579+ debugfs_create_file(file_names[i], 0600, dbgfs_root, NULL,
580+ fops[i]);
581+ dbgfs_fill_ctx_dir(dbgfs_root, dbgfs_ctxs[0]);
582+
583+ dbgfs_dirs = kmalloc_array(1, sizeof(dbgfs_root), GFP_KERNEL);
584+ if (!dbgfs_dirs) {
585+ debugfs_remove(dbgfs_root);
586+ return -ENOMEM;
587+ }
588+ dbgfs_dirs[0] = dbgfs_root;
589+
590+ return 0;
591+}
592+
593+/*
594+ * Functions for the initialization
595+ */
596+
597+static int __init damon_dbgfs_init(void)
598+{
599+ int rc;
600+
601+ dbgfs_ctxs = kmalloc(sizeof(*dbgfs_ctxs), GFP_KERNEL);
602+ if (!dbgfs_ctxs)
603+ return -ENOMEM;
604+ dbgfs_ctxs[0] = dbgfs_new_ctx();
605+ if (!dbgfs_ctxs[0]) {
606+ kfree(dbgfs_ctxs);
607+ return -ENOMEM;
608+ }
609+ dbgfs_nr_ctxs = 1;
610+
611+ rc = __damon_dbgfs_init();
612+ if (rc) {
613+ kfree(dbgfs_ctxs[0]);
614+ kfree(dbgfs_ctxs);
615+ pr_err("%s: dbgfs init failed\n", __func__);
616+ }
617+
618+ return rc;
619+}
620+
621+module_init(damon_dbgfs_init);
622+
623+#include "dbgfs-test.h"
mm/damon/vaddr-test.h+329−0
@@ -0,0 +1,329 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Data Access Monitor Unit Tests
4+ *
5+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved.
6+ *
7+ * Author: SeongJae Park <sjpark@amazon.de>
8+ */
9+
10+#ifdef CONFIG_DAMON_VADDR_KUNIT_TEST
11+
12+#ifndef _DAMON_VADDR_TEST_H
13+#define _DAMON_VADDR_TEST_H
14+
15+#include <kunit/test.h>
16+
17+static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas)
18+{
19+ int i, j;
20+ unsigned long largest_gap, gap;
21+
22+ if (!nr_vmas)
23+ return;
24+
25+ for (i = 0; i < nr_vmas - 1; i++) {
26+ vmas[i].vm_next = &vmas[i + 1];
27+
28+ vmas[i].vm_rb.rb_left = NULL;
29+ vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb;
30+
31+ largest_gap = 0;
32+ for (j = i; j < nr_vmas; j++) {
33+ if (j == 0)
34+ continue;
35+ gap = vmas[j].vm_start - vmas[j - 1].vm_end;
36+ if (gap > largest_gap)
37+ largest_gap = gap;
38+ }
39+ vmas[i].rb_subtree_gap = largest_gap;
40+ }
41+ vmas[i].vm_next = NULL;
42+ vmas[i].vm_rb.rb_right = NULL;
43+ vmas[i].rb_subtree_gap = 0;
44+}
45+
46+/*
47+ * Test __damon_va_three_regions() function
48+ *
49+ * In case of virtual memory address spaces monitoring, DAMON converts the
50+ * complex and dynamic memory mappings of each target task to three
51+ * discontiguous regions which cover every mapped areas. However, the three
52+ * regions should not include the two biggest unmapped areas in the original
53+ * mapping, because the two biggest areas are normally the areas between 1)
54+ * heap and the mmap()-ed regions, and 2) the mmap()-ed regions and stack.
55+ * Because these two unmapped areas are very huge but obviously never accessed,
56+ * covering the region is just a waste.
57+ *
58+ * '__damon_va_three_regions() receives an address space of a process. It
59+ * first identifies the start of mappings, end of mappings, and the two biggest
60+ * unmapped areas. After that, based on the information, it constructs the
61+ * three regions and returns. For more detail, refer to the comment of
62+ * 'damon_init_regions_of()' function definition in 'mm/damon.c' file.
63+ *
64+ * For example, suppose virtual address ranges of 10-20, 20-25, 200-210,
65+ * 210-220, 300-305, and 307-330 (Other comments represent this mappings in
66+ * more short form: 10-20-25, 200-210-220, 300-305, 307-330) of a process are
67+ * mapped. To cover every mappings, the three regions should start with 10,
68+ * and end with 305. The process also has three unmapped areas, 25-200,
69+ * 220-300, and 305-307. Among those, 25-200 and 220-300 are the biggest two
70+ * unmapped areas, and thus it should be converted to three regions of 10-25,
71+ * 200-220, and 300-330.
72+ */
73+static void damon_test_three_regions_in_vmas(struct kunit *test)
74+{
75+ struct damon_addr_range regions[3] = {0,};
76+ /* 10-20-25, 200-210-220, 300-305, 307-330 */
77+ struct vm_area_struct vmas[] = {
78+ (struct vm_area_struct) {.vm_start = 10, .vm_end = 20},
79+ (struct vm_area_struct) {.vm_start = 20, .vm_end = 25},
80+ (struct vm_area_struct) {.vm_start = 200, .vm_end = 210},
81+ (struct vm_area_struct) {.vm_start = 210, .vm_end = 220},
82+ (struct vm_area_struct) {.vm_start = 300, .vm_end = 305},
83+ (struct vm_area_struct) {.vm_start = 307, .vm_end = 330},
84+ };
85+
86+ __link_vmas(vmas, 6);
87+
88+ __damon_va_three_regions(&vmas[0], regions);
89+
90+ KUNIT_EXPECT_EQ(test, 10ul, regions[0].start);
91+ KUNIT_EXPECT_EQ(test, 25ul, regions[0].end);
92+ KUNIT_EXPECT_EQ(test, 200ul, regions[1].start);
93+ KUNIT_EXPECT_EQ(test, 220ul, regions[1].end);
94+ KUNIT_EXPECT_EQ(test, 300ul, regions[2].start);
95+ KUNIT_EXPECT_EQ(test, 330ul, regions[2].end);
96+}
97+
98+static struct damon_region *__nth_region_of(struct damon_target *t, int idx)
99+{
100+ struct damon_region *r;
101+ unsigned int i = 0;
102+
103+ damon_for_each_region(r, t) {
104+ if (i++ == idx)
105+ return r;
106+ }
107+
108+ return NULL;
109+}
110+
111+/*
112+ * Test 'damon_va_apply_three_regions()'
113+ *
114+ * test kunit object
115+ * regions an array containing start/end addresses of current
116+ * monitoring target regions
117+ * nr_regions the number of the addresses in 'regions'
118+ * three_regions The three regions that need to be applied now
119+ * expected start/end addresses of monitoring target regions that
120+ * 'three_regions' are applied
121+ * nr_expected the number of addresses in 'expected'
122+ *
123+ * The memory mapping of the target processes changes dynamically. To follow
124+ * the change, DAMON periodically reads the mappings, simplifies it to the
125+ * three regions, and updates the monitoring target regions to fit in the three
126+ * regions. The update of current target regions is the role of
127+ * 'damon_va_apply_three_regions()'.
128+ *
129+ * This test passes the given target regions and the new three regions that
130+ * need to be applied to the function and check whether it updates the regions
131+ * as expected.
132+ */
133+static void damon_do_test_apply_three_regions(struct kunit *test,
134+ unsigned long *regions, int nr_regions,
135+ struct damon_addr_range *three_regions,
136+ unsigned long *expected, int nr_expected)
137+{
138+ struct damon_ctx *ctx = damon_new_ctx();
139+ struct damon_target *t;
140+ struct damon_region *r;
141+ int i;
142+
143+ t = damon_new_target(42);
144+ for (i = 0; i < nr_regions / 2; i++) {
145+ r = damon_new_region(regions[i * 2], regions[i * 2 + 1]);
146+ damon_add_region(r, t);
147+ }
148+ damon_add_target(ctx, t);
149+
150+ damon_va_apply_three_regions(t, three_regions);
151+
152+ for (i = 0; i < nr_expected / 2; i++) {
153+ r = __nth_region_of(t, i);
154+ KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]);
155+ KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]);
156+ }
157+
158+ damon_destroy_ctx(ctx);
159+}
160+
161+/*
162+ * This function test most common case where the three big regions are only
163+ * slightly changed. Target regions should adjust their boundary (10-20-30,
164+ * 50-55, 70-80, 90-100) to fit with the new big regions or remove target
165+ * regions (57-79) that now out of the three regions.
166+ */
167+static void damon_test_apply_three_regions1(struct kunit *test)
168+{
169+ /* 10-20-30, 50-55-57-59, 70-80-90-100 */
170+ unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59,
171+ 70, 80, 80, 90, 90, 100};
172+ /* 5-27, 45-55, 73-104 */
173+ struct damon_addr_range new_three_regions[3] = {
174+ (struct damon_addr_range){.start = 5, .end = 27},
175+ (struct damon_addr_range){.start = 45, .end = 55},
176+ (struct damon_addr_range){.start = 73, .end = 104} };
177+ /* 5-20-27, 45-55, 73-80-90-104 */
178+ unsigned long expected[] = {5, 20, 20, 27, 45, 55,
179+ 73, 80, 80, 90, 90, 104};
180+
181+ damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions),
182+ new_three_regions, expected, ARRAY_SIZE(expected));
183+}
184+
185+/*
186+ * Test slightly bigger change. Similar to above, but the second big region
187+ * now require two target regions (50-55, 57-59) to be removed.
188+ */
189+static void damon_test_apply_three_regions2(struct kunit *test)
190+{
191+ /* 10-20-30, 50-55-57-59, 70-80-90-100 */
192+ unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59,
193+ 70, 80, 80, 90, 90, 100};
194+ /* 5-27, 56-57, 65-104 */
195+ struct damon_addr_range new_three_regions[3] = {
196+ (struct damon_addr_range){.start = 5, .end = 27},
197+ (struct damon_addr_range){.start = 56, .end = 57},
198+ (struct damon_addr_range){.start = 65, .end = 104} };
199+ /* 5-20-27, 56-57, 65-80-90-104 */
200+ unsigned long expected[] = {5, 20, 20, 27, 56, 57,
201+ 65, 80, 80, 90, 90, 104};
202+
203+ damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions),
204+ new_three_regions, expected, ARRAY_SIZE(expected));
205+}
206+
207+/*
208+ * Test a big change. The second big region has totally freed and mapped to
209+ * different area (50-59 -> 61-63). The target regions which were in the old
210+ * second big region (50-55-57-59) should be removed and new target region
211+ * covering the second big region (61-63) should be created.
212+ */
213+static void damon_test_apply_three_regions3(struct kunit *test)
214+{
215+ /* 10-20-30, 50-55-57-59, 70-80-90-100 */
216+ unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59,
217+ 70, 80, 80, 90, 90, 100};
218+ /* 5-27, 61-63, 65-104 */
219+ struct damon_addr_range new_three_regions[3] = {
220+ (struct damon_addr_range){.start = 5, .end = 27},
221+ (struct damon_addr_range){.start = 61, .end = 63},
222+ (struct damon_addr_range){.start = 65, .end = 104} };
223+ /* 5-20-27, 61-63, 65-80-90-104 */
224+ unsigned long expected[] = {5, 20, 20, 27, 61, 63,
225+ 65, 80, 80, 90, 90, 104};
226+
227+ damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions),
228+ new_three_regions, expected, ARRAY_SIZE(expected));
229+}
230+
231+/*
232+ * Test another big change. Both of the second and third big regions (50-59
233+ * and 70-100) has totally freed and mapped to different area (30-32 and
234+ * 65-68). The target regions which were in the old second and third big
235+ * regions should now be removed and new target regions covering the new second
236+ * and third big regions should be crated.
237+ */
238+static void damon_test_apply_three_regions4(struct kunit *test)
239+{
240+ /* 10-20-30, 50-55-57-59, 70-80-90-100 */
241+ unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59,
242+ 70, 80, 80, 90, 90, 100};
243+ /* 5-7, 30-32, 65-68 */
244+ struct damon_addr_range new_three_regions[3] = {
245+ (struct damon_addr_range){.start = 5, .end = 7},
246+ (struct damon_addr_range){.start = 30, .end = 32},
247+ (struct damon_addr_range){.start = 65, .end = 68} };
248+ /* expect 5-7, 30-32, 65-68 */
249+ unsigned long expected[] = {5, 7, 30, 32, 65, 68};
250+
251+ damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions),
252+ new_three_regions, expected, ARRAY_SIZE(expected));
253+}
254+
255+static void damon_test_split_evenly(struct kunit *test)
256+{
257+ struct damon_ctx *c = damon_new_ctx();
258+ struct damon_target *t;
259+ struct damon_region *r;
260+ unsigned long i;
261+
262+ KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(NULL, NULL, 5),
263+ -EINVAL);
264+
265+ t = damon_new_target(42);
266+ r = damon_new_region(0, 100);
267+ KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 0), -EINVAL);
268+
269+ damon_add_region(r, t);
270+ KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 10), 0);
271+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 10u);
272+
273+ i = 0;
274+ damon_for_each_region(r, t) {
275+ KUNIT_EXPECT_EQ(test, r->ar.start, i++ * 10);
276+ KUNIT_EXPECT_EQ(test, r->ar.end, i * 10);
277+ }
278+ damon_free_target(t);
279+
280+ t = damon_new_target(42);
281+ r = damon_new_region(5, 59);
282+ damon_add_region(r, t);
283+ KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 5), 0);
284+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u);
285+
286+ i = 0;
287+ damon_for_each_region(r, t) {
288+ if (i == 4)
289+ break;
290+ KUNIT_EXPECT_EQ(test, r->ar.start, 5 + 10 * i++);
291+ KUNIT_EXPECT_EQ(test, r->ar.end, 5 + 10 * i);
292+ }
293+ KUNIT_EXPECT_EQ(test, r->ar.start, 5 + 10 * i);
294+ KUNIT_EXPECT_EQ(test, r->ar.end, 59ul);
295+ damon_free_target(t);
296+
297+ t = damon_new_target(42);
298+ r = damon_new_region(5, 6);
299+ damon_add_region(r, t);
300+ KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(t, r, 2), -EINVAL);
301+ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1u);
302+
303+ damon_for_each_region(r, t) {
304+ KUNIT_EXPECT_EQ(test, r->ar.start, 5ul);
305+ KUNIT_EXPECT_EQ(test, r->ar.end, 6ul);
306+ }
307+ damon_free_target(t);
308+ damon_destroy_ctx(c);
309+}
310+
311+static struct kunit_case damon_test_cases[] = {
312+ KUNIT_CASE(damon_test_three_regions_in_vmas),
313+ KUNIT_CASE(damon_test_apply_three_regions1),
314+ KUNIT_CASE(damon_test_apply_three_regions2),
315+ KUNIT_CASE(damon_test_apply_three_regions3),
316+ KUNIT_CASE(damon_test_apply_three_regions4),
317+ KUNIT_CASE(damon_test_split_evenly),
318+ {},
319+};
320+
321+static struct kunit_suite damon_test_suite = {
322+ .name = "damon-primitives",
323+ .test_cases = damon_test_cases,
324+};
325+kunit_test_suite(damon_test_suite);
326+
327+#endif /* _DAMON_VADDR_TEST_H */
328+
329+#endif /* CONFIG_DAMON_VADDR_KUNIT_TEST */
mm/damon/vaddr.c+672−0
@@ -0,0 +1,672 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * DAMON Primitives for Virtual Address Spaces
4+ *
5+ * Author: SeongJae Park <sjpark@amazon.de>
6+ */
7+
8+#define pr_fmt(fmt) "damon-va: " fmt
9+
10+#include <linux/damon.h>
11+#include <linux/hugetlb.h>
12+#include <linux/mm.h>
13+#include <linux/mmu_notifier.h>
14+#include <linux/highmem.h>
15+#include <linux/page_idle.h>
16+#include <linux/pagewalk.h>
17+#include <linux/random.h>
18+#include <linux/sched/mm.h>
19+#include <linux/slab.h>
20+
21+#ifdef CONFIG_DAMON_VADDR_KUNIT_TEST
22+#undef DAMON_MIN_REGION
23+#define DAMON_MIN_REGION 1
24+#endif
25+
26+/* Get a random number in [l, r) */
27+#define damon_rand(l, r) (l + prandom_u32_max(r - l))
28+
29+/*
30+ * 't->id' should be the pointer to the relevant 'struct pid' having reference
31+ * count. Caller must put the returned task, unless it is NULL.
32+ */
33+#define damon_get_task_struct(t) \
34+ (get_pid_task((struct pid *)t->id, PIDTYPE_PID))
35+
36+/*
37+ * Get the mm_struct of the given target
38+ *
39+ * Caller _must_ put the mm_struct after use, unless it is NULL.
40+ *
41+ * Returns the mm_struct of the target on success, NULL on failure
42+ */
43+static struct mm_struct *damon_get_mm(struct damon_target *t)
44+{
45+ struct task_struct *task;
46+ struct mm_struct *mm;
47+
48+ task = damon_get_task_struct(t);
49+ if (!task)
50+ return NULL;
51+
52+ mm = get_task_mm(task);
53+ put_task_struct(task);
54+ return mm;
55+}
56+
57+/*
58+ * Functions for the initial monitoring target regions construction
59+ */
60+
61+/*
62+ * Size-evenly split a region into 'nr_pieces' small regions
63+ *
64+ * Returns 0 on success, or negative error code otherwise.
65+ */
66+static int damon_va_evenly_split_region(struct damon_target *t,
67+ struct damon_region *r, unsigned int nr_pieces)
68+{
69+ unsigned long sz_orig, sz_piece, orig_end;
70+ struct damon_region *n = NULL, *next;
71+ unsigned long start;
72+
73+ if (!r || !nr_pieces)
74+ return -EINVAL;
75+
76+ orig_end = r->ar.end;
77+ sz_orig = r->ar.end - r->ar.start;
78+ sz_piece = ALIGN_DOWN(sz_orig / nr_pieces, DAMON_MIN_REGION);
79+
80+ if (!sz_piece)
81+ return -EINVAL;
82+
83+ r->ar.end = r->ar.start + sz_piece;
84+ next = damon_next_region(r);
85+ for (start = r->ar.end; start + sz_piece <= orig_end;
86+ start += sz_piece) {
87+ n = damon_new_region(start, start + sz_piece);
88+ if (!n)
89+ return -ENOMEM;
90+ damon_insert_region(n, r, next, t);
91+ r = n;
92+ }
93+ /* complement last region for possible rounding error */
94+ if (n)
95+ n->ar.end = orig_end;
96+
97+ return 0;
98+}
99+
100+static unsigned long sz_range(struct damon_addr_range *r)
101+{
102+ return r->end - r->start;
103+}
104+
105+static void swap_ranges(struct damon_addr_range *r1,
106+ struct damon_addr_range *r2)
107+{
108+ struct damon_addr_range tmp;
109+
110+ tmp = *r1;
111+ *r1 = *r2;
112+ *r2 = tmp;
113+}
114+
115+/*
116+ * Find three regions separated by two biggest unmapped regions
117+ *
118+ * vma the head vma of the target address space
119+ * regions an array of three address ranges that results will be saved
120+ *
121+ * This function receives an address space and finds three regions in it which
122+ * separated by the two biggest unmapped regions in the space. Please refer to
123+ * below comments of '__damon_va_init_regions()' function to know why this is
124+ * necessary.
125+ *
126+ * Returns 0 if success, or negative error code otherwise.
127+ */
128+static int __damon_va_three_regions(struct vm_area_struct *vma,
129+ struct damon_addr_range regions[3])
130+{
131+ struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0};
132+ struct vm_area_struct *last_vma = NULL;
133+ unsigned long start = 0;
134+ struct rb_root rbroot;
135+
136+ /* Find two biggest gaps so that first_gap > second_gap > others */
137+ for (; vma; vma = vma->vm_next) {
138+ if (!last_vma) {
139+ start = vma->vm_start;
140+ goto next;
141+ }
142+
143+ if (vma->rb_subtree_gap <= sz_range(&second_gap)) {
144+ rbroot.rb_node = &vma->vm_rb;
145+ vma = rb_entry(rb_last(&rbroot),
146+ struct vm_area_struct, vm_rb);
147+ goto next;
148+ }
149+
150+ gap.start = last_vma->vm_end;
151+ gap.end = vma->vm_start;
152+ if (sz_range(&gap) > sz_range(&second_gap)) {
153+ swap_ranges(&gap, &second_gap);
154+ if (sz_range(&second_gap) > sz_range(&first_gap))
155+ swap_ranges(&second_gap, &first_gap);
156+ }
157+next:
158+ last_vma = vma;
159+ }
160+
161+ if (!sz_range(&second_gap) || !sz_range(&first_gap))
162+ return -EINVAL;
163+
164+ /* Sort the two biggest gaps by address */
165+ if (first_gap.start > second_gap.start)
166+ swap_ranges(&first_gap, &second_gap);
167+
168+ /* Store the result */
169+ regions[0].start = ALIGN(start, DAMON_MIN_REGION);
170+ regions[0].end = ALIGN(first_gap.start, DAMON_MIN_REGION);
171+ regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION);
172+ regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION);
173+ regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION);
174+ regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION);
175+
176+ return 0;
177+}
178+
179+/*
180+ * Get the three regions in the given target (task)
181+ *
182+ * Returns 0 on success, negative error code otherwise.
183+ */
184+static int damon_va_three_regions(struct damon_target *t,
185+ struct damon_addr_range regions[3])
186+{
187+ struct mm_struct *mm;
188+ int rc;
189+
190+ mm = damon_get_mm(t);
191+ if (!mm)
192+ return -EINVAL;
193+
194+ mmap_read_lock(mm);
195+ rc = __damon_va_three_regions(mm->mmap, regions);
196+ mmap_read_unlock(mm);
197+
198+ mmput(mm);
199+ return rc;
200+}
201+
202+/*
203+ * Initialize the monitoring target regions for the given target (task)
204+ *
205+ * t the given target
206+ *
207+ * Because only a number of small portions of the entire address space
208+ * is actually mapped to the memory and accessed, monitoring the unmapped
209+ * regions is wasteful. That said, because we can deal with small noises,
210+ * tracking every mapping is not strictly required but could even incur a high
211+ * overhead if the mapping frequently changes or the number of mappings is
212+ * high. The adaptive regions adjustment mechanism will further help to deal
213+ * with the noise by simply identifying the unmapped areas as a region that
214+ * has no access. Moreover, applying the real mappings that would have many
215+ * unmapped areas inside will make the adaptive mechanism quite complex. That
216+ * said, too huge unmapped areas inside the monitoring target should be removed
217+ * to not take the time for the adaptive mechanism.
218+ *
219+ * For the reason, we convert the complex mappings to three distinct regions
220+ * that cover every mapped area of the address space. Also the two gaps
221+ * between the three regions are the two biggest unmapped areas in the given
222+ * address space. In detail, this function first identifies the start and the
223+ * end of the mappings and the two biggest unmapped areas of the address space.
224+ * Then, it constructs the three regions as below:
225+ *
226+ * [mappings[0]->start, big_two_unmapped_areas[0]->start)
227+ * [big_two_unmapped_areas[0]->end, big_two_unmapped_areas[1]->start)
228+ * [big_two_unmapped_areas[1]->end, mappings[nr_mappings - 1]->end)
229+ *
230+ * As usual memory map of processes is as below, the gap between the heap and
231+ * the uppermost mmap()-ed region, and the gap between the lowermost mmap()-ed
232+ * region and the stack will be two biggest unmapped regions. Because these
233+ * gaps are exceptionally huge areas in usual address space, excluding these
234+ * two biggest unmapped regions will be sufficient to make a trade-off.
235+ *
236+ * <heap>
237+ * <BIG UNMAPPED REGION 1>
238+ * <uppermost mmap()-ed region>
239+ * (other mmap()-ed regions and small unmapped regions)
240+ * <lowermost mmap()-ed region>
241+ * <BIG UNMAPPED REGION 2>
242+ * <stack>
243+ */
244+static void __damon_va_init_regions(struct damon_ctx *ctx,
245+ struct damon_target *t)
246+{
247+ struct damon_region *r;
248+ struct damon_addr_range regions[3];
249+ unsigned long sz = 0, nr_pieces;
250+ int i;
251+
252+ if (damon_va_three_regions(t, regions)) {
253+ pr_err("Failed to get three regions of target %lu\n", t->id);
254+ return;
255+ }
256+
257+ for (i = 0; i < 3; i++)
258+ sz += regions[i].end - regions[i].start;
259+ if (ctx->min_nr_regions)
260+ sz /= ctx->min_nr_regions;
261+ if (sz < DAMON_MIN_REGION)
262+ sz = DAMON_MIN_REGION;
263+
264+ /* Set the initial three regions of the target */
265+ for (i = 0; i < 3; i++) {
266+ r = damon_new_region(regions[i].start, regions[i].end);
267+ if (!r) {
268+ pr_err("%d'th init region creation failed\n", i);
269+ return;
270+ }
271+ damon_add_region(r, t);
272+
273+ nr_pieces = (regions[i].end - regions[i].start) / sz;
274+ damon_va_evenly_split_region(t, r, nr_pieces);
275+ }
276+}
277+
278+/* Initialize '->regions_list' of every target (task) */
279+void damon_va_init(struct damon_ctx *ctx)
280+{
281+ struct damon_target *t;
282+
283+ damon_for_each_target(t, ctx) {
284+ /* the user may set the target regions as they want */
285+ if (!damon_nr_regions(t))
286+ __damon_va_init_regions(ctx, t);
287+ }
288+}
289+
290+/*
291+ * Functions for the dynamic monitoring target regions update
292+ */
293+
294+/*
295+ * Check whether a region is intersecting an address range
296+ *
297+ * Returns true if it is.
298+ */
299+static bool damon_intersect(struct damon_region *r, struct damon_addr_range *re)
300+{
301+ return !(r->ar.end <= re->start || re->end <= r->ar.start);
302+}
303+
304+/*
305+ * Update damon regions for the three big regions of the given target
306+ *
307+ * t the given target
308+ * bregions the three big regions of the target
309+ */
310+static void damon_va_apply_three_regions(struct damon_target *t,
311+ struct damon_addr_range bregions[3])
312+{
313+ struct damon_region *r, *next;
314+ unsigned int i = 0;
315+
316+ /* Remove regions which are not in the three big regions now */
317+ damon_for_each_region_safe(r, next, t) {
318+ for (i = 0; i < 3; i++) {
319+ if (damon_intersect(r, &bregions[i]))
320+ break;
321+ }
322+ if (i == 3)
323+ damon_destroy_region(r, t);
324+ }
325+
326+ /* Adjust intersecting regions to fit with the three big regions */
327+ for (i = 0; i < 3; i++) {
328+ struct damon_region *first = NULL, *last;
329+ struct damon_region *newr;
330+ struct damon_addr_range *br;
331+
332+ br = &bregions[i];
333+ /* Get the first and last regions which intersects with br */
334+ damon_for_each_region(r, t) {
335+ if (damon_intersect(r, br)) {
336+ if (!first)
337+ first = r;
338+ last = r;
339+ }
340+ if (r->ar.start >= br->end)
341+ break;
342+ }
343+ if (!first) {
344+ /* no damon_region intersects with this big region */
345+ newr = damon_new_region(
346+ ALIGN_DOWN(br->start,
347+ DAMON_MIN_REGION),
348+ ALIGN(br->end, DAMON_MIN_REGION));
349+ if (!newr)
350+ continue;
351+ damon_insert_region(newr, damon_prev_region(r), r, t);
352+ } else {
353+ first->ar.start = ALIGN_DOWN(br->start,
354+ DAMON_MIN_REGION);
355+ last->ar.end = ALIGN(br->end, DAMON_MIN_REGION);
356+ }
357+ }
358+}
359+
360+/*
361+ * Update regions for current memory mappings
362+ */
363+void damon_va_update(struct damon_ctx *ctx)
364+{
365+ struct damon_addr_range three_regions[3];
366+ struct damon_target *t;
367+
368+ damon_for_each_target(t, ctx) {
369+ if (damon_va_three_regions(t, three_regions))
370+ continue;
371+ damon_va_apply_three_regions(t, three_regions);
372+ }
373+}
374+
375+/*
376+ * Get an online page for a pfn if it's in the LRU list. Otherwise, returns
377+ * NULL.
378+ *
379+ * The body of this function is stolen from the 'page_idle_get_page()'. We
380+ * steal rather than reuse it because the code is quite simple.
381+ */
382+static struct page *damon_get_page(unsigned long pfn)
383+{
384+ struct page *page = pfn_to_online_page(pfn);
385+
386+ if (!page || !PageLRU(page) || !get_page_unless_zero(page))
387+ return NULL;
388+
389+ if (unlikely(!PageLRU(page))) {
390+ put_page(page);
391+ page = NULL;
392+ }
393+ return page;
394+}
395+
396+static void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm,
397+ unsigned long addr)
398+{
399+ bool referenced = false;
400+ struct page *page = damon_get_page(pte_pfn(*pte));
401+
402+ if (!page)
403+ return;
404+
405+ if (pte_young(*pte)) {
406+ referenced = true;
407+ *pte = pte_mkold(*pte);
408+ }
409+
410+#ifdef CONFIG_MMU_NOTIFIER
411+ if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE))
412+ referenced = true;
413+#endif /* CONFIG_MMU_NOTIFIER */
414+
415+ if (referenced)
416+ set_page_young(page);
417+
418+ set_page_idle(page);
419+ put_page(page);
420+}
421+
422+static void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm,
423+ unsigned long addr)
424+{
425+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
426+ bool referenced = false;
427+ struct page *page = damon_get_page(pmd_pfn(*pmd));
428+
429+ if (!page)
430+ return;
431+
432+ if (pmd_young(*pmd)) {
433+ referenced = true;
434+ *pmd = pmd_mkold(*pmd);
435+ }
436+
437+#ifdef CONFIG_MMU_NOTIFIER
438+ if (mmu_notifier_clear_young(mm, addr,
439+ addr + ((1UL) << HPAGE_PMD_SHIFT)))
440+ referenced = true;
441+#endif /* CONFIG_MMU_NOTIFIER */
442+
443+ if (referenced)
444+ set_page_young(page);
445+
446+ set_page_idle(page);
447+ put_page(page);
448+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
449+}
450+
451+static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
452+ unsigned long next, struct mm_walk *walk)
453+{
454+ pte_t *pte;
455+ spinlock_t *ptl;
456+
457+ if (pmd_huge(*pmd)) {
458+ ptl = pmd_lock(walk->mm, pmd);
459+ if (pmd_huge(*pmd)) {
460+ damon_pmdp_mkold(pmd, walk->mm, addr);
461+ spin_unlock(ptl);
462+ return 0;
463+ }
464+ spin_unlock(ptl);
465+ }
466+
467+ if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
468+ return 0;
469+ pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
470+ if (!pte_present(*pte))
471+ goto out;
472+ damon_ptep_mkold(pte, walk->mm, addr);
473+out:
474+ pte_unmap_unlock(pte, ptl);
475+ return 0;
476+}
477+
478+static struct mm_walk_ops damon_mkold_ops = {
479+ .pmd_entry = damon_mkold_pmd_entry,
480+};
481+
482+static void damon_va_mkold(struct mm_struct *mm, unsigned long addr)
483+{
484+ mmap_read_lock(mm);
485+ walk_page_range(mm, addr, addr + 1, &damon_mkold_ops, NULL);
486+ mmap_read_unlock(mm);
487+}
488+
489+/*
490+ * Functions for the access checking of the regions
491+ */
492+
493+static void damon_va_prepare_access_check(struct damon_ctx *ctx,
494+ struct mm_struct *mm, struct damon_region *r)
495+{
496+ r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
497+
498+ damon_va_mkold(mm, r->sampling_addr);
499+}
500+
501+void damon_va_prepare_access_checks(struct damon_ctx *ctx)
502+{
503+ struct damon_target *t;
504+ struct mm_struct *mm;
505+ struct damon_region *r;
506+
507+ damon_for_each_target(t, ctx) {
508+ mm = damon_get_mm(t);
509+ if (!mm)
510+ continue;
511+ damon_for_each_region(r, t)
512+ damon_va_prepare_access_check(ctx, mm, r);
513+ mmput(mm);
514+ }
515+}
516+
517+struct damon_young_walk_private {
518+ unsigned long *page_sz;
519+ bool young;
520+};
521+
522+static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr,
523+ unsigned long next, struct mm_walk *walk)
524+{
525+ pte_t *pte;
526+ spinlock_t *ptl;
527+ struct page *page;
528+ struct damon_young_walk_private *priv = walk->private;
529+
530+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
531+ if (pmd_huge(*pmd)) {
532+ ptl = pmd_lock(walk->mm, pmd);
533+ if (!pmd_huge(*pmd)) {
534+ spin_unlock(ptl);
535+ goto regular_page;
536+ }
537+ page = damon_get_page(pmd_pfn(*pmd));
538+ if (!page)
539+ goto huge_out;
540+ if (pmd_young(*pmd) || !page_is_idle(page) ||
541+ mmu_notifier_test_young(walk->mm,
542+ addr)) {
543+ *priv->page_sz = ((1UL) << HPAGE_PMD_SHIFT);
544+ priv->young = true;
545+ }
546+ put_page(page);
547+huge_out:
548+ spin_unlock(ptl);
549+ return 0;
550+ }
551+
552+regular_page:
553+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
554+
555+ if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
556+ return -EINVAL;
557+ pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
558+ if (!pte_present(*pte))
559+ goto out;
560+ page = damon_get_page(pte_pfn(*pte));
561+ if (!page)
562+ goto out;
563+ if (pte_young(*pte) || !page_is_idle(page) ||
564+ mmu_notifier_test_young(walk->mm, addr)) {
565+ *priv->page_sz = PAGE_SIZE;
566+ priv->young = true;
567+ }
568+ put_page(page);
569+out:
570+ pte_unmap_unlock(pte, ptl);
571+ return 0;
572+}
573+
574+static struct mm_walk_ops damon_young_ops = {
575+ .pmd_entry = damon_young_pmd_entry,
576+};
577+
578+static bool damon_va_young(struct mm_struct *mm, unsigned long addr,
579+ unsigned long *page_sz)
580+{
581+ struct damon_young_walk_private arg = {
582+ .page_sz = page_sz,
583+ .young = false,
584+ };
585+
586+ mmap_read_lock(mm);
587+ walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg);
588+ mmap_read_unlock(mm);
589+ return arg.young;
590+}
591+
592+/*
593+ * Check whether the region was accessed after the last preparation
594+ *
595+ * mm 'mm_struct' for the given virtual address space
596+ * r the region to be checked
597+ */
598+static void damon_va_check_access(struct damon_ctx *ctx,
599+ struct mm_struct *mm, struct damon_region *r)
600+{
601+ static struct mm_struct *last_mm;
602+ static unsigned long last_addr;
603+ static unsigned long last_page_sz = PAGE_SIZE;
604+ static bool last_accessed;
605+
606+ /* If the region is in the last checked page, reuse the result */
607+ if (mm == last_mm && (ALIGN_DOWN(last_addr, last_page_sz) ==
608+ ALIGN_DOWN(r->sampling_addr, last_page_sz))) {
609+ if (last_accessed)
610+ r->nr_accesses++;
611+ return;
612+ }
613+
614+ last_accessed = damon_va_young(mm, r->sampling_addr, &last_page_sz);
615+ if (last_accessed)
616+ r->nr_accesses++;
617+
618+ last_mm = mm;
619+ last_addr = r->sampling_addr;
620+}
621+
622+unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
623+{
624+ struct damon_target *t;
625+ struct mm_struct *mm;
626+ struct damon_region *r;
627+ unsigned int max_nr_accesses = 0;
628+
629+ damon_for_each_target(t, ctx) {
630+ mm = damon_get_mm(t);
631+ if (!mm)
632+ continue;
633+ damon_for_each_region(r, t) {
634+ damon_va_check_access(ctx, mm, r);
635+ max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
636+ }
637+ mmput(mm);
638+ }
639+
640+ return max_nr_accesses;
641+}
642+
643+/*
644+ * Functions for the target validity check and cleanup
645+ */
646+
647+bool damon_va_target_valid(void *target)
648+{
649+ struct damon_target *t = target;
650+ struct task_struct *task;
651+
652+ task = damon_get_task_struct(t);
653+ if (task) {
654+ put_task_struct(task);
655+ return true;
656+ }
657+
658+ return false;
659+}
660+
661+void damon_va_set_primitives(struct damon_ctx *ctx)
662+{
663+ ctx->primitive.init = damon_va_init;
664+ ctx->primitive.update = damon_va_update;
665+ ctx->primitive.prepare_access_checks = damon_va_prepare_access_checks;
666+ ctx->primitive.check_accesses = damon_va_check_accesses;
667+ ctx->primitive.reset_aggregated = NULL;
668+ ctx->primitive.target_valid = damon_va_target_valid;
669+ ctx->primitive.cleanup = NULL;
670+}
671+
672+#include "vaddr-test.h"
mm/early_ioremap.c+0−5
@@ -38,13 +38,8 @@ pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr, · 37 unmodified lines
3838 return prot;
3939 }
4040
41−void __init __weak early_ioremap_shutdown(void)
42−{
43−}
44−
4541 void __init early_ioremap_reset(void)
4642 {
47− early_ioremap_shutdown();
4843 after_paging_init = 1;
4944 }
5045
mm/highmem.c+1−1
@@ -436,7 +436,7 @@ EXPORT_SYMBOL(zero_user_segments); · 435 unmodified lines
436436
437437 static inline int kmap_local_idx_push(void)
438438 {
439− WARN_ON_ONCE(in_irq() && !irqs_disabled());
439+ WARN_ON_ONCE(in_hardirq() && !irqs_disabled());
440440 current->kmap_ctrl.idx += KM_INCR;
441441 BUG_ON(current->kmap_ctrl.idx >= KM_MAX_IDX);
442442 return current->kmap_ctrl.idx - 1;
mm/ioremap.c+0−25
@@ -8,33 +8,9 @@ · 7 unmodified lines
88 */
99 #include <linux/vmalloc.h>
1010 #include <linux/mm.h>
11−#include <linux/sched.h>
1211 #include <linux/io.h>
1312 #include <linux/export.h>
14−#include <asm/cacheflush.h>
1513
16−#include "pgalloc-track.h"
17−
18−#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
19−static unsigned int __ro_after_init iomap_max_page_shift = BITS_PER_LONG - 1;
20−
21−static int __init set_nohugeiomap(char *str)
22−{
23− iomap_max_page_shift = PAGE_SHIFT;
24− return 0;
25−}
26−early_param("nohugeiomap", set_nohugeiomap);
27−#else /* CONFIG_HAVE_ARCH_HUGE_VMAP */
28−static const unsigned int iomap_max_page_shift = PAGE_SHIFT;
29−#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
30−
31−int ioremap_page_range(unsigned long addr,
32− unsigned long end, phys_addr_t phys_addr, pgprot_t prot)
33−{
34− return vmap_range(addr, end, phys_addr, prot, iomap_max_page_shift);
35−}
36−
37−#ifdef CONFIG_GENERIC_IOREMAP
3814 void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot)
3915 {
4016 unsigned long offset, vaddr;
@@ -71,4 +47,3 @@ void iounmap(volatile void __iomem *addr) · 30 unmodified lines
7147 vunmap((void *)((unsigned long)addr & PAGE_MASK));
7248 }
7349 EXPORT_SYMBOL(iounmap);
74−#endif /* CONFIG_GENERIC_IOREMAP */
mm/kfence/core.c+3−0
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 #include <linux/moduleparam.h>
2121 #include <linux/random.h>
2222 #include <linux/rcupdate.h>
23+#include <linux/sched/clock.h>
2324 #include <linux/sched/sysctl.h>
2425 #include <linux/seq_file.h>
2526 #include <linux/slab.h>
@@ -196,6 +197,8 @@ static noinline void metadata_update_state(struct kfence_metadata *meta, · 170 unmodified lines
196197 */
197198 track->num_stack_entries = stack_trace_save(track->stack_entries, KFENCE_STACK_DEPTH, 1);
198199 track->pid = task_pid_nr(current);
200+ track->cpu = raw_smp_processor_id();
201+ track->ts_nsec = local_clock(); /* Same source as printk timestamps. */
199202
200203 /*
201204 * Pairs with READ_ONCE() in
mm/kfence/kfence.h+2−0
@@ -36,6 +36,8 @@ enum kfence_object_state { · 35 unmodified lines
3636 /* Alloc/free tracking information. */
3737 struct kfence_track {
3838 pid_t pid;
39+ int cpu;
40+ u64 ts_nsec;
3941 int num_stack_entries;
4042 unsigned long stack_entries[KFENCE_STACK_DEPTH];
4143 };
mm/kfence/kfence_test.c+3−0
@@ -800,6 +800,9 @@ static int test_init(struct kunit *test) · 799 unmodified lines
800800 unsigned long flags;
801801 int i;
802802
803+ if (!__kfence_pool)
804+ return -EINVAL;
805+
803806 spin_lock_irqsave(&observed.lock, flags);
804807 for (i = 0; i < ARRAY_SIZE(observed.lines); i++)
805808 observed.lines[i][0] = '\0';
mm/kfence/report.c+13−6
@@ -9,6 +9,7 @@ · 8 unmodified lines
99
1010 #include <linux/kernel.h>
1111 #include <linux/lockdep.h>
12+#include <linux/math.h>
1213 #include <linux/printk.h>
1314 #include <linux/sched/debug.h>
1415 #include <linux/seq_file.h>
@@ -100,6 +101,13 @@ static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadat · 85 unmodified lines
100101 bool show_alloc)
101102 {
102103 const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track;
104+ u64 ts_sec = track->ts_nsec;
105+ unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC);
106+
107+ /* Timestamp matches printk timestamp format. */
108+ seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus:\n",
109+ show_alloc ? "allocated" : "freed", track->pid,
110+ track->cpu, (unsigned long)ts_sec, rem_nsec / 1000);
103111
104112 if (track->num_stack_entries) {
105113 /* Skip allocation/free internals stack. */
@@ -126,15 +134,14 @@ void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *met · 20 unmodified lines
126134 return;
127135 }
128136
129− seq_con_printf(seq,
130− "kfence-#%td [0x%p-0x%p"
131− ", size=%d, cache=%s] allocated by task %d:\n",
132− meta - kfence_metadata, (void *)start, (void *)(start + size - 1), size,
133− (cache && cache->name) ? cache->name : "<destroyed>", meta->alloc_track.pid);
137+ seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n",
138+ meta - kfence_metadata, (void *)start, (void *)(start + size - 1),
139+ size, (cache && cache->name) ? cache->name : "<destroyed>");
140+
134141 kfence_print_stack(seq, meta, true);
135142
136143 if (meta->state == KFENCE_OBJECT_FREED) {
137− seq_con_printf(seq, "\nfreed by task %d:\n", meta->free_track.pid);
144+ seq_con_printf(seq, "\n");
138145 kfence_print_stack(seq, meta, false);
139146 }
140147 }
mm/kmemleak.c+1−1
@@ -598,7 +598,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size, · 597 unmodified lines
598598 object->checksum = 0;
599599
600600 /* task information */
601− if (in_irq()) {
601+ if (in_hardirq()) {
602602 object->pid = 0;
603603 strncpy(object->comm, "hardirq", sizeof(object->comm));
604604 } else if (in_serving_softirq()) {
mm/memory_hotplug.c+341−33
@@ -52,6 +52,73 @@ module_param(memmap_on_memory, bool, 0444); · 51 unmodified lines
5252 MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug");
5353 #endif
5454
55+enum {
56+ ONLINE_POLICY_CONTIG_ZONES = 0,
57+ ONLINE_POLICY_AUTO_MOVABLE,
58+};
59+
60+const char *online_policy_to_str[] = {
61+ [ONLINE_POLICY_CONTIG_ZONES] = "contig-zones",
62+ [ONLINE_POLICY_AUTO_MOVABLE] = "auto-movable",
63+};
64+
65+static int set_online_policy(const char *val, const struct kernel_param *kp)
66+{
67+ int ret = sysfs_match_string(online_policy_to_str, val);
68+
69+ if (ret < 0)
70+ return ret;
71+ *((int *)kp->arg) = ret;
72+ return 0;
73+}
74+
75+static int get_online_policy(char *buffer, const struct kernel_param *kp)
76+{
77+ return sprintf(buffer, "%s\n", online_policy_to_str[*((int *)kp->arg)]);
78+}
79+
80+/*
81+ * memory_hotplug.online_policy: configure online behavior when onlining without
82+ * specifying a zone (MMOP_ONLINE)
83+ *
84+ * "contig-zones": keep zone contiguous
85+ * "auto-movable": online memory to ZONE_MOVABLE if the configuration
86+ * (auto_movable_ratio, auto_movable_numa_aware) allows for it
87+ */
88+static int online_policy __read_mostly = ONLINE_POLICY_CONTIG_ZONES;
89+static const struct kernel_param_ops online_policy_ops = {
90+ .set = set_online_policy,
91+ .get = get_online_policy,
92+};
93+module_param_cb(online_policy, &online_policy_ops, &online_policy, 0644);
94+MODULE_PARM_DESC(online_policy,
95+ "Set the online policy (\"contig-zones\", \"auto-movable\") "
96+ "Default: \"contig-zones\"");
97+
98+/*
99+ * memory_hotplug.auto_movable_ratio: specify maximum MOVABLE:KERNEL ratio
100+ *
101+ * The ratio represent an upper limit and the kernel might decide to not
102+ * online some memory to ZONE_MOVABLE -- e.g., because hotplugged KERNEL memory
103+ * doesn't allow for more MOVABLE memory.
104+ */
105+static unsigned int auto_movable_ratio __read_mostly = 301;
106+module_param(auto_movable_ratio, uint, 0644);
107+MODULE_PARM_DESC(auto_movable_ratio,
108+ "Set the maximum ratio of MOVABLE:KERNEL memory in the system "
109+ "in percent for \"auto-movable\" online policy. Default: 301");
110+
111+/*
112+ * memory_hotplug.auto_movable_numa_aware: consider numa node stats
113+ */
114+#ifdef CONFIG_NUMA
115+static bool auto_movable_numa_aware __read_mostly = true;
116+module_param(auto_movable_numa_aware, bool, 0644);
117+MODULE_PARM_DESC(auto_movable_numa_aware,
118+ "Consider numa node stats in addition to global stats in "
119+ "\"auto-movable\" online policy. Default: true");
120+#endif /* CONFIG_NUMA */
121+
55122 /*
56123 * online_page_callback contains pointer to current page onlining function.
57124 * Initially it is generic_online_page(). If it is required it could be
@@ -410,15 +477,13 @@ void __ref remove_pfn_range_from_zone(struct zone *zone, · 352 unmodified lines
410477 sizeof(struct page) * cur_nr_pages);
411478 }
412479
413−#ifdef CONFIG_ZONE_DEVICE
414480 /*
415481 * Zone shrinking code cannot properly deal with ZONE_DEVICE. So
416482 * we will not try to shrink the zones - which is okay as
417483 * set_zone_contiguous() cannot deal with ZONE_DEVICE either way.
418484 */
419− if (zone_idx(zone) == ZONE_DEVICE)
485+ if (zone_is_zone_device(zone))
420486 return;
421−#endif
422487
423488 clear_zone_contiguous(zone);
424489
@@ -663,6 +728,109 @@ void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, · 238 unmodified lines
663728 set_zone_contiguous(zone);
664729 }
665730
731+struct auto_movable_stats {
732+ unsigned long kernel_early_pages;
733+ unsigned long movable_pages;
734+};
735+
736+static void auto_movable_stats_account_zone(struct auto_movable_stats *stats,
737+ struct zone *zone)
738+{
739+ if (zone_idx(zone) == ZONE_MOVABLE) {
740+ stats->movable_pages += zone->present_pages;
741+ } else {
742+ stats->kernel_early_pages += zone->present_early_pages;
743+#ifdef CONFIG_CMA
744+ /*
745+ * CMA pages (never on hotplugged memory) behave like
746+ * ZONE_MOVABLE.
747+ */
748+ stats->movable_pages += zone->cma_pages;
749+ stats->kernel_early_pages -= zone->cma_pages;
750+#endif /* CONFIG_CMA */
751+ }
752+}
753+struct auto_movable_group_stats {
754+ unsigned long movable_pages;
755+ unsigned long req_kernel_early_pages;
756+};
757+
758+static int auto_movable_stats_account_group(struct memory_group *group,
759+ void *arg)
760+{
761+ const int ratio = READ_ONCE(auto_movable_ratio);
762+ struct auto_movable_group_stats *stats = arg;
763+ long pages;
764+
765+ /*
766+ * We don't support modifying the config while the auto-movable online
767+ * policy is already enabled. Just avoid the division by zero below.
768+ */
769+ if (!ratio)
770+ return 0;
771+
772+ /*
773+ * Calculate how many early kernel pages this group requires to
774+ * satisfy the configured zone ratio.
775+ */
776+ pages = group->present_movable_pages * 100 / ratio;
777+ pages -= group->present_kernel_pages;
778+
779+ if (pages > 0)
780+ stats->req_kernel_early_pages += pages;
781+ stats->movable_pages += group->present_movable_pages;
782+ return 0;
783+}
784+
785+static bool auto_movable_can_online_movable(int nid, struct memory_group *group,
786+ unsigned long nr_pages)
787+{
788+ unsigned long kernel_early_pages, movable_pages;
789+ struct auto_movable_group_stats group_stats = {};
790+ struct auto_movable_stats stats = {};
791+ pg_data_t *pgdat = NODE_DATA(nid);
792+ struct zone *zone;
793+ int i;
794+
795+ /* Walk all relevant zones and collect MOVABLE vs. KERNEL stats. */
796+ if (nid == NUMA_NO_NODE) {
797+ /* TODO: cache values */
798+ for_each_populated_zone(zone)
799+ auto_movable_stats_account_zone(&stats, zone);
800+ } else {
801+ for (i = 0; i < MAX_NR_ZONES; i++) {
802+ zone = pgdat->node_zones + i;
803+ if (populated_zone(zone))
804+ auto_movable_stats_account_zone(&stats, zone);
805+ }
806+ }
807+
808+ kernel_early_pages = stats.kernel_early_pages;
809+ movable_pages = stats.movable_pages;
810+
811+ /*
812+ * Kernel memory inside dynamic memory group allows for more MOVABLE
813+ * memory within the same group. Remove the effect of all but the
814+ * current group from the stats.
815+ */
816+ walk_dynamic_memory_groups(nid, auto_movable_stats_account_group,
817+ group, &group_stats);
818+ if (kernel_early_pages <= group_stats.req_kernel_early_pages)
819+ return false;
820+ kernel_early_pages -= group_stats.req_kernel_early_pages;
821+ movable_pages -= group_stats.movable_pages;
822+
823+ if (group && group->is_dynamic)
824+ kernel_early_pages += group->present_kernel_pages;
825+
826+ /*
827+ * Test if we could online the given number of pages to ZONE_MOVABLE
828+ * and still stay in the configured ratio.
829+ */
830+ movable_pages += nr_pages;
831+ return movable_pages <= (auto_movable_ratio * kernel_early_pages) / 100;
832+}
833+
666834 /*
667835 * Returns a default kernel memory zone for the given pfn range.
668836 * If no kernel zone covers this pfn range it will automatically go
@@ -684,6 +852,117 @@ static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn · 15 unmodified lines
684852 return &pgdat->node_zones[ZONE_NORMAL];
685853 }
686854
855+/*
856+ * Determine to which zone to online memory dynamically based on user
857+ * configuration and system stats. We care about the following ratio:
858+ *
859+ * MOVABLE : KERNEL
860+ *
861+ * Whereby MOVABLE is memory in ZONE_MOVABLE and KERNEL is memory in
862+ * one of the kernel zones. CMA pages inside one of the kernel zones really
863+ * behaves like ZONE_MOVABLE, so we treat them accordingly.
864+ *
865+ * We don't allow for hotplugged memory in a KERNEL zone to increase the
866+ * amount of MOVABLE memory we can have, so we end up with:
867+ *
868+ * MOVABLE : KERNEL_EARLY
869+ *
870+ * Whereby KERNEL_EARLY is memory in one of the kernel zones, available sinze
871+ * boot. We base our calculation on KERNEL_EARLY internally, because:
872+ *
873+ * a) Hotplugged memory in one of the kernel zones can sometimes still get
874+ * hotunplugged, especially when hot(un)plugging individual memory blocks.
875+ * There is no coordination across memory devices, therefore "automatic"
876+ * hotunplugging, as implemented in hypervisors, could result in zone
877+ * imbalances.
878+ * b) Early/boot memory in one of the kernel zones can usually not get
879+ * hotunplugged again (e.g., no firmware interface to unplug, fragmented
880+ * with unmovable allocations). While there are corner cases where it might
881+ * still work, it is barely relevant in practice.
882+ *
883+ * Exceptions are dynamic memory groups, which allow for more MOVABLE
884+ * memory within the same memory group -- because in that case, there is
885+ * coordination within the single memory device managed by a single driver.
886+ *
887+ * We rely on "present pages" instead of "managed pages", as the latter is
888+ * highly unreliable and dynamic in virtualized environments, and does not
889+ * consider boot time allocations. For example, memory ballooning adjusts the
890+ * managed pages when inflating/deflating the balloon, and balloon compaction
891+ * can even migrate inflated pages between zones.
892+ *
893+ * Using "present pages" is better but some things to keep in mind are:
894+ *
895+ * a) Some memblock allocations, such as for the crashkernel area, are
896+ * effectively unused by the kernel, yet they account to "present pages".
897+ * Fortunately, these allocations are comparatively small in relevant setups
898+ * (e.g., fraction of system memory).
899+ * b) Some hotplugged memory blocks in virtualized environments, esecially
900+ * hotplugged by virtio-mem, look like they are completely present, however,
901+ * only parts of the memory block are actually currently usable.
902+ * "present pages" is an upper limit that can get reached at runtime. As
903+ * we base our calculations on KERNEL_EARLY, this is not an issue.
904+ */
905+static struct zone *auto_movable_zone_for_pfn(int nid,
906+ struct memory_group *group,
907+ unsigned long pfn,
908+ unsigned long nr_pages)
909+{
910+ unsigned long online_pages = 0, max_pages, end_pfn;
911+ struct page *page;
912+
913+ if (!auto_movable_ratio)
914+ goto kernel_zone;
915+
916+ if (group && !group->is_dynamic) {
917+ max_pages = group->s.max_pages;
918+ online_pages = group->present_movable_pages;
919+
920+ /* If anything is !MOVABLE online the rest !MOVABLE. */
921+ if (group->present_kernel_pages)
922+ goto kernel_zone;
923+ } else if (!group || group->d.unit_pages == nr_pages) {
924+ max_pages = nr_pages;
925+ } else {
926+ max_pages = group->d.unit_pages;
927+ /*
928+ * Take a look at all online sections in the current unit.
929+ * We can safely assume that all pages within a section belong
930+ * to the same zone, because dynamic memory groups only deal
931+ * with hotplugged memory.
932+ */
933+ pfn = ALIGN_DOWN(pfn, group->d.unit_pages);
934+ end_pfn = pfn + group->d.unit_pages;
935+ for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
936+ page = pfn_to_online_page(pfn);
937+ if (!page)
938+ continue;
939+ /* If anything is !MOVABLE online the rest !MOVABLE. */
940+ if (page_zonenum(page) != ZONE_MOVABLE)
941+ goto kernel_zone;
942+ online_pages += PAGES_PER_SECTION;
943+ }
944+ }
945+
946+ /*
947+ * Online MOVABLE if we could *currently* online all remaining parts
948+ * MOVABLE. We expect to (add+) online them immediately next, so if
949+ * nobody interferes, all will be MOVABLE if possible.
950+ */
951+ nr_pages = max_pages - online_pages;
952+ if (!auto_movable_can_online_movable(NUMA_NO_NODE, group, nr_pages))
953+ goto kernel_zone;
954+
955+#ifdef CONFIG_NUMA
956+ if (auto_movable_numa_aware &&
957+ !auto_movable_can_online_movable(nid, group, nr_pages))
958+ goto kernel_zone;
959+#endif /* CONFIG_NUMA */
960+
961+ return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
962+kernel_zone:
963+ return default_kernel_zone_for_pfn(nid, pfn, nr_pages);
964+}
965+
687966 static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,
688967 unsigned long nr_pages)
689968 {
@@ -708,7 +987,8 @@ static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn · 18 unmodified lines
708987 return movable_node_enabled ? movable_zone : kernel_zone;
709988 }
710989
711−struct zone *zone_for_pfn_range(int online_type, int nid, unsigned start_pfn,
990+struct zone *zone_for_pfn_range(int online_type, int nid,
991+ struct memory_group *group, unsigned long start_pfn,
712992 unsigned long nr_pages)
713993 {
714994 if (online_type == MMOP_ONLINE_KERNEL)
@@ -717,6 +997,9 @@ struct zone *zone_for_pfn_range(int online_type, int nid, unsigned start_pfn, · 2 unmodified lines
717997 if (online_type == MMOP_ONLINE_MOVABLE)
718998 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
719999
1000+ if (online_policy == ONLINE_POLICY_AUTO_MOVABLE)
1001+ return auto_movable_zone_for_pfn(nid, group, start_pfn, nr_pages);
1002+
7201003 return default_zone_for_pfn(nid, start_pfn, nr_pages);
7211004 }
7221005
@@ -724,10 +1007,25 @@ struct zone *zone_for_pfn_range(int online_type, int nid, unsigned start_pfn, · 1 unmodified line
7241007 * This function should only be called by memory_block_{online,offline},
7251008 * and {online,offline}_pages.
7261009 */
727−void adjust_present_page_count(struct zone *zone, long nr_pages)
1010+void adjust_present_page_count(struct page *page, struct memory_group *group,
1011+ long nr_pages)
7281012 {
1013+ struct zone *zone = page_zone(page);
1014+ const bool movable = zone_idx(zone) == ZONE_MOVABLE;
1015+
1016+ /*
1017+ * We only support onlining/offlining/adding/removing of complete
1018+ * memory blocks; therefore, either all is either early or hotplugged.
1019+ */
1020+ if (early_section(__pfn_to_section(page_to_pfn(page))))
1021+ zone->present_early_pages += nr_pages;
7291022 zone->present_pages += nr_pages;
7301023 zone->zone_pgdat->node_present_pages += nr_pages;
1024+
1025+ if (group && movable)
1026+ group->present_movable_pages += nr_pages;
1027+ else if (group && !movable)
1028+ group->present_kernel_pages += nr_pages;
7311029 }
7321030
7331031 int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages,
@@ -773,7 +1071,8 @@ void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages) · 39 unmodified lines
7731071 kasan_remove_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages));
7741072 }
7751073
776−int __ref online_pages(unsigned long pfn, unsigned long nr_pages, struct zone *zone)
1074+int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
1075+ struct zone *zone, struct memory_group *group)
7771076 {
7781077 unsigned long flags;
7791078 int need_zonelists_rebuild = 0;
@@ -826,7 +1125,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, struct zone *z · 46 unmodified lines
8261125 }
8271126
8281127 online_pages_range(pfn, nr_pages);
829− adjust_present_page_count(zone, nr_pages);
1128+ adjust_present_page_count(pfn_to_page(pfn), group, nr_pages);
8301129
8311130 node_states_set_node(nid, &arg);
8321131 if (need_zonelists_rebuild)
@@ -1059,6 +1358,7 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) · 226 unmodified lines
10591358 {
10601359 struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) };
10611360 struct vmem_altmap mhp_altmap = {};
1361+ struct memory_group *group = NULL;
10621362 u64 start, size;
10631363 bool new_node = false;
10641364 int ret;
@@ -1070,6 +1370,13 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) · 5 unmodified lines
10701370 if (ret)
10711371 return ret;
10721372
1373+ if (mhp_flags & MHP_NID_IS_MGID) {
1374+ group = memory_group_find_by_id(nid);
1375+ if (!group)
1376+ return -EINVAL;
1377+ nid = group->nid;
1378+ }
1379+
10731380 if (!node_possible(nid)) {
10741381 WARN(1, "node %d was absent from the node_possible_map\n", nid);
10751382 return -EINVAL;
@@ -1104,9 +1411,10 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) · 28 unmodified lines
11041411 goto error;
11051412
11061413 /* create memory block devices after memory was added */
1107− ret = create_memory_block_devices(start, size, mhp_altmap.alloc);
1414+ ret = create_memory_block_devices(start, size, mhp_altmap.alloc,
1415+ group);
11081416 if (ret) {
1109− arch_remove_memory(nid, start, size, NULL);
1417+ arch_remove_memory(start, size, NULL);
11101418 goto error;
11111419 }
11121420
@@ -1298,7 +1606,7 @@ struct zone *test_pages_in_a_zone(unsigned long start_pfn, · 185 unmodified lines
12981606 unsigned long pfn, sec_end_pfn;
12991607 struct zone *zone = NULL;
13001608 struct page *page;
1301− int i;
1609+
13021610 for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1);
13031611 pfn < end_pfn;
13041612 pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) {
@@ -1307,17 +1615,10 @@ struct zone *test_pages_in_a_zone(unsigned long start_pfn, · 2 unmodified lines
13071615 continue;
13081616 for (; pfn < sec_end_pfn && pfn < end_pfn;
13091617 pfn += MAX_ORDER_NR_PAGES) {
1310− i = 0;
1311− /* This is just a CONFIG_HOLES_IN_ZONE check.*/
1312− while ((i < MAX_ORDER_NR_PAGES) &&
1313− !pfn_valid_within(pfn + i))
1314− i++;
1315− if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn)
1316− continue;
13171618 /* Check if we got outside of the zone */
1318− if (zone && !zone_spans_pfn(zone, pfn + i))
1619+ if (zone && !zone_spans_pfn(zone, pfn))
13191620 return NULL;
1320− page = pfn_to_page(pfn + i);
1621+ page = pfn_to_page(pfn);
13211622 if (zone && page_zone(page) != zone)
13221623 return NULL;
13231624 zone = page_zone(page);
@@ -1568,7 +1869,8 @@ static int count_system_ram_pages_cb(unsigned long start_pfn, · 244 unmodified lines
15681869 return 0;
15691870 }
15701871
1571−int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages)
1872+int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages,
1873+ struct memory_group *group)
15721874 {
15731875 const unsigned long end_pfn = start_pfn + nr_pages;
15741876 unsigned long pfn, system_ram_pages = 0;
@@ -1704,7 +2006,7 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) · 129 unmodified lines
17042006
17052007 /* removal success */
17062008 adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages);
1707− adjust_present_page_count(zone, -nr_pages);
2009+ adjust_present_page_count(pfn_to_page(start_pfn), group, -nr_pages);
17082010
17092011 /* reinitialise watermarks and update pcp limits */
17102012 init_per_zone_wmark_min();
@@ -1746,7 +2048,9 @@ failed_removal: · 35 unmodified lines
17462048 static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
17472049 {
17482050 int ret = !is_memblock_offlined(mem);
2051+ int *nid = arg;
17492052
2053+ *nid = mem->nid;
17502054 if (unlikely(ret)) {
17512055 phys_addr_t beginpa, endpa;
17522056
@@ -1839,12 +2143,12 @@ void try_offline_node(int nid) · 86 unmodified lines
18392143 }
18402144 EXPORT_SYMBOL(try_offline_node);
18412145
1842−static int __ref try_remove_memory(int nid, u64 start, u64 size)
2146+static int __ref try_remove_memory(u64 start, u64 size)
18432147 {
1844− int rc = 0;
18452148 struct vmem_altmap mhp_altmap = {};
18462149 struct vmem_altmap *altmap = NULL;
18472150 unsigned long nr_vmemmap_pages;
2151+ int rc = 0, nid = NUMA_NO_NODE;
18482152
18492153 BUG_ON(check_hotplug_memory_range(start, size));
18502154
@@ -1852,8 +2156,12 @@ static int __ref try_remove_memory(int nid, u64 start, u64 size) · 1 unmodified line
18522156 * All memory blocks must be offlined before removing memory. Check
18532157 * whether all memory blocks in question are offline and return error
18542158 * if this is not the case.
2159+ *
2160+ * While at it, determine the nid. Note that if we'd have mixed nodes,
2161+ * we'd only try to offline the last determined one -- which is good
2162+ * enough for the cases we care about.
18552163 */
1856− rc = walk_memory_blocks(start, size, NULL, check_memblock_offlined_cb);
2164+ rc = walk_memory_blocks(start, size, &nid, check_memblock_offlined_cb);
18572165 if (rc)
18582166 return rc;
18592167
@@ -1893,7 +2201,7 @@ static int __ref try_remove_memory(int nid, u64 start, u64 size) · 33 unmodified lines
18932201
18942202 mem_hotplug_begin();
18952203
1896− arch_remove_memory(nid, start, size, altmap);
2204+ arch_remove_memory(start, size, altmap);
18972205
18982206 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) {
18992207 memblock_free(start, size);
@@ -1902,7 +2210,8 @@ static int __ref try_remove_memory(int nid, u64 start, u64 size) · 2 unmodified lines
19022210
19032211 release_mem_region_adjustable(start, size);
19042212
1905− try_offline_node(nid);
2213+ if (nid != NUMA_NO_NODE)
2214+ try_offline_node(nid);
19062215
19072216 mem_hotplug_done();
19082217 return 0;
@@ -1910,7 +2219,6 @@ static int __ref try_remove_memory(int nid, u64 start, u64 size) · 1 unmodified line
19102219
19112220 /**
19122221 * __remove_memory - Remove memory if every memory block is offline
1913− * @nid: the node ID
19142222 * @start: physical address of the region to remove
19152223 * @size: size of the region to remove
19162224 *
@@ -1918,14 +2226,14 @@ static int __ref try_remove_memory(int nid, u64 start, u64 size) · 1 unmodified line
19182226 * and online/offline operations before this call, as required by
19192227 * try_offline_node().
19202228 */
1921−void __remove_memory(int nid, u64 start, u64 size)
2229+void __remove_memory(u64 start, u64 size)
19222230 {
19232231
19242232 /*
19252233 * trigger BUG() if some memory is not offlined prior to calling this
19262234 * function
19272235 */
1928− if (try_remove_memory(nid, start, size))
2236+ if (try_remove_memory(start, size))
19292237 BUG();
19302238 }
19312239
@@ -1933,12 +2241,12 @@ void __remove_memory(int nid, u64 start, u64 size) · 1 unmodified line
19332241 * Remove memory if every memory block is offline, otherwise return -EBUSY is
19342242 * some memory is not offline
19352243 */
1936−int remove_memory(int nid, u64 start, u64 size)
2244+int remove_memory(u64 start, u64 size)
19372245 {
19382246 int rc;
19392247
19402248 lock_device_hotplug();
1941− rc = try_remove_memory(nid, start, size);
2249+ rc = try_remove_memory(start, size);
19422250 unlock_device_hotplug();
19432251
19442252 return rc;
@@ -1998,7 +2306,7 @@ static int try_reonline_memory_block(struct memory_block *mem, void *arg) · 53 unmodified lines
19982306 * unplugged all memory (so it's no longer in use) and want to offline + remove
19992307 * that memory.
20002308 */
2001−int offline_and_remove_memory(int nid, u64 start, u64 size)
2309+int offline_and_remove_memory(u64 start, u64 size)
20022310 {
20032311 const unsigned long mb_count = size / memory_block_size_bytes();
20042312 uint8_t *online_types, *tmp;
@@ -2034,7 +2342,7 @@ int offline_and_remove_memory(int nid, u64 start, u64 size) · 29 unmodified lines
20342342 * This cannot fail as it cannot get onlined in the meantime.
20352343 */
20362344 if (!rc) {
2037− rc = try_remove_memory(nid, start, size);
2345+ rc = try_remove_memory(start, size);
20382346 if (rc)
20392347 pr_err("%s: Failed to remove memory: %d", __func__, rc);
20402348 }
mm/memremap.c+1−4
@@ -140,14 +140,11 @@ static void pageunmap_range(struct dev_pagemap *pgmap, int range_id) · 139 unmodified lines
140140 {
141141 struct range *range = &pgmap->ranges[range_id];
142142 struct page *first_page;
143− int nid;
144143
145144 /* make sure to access a memmap that was actually initialized */
146145 first_page = pfn_to_page(pfn_first(pgmap, range_id));
147146
148147 /* pages are dead and unused, undo the arch mapping */
149− nid = page_to_nid(first_page);
150−
151148 mem_hotplug_begin();
152149 remove_pfn_range_from_zone(page_zone(first_page), PHYS_PFN(range->start),
153150 PHYS_PFN(range_len(range)));
@@ -155,7 +152,7 @@ static void pageunmap_range(struct dev_pagemap *pgmap, int range_id) · 1 unmodified line
155152 __remove_pages(PHYS_PFN(range->start),
156153 PHYS_PFN(range_len(range)), NULL);
157154 } else {
158− arch_remove_memory(nid, range->start, range_len(range),
155+ arch_remove_memory(range->start, range_len(range),
159156 pgmap_altmap(pgmap));
160157 kasan_remove_zero_shadow(__va(range->start), range_len(range));
161158 }
mm/page_alloc.c+5−22
@@ -594,8 +594,6 @@ static int page_outside_zone_boundaries(struct zone *zone, struct page *page) · 593 unmodified lines
594594
595595 static int page_is_consistent(struct zone *zone, struct page *page)
596596 {
597− if (!pfn_valid_within(page_to_pfn(page)))
598− return 0;
599597 if (zone != page_zone(page))
600598 return 0;
601599
@@ -1025,16 +1023,12 @@ buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn, · 423 unmodified lines
10251023 if (order >= MAX_ORDER - 2)
10261024 return false;
10271025
1028− if (!pfn_valid_within(buddy_pfn))
1029− return false;
1030−
10311026 combined_pfn = buddy_pfn & pfn;
10321027 higher_page = page + (combined_pfn - pfn);
10331028 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
10341029 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
10351030
1036− return pfn_valid_within(buddy_pfn) &&
1037− page_is_buddy(higher_page, higher_buddy, order + 1);
1031+ return page_is_buddy(higher_page, higher_buddy, order + 1);
10381032 }
10391033
10401034 /*
@@ -1095,8 +1089,6 @@ continue_merging: · 54 unmodified lines
10951089 buddy_pfn = __find_buddy_pfn(pfn, order);
10961090 buddy = page + (buddy_pfn - pfn);
10971091
1098− if (!pfn_valid_within(buddy_pfn))
1099− goto done_merging;
11001092 if (!page_is_buddy(page, buddy, order))
11011093 goto done_merging;
11021094 /*
@@ -1754,9 +1746,7 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn, · 651 unmodified lines
17541746 /*
17551747 * Check that the whole (or subset of) a pageblock given by the interval of
17561748 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1757− * with the migration of free compaction scanner. The scanners then need to
1758− * use only pfn_valid_within() check for arches that allow holes within
1759− * pageblocks.
1749+ * with the migration of free compaction scanner.
17601750 *
17611751 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
17621752 *
@@ -1872,8 +1862,6 @@ static inline void __init pgdat_init_report_one_done(void) · 109 unmodified lines
18721862 */
18731863 static inline bool __init deferred_pfn_valid(unsigned long pfn)
18741864 {
1875− if (!pfn_valid_within(pfn))
1876− return false;
18771865 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
18781866 return false;
18791867 return true;
@@ -2520,11 +2508,6 @@ static int move_freepages(struct zone *zone, · 640 unmodified lines
25202508 int pages_moved = 0;
25212509
25222510 for (pfn = start_pfn; pfn <= end_pfn;) {
2523− if (!pfn_valid_within(pfn)) {
2524− pfn++;
2525− continue;
2526− }
2527−
25282511 page = pfn_to_page(pfn);
25292512 if (!PageBuddy(page)) {
25302513 /*
@@ -7271,6 +7254,9 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, · 4740 unmodified lines
72717254 zone->zone_start_pfn = 0;
72727255 zone->spanned_pages = size;
72737256 zone->present_pages = real_size;
7257+#if defined(CONFIG_MEMORY_HOTPLUG)
7258+ zone->present_early_pages = real_size;
7259+#endif
72747260
72757261 totalpages += size;
72767262 realtotalpages += real_size;
@@ -8828,9 +8814,6 @@ struct page *has_unmovable_pages(struct zone *zone, struct page *page, · 1551 unmodified lines
88288814 }
88298815
88308816 for (; iter < pageblock_nr_pages - offset; iter++) {
8831− if (!pfn_valid_within(pfn + iter))
8832− continue;
8833−
88348817 page = pfn_to_page(pfn + iter);
88358818
88368819 /*
mm/page_ext.c+11−1
@@ -58,11 +58,21 @@ · 57 unmodified lines
5858 * can utilize this callback to initialize the state of it correctly.
5959 */
6060
61+#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
62+static bool need_page_idle(void)
63+{
64+ return true;
65+}
66+struct page_ext_operations page_idle_ops = {
67+ .need = need_page_idle,
68+};
69+#endif
70+
6171 static struct page_ext_operations *page_ext_ops[] = {
6272 #ifdef CONFIG_PAGE_OWNER
6373 &page_owner_ops,
6474 #endif
65−#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
75+#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT)
6676 &page_idle_ops,
6777 #endif
6878 };
mm/page_idle.c+0−10
@@ -207,16 +207,6 @@ static const struct attribute_group page_idle_attr_group = { · 206 unmodified lines
207207 .name = "page_idle",
208208 };
209209
210−#ifndef CONFIG_64BIT
211−static bool need_page_idle(void)
212−{
213− return true;
214−}
215−struct page_ext_operations page_idle_ops = {
216− .need = need_page_idle,
217−};
218−#endif
219−
220210 static int __init page_idle_init(void)
221211 {
222212 int err;
mm/page_isolation.c+1−6
@@ -93,8 +93,7 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype) · 92 unmodified lines
9393 buddy_pfn = __find_buddy_pfn(pfn, order);
9494 buddy = page + (buddy_pfn - pfn);
9595
96− if (pfn_valid_within(buddy_pfn) &&
97− !is_migrate_isolate_page(buddy)) {
96+ if (!is_migrate_isolate_page(buddy)) {
9897 __isolate_free_page(page, order);
9998 isolated_page = true;
10099 }
@@ -250,10 +249,6 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, · 149 unmodified lines
250249 struct page *page;
251250
252251 while (pfn < end_pfn) {
253− if (!pfn_valid_within(pfn)) {
254− pfn++;
255− continue;
256− }
257252 page = pfn_to_page(pfn);
258253 if (PageBuddy(page))
259254 /*
mm/page_owner.c+1−13
@@ -276,9 +276,6 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, · 275 unmodified lines
276276 pageblock_mt = get_pageblock_migratetype(page);
277277
278278 for (; pfn < block_end_pfn; pfn++) {
279− if (!pfn_valid_within(pfn))
280− continue;
281−
282279 /* The pageblock is online, no need to recheck. */
283280 page = pfn_to_page(pfn);
284281
@@ -479,10 +476,6 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) · 194 unmodified lines
479476 continue;
480477 }
481478
482− /* Check for holes within a MAX_ORDER area */
483− if (!pfn_valid_within(pfn))
484− continue;
485−
486479 page = pfn_to_page(pfn);
487480 if (PageBuddy(page)) {
488481 unsigned long freepage_order = buddy_order_unsafe(page);
@@ -560,14 +553,9 @@ static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) · 71 unmodified lines
560553 block_end_pfn = min(block_end_pfn, end_pfn);
561554
562555 for (; pfn < block_end_pfn; pfn++) {
563− struct page *page;
556+ struct page *page = pfn_to_page(pfn);
564557 struct page_ext *page_ext;
565558
566− if (!pfn_valid_within(pfn))
567− continue;
568−
569− page = pfn_to_page(pfn);
570−
571559 if (page_zone(page) != zone)
572560 continue;
573561
mm/percpu.c+0−1
@@ -146,7 +146,6 @@ static unsigned int pcpu_high_unit_cpu __ro_after_init; · 145 unmodified lines
146146
147147 /* the address of the first chunk which starts with the kernel static area */
148148 void *pcpu_base_addr __ro_after_init;
149−EXPORT_SYMBOL_GPL(pcpu_base_addr);
150149
151150 static const int *pcpu_unit_map __ro_after_init; /* cpu -> unit */
152151 const unsigned long *pcpu_unit_offsets __ro_after_init; /* cpu -> unit offset */
mm/rmap.c+4−2
@@ -1231,11 +1231,13 @@ void page_add_file_rmap(struct page *page, bool compound) · 1230 unmodified lines
12311231 nr_pages);
12321232 } else {
12331233 if (PageTransCompound(page) && page_mapping(page)) {
1234+ struct page *head = compound_head(page);
1235+
12341236 VM_WARN_ON_ONCE(!PageLocked(page));
12351237
1236− SetPageDoubleMap(compound_head(page));
1238+ SetPageDoubleMap(head);
12371239 if (PageMlocked(page))
1238− clear_page_mlock(compound_head(page));
1240+ clear_page_mlock(head);
12391241 }
12401242 if (!atomic_inc_and_test(&page->_mapcount))
12411243 goto out;
mm/secretmem.c+5−4
@@ -18,6 +18,7 @@ · 17 unmodified lines
1818 #include <linux/secretmem.h>
1919 #include <linux/set_memory.h>
2020 #include <linux/sched/signal.h>
21+#include <linux/refcount.h>
2122
2223 #include <uapi/linux/magic.h>
2324
@@ -40,11 +41,11 @@ module_param_named(enable, secretmem_enable, bool, 0400); · 16 unmodified lines
4041 MODULE_PARM_DESC(secretmem_enable,
4142 "Enable secretmem and memfd_secret(2) system call");
4243
43−static atomic_t secretmem_users;
44+static refcount_t secretmem_users;
4445
4546 bool secretmem_active(void)
4647 {
47− return !!atomic_read(&secretmem_users);
48+ return !!refcount_read(&secretmem_users);
4849 }
4950
5051 static vm_fault_t secretmem_fault(struct vm_fault *vmf)
@@ -103,7 +104,7 @@ static const struct vm_operations_struct secretmem_vm_ops = { · 52 unmodified lines
103104
104105 static int secretmem_release(struct inode *inode, struct file *file)
105106 {
106− atomic_dec(&secretmem_users);
107+ refcount_dec(&secretmem_users);
107108 return 0;
108109 }
109110
@@ -217,7 +218,7 @@ SYSCALL_DEFINE1(memfd_secret, unsigned int, flags) · 107 unmodified lines
217218 file->f_flags |= O_LARGEFILE;
218219
219220 fd_install(fd, file);
220− atomic_inc(&secretmem_users);
221+ refcount_inc(&secretmem_users);
221222 return fd;
222223
223224 err_put_fd:
mm/vmalloc.c+17−5
@@ -44,6 +44,19 @@ · 43 unmodified lines
4444 #include "internal.h"
4545 #include "pgalloc-track.h"
4646
47+#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
48+static unsigned int __ro_after_init ioremap_max_page_shift = BITS_PER_LONG - 1;
49+
50+static int __init set_nohugeiomap(char *str)
51+{
52+ ioremap_max_page_shift = PAGE_SHIFT;
53+ return 0;
54+}
55+early_param("nohugeiomap", set_nohugeiomap);
56+#else /* CONFIG_HAVE_ARCH_HUGE_VMAP */
57+static const unsigned int ioremap_max_page_shift = PAGE_SHIFT;
58+#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
59+
4760 #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
4861 static bool __ro_after_init vmap_allow_huge = true;
4962
@@ -298,15 +311,14 @@ static int vmap_range_noflush(unsigned long addr, unsigned long end, · 248 unmodified lines
298311 return err;
299312 }
300313
301−int vmap_range(unsigned long addr, unsigned long end,
302− phys_addr_t phys_addr, pgprot_t prot,
303− unsigned int max_page_shift)
314+int ioremap_page_range(unsigned long addr, unsigned long end,
315+ phys_addr_t phys_addr, pgprot_t prot)
304316 {
305317 int err;
306318
307− err = vmap_range_noflush(addr, end, phys_addr, prot, max_page_shift);
319+ err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot),
320+ ioremap_max_page_shift);
308321 flush_cache_vmap(addr, end);
309−
310322 return err;
311323 }
312324
mm/workingset.c+1−1
@@ -249,7 +249,7 @@ void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages) · 248 unmodified lines
249249 * @target_memcg: the cgroup that is causing the reclaim
250250 * @page: the page being evicted
251251 *
252− * Returns a shadow entry to be stored in @page->mapping->i_pages in place
252+ * Return: a shadow entry to be stored in @page->mapping->i_pages in place
253253 * of the evicted @page so that a later refault can be detected.
254254 */
255255 void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg)
scripts/check_extable.sh+1−1
@@ -4,7 +4,7 @@ · 3 unmodified lines
44
55 obj=$1
66
7−file ${obj} | grep -q ELF || (echo "${obj} is not and ELF file." 1>&2 ; exit 0)
7+file ${obj} | grep -q ELF || (echo "${obj} is not an ELF file." 1>&2 ; exit 0)
88
99 # Bail out early if there isn't an __ex_table section in this object file.
1010 objdump -hj __ex_table ${obj} 2> /dev/null > /dev/null
scripts/checkpatch.pl+57−36
@@ -501,7 +501,7 @@ our $Binary = qr{(?i)0b[01]+$Int_type?}; · 500 unmodified lines
501501 our $Hex = qr{(?i)0x[0-9a-f]+$Int_type?};
502502 our $Int = qr{[0-9]+$Int_type?};
503503 our $Octal = qr{0[0-7]+$Int_type?};
504−our $String = qr{"[X\t]*"};
504+our $String = qr{(?:\b[Lu])?"[X\t]*"};
505505 our $Float_hex = qr{(?i)0x[0-9a-f]+p-?[0-9]+[fl]?};
506506 our $Float_dec = qr{(?i)(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:e-?[0-9]+)?[fl]?};
507507 our $Float_int = qr{(?i)[0-9]+e-?[0-9]+[fl]?};
@@ -1181,7 +1181,8 @@ sub git_commit_info { · 673 unmodified lines
11811181 # git log --format='%H %s' -1 $line |
11821182 # echo "commit $(cut -c 1-12,41-)"
11831183 # done
1184− } elsif ($lines[0] =~ /^fatal: ambiguous argument '$commit': unknown revision or path not in the working tree\./) {
1184+ } elsif ($lines[0] =~ /^fatal: ambiguous argument '$commit': unknown revision or path not in the working tree\./ ||
1185+ $lines[0] =~ /^fatal: bad object $commit/) {
11851186 $id = undef;
11861187 } else {
11871188 $id = substr($lines[0], 0, 12);
@@ -2587,6 +2588,8 @@ sub process { · 1399 unmodified lines
25872588 my $reported_maintainer_file = 0;
25882589 my $non_utf8_charset = 0;
25892590
2591+ my $last_git_commit_id_linenr = -1;
2592+
25902593 my $last_blank_line = 0;
25912594 my $last_coalesced_string_linenr = -1;
25922595
@@ -2909,10 +2912,10 @@ sub process { · 316 unmodified lines
29092912 my ($email_name, $email_comment, $email_address, $comment1) = parse_email($ctx);
29102913 my ($author_name, $author_comment, $author_address, $comment2) = parse_email($author);
29112914
2912− if ($email_address eq $author_address && $email_name eq $author_name) {
2915+ if (lc $email_address eq lc $author_address && $email_name eq $author_name) {
29132916 $author_sob = $ctx;
29142917 $authorsignoff = 2;
2915− } elsif ($email_address eq $author_address) {
2918+ } elsif (lc $email_address eq lc $author_address) {
29162919 $author_sob = $ctx;
29172920 $authorsignoff = 3;
29182921 } elsif ($email_name eq $author_name) {
@@ -3170,10 +3173,20 @@ sub process { · 251 unmodified lines
31703173 }
31713174
31723175 # Check for git id commit length and improperly formed commit descriptions
3173− if ($in_commit_log && !$commit_log_possible_stack_dump &&
3176+# A correctly formed commit description is:
3177+# commit <SHA-1 hash length 12+ chars> ("Complete commit subject")
3178+# with the commit subject '("' prefix and '")' suffix
3179+# This is a fairly compilicated block as it tests for what appears to be
3180+# bare SHA-1 hash with minimum length of 5. It also avoids several types of
3181+# possible SHA-1 matches.
3182+# A commit match can span multiple lines so this block attempts to find a
3183+# complete typical commit on a maximum of 3 lines
3184+ if ($perl_version_ok &&
3185+ $in_commit_log && !$commit_log_possible_stack_dump &&
31743186 $line !~ /^\s*(?:Link|Patchwork|http|https|BugLink|base-commit):/i &&
31753187 $line !~ /^This reverts commit [0-9a-f]{7,40}/ &&
3176− ($line =~ /\bcommit\s+[0-9a-f]{5,}\b/i ||
3188+ (($line =~ /\bcommit\s+[0-9a-f]{5,}\b/i ||
3189+ ($line =~ /\bcommit\s*$/i && defined($rawlines[$linenr]) && $rawlines[$linenr] =~ /^\s*[0-9a-f]{5,}\b/i)) ||
31773190 ($line =~ /(?:\s|^)[0-9a-f]{12,40}(?:[\s"'\(\[]|$)/i &&
31783191 $line !~ /[\<\[][0-9a-f]{12,40}[\>\]]/i &&
31793192 $line !~ /\bfixes:\s*[0-9a-f]{12,40}/i))) {
@@ -3183,49 +3196,56 @@ sub process { · 3 unmodified lines
31833196 my $long = 0;
31843197 my $case = 1;
31853198 my $space = 1;
3186− my $hasdesc = 0;
3187− my $hasparens = 0;
31883199 my $id = '0123456789ab';
31893200 my $orig_desc = "commit description";
31903201 my $description = "";
3202+ my $herectx = $herecurr;
3203+ my $has_parens = 0;
3204+ my $has_quotes = 0;
3205+
3206+ my $input = $line;
3207+ if ($line =~ /(?:\bcommit\s+[0-9a-f]{5,}|\bcommit\s*$)/i) {
3208+ for (my $n = 0; $n < 2; $n++) {
3209+ if ($input =~ /\bcommit\s+[0-9a-f]{5,}\s*($balanced_parens)/i) {
3210+ $orig_desc = $1;
3211+ $has_parens = 1;
3212+ # Always strip leading/trailing parens then double quotes if existing
3213+ $orig_desc = substr($orig_desc, 1, -1);
3214+ if ($orig_desc =~ /^".*"$/) {
3215+ $orig_desc = substr($orig_desc, 1, -1);
3216+ $has_quotes = 1;
3217+ }
3218+ last;
3219+ }
3220+ last if ($#lines < $linenr + $n);
3221+ $input .= " " . trim($rawlines[$linenr + $n]);
3222+ $herectx .= "$rawlines[$linenr + $n]\n";
3223+ }
3224+ $herectx = $herecurr if (!$has_parens);
3225+ }
31913226
3192− if ($line =~ /\b(c)ommit\s+([0-9a-f]{5,})\b/i) {
3227+ if ($input =~ /\b(c)ommit\s+([0-9a-f]{5,})\b/i) {
31933228 $init_char = $1;
31943229 $orig_commit = lc($2);
3195− } elsif ($line =~ /\b([0-9a-f]{12,40})\b/i) {
3230+ $short = 0 if ($input =~ /\bcommit\s+[0-9a-f]{12,40}/i);
3231+ $long = 1 if ($input =~ /\bcommit\s+[0-9a-f]{41,}/i);
3232+ $space = 0 if ($input =~ /\bcommit [0-9a-f]/i);
3233+ $case = 0 if ($input =~ /\b[Cc]ommit\s+[0-9a-f]{5,40}[^A-F]/);
3234+ } elsif ($input =~ /\b([0-9a-f]{12,40})\b/i) {
31963235 $orig_commit = lc($1);
31973236 }
31983237
3199− $short = 0 if ($line =~ /\bcommit\s+[0-9a-f]{12,40}/i);
3200− $long = 1 if ($line =~ /\bcommit\s+[0-9a-f]{41,}/i);
3201− $space = 0 if ($line =~ /\bcommit [0-9a-f]/i);
3202− $case = 0 if ($line =~ /\b[Cc]ommit\s+[0-9a-f]{5,40}[^A-F]/);
3203− if ($line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("([^"]+)"\)/i) {
3204− $orig_desc = $1;
3205− $hasparens = 1;
3206− } elsif ($line =~ /\bcommit\s+[0-9a-f]{5,}\s*$/i &&
3207− defined $rawlines[$linenr] &&
3208− $rawlines[$linenr] =~ /^\s*\("([^"]+)"\)/) {
3209− $orig_desc = $1;
3210− $hasparens = 1;
3211− } elsif ($line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("[^"]+$/i &&
3212− defined $rawlines[$linenr] &&
3213− $rawlines[$linenr] =~ /^\s*[^"]+"\)/) {
3214− $line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("([^"]+)$/i;
3215− $orig_desc = $1;
3216− $rawlines[$linenr] =~ /^\s*([^"]+)"\)/;
3217− $orig_desc .= " " . $1;
3218− $hasparens = 1;
3219− }
3220−
32213238 ($id, $description) = git_commit_info($orig_commit,
32223239 $id, $orig_desc);
32233240
32243241 if (defined($id) &&
3225− ($short || $long || $space || $case || ($orig_desc ne $description) || !$hasparens)) {
3242+ ($short || $long || $space || $case || ($orig_desc ne $description) || !$has_quotes) &&
3243+ $last_git_commit_id_linenr != $linenr - 1) {
32263244 ERROR("GIT_COMMIT_ID",
3227− "Please use git commit description style 'commit <12+ chars of sha1> (\"<title line>\")' - ie: '${init_char}ommit $id (\"$description\")'\n" . $herecurr);
3245+ "Please use git commit description style 'commit <12+ chars of sha1> (\"<title line>\")' - ie: '${init_char}ommit $id (\"$description\")'\n" . $herectx);
32283246 }
3247+ #don't report the next line if this line ends in commit and the sha1 hash is the next line
3248+ $last_git_commit_id_linenr = $linenr if ($line =~ /\bcommit\s*$/i);
32293249 }
32303250
32313251 # Check for added, moved or deleted files
@@ -6132,7 +6152,8 @@ sub process { · 2900 unmodified lines
61326152 }
61336153
61346154 # concatenated string without spaces between elements
6135− if ($line =~ /$String[A-Za-z0-9_]/ || $line =~ /[A-Za-z0-9_]$String/) {
6155+ if ($line =~ /$String[A-Z_]/ ||
6156+ ($line =~ /([A-Za-z0-9_]+)$String/ && $1 !~ /^[Lu]$/)) {
61366157 if (CHK("CONCATENATED_STRING",
61376158 "Concatenated strings should use spaces between elements\n" . $herecurr) &&
61386159 $fix) {
@@ -6145,7 +6166,7 @@ sub process { · 6 unmodified lines
61456166 }
61466167
61476168 # uncoalesced string fragments
6148− if ($line =~ /$String\s*"/) {
6169+ if ($line =~ /$String\s*[Lu]?"/) {
61496170 if (WARN("STRING_FRAGMENTS",
61506171 "Consecutive strings are generally better as a single string\n" . $herecurr) &&
61516172 $fix) {
tools/include/linux/bitmap.h+2−2
@@ -111,10 +111,10 @@ static inline int test_and_clear_bit(int nr, unsigned long *addr) · 110 unmodified lines
111111 }
112112
113113 /**
114− * bitmap_alloc - Allocate bitmap
114+ * bitmap_zalloc - Allocate bitmap
115115 * @nbits: Number of bits
116116 */
117−static inline unsigned long *bitmap_alloc(int nbits)
117+static inline unsigned long *bitmap_zalloc(int nbits)
118118 {
119119 return calloc(1, BITS_TO_LONGS(nbits) * sizeof(unsigned long));
120120 }
tools/perf/bench/find-bit-bench.c+1−1
@@ -54,7 +54,7 @@ static bool asm_test_bit(long nr, const unsigned long *addr) · 53 unmodified lines
5454
5555 static int do_for_each_set_bit(unsigned int num_bits)
5656 {
57− unsigned long *to_test = bitmap_alloc(num_bits);
57+ unsigned long *to_test = bitmap_zalloc(num_bits);
5858 struct timeval start, end, diff;
5959 u64 runtime_us;
6060 struct stats fb_time_stats, tb_time_stats;
tools/perf/builtin-c2c.c+3−3
@@ -139,11 +139,11 @@ static void *c2c_he_zalloc(size_t size) · 138 unmodified lines
139139 if (!c2c_he)
140140 return NULL;
141141
142− c2c_he->cpuset = bitmap_alloc(c2c.cpus_cnt);
142+ c2c_he->cpuset = bitmap_zalloc(c2c.cpus_cnt);
143143 if (!c2c_he->cpuset)
144144 return NULL;
145145
146− c2c_he->nodeset = bitmap_alloc(c2c.nodes_cnt);
146+ c2c_he->nodeset = bitmap_zalloc(c2c.nodes_cnt);
147147 if (!c2c_he->nodeset)
148148 return NULL;
149149
@@ -2047,7 +2047,7 @@ static int setup_nodes(struct perf_session *session) · 1897 unmodified lines
20472047 struct perf_cpu_map *map = n[node].map;
20482048 unsigned long *set;
20492049
2050− set = bitmap_alloc(c2c.cpus_cnt);
2050+ set = bitmap_zalloc(c2c.cpus_cnt);
20512051 if (!set)
20522052 return -ENOMEM;
20532053
tools/perf/builtin-record.c+1−1
@@ -2757,7 +2757,7 @@ int cmd_record(int argc, const char **argv) · 2756 unmodified lines
27572757
27582758 if (rec->opts.affinity != PERF_AFFINITY_SYS) {
27592759 rec->affinity_mask.nbits = cpu__max_cpu();
2760− rec->affinity_mask.bits = bitmap_alloc(rec->affinity_mask.nbits);
2760+ rec->affinity_mask.bits = bitmap_zalloc(rec->affinity_mask.nbits);
27612761 if (!rec->affinity_mask.bits) {
27622762 pr_err("Failed to allocate thread mask for %zd cpus\n", rec->affinity_mask.nbits);
27632763 err = -ENOMEM;
tools/perf/tests/bitmap.c+1−1
@@ -14,7 +14,7 @@ static unsigned long *get_bitmap(const char *str, int nbits) · 13 unmodified lines
1414 unsigned long *bm = NULL;
1515 int i;
1616
17− bm = bitmap_alloc(nbits);
17+ bm = bitmap_zalloc(nbits);
1818
1919 if (map && bm) {
2020 for (i = 0; i < map->nr; i++)
tools/perf/tests/mem2node.c+1−1
@@ -27,7 +27,7 @@ static unsigned long *get_bitmap(const char *str, int nbits) · 26 unmodified lines
2727 unsigned long *bm = NULL;
2828 int i;
2929
30− bm = bitmap_alloc(nbits);
30+ bm = bitmap_zalloc(nbits);
3131
3232 if (map && bm) {
3333 for (i = 0; i < map->nr; i++) {
tools/perf/util/affinity.c+2−2
@@ -25,11 +25,11 @@ int affinity__setup(struct affinity *a) · 24 unmodified lines
2525 {
2626 int cpu_set_size = get_cpu_set_size();
2727
28− a->orig_cpus = bitmap_alloc(cpu_set_size * 8);
28+ a->orig_cpus = bitmap_zalloc(cpu_set_size * 8);
2929 if (!a->orig_cpus)
3030 return -1;
3131 sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
32− a->sched_cpus = bitmap_alloc(cpu_set_size * 8);
32+ a->sched_cpus = bitmap_zalloc(cpu_set_size * 8);
3333 if (!a->sched_cpus) {
3434 zfree(&a->orig_cpus);
3535 return -1;
tools/perf/util/header.c+2−2
@@ -278,7 +278,7 @@ static int do_read_bitmap(struct feat_fd *ff, unsigned long **pset, u64 *psize) · 277 unmodified lines
278278 if (ret)
279279 return ret;
280280
281− set = bitmap_alloc(size);
281+ set = bitmap_zalloc(size);
282282 if (!set)
283283 return -ENOMEM;
284284
@@ -1294,7 +1294,7 @@ static int memory_node__read(struct memory_node *n, unsigned long idx) · 1009 unmodified lines
12941294
12951295 size++;
12961296
1297− n->set = bitmap_alloc(size);
1297+ n->set = bitmap_zalloc(size);
12981298 if (!n->set) {
12991299 closedir(dir);
13001300 return -ENOMEM;
tools/perf/util/metricgroup.c+1−1
@@ -313,7 +313,7 @@ static int metricgroup__setup_events(struct list_head *groups, · 312 unmodified lines
313313 struct evsel *evsel, *tmp;
314314 unsigned long *evlist_used;
315315
316− evlist_used = bitmap_alloc(perf_evlist->core.nr_entries);
316+ evlist_used = bitmap_zalloc(perf_evlist->core.nr_entries);
317317 if (!evlist_used)
318318 return -ENOMEM;
319319
tools/perf/util/mmap.c+2−2
@@ -106,7 +106,7 @@ static int perf_mmap__aio_bind(struct mmap *map, int idx, int cpu, int affinity) · 105 unmodified lines
106106 data = map->aio.data[idx];
107107 mmap_len = mmap__mmap_len(map);
108108 node_index = cpu__get_node(cpu);
109− node_mask = bitmap_alloc(node_index + 1);
109+ node_mask = bitmap_zalloc(node_index + 1);
110110 if (!node_mask) {
111111 pr_err("Failed to allocate node mask for mbind: error %m\n");
112112 return -1;
@@ -258,7 +258,7 @@ static void build_node_mask(int node, struct mmap_cpu_mask *mask) · 145 unmodified lines
258258 static int perf_mmap__setup_affinity_mask(struct mmap *map, struct mmap_params *mp)
259259 {
260260 map->affinity_mask.nbits = cpu__max_cpu();
261− map->affinity_mask.bits = bitmap_alloc(map->affinity_mask.nbits);
261+ map->affinity_mask.bits = bitmap_zalloc(map->affinity_mask.nbits);
262262 if (!map->affinity_mask.bits)
263263 return -1;
264264
tools/testing/selftests/damon/Makefile+7−0
@@ -0,0 +1,7 @@
1+# SPDX-License-Identifier: GPL-2.0
2+# Makefile for damon selftests
3+
4+TEST_FILES = _chk_dependency.sh
5+TEST_PROGS = debugfs_attrs.sh
6+
7+include ../lib.mk
tools/testing/selftests/damon/_chk_dependency.sh+28−0
@@ -0,0 +1,28 @@
1+#!/bin/bash
2+# SPDX-License-Identifier: GPL-2.0
3+
4+# Kselftest framework requirement - SKIP code is 4.
5+ksft_skip=4
6+
7+DBGFS=/sys/kernel/debug/damon
8+
9+if [ $EUID -ne 0 ];
10+then
11+ echo "Run as root"
12+ exit $ksft_skip
13+fi
14+
15+if [ ! -d "$DBGFS" ]
16+then
17+ echo "$DBGFS not found"
18+ exit $ksft_skip
19+fi
20+
21+for f in attrs target_ids monitor_on
22+do
23+ if [ ! -f "$DBGFS/$f" ]
24+ then
25+ echo "$f not found"
26+ exit 1
27+ fi
28+done
tools/testing/selftests/damon/debugfs_attrs.sh+75−0
@@ -0,0 +1,75 @@
1+#!/bin/bash
2+# SPDX-License-Identifier: GPL-2.0
3+
4+test_write_result() {
5+ file=$1
6+ content=$2
7+ orig_content=$3
8+ expect_reason=$4
9+ expected=$5
10+
11+ echo "$content" > "$file"
12+ if [ $? -ne "$expected" ]
13+ then
14+ echo "writing $content to $file doesn't return $expected"
15+ echo "expected because: $expect_reason"
16+ echo "$orig_content" > "$file"
17+ exit 1
18+ fi
19+}
20+
21+test_write_succ() {
22+ test_write_result "$1" "$2" "$3" "$4" 0
23+}
24+
25+test_write_fail() {
26+ test_write_result "$1" "$2" "$3" "$4" 1
27+}
28+
29+test_content() {
30+ file=$1
31+ orig_content=$2
32+ expected=$3
33+ expect_reason=$4
34+
35+ content=$(cat "$file")
36+ if [ "$content" != "$expected" ]
37+ then
38+ echo "reading $file expected $expected but $content"
39+ echo "expected because: $expect_reason"
40+ echo "$orig_content" > "$file"
41+ exit 1
42+ fi
43+}
44+
45+source ./_chk_dependency.sh
46+
47+# Test attrs file
48+# ===============
49+
50+file="$DBGFS/attrs"
51+orig_content=$(cat "$file")
52+
53+test_write_succ "$file" "1 2 3 4 5" "$orig_content" "valid input"
54+test_write_fail "$file" "1 2 3 4" "$orig_content" "no enough fields"
55+test_write_fail "$file" "1 2 3 5 4" "$orig_content" \
56+ "min_nr_regions > max_nr_regions"
57+test_content "$file" "$orig_content" "1 2 3 4 5" "successfully written"
58+echo "$orig_content" > "$file"
59+
60+# Test target_ids file
61+# ====================
62+
63+file="$DBGFS/target_ids"
64+orig_content=$(cat "$file")
65+
66+test_write_succ "$file" "1 2 3 4" "$orig_content" "valid input"
67+test_write_succ "$file" "1 2 abc 4" "$orig_content" "still valid input"
68+test_content "$file" "$orig_content" "1 2" "non-integer was there"
69+test_write_succ "$file" "abc 2 3" "$orig_content" "the file allows wrong input"
70+test_content "$file" "$orig_content" "" "wrong input written"
71+test_write_succ "$file" "" "$orig_content" "empty input"
72+test_content "$file" "$orig_content" "" "empty input written"
73+echo "$orig_content" > "$file"
74+
75+echo "PASS"
tools/testing/selftests/kvm/dirty_log_perf_test.c+1−1
@@ -171,7 +171,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 170 unmodified lines
171171 guest_num_pages = (nr_vcpus * guest_percpu_mem_size) >> vm_get_page_shift(vm);
172172 guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages);
173173 host_num_pages = vm_num_host_pages(mode, guest_num_pages);
174− bmap = bitmap_alloc(host_num_pages);
174+ bmap = bitmap_zalloc(host_num_pages);
175175
176176 if (dirty_log_manual_caps) {
177177 cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2;
tools/testing/selftests/kvm/dirty_log_test.c+2−2
@@ -749,8 +749,8 @@ static void run_test(enum vm_guest_mode mode, void *arg) · 748 unmodified lines
749749
750750 pr_info("guest physical test memory offset: 0x%lx\n", guest_test_phys_mem);
751751
752− bmap = bitmap_alloc(host_num_pages);
753− host_bmap_track = bitmap_alloc(host_num_pages);
752+ bmap = bitmap_zalloc(host_num_pages);
753+ host_bmap_track = bitmap_zalloc(host_num_pages);
754754
755755 /* Add an extra memory slot for testing dirty logging */
756756 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c+1−1
@@ -111,7 +111,7 @@ int main(int argc, char *argv[]) · 110 unmodified lines
111111 nested_map(vmx, vm, NESTED_TEST_MEM1, GUEST_TEST_MEM, 4096);
112112 nested_map(vmx, vm, NESTED_TEST_MEM2, GUEST_TEST_MEM, 4096);
113113
114− bmap = bitmap_alloc(TEST_MEM_PAGES);
114+ bmap = bitmap_zalloc(TEST_MEM_PAGES);
115115 host_test_mem = addr_gpa2hva(vm, GUEST_TEST_MEM);
116116
117117 while (!done) {
tools/testing/selftests/memfd/memfd_test.c+1−1
@@ -56,7 +56,7 @@ static int mfd_assert_new(const char *name, loff_t sz, unsigned int flags) · 55 unmodified lines
5656
5757 static int mfd_assert_reopen_fd(int fd_in)
5858 {
59− int r, fd;
59+ int fd;
6060 char path[100];
6161
6262 sprintf(path, "/proc/self/fd/%d", fd_in);