Merge tag 'mm-stable-2022-10-08' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

Pull MM updates from Andrew Morton: - Yu Zhao's Multi-Gen LRU patches are here. They've been under test in linux-next for a couple of months without, to my knowledge, any negative reports (or any positive ones, come to that). - Also the Maple Tree from Liam Howlett. An overlapping range-based tree for vmas. It it apparently slightly more efficient in its own right, but is mainly targeted at enabling work to reduce mmap_lock contention. Liam has identified a number of other tree users in the kernel which could be beneficially onverted to mapletrees. Yu Zhao has identified a hard-to-hit but "easy to fix" lockdep splat at [1]. This has yet to be addressed due to Liam's unfortunately timed vacation. He is now back and we'll get this fixed up. - Dmitry Vyukov introduces KMSAN: the Kernel Memory Sanitizer. It uses clang-generated instrumentation to detect used-unintialized bugs down to the single bit level. KMSAN keeps finding bugs. New ones, as well as the legacy ones. - Yang Shi adds a userspace mechanism (madvise) to induce a collapse of memory into THPs. - Zach O'Keefe has expanded Yang Shi's madvise(MADV_COLLAPSE) to support file/shmem-backed pages. - userfaultfd updates from Axel Rasmussen - zsmalloc cleanups from Alexey Romanov - cleanups from Miaohe Lin: vmscan, hugetlb_cgroup, hugetlb and memory-failure - Huang Ying adds enhancements to NUMA balancing memory tiering mode's page promotion, with a new way of detecting hot pages. - memcg updates from Shakeel Butt: charging optimizations and reduced memory consumption. - memcg cleanups from Kairui Song. - memcg fixes and cleanups from Johannes Weiner. - Vishal Moola provides more folio conversions - Zhang Yi removed ll_rw_block() :( - migration enhancements from Peter Xu - migration error-path bugfixes from Huang Ying - Aneesh Kumar added ability for a device driver to alter the memory tiering promotion paths. For optimizations by PMEM drivers, DRM drivers, etc. - vma merging improvements from Jakub Matěn. - NUMA hinting cleanups from David Hildenbrand. - xu xin added aditional userspace visibility into KSM merging activity. - THP & KSM code consolidation from Qi Zheng. - more folio work from Matthew Wilcox. - KASAN updates from Andrey Konovalov. - DAMON cleanups from Kaixu Xia. - DAMON work from SeongJae Park: fixes, cleanups. - hugetlb sysfs cleanups from Muchun Song. - Mike Kravetz fixes locking issues in hugetlbfs and in hugetlb core. Link: https://lkml.kernel.org/r/CAOUHufZabH85CeUN-MEMgL8gJGzJEWUrkiM58JkTbBhh-jew0Q@mail.gmail.com [1] * tag 'mm-stable-2022-10-08' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (555 commits) hugetlb: allocate vma lock for all sharable vmas hugetlb: take hugetlb vma_lock when clearing vma_lock->vma pointer hugetlb: fix vma lock handling during split vma and range unmapping mglru: mm/vmscan.c: fix imprecise comments mm/mglru: don't sync disk for each aging cycle mm: memcontrol: drop dead CONFIG_MEMCG_SWAP config symbol mm: memcontrol: use do_memsw_account() in a few more places mm: memcontrol: deprecate swapaccounting=0 mode mm: memcontrol: don't allocate cgroup swap arrays when memcg is disabled mm/secretmem: remove reduntant return value mm/hugetlb: add available_huge_pages() func mm: remove unused inline functions from include/linux/mm_inline.h selftests/vm: add selftest for MADV_COLLAPSE of uffd-minor memory selftests/vm: add file/shmem MADV_COLLAPSE selftest for cleared pmd selftests/vm: add thp collapse shmem testing selftests/vm: add thp collapse file and tmpfs testing selftests/vm: modularize thp collapse memory operations selftests/vm: dedup THP helpers mm/khugepaged: add tracepoint to hpage_collapse_scan_file() mm/madvise: add file and shmem support to MADV_COLLAPSE ...

Linus Torvalds committed 4 years ago

Commit 27bc50f2 parents 70442fc bbff39c409 files · +65792 −8034Browse files

Showing the changes to kernel (21 of 409 files). Show all

Unified · Split

21 files

kernel/Makefile +1 −0
kernel/acct.c +5 −6
kernel/bounds.c +7 −0
kernel/bpf/core.c +1 −1
kernel/bpf/task_iter.c +5 −5
kernel/cgroup/cgroup-internal.h +0 −1
kernel/debug/debug_core.c +0 −12
kernel/delayacct.c +11 −2
kernel/dma/mapping.c +7 −3
kernel/entry/common.c +5 −0
kernel/events/core.c +2 −1
kernel/events/uprobes.c +22 −17
kernel/exit.c +3 −0
kernel/fork.c +43 −36
kernel/kcov.c +7 −0
kernel/locking/Makefile +2 −1
kernel/sched/core.c +15 −0
kernel/sched/debug.c +1 −0
kernel/sched/fair.c +172 −3
kernel/sched/sched.h +1 −0
kernel/sysctl.c +8 −0

kernel/Makefile+1−0

@@ -38,6 +38,7 @@ KCOV_INSTRUMENT_kcov.o := n · 37 unmodified lines
38	38	KASAN_SANITIZE_kcov.o := n
39	39	KCSAN_SANITIZE_kcov.o := n
40	40	UBSAN_SANITIZE_kcov.o := n
	41	+KMSAN_SANITIZE_kcov.o := n
41	42	CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector
42	43
43	44	# Don't instrument error handlers

kernel/acct.c+5−6

@@ -555,15 +555,14 @@ void acct_collect(long exitcode, int group_dead) · 554 unmodified lines
555	555	unsigned long vsize = 0;
556	556
557	557	if (group_dead && current->mm) {
	558	+ struct mm_struct *mm = current->mm;
	559	+ VMA_ITERATOR(vmi, mm, 0);
558	560	struct vm_area_struct *vma;
559	561
560		− mmap_read_lock(current->mm);
561		− vma = current->mm->mmap;
562		− while (vma) {
	562	+ mmap_read_lock(mm);
	563	+ for_each_vma(vmi, vma)
563	564	vsize += vma->vm_end - vma->vm_start;
564		− vma = vma->vm_next;
565		− }
566		− mmap_read_unlock(current->mm);
	565	+ mmap_read_unlock(mm);
567	566	}
568	567
569	568	spin_lock_irq(&current->sighand->siglock);

kernel/bounds.c+7−0

@@ -22,6 +22,13 @@ int main(void) · 21 unmodified lines
22	22	DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
23	23	#endif
24	24	DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
	25	+#ifdef CONFIG_LRU_GEN
	26	+ DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1));
	27	+ DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2);
	28	+#else
	29	+ DEFINE(LRU_GEN_WIDTH, 0);
	30	+ DEFINE(__LRU_REFS_WIDTH, 0);
	31	+#endif
25	32	/* End of constants */
26	33
27	34	return 0;

kernel/bpf/core.c+1−1

@@ -2007,7 +2007,7 @@ out: · 2006 unmodified lines
2007	2007	static unsigned int PROG_NAME(stack_size)(const void ctx, const struct bpf_insn insn) \
2008	2008	{ \
2009	2009	u64 stack[stack_size / sizeof(u64)]; \
2010		− u64 regs[MAX_BPF_EXT_REG]; \
	2010	+ u64 regs[MAX_BPF_EXT_REG] = {}; \
2011	2011	\
2012	2012	FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
2013	2013	ARG1 = (u64) (unsigned long) ctx; \

kernel/bpf/task_iter.c+5−5

@@ -445,8 +445,8 @@ struct bpf_iter_seq_task_vma_info { · 444 unmodified lines
445	445	};
446	446
447	447	enum bpf_task_vma_iter_find_op {
448		− task_vma_iter_first_vma, /* use mm->mmap */
449		− task_vma_iter_next_vma, /* use curr_vma->vm_next */
	448	+ task_vma_iter_first_vma, /* use find_vma() with addr 0 */
	449	+ task_vma_iter_next_vma, /* use vma_next() with curr_vma */
450	450	task_vma_iter_find_vma, /* use find_vma() to find next vma */
451	451	};
452	452
@@ -544,10 +544,10 @@ again: · 91 unmodified lines
544	544
545	545	switch (op) {
546	546	case task_vma_iter_first_vma:
547		− curr_vma = curr_task->mm->mmap;
	547	+ curr_vma = find_vma(curr_task->mm, 0);
548	548	break;
549	549	case task_vma_iter_next_vma:
550		− curr_vma = curr_vma->vm_next;
	550	+ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end);
551	551	break;
552	552	case task_vma_iter_find_vma:
553	553	/* We dropped mmap_lock so it is necessary to use find_vma
@@ -561,7 +561,7 @@ again: · 7 unmodified lines
561	561	if (curr_vma &&
562	562	curr_vma->vm_start == info->prev_vm_start &&
563	563	curr_vma->vm_end == info->prev_vm_end)
564		− curr_vma = curr_vma->vm_next;
	564	+ curr_vma = find_vma(curr_task->mm, curr_vma->vm_end);
565	565	break;
566	566	}
567	567	if (!curr_vma) {

kernel/cgroup/cgroup-internal.h+0−1

@@ -164,7 +164,6 @@ struct cgroup_mgctx { · 163 unmodified lines
164	164	#define DEFINE_CGROUP_MGCTX(name) \
165	165	struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name)
166	166
167		−extern struct mutex cgroup_mutex;
168	167	extern spinlock_t css_set_lock;
169	168	extern struct cgroup_subsys *cgroup_subsys[];
170	169	extern struct list_head cgroup_roots;

kernel/debug/debug_core.c+0−12

@@ -50,7 +50,6 @@ · 49 unmodified lines
50	50	#include <linux/pid.h>
51	51	#include <linux/smp.h>
52	52	#include <linux/mm.h>
53		−#include <linux/vmacache.h>
54	53	#include <linux/rcupdate.h>
55	54	#include <linux/irq.h>
56	55	#include <linux/security.h>
@@ -283,17 +282,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr) · 226 unmodified lines
283	282	if (!CACHE_FLUSH_IS_SAFE)
284	283	return;
285	284
286		− if (current->mm) {
287		− int i;
288		−
289		− for (i = 0; i < VMACACHE_SIZE; i++) {
290		− if (!current->vmacache.vmas[i])
291		− continue;
292		− flush_cache_range(current->vmacache.vmas[i],
293		− addr, addr + BREAK_INSTR_SIZE);
294		− }
295		− }
296		−
297	285	/* Force flush instruction cache if it was outside the mm */
298	286	flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
299	287	}

kernel/delayacct.c+11−2

@@ -214,13 +214,22 @@ void __delayacct_freepages_end(void) · 213 unmodified lines
214	214	&current->delays->freepages_count);
215	215	}
216	216
217		−void __delayacct_thrashing_start(void)
	217	+void __delayacct_thrashing_start(bool *in_thrashing)
218	218	{
	219	+ *in_thrashing = !!current->in_thrashing;
	220	+ if (*in_thrashing)
	221	+ return;
	222	+
	223	+ current->in_thrashing = 1;
219	224	current->delays->thrashing_start = local_clock();
220	225	}
221	226
222		−void __delayacct_thrashing_end(void)
	227	+void __delayacct_thrashing_end(bool *in_thrashing)
223	228	{
	229	+ if (*in_thrashing)
	230	+ return;
	231	+
	232	+ current->in_thrashing = 0;
224	233	delayacct_end(&current->delays->lock,
225	234	&current->delays->thrashing_start,
226	235	&current->delays->thrashing_delay,

kernel/dma/mapping.c+7−3

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/dma-map-ops.h>
11	11	#include <linux/export.h>
12	12	#include <linux/gfp.h>
	13	+#include <linux/kmsan.h>
13	14	#include <linux/of_device.h>
14	15	#include <linux/slab.h>
15	16	#include <linux/vmalloc.h>
@@ -156,6 +157,7 @@ dma_addr_t dma_map_page_attrs(struct device dev, struct page page, · 140 unmodified lines
156	157	addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
157	158	else
158	159	addr = ops->map_page(dev, page, offset, size, dir, attrs);
	160	+ kmsan_handle_dma(page, offset, size, dir);
159	161	debug_dma_map_page(dev, page, offset, size, dir, addr, attrs);
160	162
161	163	return addr;
@@ -194,11 +196,13 @@ static int __dma_map_sg_attrs(struct device dev, struct scatterlist sg, · 32 unmodified lines
194	196	else
195	197	ents = ops->map_sg(dev, sg, nents, dir, attrs);
196	198
197		− if (ents > 0)
	199	+ if (ents > 0) {
	200	+ kmsan_handle_dma_sg(sg, nents, dir);
198	201	debug_dma_map_sg(dev, sg, nents, ents, dir, attrs);
199		− else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
200		− ents != -EIO && ents != -EREMOTEIO))
	202	+ } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
	203	+ ents != -EIO && ents != -EREMOTEIO)) {
201	204	return -EIO;
	205	+ }
202	206
203	207	return ents;
204	208	}

kernel/entry/common.c+5−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	#include <linux/resume_user_mode.h>
6	6	#include <linux/highmem.h>
7	7	#include <linux/jump_label.h>
	8	+#include <linux/kmsan.h>
8	9	#include <linux/livepatch.h>
9	10	#include <linux/audit.h>
10	11	#include <linux/tick.h>
@@ -24,6 +25,7 @@ static __always_inline void __enter_from_user_mode(struct pt_regs *regs) · 13 unmodified lines
24	25	user_exit_irqoff();
25	26
26	27	instrumentation_begin();
	28	+ kmsan_unpoison_entry_regs(regs);
27	29	trace_hardirqs_off_finish();
28	30	instrumentation_end();
29	31	}
@@ -352,6 +354,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) · 322 unmodified lines
352	354	lockdep_hardirqs_off(CALLER_ADDR0);
353	355	ct_irq_enter();
354	356	instrumentation_begin();
	357	+ kmsan_unpoison_entry_regs(regs);
355	358	trace_hardirqs_off_finish();
356	359	instrumentation_end();
357	360
@@ -367,6 +370,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) · 9 unmodified lines
367	370	*/
368	371	lockdep_hardirqs_off(CALLER_ADDR0);
369	372	instrumentation_begin();
	373	+ kmsan_unpoison_entry_regs(regs);
370	374	rcu_irq_enter_check_tick();
371	375	trace_hardirqs_off_finish();
372	376	instrumentation_end();
@@ -452,6 +456,7 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) · 79 unmodified lines
452	456	ct_nmi_enter();
453	457
454	458	instrumentation_begin();
	459	+ kmsan_unpoison_entry_regs(regs);
455	460	trace_hardirqs_off_finish();
456	461	ftrace_nmi_enter();
457	462	instrumentation_end();

kernel/events/core.c+2−1

@@ -10270,8 +10270,9 @@ static void perf_addr_filter_apply(struct perf_addr_filter *filter, · 10269 unmodified lines
10270	10270	struct perf_addr_filter_range *fr)
10271	10271	{
10272	10272	struct vm_area_struct *vma;
	10273	+ VMA_ITERATOR(vmi, mm, 0);
10273	10274
10274		− for (vma = mm->mmap; vma; vma = vma->vm_next) {
	10275	+ for_each_vma(vmi, vma) {
10275	10276	if (!vma->vm_file)
10276	10277	continue;
10277	10278

kernel/events/uprobes.c+22−17

@@ -19,7 +19,7 @@ · 18 unmodified lines
19	19	#include <linux/export.h>
20	20	#include <linux/rmap.h> /* anon_vma_prepare */
21	21	#include <linux/mmu_notifier.h> /* set_pte_at_notify */
22		−#include <linux/swap.h> /* try_to_free_swap */
	22	+#include <linux/swap.h> /* folio_free_swap */
23	23	#include <linux/ptrace.h> /* user_enable_single_step */
24	24	#include <linux/kdebug.h> /* notifier mechanism */
25	25	#include "../../mm/internal.h" /* munlock_vma_page */
@@ -154,8 +154,10 @@ static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) · 128 unmodified lines
154	154	static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
155	155	struct page old_page, struct page new_page)
156	156	{
	157	+ struct folio *old_folio = page_folio(old_page);
	158	+ struct folio *new_folio;
157	159	struct mm_struct *mm = vma->vm_mm;
158		− DEFINE_FOLIO_VMA_WALK(pvmw, page_folio(old_page), vma, addr, 0);
	160	+ DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0);
159	161	int err;
160	162	struct mmu_notifier_range range;
161	163
@@ -163,14 +165,14 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 1 unmodified line
163	165	addr + PAGE_SIZE);
164	166
165	167	if (new_page) {
166		− err = mem_cgroup_charge(page_folio(new_page), vma->vm_mm,
167		− GFP_KERNEL);
	168	+ new_folio = page_folio(new_page);
	169	+ err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL);
168	170	if (err)
169	171	return err;
170	172	}
171	173
172		− /* For try_to_free_swap() below */
173		− lock_page(old_page);
	174	+ /* For folio_free_swap() below */
	175	+ folio_lock(old_folio);
174	176
175	177	mmu_notifier_invalidate_range_start(&range);
176	178	err = -EAGAIN;
@@ -179,14 +181,14 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 2 unmodified lines
179	181	VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
180	182
181	183	if (new_page) {
182		− get_page(new_page);
	184	+ folio_get(new_folio);
183	185	page_add_new_anon_rmap(new_page, vma, addr);
184		− lru_cache_add_inactive_or_unevictable(new_page, vma);
	186	+ folio_add_lru_vma(new_folio, vma);
185	187	} else
186	188	/* no new page, just dec_mm_counter for old_page */
187	189	dec_mm_counter(mm, MM_ANONPAGES);
188	190
189		− if (!PageAnon(old_page)) {
	191	+ if (!folio_test_anon(old_folio)) {
190	192	dec_mm_counter(mm, mm_counter_file(old_page));
191	193	inc_mm_counter(mm, MM_ANONPAGES);
192	194	}
@@ -198,15 +200,15 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, · 5 unmodified lines
198	200	mk_pte(new_page, vma->vm_page_prot));
199	201
200	202	page_remove_rmap(old_page, vma, false);
201		− if (!page_mapped(old_page))
202		− try_to_free_swap(old_page);
	203	+ if (!folio_mapped(old_folio))
	204	+ folio_free_swap(old_folio);
203	205	page_vma_mapped_walk_done(&pvmw);
204		− put_page(old_page);
	206	+ folio_put(old_folio);
205	207
206	208	err = 0;
207	209	unlock:
208	210	mmu_notifier_invalidate_range_end(&range);
209		− unlock_page(old_page);
	211	+ folio_unlock(old_folio);
210	212	return err;
211	213	}
212	214
@@ -349,9 +351,10 @@ static bool valid_ref_ctr_vma(struct uprobe *uprobe, · 136 unmodified lines
349	351	static struct vm_area_struct *
350	352	find_ref_ctr_vma(struct uprobe uprobe, struct mm_struct mm)
351	353	{
	354	+ VMA_ITERATOR(vmi, mm, 0);
352	355	struct vm_area_struct *tmp;
353	356
354		− for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
	357	+ for_each_vma(vmi, tmp)
355	358	if (valid_ref_ctr_vma(uprobe, tmp))
356	359	return tmp;
357	360
@@ -552,7 +555,7 @@ put_old: · 194 unmodified lines
552	555
553	556	/* try collapse pmd for compound page */
554	557	if (!ret && orig_page_huge)
555		− collapse_pte_mapped_thp(mm, vaddr);
	558	+ collapse_pte_mapped_thp(mm, vaddr, false);
556	559
557	560	return ret;
558	561	}
@@ -1231,11 +1234,12 @@ int uprobe_apply(struct inode *inode, loff_t offset, · 672 unmodified lines
1231	1234
1232	1235	static int unapply_uprobe(struct uprobe uprobe, struct mm_struct mm)
1233	1236	{
	1237	+ VMA_ITERATOR(vmi, mm, 0);
1234	1238	struct vm_area_struct *vma;
1235	1239	int err = 0;
1236	1240
1237	1241	mmap_read_lock(mm);
1238		− for (vma = mm->mmap; vma; vma = vma->vm_next) {
	1242	+ for_each_vma(vmi, vma) {
1239	1243	unsigned long vaddr;
1240	1244	loff_t offset;
1241	1245
@@ -1983,9 +1987,10 @@ bool uprobe_deny_signal(void) · 741 unmodified lines
1983	1987
1984	1988	static void mmf_recalc_uprobes(struct mm_struct *mm)
1985	1989	{
	1990	+ VMA_ITERATOR(vmi, mm, 0);
1986	1991	struct vm_area_struct *vma;
1987	1992
1988		− for (vma = mm->mmap; vma; vma = vma->vm_next) {
	1993	+ for_each_vma(vmi, vma) {
1989	1994	if (!valid_vma(vma, false))
1990	1995	continue;
1991	1996	/*

kernel/exit.c+3−0

@@ -60,6 +60,7 @@ · 59 unmodified lines
60	60	#include <linux/writeback.h>
61	61	#include <linux/shm.h>
62	62	#include <linux/kcov.h>
	63	+#include <linux/kmsan.h>
63	64	#include <linux/random.h>
64	65	#include <linux/rcuwait.h>
65	66	#include <linux/compat.h>
@@ -466,6 +467,7 @@ assign_new_owner: · 400 unmodified lines
466	467	goto retry;
467	468	}
468	469	WRITE_ONCE(mm->owner, c);
	470	+ lru_gen_migrate_mm(mm);
469	471	task_unlock(c);
470	472	put_task_struct(c);
471	473	}
@@ -759,6 +761,7 @@ void __noreturn do_exit(long code) · 287 unmodified lines
759	761	WARN_ON(tsk->plug);
760	762
761	763	kcov_task_exit(tsk);
	764	+ kmsan_task_exit(tsk);
762	765
763	766	coredump_task_exit(tsk);
764	767	ptrace_event(PTRACE_EVENT_EXIT, code);

kernel/fork.c+43−36

@@ -37,13 +37,13 @@ · 36 unmodified lines
37	37	#include <linux/fdtable.h>
38	38	#include <linux/iocontext.h>
39	39	#include <linux/key.h>
	40	+#include <linux/kmsan.h>
40	41	#include <linux/binfmts.h>
41	42	#include <linux/mman.h>
42	43	#include <linux/mmu_notifier.h>
43	44	#include <linux/fs.h>
44	45	#include <linux/mm.h>
45	46	#include <linux/mm_inline.h>
46		−#include <linux/vmacache.h>
47	47	#include <linux/nsproxy.h>
48	48	#include <linux/capability.h>
49	49	#include <linux/cpu.h>
@@ -475,7 +475,6 @@ struct vm_area_struct vm_area_dup(struct vm_area_struct orig) · 425 unmodified lines
475	475	*/
476	476	new = data_race(orig);
477	477	INIT_LIST_HEAD(&new->anon_vma_chain);
478		− new->vm_next = new->vm_prev = NULL;
479	478	dup_anon_vma_name(orig, new);
480	479	}
481	480	return new;
@@ -580,11 +579,12 @@ static void dup_mm_exe_file(struct mm_struct mm, struct mm_struct oldmm) · 98 unmodified lines
580	579	static __latent_entropy int dup_mmap(struct mm_struct *mm,
581	580	struct mm_struct *oldmm)
582	581	{
583		− struct vm_area_struct mpnt, tmp, prev, *pprev;
584		− struct rb_node *rb_link, rb_parent;
	582	+ struct vm_area_struct mpnt, tmp;
585	583	int retval;
586		− unsigned long charge;
	584	+ unsigned long charge = 0;
587	585	LIST_HEAD(uf);
	586	+ MA_STATE(old_mas, &oldmm->mm_mt, 0, 0);
	587	+ MA_STATE(mas, &mm->mm_mt, 0, 0);
588	588
589	589	uprobe_start_dup_mmap();
590	590	if (mmap_write_lock_killable(oldmm)) {
@@ -606,16 +606,16 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, · 15 unmodified lines
606	606	mm->exec_vm = oldmm->exec_vm;
607	607	mm->stack_vm = oldmm->stack_vm;
608	608
609		− rb_link = &mm->mm_rb.rb_node;
610		− rb_parent = NULL;
611		− pprev = &mm->mmap;
612	609	retval = ksm_fork(mm, oldmm);
613	610	if (retval)
614	611	goto out;
615	612	khugepaged_fork(mm, oldmm);
616	613
617		− prev = NULL;
618		− for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
	614	+ retval = mas_expected_entries(&mas, oldmm->map_count);
	615	+ if (retval)
	616	+ goto out;
	617	+
	618	+ mas_for_each(&old_mas, mpnt, ULONG_MAX) {
619	619	struct file *file;
620	620
621	621	if (mpnt->vm_flags & VM_DONTCOPY) {
@@ -629,7 +629,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, · 7 unmodified lines
629	629	*/
630	630	if (fatal_signal_pending(current)) {
631	631	retval = -EINTR;
632		− goto out;
	632	+ goto loop_out;
633	633	}
634	634	if (mpnt->vm_flags & VM_ACCOUNT) {
635	635	unsigned long len = vma_pages(mpnt);
@@ -675,24 +675,17 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, · 39 unmodified lines
675	675	}
676	676
677	677	/*
678		− * Clear hugetlb-related page reserves for children. This only
679		− * affects MAP_PRIVATE mappings. Faults generated by the child
680		− * are not guaranteed to succeed, even if read-only
	678	+ * Copy/update hugetlb private vma information.
681	679	*/
682	680	if (is_vm_hugetlb_page(tmp))
683		− reset_vma_resv_huge_pages(tmp);
684		−
685		− /*
686		− * Link in the new vma and copy the page table entries.
687		− */
688		− *pprev = tmp;
689		− pprev = &tmp->vm_next;
690		− tmp->vm_prev = prev;
691		− prev = tmp;
	681	+ hugetlb_dup_vma_private(tmp);
692	682
693		− __vma_link_rb(mm, tmp, rb_link, rb_parent);
694		− rb_link = &tmp->vm_rb.rb_right;
695		− rb_parent = &tmp->vm_rb;
	683	+ /* Link the vma into the MT */
	684	+ mas.index = tmp->vm_start;
	685	+ mas.last = tmp->vm_end - 1;
	686	+ mas_store(&mas, tmp);
	687	+ if (mas_is_err(&mas))
	688	+ goto fail_nomem_mas_store;
696	689
697	690	mm->map_count++;
698	691	if (!(tmp->vm_flags & VM_WIPEONFORK))
@@ -702,10 +695,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, · 3 unmodified lines
702	695	tmp->vm_ops->open(tmp);
703	696
704	697	if (retval)
705		− goto out;
	698	+ goto loop_out;
706	699	}
707	700	/* a new mm has just been created */
708	701	retval = arch_dup_mmap(oldmm, mm);
	702	+loop_out:
	703	+ mas_destroy(&mas);
709	704	out:
710	705	mmap_write_unlock(mm);
711	706	flush_tlb_mm(oldmm);
@@ -714,6 +709,9 @@ out: · 2 unmodified lines
714	709	fail_uprobe_end:
715	710	uprobe_end_dup_mmap();
716	711	return retval;
	712	+
	713	+fail_nomem_mas_store:
	714	+ unlink_anon_vmas(tmp);
717	715	fail_nomem_anon_vma_fork:
718	716	mpol_put(vma_policy(tmp));
719	717	fail_nomem_policy:
@@ -721,7 +719,7 @@ fail_nomem_policy: · 1 unmodified line
721	719	fail_nomem:
722	720	retval = -ENOMEM;
723	721	vm_unacct_memory(charge);
724		− goto out;
	722	+ goto loop_out;
725	723	}
726	724
727	725	static inline int mm_alloc_pgd(struct mm_struct *mm)
@@ -1026,6 +1024,7 @@ static struct task_struct dup_task_struct(struct task_struct orig, int node) · 298 unmodified lines
1026	1024	tsk->worker_private = NULL;
1027	1025
1028	1026	kcov_task_init(tsk);
	1027	+ kmsan_task_create(tsk);
1029	1028	kmap_local_fork(tsk);
1030	1029
1031	1030	#ifdef CONFIG_FAULT_INJECTION
@@ -1109,9 +1108,8 @@ static void mm_init_uprobes_state(struct mm_struct *mm) · 77 unmodified lines
1109	1108	static struct mm_struct mm_init(struct mm_struct mm, struct task_struct *p,
1110	1109	struct user_namespace *user_ns)
1111	1110	{
1112		− mm->mmap = NULL;
1113		− mm->mm_rb = RB_ROOT;
1114		− mm->vmacache_seqnum = 0;
	1111	+ mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
	1112	+ mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
1115	1113	atomic_set(&mm->mm_users, 1);
1116	1114	atomic_set(&mm->mm_count, 1);
1117	1115	seqcount_init(&mm->write_protect_seq);
@@ -1152,6 +1150,7 @@ static struct mm_struct mm_init(struct mm_struct mm, struct task_struct *p, · 34 unmodified lines
1152	1150	goto fail_nocontext;
1153	1151
1154	1152	mm->user_ns = get_user_ns(user_ns);
	1153	+ lru_gen_init_mm(mm);
1155	1154	return mm;
1156	1155
1157	1156	fail_nocontext:
@@ -1194,6 +1193,7 @@ static inline void __mmput(struct mm_struct *mm) · 36 unmodified lines
1194	1193	}
1195	1194	if (mm->binfmt)
1196	1195	module_put(mm->binfmt->module);
	1196	+ lru_gen_del_mm(mm);
1197	1197	mmdrop(mm);
1198	1198	}
1199	1199
@@ -1285,13 +1285,16 @@ int replace_mm_exe_file(struct mm_struct mm, struct file new_exe_file) · 85 unmodified lines
1285	1285	/* Forbid mm->exe_file change if old file still mapped. */
1286	1286	old_exe_file = get_mm_exe_file(mm);
1287	1287	if (old_exe_file) {
	1288	+ VMA_ITERATOR(vmi, mm, 0);
1288	1289	mmap_read_lock(mm);
1289		− for (vma = mm->mmap; vma && !ret; vma = vma->vm_next) {
	1290	+ for_each_vma(vmi, vma) {
1290	1291	if (!vma->vm_file)
1291	1292	continue;
1292	1293	if (path_equal(&vma->vm_file->f_path,
1293		− &old_exe_file->f_path))
	1294	+ &old_exe_file->f_path)) {
1294	1295	ret = -EBUSY;
	1296	+ break;
	1297	+ }
1295	1298	}
1296	1299	mmap_read_unlock(mm);
1297	1300	fput(old_exe_file);
@@ -1566,9 +1569,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) · 268 unmodified lines
1566	1569	if (!oldmm)
1567	1570	return 0;
1568	1571
1569		− /* initialize the new vmacache entries */
1570		− vmacache_flush(tsk);
1571		−
1572	1572	if (clone_flags & CLONE_VM) {
1573	1573	mmget(oldmm);
1574	1574	mm = oldmm;
@@ -2693,6 +2693,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) · 1118 unmodified lines
2693	2693	get_task_struct(p);
2694	2694	}
2695	2695
	2696	+ if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) {
	2697	+ /* lock the task to synchronize with memcg migration */
	2698	+ task_lock(p);
	2699	+ lru_gen_add_mm(p->mm);
	2700	+ task_unlock(p);
	2701	+ }
	2702	+
2696	2703	wake_up_new_task(p);
2697	2704
2698	2705	/* forking complete and child started to run, tell ptracer */

kernel/kcov.c+7−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <linux/fs.h>
12	12	#include <linux/hashtable.h>
13	13	#include <linux/init.h>
	14	+#include <linux/kmsan-checks.h>
14	15	#include <linux/mm.h>
15	16	#include <linux/preempt.h>
16	17	#include <linux/printk.h>
@@ -152,6 +153,12 @@ static void kcov_remote_area_put(struct kcov_remote_area *area, · 135 unmodified lines
152	153	INIT_LIST_HEAD(&area->list);
153	154	area->size = size;
154	155	list_add(&area->list, &kcov_remote_areas);
	156	+ /*
	157	+ * KMSAN doesn't instrument this file, so it may not know area->list
	158	+ * is initialized. Unpoison it explicitly to avoid reports in
	159	+ * kcov_remote_area_get().
	160	+ */
	161	+ kmsan_unpoison_memory(&area->list, sizeof(area->list));
155	162	}
156	163
157	164	static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)

kernel/locking/Makefile+2−1

@@ -5,8 +5,9 @@ KCOV_INSTRUMENT := n · 4 unmodified lines
5	5
6	6	obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
7	7
8		−# Avoid recursion lockdep -> KCSAN -> ... -> lockdep.
	8	+# Avoid recursion lockdep -> sanitizer -> ... -> lockdep.
9	9	KCSAN_SANITIZE_lockdep.o := n
	10	+KMSAN_SANITIZE_lockdep.o := n
10	11
11	12	ifdef CONFIG_FUNCTION_TRACER
12	13	CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE)

kernel/sched/core.c+15−0

@@ -4389,6 +4389,17 @@ void set_numabalancing_state(bool enabled) · 4388 unmodified lines
4389	4389	}
4390	4390
4391	4391	#ifdef CONFIG_PROC_SYSCTL
	4392	+static void reset_memory_tiering(void)
	4393	+{
	4394	+ struct pglist_data *pgdat;
	4395	+
	4396	+ for_each_online_pgdat(pgdat) {
	4397	+ pgdat->nbp_threshold = 0;
	4398	+ pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
	4399	+ pgdat->nbp_th_start = jiffies_to_msecs(jiffies);
	4400	+ }
	4401	+}
	4402	+
4392	4403	int sysctl_numa_balancing(struct ctl_table *table, int write,
4393	4404	void buffer, size_t lenp, loff_t *ppos)
4394	4405	{
@@ -4405,6 +4416,9 @@ int sysctl_numa_balancing(struct ctl_table *table, int write, · 10 unmodified lines
4405	4416	if (err < 0)
4406	4417	return err;
4407	4418	if (write) {
	4419	+ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
	4420	+ (state & NUMA_BALANCING_MEMORY_TIERING))
	4421	+ reset_memory_tiering();
4408	4422	sysctl_numa_balancing_mode = state;
4409	4423	__set_numabalancing_state(state);
4410	4424	}
@@ -5159,6 +5173,7 @@ context_switch(struct rq rq, struct task_struct prev, · 748 unmodified lines
5159	5173	* finish_task_switch()'s mmdrop().
5160	5174	*/
5161	5175	switch_mm_irqs_off(prev->active_mm, next->mm, next);
	5176	+ lru_gen_use_mm(next->mm);
5162	5177
5163	5178	if (!prev->mm) { // from kernel
5164	5179	/* will mmdrop() in finish_task_switch(). */

kernel/sched/debug.c+1−0

@@ -333,6 +333,7 @@ static __init int sched_init_debug(void) · 332 unmodified lines
333	333	debugfs_create_u32("scan_period_min_ms", 0644, numa, &sysctl_numa_balancing_scan_period_min);
334	334	debugfs_create_u32("scan_period_max_ms", 0644, numa, &sysctl_numa_balancing_scan_period_max);
335	335	debugfs_create_u32("scan_size_mb", 0644, numa, &sysctl_numa_balancing_scan_size);
	336	+ debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
336	337	#endif
337	338
338	339	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);

kernel/sched/fair.c+172−3

@@ -40,6 +40,7 @@ · 39 unmodified lines
40	40
41	41	#include <linux/cpuidle.h>
42	42	#include <linux/interrupt.h>
	43	+#include <linux/memory-tiers.h>
43	44	#include <linux/mempolicy.h>
44	45	#include <linux/mutex_api.h>
45	46	#include <linux/profile.h>
@@ -1090,6 +1091,12 @@ unsigned int sysctl_numa_balancing_scan_size = 256; · 1044 unmodified lines
1090	1091	/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
1091	1092	unsigned int sysctl_numa_balancing_scan_delay = 1000;
1092	1093
	1094	+/* The page with hint page fault latency < threshold in ms is considered hot */
	1095	+unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC;
	1096	+
	1097	+/* Restrict the NUMA promotion throughput (MB/s) for each target node. */
	1098	+unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
	1099	+
1093	1100	struct numa_group {
1094	1101	refcount_t refcount;
1095	1102
@@ -1432,6 +1439,120 @@ static inline unsigned long group_weight(struct task_struct *p, int nid, · 336 unmodified lines
1432	1439	return 1000 * faults / total_faults;
1433	1440	}
1434	1441
	1442	+/*
	1443	+ * If memory tiering mode is enabled, cpupid of slow memory page is
	1444	+ * used to record scan time instead of CPU and PID. When tiering mode
	1445	+ * is disabled at run time, the scan time (in cpupid) will be
	1446	+ * interpreted as CPU and PID. So CPU needs to be checked to avoid to
	1447	+ * access out of array bound.
	1448	+ */
	1449	+static inline bool cpupid_valid(int cpupid)
	1450	+{
	1451	+ return cpupid_to_cpu(cpupid) < nr_cpu_ids;
	1452	+}
	1453	+
	1454	+/*
	1455	+ * For memory tiering mode, if there are enough free pages (more than
	1456	+ * enough watermark defined here) in fast memory node, to take full
	1457	+ * advantage of fast memory capacity, all recently accessed slow
	1458	+ * memory pages will be migrated to fast memory node without
	1459	+ * considering hot threshold.
	1460	+ */
	1461	+static bool pgdat_free_space_enough(struct pglist_data *pgdat)
	1462	+{
	1463	+ int z;
	1464	+ unsigned long enough_wmark;
	1465	+
	1466	+ enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT,
	1467	+ pgdat->node_present_pages >> 4);
	1468	+ for (z = pgdat->nr_zones - 1; z >= 0; z--) {
	1469	+ struct zone *zone = pgdat->node_zones + z;
	1470	+
	1471	+ if (!populated_zone(zone))
	1472	+ continue;
	1473	+
	1474	+ if (zone_watermark_ok(zone, 0,
	1475	+ wmark_pages(zone, WMARK_PROMO) + enough_wmark,
	1476	+ ZONE_MOVABLE, 0))
	1477	+ return true;
	1478	+ }
	1479	+ return false;
	1480	+}
	1481	+
	1482	+/*
	1483	+ * For memory tiering mode, when page tables are scanned, the scan
	1484	+ * time will be recorded in struct page in addition to make page
	1485	+ * PROT_NONE for slow memory page. So when the page is accessed, in
	1486	+ * hint page fault handler, the hint page fault latency is calculated
	1487	+ * via,
	1488	+ *
	1489	+ * hint page fault latency = hint page fault time - scan time
	1490	+ *
	1491	+ * The smaller the hint page fault latency, the higher the possibility
	1492	+ * for the page to be hot.
	1493	+ */
	1494	+static int numa_hint_fault_latency(struct page *page)
	1495	+{
	1496	+ int last_time, time;
	1497	+
	1498	+ time = jiffies_to_msecs(jiffies);
	1499	+ last_time = xchg_page_access_time(page, time);
	1500	+
	1501	+ return (time - last_time) & PAGE_ACCESS_TIME_MASK;
	1502	+}
	1503	+
	1504	+/*
	1505	+ * For memory tiering mode, too high promotion/demotion throughput may
	1506	+ * hurt application latency. So we provide a mechanism to rate limit
	1507	+ * the number of pages that are tried to be promoted.
	1508	+ */
	1509	+static bool numa_promotion_rate_limit(struct pglist_data *pgdat,
	1510	+ unsigned long rate_limit, int nr)
	1511	+{
	1512	+ unsigned long nr_cand;
	1513	+ unsigned int now, start;
	1514	+
	1515	+ now = jiffies_to_msecs(jiffies);
	1516	+ mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr);
	1517	+ nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
	1518	+ start = pgdat->nbp_rl_start;
	1519	+ if (now - start > MSEC_PER_SEC &&
	1520	+ cmpxchg(&pgdat->nbp_rl_start, start, now) == start)
	1521	+ pgdat->nbp_rl_nr_cand = nr_cand;
	1522	+ if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit)
	1523	+ return true;
	1524	+ return false;
	1525	+}
	1526	+
	1527	+#define NUMA_MIGRATION_ADJUST_STEPS 16
	1528	+
	1529	+static void numa_promotion_adjust_threshold(struct pglist_data *pgdat,
	1530	+ unsigned long rate_limit,
	1531	+ unsigned int ref_th)
	1532	+{
	1533	+ unsigned int now, start, th_period, unit_th, th;
	1534	+ unsigned long nr_cand, ref_cand, diff_cand;
	1535	+
	1536	+ now = jiffies_to_msecs(jiffies);
	1537	+ th_period = sysctl_numa_balancing_scan_period_max;
	1538	+ start = pgdat->nbp_th_start;
	1539	+ if (now - start > th_period &&
	1540	+ cmpxchg(&pgdat->nbp_th_start, start, now) == start) {
	1541	+ ref_cand = rate_limit *
	1542	+ sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC;
	1543	+ nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
	1544	+ diff_cand = nr_cand - pgdat->nbp_th_nr_cand;
	1545	+ unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS;
	1546	+ th = pgdat->nbp_threshold ? : ref_th;
	1547	+ if (diff_cand > ref_cand * 11 / 10)
	1548	+ th = max(th - unit_th, unit_th);
	1549	+ else if (diff_cand < ref_cand * 9 / 10)
	1550	+ th = min(th + unit_th, ref_th * 2);
	1551	+ pgdat->nbp_th_nr_cand = nr_cand;
	1552	+ pgdat->nbp_threshold = th;
	1553	+ }
	1554	+}
	1555	+
1435	1556	bool should_numa_migrate_memory(struct task_struct p, struct page page,
1436	1557	int src_nid, int dst_cpu)
1437	1558	{
@@ -1439,9 +1560,44 @@ bool should_numa_migrate_memory(struct task_struct p, struct page page, · 1 unmodified line
1439	1560	int dst_nid = cpu_to_node(dst_cpu);
1440	1561	int last_cpupid, this_cpupid;
1441	1562
	1563	+ /*
	1564	+ * The pages in slow memory node should be migrated according
	1565	+ * to hot/cold instead of private/shared.
	1566	+ */
	1567	+ if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
	1568	+ !node_is_toptier(src_nid)) {
	1569	+ struct pglist_data *pgdat;
	1570	+ unsigned long rate_limit;
	1571	+ unsigned int latency, th, def_th;
	1572	+
	1573	+ pgdat = NODE_DATA(dst_nid);
	1574	+ if (pgdat_free_space_enough(pgdat)) {
	1575	+ /* workload changed, reset hot threshold */
	1576	+ pgdat->nbp_threshold = 0;
	1577	+ return true;
	1578	+ }
	1579	+
	1580	+ def_th = sysctl_numa_balancing_hot_threshold;
	1581	+ rate_limit = sysctl_numa_balancing_promote_rate_limit << \
	1582	+ (20 - PAGE_SHIFT);
	1583	+ numa_promotion_adjust_threshold(pgdat, rate_limit, def_th);
	1584	+
	1585	+ th = pgdat->nbp_threshold ? : def_th;
	1586	+ latency = numa_hint_fault_latency(page);
	1587	+ if (latency >= th)
	1588	+ return false;
	1589	+
	1590	+ return !numa_promotion_rate_limit(pgdat, rate_limit,
	1591	+ thp_nr_pages(page));
	1592	+ }
	1593	+
1442	1594	this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
1443	1595	last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
1444	1596
	1597	+ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
	1598	+ !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid))
	1599	+ return false;
	1600	+
1445	1601	/*
1446	1602	* Allow first faults or private faults to migrate immediately early in
1447	1603	* the lifetime of a task. The magic number 4 is based on waiting for
@@ -2681,6 +2837,15 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) · 1233 unmodified lines
2681	2837	if (!p->mm)
2682	2838	return;
2683	2839
	2840	+ /*
	2841	+ * NUMA faults statistics are unnecessary for the slow memory
	2842	+ * node for memory tiering mode.
	2843	+ */
	2844	+ if (!node_is_toptier(mem_node) &&
	2845	+ (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING \|\|
	2846	+ !cpupid_valid(last_cpupid)))
	2847	+ return;
	2848	+
2684	2849	/* Allocate buffer to track faults on a per-node basis */
2685	2850	if (unlikely(!p->numa_faults)) {
2686	2851	int size = sizeof(p->numa_faults)
@@ -2761,6 +2926,7 @@ static void task_numa_work(struct callback_head *work) · 74 unmodified lines
2761	2926	struct task_struct *p = current;
2762	2927	struct mm_struct *mm = p->mm;
2763	2928	u64 runtime = p->se.sum_exec_runtime;
	2929	+ MA_STATE(mas, &mm->mm_mt, 0, 0);
2764	2930	struct vm_area_struct *vma;
2765	2931	unsigned long start, end;
2766	2932	unsigned long nr_pte_updates = 0;
@@ -2817,13 +2983,16 @@ static void task_numa_work(struct callback_head *work) · 50 unmodified lines
2817	2983
2818	2984	if (!mmap_read_trylock(mm))
2819	2985	return;
2820		− vma = find_vma(mm, start);
	2986	+ mas_set(&mas, start);
	2987	+ vma = mas_find(&mas, ULONG_MAX);
2821	2988	if (!vma) {
2822	2989	reset_ptenuma_scan(p);
2823	2990	start = 0;
2824		− vma = mm->mmap;
	2991	+ mas_set(&mas, start);
	2992	+ vma = mas_find(&mas, ULONG_MAX);
2825	2993	}
2826		− for (; vma; vma = vma->vm_next) {
	2994	+
	2995	+ for (; vma; vma = mas_find(&mas, ULONG_MAX)) {
2827	2996	if (!vma_migratable(vma) \|\| !vma_policy_mof(vma) \|\|
2828	2997	is_vm_hugetlb_page(vma) \|\| (vma->vm_flags & VM_MIXEDMAP)) {
2829	2998	continue;

kernel/sched/sched.h+1−0

@@ -2446,6 +2446,7 @@ extern unsigned int sysctl_numa_balancing_scan_delay; · 2445 unmodified lines
2446	2446	extern unsigned int sysctl_numa_balancing_scan_period_min;
2447	2447	extern unsigned int sysctl_numa_balancing_scan_period_max;
2448	2448	extern unsigned int sysctl_numa_balancing_scan_size;
	2449	+extern unsigned int sysctl_numa_balancing_hot_threshold;
2449	2450	#endif
2450	2451
2451	2452	#ifdef CONFIG_SCHED_HRTICK

kernel/sysctl.c+8−0

@@ -1643,6 +1643,14 @@ static struct ctl_table kern_table[] = { · 1642 unmodified lines
1643	1643	.extra1 = SYSCTL_ZERO,
1644	1644	.extra2 = SYSCTL_FOUR,
1645	1645	},
	1646	+ {
	1647	+ .procname = "numa_balancing_promote_rate_limit_MBps",
	1648	+ .data = &sysctl_numa_balancing_promote_rate_limit,
	1649	+ .maxlen = sizeof(unsigned int),
	1650	+ .mode = 0644,
	1651	+ .proc_handler = proc_dointvec_minmax,
	1652	+ .extra1 = SYSCTL_ZERO,
	1653	+ },
1646	1654	#endif /* CONFIG_NUMA_BALANCING */
1647	1655	{
1648	1656	.procname = "panic",

Merge tag 'mm-stable-2022-10-08' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code