Merge tag 'mm-stable-2022-10-08' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

Pull MM updates from Andrew Morton: - Yu Zhao's Multi-Gen LRU patches are here. They've been under test in linux-next for a couple of months without, to my knowledge, any negative reports (or any positive ones, come to that). - Also the Maple Tree from Liam Howlett. An overlapping range-based tree for vmas. It it apparently slightly more efficient in its own right, but is mainly targeted at enabling work to reduce mmap_lock contention. Liam has identified a number of other tree users in the kernel which could be beneficially onverted to mapletrees. Yu Zhao has identified a hard-to-hit but "easy to fix" lockdep splat at [1]. This has yet to be addressed due to Liam's unfortunately timed vacation. He is now back and we'll get this fixed up. - Dmitry Vyukov introduces KMSAN: the Kernel Memory Sanitizer. It uses clang-generated instrumentation to detect used-unintialized bugs down to the single bit level. KMSAN keeps finding bugs. New ones, as well as the legacy ones. - Yang Shi adds a userspace mechanism (madvise) to induce a collapse of memory into THPs. - Zach O'Keefe has expanded Yang Shi's madvise(MADV_COLLAPSE) to support file/shmem-backed pages. - userfaultfd updates from Axel Rasmussen - zsmalloc cleanups from Alexey Romanov - cleanups from Miaohe Lin: vmscan, hugetlb_cgroup, hugetlb and memory-failure - Huang Ying adds enhancements to NUMA balancing memory tiering mode's page promotion, with a new way of detecting hot pages. - memcg updates from Shakeel Butt: charging optimizations and reduced memory consumption. - memcg cleanups from Kairui Song. - memcg fixes and cleanups from Johannes Weiner. - Vishal Moola provides more folio conversions - Zhang Yi removed ll_rw_block() :( - migration enhancements from Peter Xu - migration error-path bugfixes from Huang Ying - Aneesh Kumar added ability for a device driver to alter the memory tiering promotion paths. For optimizations by PMEM drivers, DRM drivers, etc. - vma merging improvements from Jakub Matěn. - NUMA hinting cleanups from David Hildenbrand. - xu xin added aditional userspace visibility into KSM merging activity. - THP & KSM code consolidation from Qi Zheng. - more folio work from Matthew Wilcox. - KASAN updates from Andrey Konovalov. - DAMON cleanups from Kaixu Xia. - DAMON work from SeongJae Park: fixes, cleanups. - hugetlb sysfs cleanups from Muchun Song. - Mike Kravetz fixes locking issues in hugetlbfs and in hugetlb core. Link: https://lkml.kernel.org/r/CAOUHufZabH85CeUN-MEMgL8gJGzJEWUrkiM58JkTbBhh-jew0Q@mail.gmail.com [1] * tag 'mm-stable-2022-10-08' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (555 commits) hugetlb: allocate vma lock for all sharable vmas hugetlb: take hugetlb vma_lock when clearing vma_lock->vma pointer hugetlb: fix vma lock handling during split vma and range unmapping mglru: mm/vmscan.c: fix imprecise comments mm/mglru: don't sync disk for each aging cycle mm: memcontrol: drop dead CONFIG_MEMCG_SWAP config symbol mm: memcontrol: use do_memsw_account() in a few more places mm: memcontrol: deprecate swapaccounting=0 mode mm: memcontrol: don't allocate cgroup swap arrays when memcg is disabled mm/secretmem: remove reduntant return value mm/hugetlb: add available_huge_pages() func mm: remove unused inline functions from include/linux/mm_inline.h selftests/vm: add selftest for MADV_COLLAPSE of uffd-minor memory selftests/vm: add file/shmem MADV_COLLAPSE selftest for cleared pmd selftests/vm: add thp collapse shmem testing selftests/vm: add thp collapse file and tmpfs testing selftests/vm: modularize thp collapse memory operations selftests/vm: dedup THP helpers mm/khugepaged: add tracepoint to hpage_collapse_scan_file() mm/madvise: add file and shmem support to MADV_COLLAPSE ...

Linus Torvalds committed 4 years ago

Commit 27bc50f2 parents 70442fc bbff39c409 files · +65792 −8034Browse files

Showing the changes to kernel (21 of 409 files). Show all

Unified · Split

21 files

kernel/Makefile +1 −0
kernel/acct.c +5 −6
kernel/bounds.c +7 −0
kernel/bpf/core.c +1 −1
kernel/bpf/task_iter.c +5 −5
kernel/cgroup/cgroup-internal.h +0 −1
kernel/debug/debug_core.c +0 −12
kernel/delayacct.c +11 −2
kernel/dma/mapping.c +7 −3
kernel/entry/common.c +5 −0
kernel/events/core.c +2 −1
kernel/events/uprobes.c +22 −17
kernel/exit.c +3 −0
kernel/fork.c +43 −36
kernel/kcov.c +7 −0
kernel/locking/Makefile +2 −1
kernel/sched/core.c +15 −0
kernel/sched/debug.c +1 −0
kernel/sched/fair.c +172 −3
kernel/sched/sched.h +1 −0
kernel/sysctl.c +8 −0

kernel/Makefile+1−0

@@ -38,6 +38,7 @@ KCOV_INSTRUMENT_kcov.o := n · 37 unmodified lines
38	KASAN_SANITIZE_kcov.o := n	38	KASAN_SANITIZE_kcov.o := n
39	KCSAN_SANITIZE_kcov.o := n	39	KCSAN_SANITIZE_kcov.o := n
40	UBSAN_SANITIZE_kcov.o := n	40	UBSAN_SANITIZE_kcov.o := n
		41	KMSAN_SANITIZE_kcov.o := n
41	CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector	42	CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector
42		43
43	# Don't instrument error handlers	44	# Don't instrument error handlers

kernel/acct.c+5−6

 	unsigned long vsize = 0;
 	if (group_dead && current->mm) {
+		struct mm_struct *mm = current->mm;
+		VMA_ITERATOR(vmi, mm, 0);
 		struct vm_area_struct *vma;
-		mmap_read_lock(current->mm);
+		mmap_read_lock(mm);
-		vma = current->mm->mmap;
+		for_each_vma(vmi, vma)
-		while (vma) {
 			vsize += vma->vm_end - vma->vm_start;
-			vma = vma->vm_next;
+		mmap_read_unlock(mm);
-		}
-		mmap_read_unlock(current->mm);
 	}
 	spin_lock_irq(&current->sighand->siglock);

kernel/bounds.c+7−0

@@ -22,6 +22,13 @@ int main(void) · 21 unmodified lines
22	DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));	22	DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS));
23	#endif	23	#endif
24	DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));	24	DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t));
		25	#ifdef CONFIG_LRU_GEN
		26	DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1));
		27	DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2);
		28	#else
		29	DEFINE(LRU_GEN_WIDTH, 0);
		30	DEFINE(__LRU_REFS_WIDTH, 0);
		31	#endif
25	/* End of constants */	32	/* End of constants */
26		33
27	return 0;	34	return 0;

kernel/bpf/core.c+1−1

@@ -2007,7 +2007,7 @@ out: · 2006 unmodified lines
2007	static unsigned int PROG_NAME(stack_size)(const void ctx, const struct bpf_insn insn) \	2007	static unsigned int PROG_NAME(stack_size)(const void ctx, const struct bpf_insn insn) \
2008	{ \	2008	{ \
2009	u64 stack[stack_size / sizeof(u64)]; \	2009	u64 stack[stack_size / sizeof(u64)]; \
2010	u64 regs[MAX_BPF_EXT_REG]; \	2010	u64 regs[MAX_BPF_EXT_REG] = {}; \
2011	\	2011	\
2012	FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \	2012	FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \
2013	ARG1 = (u64) (unsigned long) ctx; \	2013	ARG1 = (u64) (unsigned long) ctx; \

kernel/bpf/task_iter.c+5−5

@@ -445,8 +445,8 @@ struct bpf_iter_seq_task_vma_info { · 444 unmodified lines
445	};	445	};
446		446
447	enum bpf_task_vma_iter_find_op {	447	enum bpf_task_vma_iter_find_op {
448	task_vma_iter_first_vma, /* use mm->mmap */	448	task_vma_iter_first_vma, /* use find_vma() with addr 0 */
449	task_vma_iter_next_vma, /* use curr_vma->vm_next */	449	task_vma_iter_next_vma, /* use vma_next() with curr_vma */
450	task_vma_iter_find_vma, /* use find_vma() to find next vma */	450	task_vma_iter_find_vma, /* use find_vma() to find next vma */
451	};	451	};
452		452
@@ -544,10 +544,10 @@ again: · 91 unmodified lines
544		544
545	switch (op) {	545	switch (op) {
546	case task_vma_iter_first_vma:	546	case task_vma_iter_first_vma:
547	curr_vma = curr_task->mm->mmap;	547	curr_vma = find_vma(curr_task->mm, 0);
548	break;	548	break;
549	case task_vma_iter_next_vma:	549	case task_vma_iter_next_vma:
550	curr_vma = curr_vma->vm_next;	550	curr_vma = find_vma(curr_task->mm, curr_vma->vm_end);
551	break;	551	break;
552	case task_vma_iter_find_vma:	552	case task_vma_iter_find_vma:
553	/* We dropped mmap_lock so it is necessary to use find_vma	553	/* We dropped mmap_lock so it is necessary to use find_vma
@@ -561,7 +561,7 @@ again: · 7 unmodified lines
561	if (curr_vma &&	561	if (curr_vma &&
562	curr_vma->vm_start == info->prev_vm_start &&	562	curr_vma->vm_start == info->prev_vm_start &&
563	curr_vma->vm_end == info->prev_vm_end)	563	curr_vma->vm_end == info->prev_vm_end)
564	curr_vma = curr_vma->vm_next;	564	curr_vma = find_vma(curr_task->mm, curr_vma->vm_end);
565	break;	565	break;
566	}	566	}
567	if (!curr_vma) {	567	if (!curr_vma) {

kernel/cgroup/cgroup-internal.h+0−1

@@ -164,7 +164,6 @@ struct cgroup_mgctx { · 163 unmodified lines
164	#define DEFINE_CGROUP_MGCTX(name) \	164	#define DEFINE_CGROUP_MGCTX(name) \
165	struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name)	165	struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name)
166		166
167	extern struct mutex cgroup_mutex;
168	extern spinlock_t css_set_lock;	167	extern spinlock_t css_set_lock;
169	extern struct cgroup_subsys *cgroup_subsys[];	168	extern struct cgroup_subsys *cgroup_subsys[];
170	extern struct list_head cgroup_roots;	169	extern struct list_head cgroup_roots;

kernel/debug/debug_core.c+0−12

 #include <linux/pid.h>
 #include <linux/smp.h>
 #include <linux/mm.h>
-#include <linux/vmacache.h>
 #include <linux/rcupdate.h>
 #include <linux/irq.h>
 #include <linux/security.h>
 	if (!CACHE_FLUSH_IS_SAFE)
 		return;
-	if (current->mm) {
-		int i;
-		for (i = 0; i < VMACACHE_SIZE; i++) {
-			if (!current->vmacache.vmas[i])
-				continue;
-			flush_cache_range(current->vmacache.vmas[i],
-					  addr, addr + BREAK_INSTR_SIZE);
-		}
-	}
 	/* Force flush instruction cache if it was outside the mm */
 	flush_icache_range(addr, addr + BREAK_INSTR_SIZE);
 }

kernel/delayacct.c+11−2

 		      &current->delays->freepages_count);
 }
-void __delayacct_thrashing_start(void)
+void __delayacct_thrashing_start(bool *in_thrashing)
 {
+	*in_thrashing = !!current->in_thrashing;
+	if (*in_thrashing)
+		return;
+	current->in_thrashing = 1;
 	current->delays->thrashing_start = local_clock();
 }
-void __delayacct_thrashing_end(void)
+void __delayacct_thrashing_end(bool *in_thrashing)
 {
+	if (*in_thrashing)
+		return;
+	current->in_thrashing = 0;
 	delayacct_end(&current->delays->lock,
 		      &current->delays->thrashing_start,
 		      &current->delays->thrashing_delay,

kernel/dma/mapping.c+7−3

 #include <linux/dma-map-ops.h>
 #include <linux/export.h>
 #include <linux/gfp.h>
+#include <linux/kmsan.h>
 #include <linux/of_device.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
 	else
 		addr = ops->map_page(dev, page, offset, size, dir, attrs);
+	kmsan_handle_dma(page, offset, size, dir);
 	debug_dma_map_page(dev, page, offset, size, dir, addr, attrs);
 	return addr;
 	else
 		ents = ops->map_sg(dev, sg, nents, dir, attrs);
-	if (ents > 0)
+	if (ents > 0) {
+		kmsan_handle_dma_sg(sg, nents, dir);
 		debug_dma_map_sg(dev, sg, nents, ents, dir, attrs);
-	else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
+	} else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
-			      ents != -EIO && ents != -EREMOTEIO))
+				ents != -EIO && ents != -EREMOTEIO)) {
 		return -EIO;
+	}
 	return ents;
 }

kernel/entry/common.c+5−0

 #include <linux/resume_user_mode.h>
 #include <linux/highmem.h>
 #include <linux/jump_label.h>
+#include <linux/kmsan.h>
 #include <linux/livepatch.h>
 #include <linux/audit.h>
 #include <linux/tick.h>
 	user_exit_irqoff();
 	instrumentation_begin();
+	kmsan_unpoison_entry_regs(regs);
 	trace_hardirqs_off_finish();
 	instrumentation_end();
 }
 		lockdep_hardirqs_off(CALLER_ADDR0);
 		ct_irq_enter();
 		instrumentation_begin();
+		kmsan_unpoison_entry_regs(regs);
 		trace_hardirqs_off_finish();
 		instrumentation_end();
 	 */
 	lockdep_hardirqs_off(CALLER_ADDR0);
 	instrumentation_begin();
+	kmsan_unpoison_entry_regs(regs);
 	rcu_irq_enter_check_tick();
 	trace_hardirqs_off_finish();
 	instrumentation_end();
 	ct_nmi_enter();
 	instrumentation_begin();
+	kmsan_unpoison_entry_regs(regs);
 	trace_hardirqs_off_finish();
 	ftrace_nmi_enter();
 	instrumentation_end();

kernel/events/core.c+2−1

@@ -10270,8 +10270,9 @@ static void perf_addr_filter_apply(struct perf_addr_filter *filter, · 10269 unmodified lines
10270	struct perf_addr_filter_range *fr)	10270	struct perf_addr_filter_range *fr)
10271	{	10271	{
10272	struct vm_area_struct *vma;	10272	struct vm_area_struct *vma;
		10273	VMA_ITERATOR(vmi, mm, 0);
10273		10274
10274	for (vma = mm->mmap; vma; vma = vma->vm_next) {	10275	for_each_vma(vmi, vma) {
10275	if (!vma->vm_file)	10276	if (!vma->vm_file)
10276	continue;	10277	continue;
10277		10278

kernel/events/uprobes.c+22−17

 #include <linux/export.h>
 #include <linux/rmap.h>		/* anon_vma_prepare */
 #include <linux/mmu_notifier.h>	/* set_pte_at_notify */
-#include <linux/swap.h>		/* try_to_free_swap */
+#include <linux/swap.h>		/* folio_free_swap */
 #include <linux/ptrace.h>	/* user_enable_single_step */
 #include <linux/kdebug.h>	/* notifier mechanism */
 #include "../../mm/internal.h"	/* munlock_vma_page */
 static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 				struct page *old_page, struct page *new_page)
 {
+	struct folio *old_folio = page_folio(old_page);
+	struct folio *new_folio;
 	struct mm_struct *mm = vma->vm_mm;
-	DEFINE_FOLIO_VMA_WALK(pvmw, page_folio(old_page), vma, addr, 0);
+	DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0);
 	int err;
 	struct mmu_notifier_range range;
 				addr + PAGE_SIZE);
 	if (new_page) {
-		err = mem_cgroup_charge(page_folio(new_page), vma->vm_mm,
+		new_folio = page_folio(new_page);
-					GFP_KERNEL);
+		err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL);
 		if (err)
 			return err;
 	}
-	/* For try_to_free_swap() below */
+	/* For folio_free_swap() below */
-	lock_page(old_page);
+	folio_lock(old_folio);
 	mmu_notifier_invalidate_range_start(&range);
 	err = -EAGAIN;
 	VM_BUG_ON_PAGE(addr != pvmw.address, old_page);
 	if (new_page) {
-		get_page(new_page);
+		folio_get(new_folio);
 		page_add_new_anon_rmap(new_page, vma, addr);
-		lru_cache_add_inactive_or_unevictable(new_page, vma);
+		folio_add_lru_vma(new_folio, vma);
 	} else
 		/* no new page, just dec_mm_counter for old_page */
 		dec_mm_counter(mm, MM_ANONPAGES);
-	if (!PageAnon(old_page)) {
+	if (!folio_test_anon(old_folio)) {
 		dec_mm_counter(mm, mm_counter_file(old_page));
 		inc_mm_counter(mm, MM_ANONPAGES);
 	}
 				  mk_pte(new_page, vma->vm_page_prot));
 	page_remove_rmap(old_page, vma, false);
-	if (!page_mapped(old_page))
+	if (!folio_mapped(old_folio))
-		try_to_free_swap(old_page);
+		folio_free_swap(old_folio);
 	page_vma_mapped_walk_done(&pvmw);
-	put_page(old_page);
+	folio_put(old_folio);
 	err = 0;
  unlock:
 	mmu_notifier_invalidate_range_end(&range);
-	unlock_page(old_page);
+	folio_unlock(old_folio);
 	return err;
 }
 static struct vm_area_struct *
 find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm)
 {
+	VMA_ITERATOR(vmi, mm, 0);
 	struct vm_area_struct *tmp;
-	for (tmp = mm->mmap; tmp; tmp = tmp->vm_next)
+	for_each_vma(vmi, tmp)
 		if (valid_ref_ctr_vma(uprobe, tmp))
 			return tmp;
 	/* try collapse pmd for compound page */
 	if (!ret && orig_page_huge)
-		collapse_pte_mapped_thp(mm, vaddr);
+		collapse_pte_mapped_thp(mm, vaddr, false);
 	return ret;
 }
 static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
 {
+	VMA_ITERATOR(vmi, mm, 0);
 	struct vm_area_struct *vma;
 	int err = 0;
 	mmap_read_lock(mm);
-	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+	for_each_vma(vmi, vma) {
 		unsigned long vaddr;
 		loff_t offset;
 static void mmf_recalc_uprobes(struct mm_struct *mm)
 {
+	VMA_ITERATOR(vmi, mm, 0);
 	struct vm_area_struct *vma;
-	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+	for_each_vma(vmi, vma) {
 		if (!valid_vma(vma, false))
 			continue;
 		/*

kernel/exit.c+3−0

 #include <linux/writeback.h>
 #include <linux/shm.h>
 #include <linux/kcov.h>
+#include <linux/kmsan.h>
 #include <linux/random.h>
 #include <linux/rcuwait.h>
 #include <linux/compat.h>
 		goto retry;
 	}
 	WRITE_ONCE(mm->owner, c);
+	lru_gen_migrate_mm(mm);
 	task_unlock(c);
 	put_task_struct(c);
 }
 	WARN_ON(tsk->plug);
 	kcov_task_exit(tsk);
+	kmsan_task_exit(tsk);
 	coredump_task_exit(tsk);
 	ptrace_event(PTRACE_EVENT_EXIT, code);

kernel/fork.c+43−36

 #include <linux/fdtable.h>
 #include <linux/iocontext.h>
 #include <linux/key.h>
+#include <linux/kmsan.h>
 #include <linux/binfmts.h>
 #include <linux/mman.h>
 #include <linux/mmu_notifier.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/mm_inline.h>
-#include <linux/vmacache.h>
 #include <linux/nsproxy.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
 		 */
 		*new = data_race(*orig);
 		INIT_LIST_HEAD(&new->anon_vma_chain);
-		new->vm_next = new->vm_prev = NULL;
 		dup_anon_vma_name(orig, new);
 	}
 	return new;
 static __latent_entropy int dup_mmap(struct mm_struct *mm,
 					struct mm_struct *oldmm)
 {
-	struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
+	struct vm_area_struct *mpnt, *tmp;
-	struct rb_node **rb_link, *rb_parent;
 	int retval;
-	unsigned long charge;
+	unsigned long charge = 0;
 	LIST_HEAD(uf);
+	MA_STATE(old_mas, &oldmm->mm_mt, 0, 0);
+	MA_STATE(mas, &mm->mm_mt, 0, 0);
 	uprobe_start_dup_mmap();
 	if (mmap_write_lock_killable(oldmm)) {
 	mm->exec_vm = oldmm->exec_vm;
 	mm->stack_vm = oldmm->stack_vm;
-	rb_link = &mm->mm_rb.rb_node;
-	rb_parent = NULL;
-	pprev = &mm->mmap;
 	retval = ksm_fork(mm, oldmm);
 	if (retval)
 		goto out;
 	khugepaged_fork(mm, oldmm);
-	prev = NULL;
+	retval = mas_expected_entries(&mas, oldmm->map_count);
-	for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
+	if (retval)
+		goto out;
+	mas_for_each(&old_mas, mpnt, ULONG_MAX) {
 		struct file *file;
 		if (mpnt->vm_flags & VM_DONTCOPY) {
 		 */
 		if (fatal_signal_pending(current)) {
 			retval = -EINTR;
-			goto out;
+			goto loop_out;
 		}
 		if (mpnt->vm_flags & VM_ACCOUNT) {
 			unsigned long len = vma_pages(mpnt);
 		}
 		/*
-		 * Clear hugetlb-related page reserves for children. This only
+		 * Copy/update hugetlb private vma information.
-		 * affects MAP_PRIVATE mappings. Faults generated by the child
-		 * are not guaranteed to succeed, even if read-only
 		 */
 		if (is_vm_hugetlb_page(tmp))
-			reset_vma_resv_huge_pages(tmp);
+			hugetlb_dup_vma_private(tmp);
-		/*
-		 * Link in the new vma and copy the page table entries.
-		 */
-		*pprev = tmp;
-		pprev = &tmp->vm_next;
-		tmp->vm_prev = prev;
-		prev = tmp;
-		__vma_link_rb(mm, tmp, rb_link, rb_parent);
+		/* Link the vma into the MT */
-		rb_link = &tmp->vm_rb.rb_right;
+		mas.index = tmp->vm_start;
-		rb_parent = &tmp->vm_rb;
+		mas.last = tmp->vm_end - 1;
+		mas_store(&mas, tmp);
+		if (mas_is_err(&mas))
+			goto fail_nomem_mas_store;
 		mm->map_count++;
 		if (!(tmp->vm_flags & VM_WIPEONFORK))
 			tmp->vm_ops->open(tmp);
 		if (retval)
-			goto out;
+			goto loop_out;
 	}
 	/* a new mm has just been created */
 	retval = arch_dup_mmap(oldmm, mm);
+loop_out:
+	mas_destroy(&mas);
 out:
 	mmap_write_unlock(mm);
 	flush_tlb_mm(oldmm);
 fail_uprobe_end:
 	uprobe_end_dup_mmap();
 	return retval;
+fail_nomem_mas_store:
+	unlink_anon_vmas(tmp);
 fail_nomem_anon_vma_fork:
 	mpol_put(vma_policy(tmp));
 fail_nomem_policy:
 fail_nomem:
 	retval = -ENOMEM;
 	vm_unacct_memory(charge);
-	goto out;
+	goto loop_out;
 }
 static inline int mm_alloc_pgd(struct mm_struct *mm)
 	tsk->worker_private = NULL;
 	kcov_task_init(tsk);
+	kmsan_task_create(tsk);
 	kmap_local_fork(tsk);
 #ifdef CONFIG_FAULT_INJECTION
 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 	struct user_namespace *user_ns)
 {
-	mm->mmap = NULL;
+	mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
-	mm->mm_rb = RB_ROOT;
+	mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
-	mm->vmacache_seqnum = 0;
 	atomic_set(&mm->mm_users, 1);
 	atomic_set(&mm->mm_count, 1);
 	seqcount_init(&mm->write_protect_seq);
 		goto fail_nocontext;
 	mm->user_ns = get_user_ns(user_ns);
+	lru_gen_init_mm(mm);
 	return mm;
 fail_nocontext:
 	}
 	if (mm->binfmt)
 		module_put(mm->binfmt->module);
+	lru_gen_del_mm(mm);
 	mmdrop(mm);
 }
 	/* Forbid mm->exe_file change if old file still mapped. */
 	old_exe_file = get_mm_exe_file(mm);
 	if (old_exe_file) {
+		VMA_ITERATOR(vmi, mm, 0);
 		mmap_read_lock(mm);
-		for (vma = mm->mmap; vma && !ret; vma = vma->vm_next) {
+		for_each_vma(vmi, vma) {
 			if (!vma->vm_file)
 				continue;
 			if (path_equal(&vma->vm_file->f_path,
-				       &old_exe_file->f_path))
+				       &old_exe_file->f_path)) {
 				ret = -EBUSY;
+				break;
+			}
 		}
 		mmap_read_unlock(mm);
 		fput(old_exe_file);
 	if (!oldmm)
 		return 0;
-	/* initialize the new vmacache entries */
-	vmacache_flush(tsk);
 	if (clone_flags & CLONE_VM) {
 		mmget(oldmm);
 		mm = oldmm;
 		get_task_struct(p);
 	}
+	if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) {
+		/* lock the task to synchronize with memcg migration */
+		task_lock(p);
+		lru_gen_add_mm(p->mm);
+		task_unlock(p);
+	}
 	wake_up_new_task(p);
 	/* forking complete and child started to run, tell ptracer */

kernel/kcov.c+7−0

 #include <linux/fs.h>
 #include <linux/hashtable.h>
 #include <linux/init.h>
+#include <linux/kmsan-checks.h>
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/printk.h>
 	INIT_LIST_HEAD(&area->list);
 	area->size = size;
 	list_add(&area->list, &kcov_remote_areas);
+	/*
+	 * KMSAN doesn't instrument this file, so it may not know area->list
+	 * is initialized. Unpoison it explicitly to avoid reports in
+	 * kcov_remote_area_get().
+	 */
+	kmsan_unpoison_memory(&area->list, sizeof(area->list));
 }
 static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)

kernel/locking/Makefile+2−1

@@ -5,8 +5,9 @@ KCOV_INSTRUMENT := n · 4 unmodified lines
5		5
6	obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o	6	obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
7		7
8	# Avoid recursion lockdep -> KCSAN -> ... -> lockdep.	8	# Avoid recursion lockdep -> sanitizer -> ... -> lockdep.
9	KCSAN_SANITIZE_lockdep.o := n	9	KCSAN_SANITIZE_lockdep.o := n
		10	KMSAN_SANITIZE_lockdep.o := n
10		11
11	ifdef CONFIG_FUNCTION_TRACER	12	ifdef CONFIG_FUNCTION_TRACER
12	CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE)	13	CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE)

kernel/sched/core.c+15−0

 }
 #ifdef CONFIG_PROC_SYSCTL
+static void reset_memory_tiering(void)
+{
+	struct pglist_data *pgdat;
+	for_each_online_pgdat(pgdat) {
+		pgdat->nbp_threshold = 0;
+		pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
+		pgdat->nbp_th_start = jiffies_to_msecs(jiffies);
+	}
+}
 int sysctl_numa_balancing(struct ctl_table *table, int write,
 			  void *buffer, size_t *lenp, loff_t *ppos)
 {
 	if (err < 0)
 		return err;
 	if (write) {
+		if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
+		    (state & NUMA_BALANCING_MEMORY_TIERING))
+			reset_memory_tiering();
 		sysctl_numa_balancing_mode = state;
 		__set_numabalancing_state(state);
 	}
 		 * finish_task_switch()'s mmdrop().
 		 */
 		switch_mm_irqs_off(prev->active_mm, next->mm, next);
+		lru_gen_use_mm(next->mm);
 		if (!prev->mm) {                        // from kernel
 			/* will mmdrop() in finish_task_switch(). */

kernel/sched/debug.c+1−0

@@ -333,6 +333,7 @@ static __init int sched_init_debug(void) · 332 unmodified lines
333	debugfs_create_u32("scan_period_min_ms", 0644, numa, &sysctl_numa_balancing_scan_period_min);	333	debugfs_create_u32("scan_period_min_ms", 0644, numa, &sysctl_numa_balancing_scan_period_min);
334	debugfs_create_u32("scan_period_max_ms", 0644, numa, &sysctl_numa_balancing_scan_period_max);	334	debugfs_create_u32("scan_period_max_ms", 0644, numa, &sysctl_numa_balancing_scan_period_max);
335	debugfs_create_u32("scan_size_mb", 0644, numa, &sysctl_numa_balancing_scan_size);	335	debugfs_create_u32("scan_size_mb", 0644, numa, &sysctl_numa_balancing_scan_size);
		336	debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
336	#endif	337	#endif
337		338
338	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);	339	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);

kernel/sched/fair.c+172−3

@@ -40,6 +40,7 @@ · 39 unmodified lines
40		40
41	#include <linux/cpuidle.h>	41	#include <linux/cpuidle.h>
42	#include <linux/interrupt.h>	42	#include <linux/interrupt.h>
		43	#include <linux/memory-tiers.h>
43	#include <linux/mempolicy.h>	44	#include <linux/mempolicy.h>
44	#include <linux/mutex_api.h>	45	#include <linux/mutex_api.h>
45	#include <linux/profile.h>	46	#include <linux/profile.h>
@@ -1090,6 +1091,12 @@ unsigned int sysctl_numa_balancing_scan_size = 256; · 1044 unmodified lines
1090	/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */	1091	/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
1091	unsigned int sysctl_numa_balancing_scan_delay = 1000;	1092	unsigned int sysctl_numa_balancing_scan_delay = 1000;
1092		1093
		1094	/* The page with hint page fault latency < threshold in ms is considered hot */
		1095	unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC;
		1096
		1097	/* Restrict the NUMA promotion throughput (MB/s) for each target node. */
		1098	unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
		1099
1093	struct numa_group {	1100	struct numa_group {
1094	refcount_t refcount;	1101	refcount_t refcount;
1095		1102
@@ -1432,6 +1439,120 @@ static inline unsigned long group_weight(struct task_struct *p, int nid, · 336 unmodified lines
1432	return 1000 * faults / total_faults;	1439	return 1000 * faults / total_faults;
1433	}	1440	}
1434		1441
		1442	/*
		1443	* If memory tiering mode is enabled, cpupid of slow memory page is
		1444	* used to record scan time instead of CPU and PID. When tiering mode
		1445	* is disabled at run time, the scan time (in cpupid) will be
		1446	* interpreted as CPU and PID. So CPU needs to be checked to avoid to
		1447	* access out of array bound.
		1448	*/
		1449	static inline bool cpupid_valid(int cpupid)
		1450	{
		1451	return cpupid_to_cpu(cpupid) < nr_cpu_ids;
		1452	}
		1453
		1454	/*
		1455	* For memory tiering mode, if there are enough free pages (more than
		1456	* enough watermark defined here) in fast memory node, to take full
		1457	* advantage of fast memory capacity, all recently accessed slow
		1458	* memory pages will be migrated to fast memory node without
		1459	* considering hot threshold.
		1460	*/
		1461	static bool pgdat_free_space_enough(struct pglist_data *pgdat)
		1462	{
		1463	int z;
		1464	unsigned long enough_wmark;
		1465
		1466	enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT,
		1467	pgdat->node_present_pages >> 4);
		1468	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
		1469	struct zone *zone = pgdat->node_zones + z;
		1470
		1471	if (!populated_zone(zone))
		1472	continue;
		1473
		1474	if (zone_watermark_ok(zone, 0,
		1475	wmark_pages(zone, WMARK_PROMO) + enough_wmark,
		1476	ZONE_MOVABLE, 0))
		1477	return true;
		1478	}
		1479	return false;
		1480	}
		1481
		1482	/*
		1483	* For memory tiering mode, when page tables are scanned, the scan
		1484	* time will be recorded in struct page in addition to make page
		1485	* PROT_NONE for slow memory page. So when the page is accessed, in
		1486	* hint page fault handler, the hint page fault latency is calculated
		1487	* via,
		1488	*
		1489	* hint page fault latency = hint page fault time - scan time
		1490	*
		1491	* The smaller the hint page fault latency, the higher the possibility
		1492	* for the page to be hot.
		1493	*/
		1494	static int numa_hint_fault_latency(struct page *page)
		1495	{
		1496	int last_time, time;
		1497
		1498	time = jiffies_to_msecs(jiffies);
		1499	last_time = xchg_page_access_time(page, time);
		1500
		1501	return (time - last_time) & PAGE_ACCESS_TIME_MASK;
		1502	}
		1503
		1504	/*
		1505	* For memory tiering mode, too high promotion/demotion throughput may
		1506	* hurt application latency. So we provide a mechanism to rate limit
		1507	* the number of pages that are tried to be promoted.
		1508	*/
		1509	static bool numa_promotion_rate_limit(struct pglist_data *pgdat,
		1510	unsigned long rate_limit, int nr)
		1511	{
		1512	unsigned long nr_cand;
		1513	unsigned int now, start;
		1514
		1515	now = jiffies_to_msecs(jiffies);
		1516	mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr);
		1517	nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
		1518	start = pgdat->nbp_rl_start;
		1519	if (now - start > MSEC_PER_SEC &&
		1520	cmpxchg(&pgdat->nbp_rl_start, start, now) == start)
		1521	pgdat->nbp_rl_nr_cand = nr_cand;
		1522	if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit)
		1523	return true;
		1524	return false;
		1525	}
		1526
		1527	#define NUMA_MIGRATION_ADJUST_STEPS 16
		1528
		1529	static void numa_promotion_adjust_threshold(struct pglist_data *pgdat,
		1530	unsigned long rate_limit,
		1531	unsigned int ref_th)
		1532	{
		1533	unsigned int now, start, th_period, unit_th, th;
		1534	unsigned long nr_cand, ref_cand, diff_cand;
		1535
		1536	now = jiffies_to_msecs(jiffies);
		1537	th_period = sysctl_numa_balancing_scan_period_max;
		1538	start = pgdat->nbp_th_start;
		1539	if (now - start > th_period &&
		1540	cmpxchg(&pgdat->nbp_th_start, start, now) == start) {
		1541	ref_cand = rate_limit *
		1542	sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC;
		1543	nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE);
		1544	diff_cand = nr_cand - pgdat->nbp_th_nr_cand;
		1545	unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS;
		1546	th = pgdat->nbp_threshold ? : ref_th;
		1547	if (diff_cand > ref_cand * 11 / 10)
		1548	th = max(th - unit_th, unit_th);
		1549	else if (diff_cand < ref_cand * 9 / 10)
		1550	th = min(th + unit_th, ref_th * 2);
		1551	pgdat->nbp_th_nr_cand = nr_cand;
		1552	pgdat->nbp_threshold = th;
		1553	}
		1554	}
		1555
1435	bool should_numa_migrate_memory(struct task_struct p, struct page page,	1556	bool should_numa_migrate_memory(struct task_struct p, struct page page,
1436	int src_nid, int dst_cpu)	1557	int src_nid, int dst_cpu)
1437	{	1558	{
@@ -1439,9 +1560,44 @@ bool should_numa_migrate_memory(struct task_struct p, struct page page, · 1 unmodified line
1439	int dst_nid = cpu_to_node(dst_cpu);	1560	int dst_nid = cpu_to_node(dst_cpu);
1440	int last_cpupid, this_cpupid;	1561	int last_cpupid, this_cpupid;
1441		1562
		1563	/*
		1564	* The pages in slow memory node should be migrated according
		1565	* to hot/cold instead of private/shared.
		1566	*/
		1567	if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
		1568	!node_is_toptier(src_nid)) {
		1569	struct pglist_data *pgdat;
		1570	unsigned long rate_limit;
		1571	unsigned int latency, th, def_th;
		1572
		1573	pgdat = NODE_DATA(dst_nid);
		1574	if (pgdat_free_space_enough(pgdat)) {
		1575	/* workload changed, reset hot threshold */
		1576	pgdat->nbp_threshold = 0;
		1577	return true;
		1578	}
		1579
		1580	def_th = sysctl_numa_balancing_hot_threshold;
		1581	rate_limit = sysctl_numa_balancing_promote_rate_limit << \
		1582	(20 - PAGE_SHIFT);
		1583	numa_promotion_adjust_threshold(pgdat, rate_limit, def_th);
		1584
		1585	th = pgdat->nbp_threshold ? : def_th;
		1586	latency = numa_hint_fault_latency(page);
		1587	if (latency >= th)
		1588	return false;
		1589
		1590	return !numa_promotion_rate_limit(pgdat, rate_limit,
		1591	thp_nr_pages(page));
		1592	}
		1593
1442	this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);	1594	this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid);
1443	last_cpupid = page_cpupid_xchg_last(page, this_cpupid);	1595	last_cpupid = page_cpupid_xchg_last(page, this_cpupid);
1444		1596
		1597	if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
		1598	!node_is_toptier(src_nid) && !cpupid_valid(last_cpupid))
		1599	return false;
		1600
1445	/*	1601	/*
1446	* Allow first faults or private faults to migrate immediately early in	1602	* Allow first faults or private faults to migrate immediately early in
1447	* the lifetime of a task. The magic number 4 is based on waiting for	1603	* the lifetime of a task. The magic number 4 is based on waiting for
@@ -2681,6 +2837,15 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) · 1233 unmodified lines
2681	if (!p->mm)	2837	if (!p->mm)
2682	return;	2838	return;
2683		2839
		2840	/*
		2841	* NUMA faults statistics are unnecessary for the slow memory
		2842	* node for memory tiering mode.
		2843	*/
		2844	if (!node_is_toptier(mem_node) &&
		2845	(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING \|\|
		2846	!cpupid_valid(last_cpupid)))
		2847	return;
		2848
2684	/* Allocate buffer to track faults on a per-node basis */	2849	/* Allocate buffer to track faults on a per-node basis */
2685	if (unlikely(!p->numa_faults)) {	2850	if (unlikely(!p->numa_faults)) {
2686	int size = sizeof(p->numa_faults)	2851	int size = sizeof(p->numa_faults)
@@ -2761,6 +2926,7 @@ static void task_numa_work(struct callback_head *work) · 74 unmodified lines
2761	struct task_struct *p = current;	2926	struct task_struct *p = current;
2762	struct mm_struct *mm = p->mm;	2927	struct mm_struct *mm = p->mm;
2763	u64 runtime = p->se.sum_exec_runtime;	2928	u64 runtime = p->se.sum_exec_runtime;
		2929	MA_STATE(mas, &mm->mm_mt, 0, 0);
2764	struct vm_area_struct *vma;	2930	struct vm_area_struct *vma;
2765	unsigned long start, end;	2931	unsigned long start, end;
2766	unsigned long nr_pte_updates = 0;	2932	unsigned long nr_pte_updates = 0;
@@ -2817,13 +2983,16 @@ static void task_numa_work(struct callback_head *work) · 50 unmodified lines
2817		2983
2818	if (!mmap_read_trylock(mm))	2984	if (!mmap_read_trylock(mm))
2819	return;	2985	return;
2820	vma = find_vma(mm, start);	2986	mas_set(&mas, start);
		2987	vma = mas_find(&mas, ULONG_MAX);
2821	if (!vma) {	2988	if (!vma) {
2822	reset_ptenuma_scan(p);	2989	reset_ptenuma_scan(p);
2823	start = 0;	2990	start = 0;
2824	vma = mm->mmap;	2991	mas_set(&mas, start);
		2992	vma = mas_find(&mas, ULONG_MAX);
2825	}	2993	}
2826	for (; vma; vma = vma->vm_next) {	2994
		2995	for (; vma; vma = mas_find(&mas, ULONG_MAX)) {
2827	if (!vma_migratable(vma) \|\| !vma_policy_mof(vma) \|\|	2996	if (!vma_migratable(vma) \|\| !vma_policy_mof(vma) \|\|
2828	is_vm_hugetlb_page(vma) \|\| (vma->vm_flags & VM_MIXEDMAP)) {	2997	is_vm_hugetlb_page(vma) \|\| (vma->vm_flags & VM_MIXEDMAP)) {
2829	continue;	2998	continue;

kernel/sched/sched.h+1−0

@@ -2446,6 +2446,7 @@ extern unsigned int sysctl_numa_balancing_scan_delay; · 2445 unmodified lines
2446	extern unsigned int sysctl_numa_balancing_scan_period_min;	2446	extern unsigned int sysctl_numa_balancing_scan_period_min;
2447	extern unsigned int sysctl_numa_balancing_scan_period_max;	2447	extern unsigned int sysctl_numa_balancing_scan_period_max;
2448	extern unsigned int sysctl_numa_balancing_scan_size;	2448	extern unsigned int sysctl_numa_balancing_scan_size;
		2449	extern unsigned int sysctl_numa_balancing_hot_threshold;
2449	#endif	2450	#endif
2450		2451
2451	#ifdef CONFIG_SCHED_HRTICK	2452	#ifdef CONFIG_SCHED_HRTICK

kernel/sysctl.c+8−0

@@ -1643,6 +1643,14 @@ static struct ctl_table kern_table[] = { · 1642 unmodified lines
1643	.extra1 = SYSCTL_ZERO,	1643	.extra1 = SYSCTL_ZERO,
1644	.extra2 = SYSCTL_FOUR,	1644	.extra2 = SYSCTL_FOUR,
1645	},	1645	},
		1646	{
		1647	.procname = "numa_balancing_promote_rate_limit_MBps",
		1648	.data = &sysctl_numa_balancing_promote_rate_limit,
		1649	.maxlen = sizeof(unsigned int),
		1650	.mode = 0644,
		1651	.proc_handler = proc_dointvec_minmax,
		1652	.extra1 = SYSCTL_ZERO,
		1653	},
1646	#endif /* CONFIG_NUMA_BALANCING */	1654	#endif /* CONFIG_NUMA_BALANCING */
1647	{	1655	{
1648	.procname = "panic",	1656	.procname = "panic",

Merge tag 'mm-stable-2022-10-08' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code