Merge branch 'akpm' (patches from Andrew)

Merge misc fixes from Andrew Morton: "21 fixes" * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (21 commits) userfaultfd: replace ENOSPC with ESRCH in case mm has gone during copy/zeropage zram: rework copy of compressor name in comp_algorithm_store() rmap: do not call mmu_notifier_invalidate_page() under ptl mm: fix list corruptions on shmem shrinklist mm/balloon_compaction.c: don't zero ballooned pages MAINTAINERS: copy virtio on balloon_compaction.c mm: fix KSM data corruption mm: fix MADV_[FREE|DONTNEED] TLB flush miss problem mm: make tlb_flush_pending global mm: refactor TLB gathering API Revert "mm: numa: defer TLB flush for THP migration as long as possible" mm: migrate: fix barriers around tlb_flush_pending mm: migrate: prevent racy access to tlb_flush_pending fault-inject: fix wrong should_fail() decision in task context test_kmod: fix small memory leak on filesystem tests test_kmod: fix the lock in register_test_dev_kmod() test_kmod: fix bug which allows negative values on two config options test_kmod: fix spelling mistake: "EMTPY" -> "EMPTY" userfaultfd: hugetlbfs: remove superfluous page unlock in VM_SHARED case mm: ratelimit PFNs busy info message ...

Linus Torvalds committed 9 years ago

Commit 27df7042 parents 4e082e9 e86b29828 files · +208 −121Browse files

Unified · Split

28 files

MAINTAINERS +1 −0
arch/arm/include/asm/tlb.h +9 −2
arch/ia64/include/asm/tlb.h +6 −2
arch/s390/include/asm/tlb.h +11 −6
arch/sh/include/asm/tlb.h +5 −3
arch/um/include/asm/tlb.h +10 −3
drivers/block/zram/zram_drv.c +2 −2
fs/proc/meminfo.c +4 −4
fs/proc/task_mmu.c +5 −2
fs/userfaultfd.c +2 −2
include/asm-generic/tlb.h +4 −3
include/linux/mm_types.h +38 −26
kernel/fork.c +1 −1
kernel/power/snapshot.c +1 −1
lib/fault-inject.c +5 −3
lib/test_kmod.c +8 −8
mm/balloon_compaction.c +1 −1
mm/debug.c +1 −5
mm/huge_memory.c +7 −0
mm/hugetlb.c +1 −1
mm/ksm.c +2 −1
mm/memory.c +35 −7
mm/migrate.c +0 −6
mm/mprotect.c +2 −2
mm/page_alloc.c +6 −5
mm/rmap.c +30 −22
mm/shmem.c +10 −2
mm/util.c +1 −1

MAINTAINERS+1−0

@@ -14004,6 +14004,7 @@ F: drivers/block/virtio_blk.c · 14003 unmodified lines
14004	14004	F: include/linux/virtio*.h
14005	14005	F: include/uapi/linux/virtio_*.h
14006	14006	F: drivers/crypto/virtio/
	14007	+F: mm/balloon_compaction.c
14007	14008
14008	14009	VIRTIO CRYPTO DRIVER
14009	14010	M: Gonglei <arei.gonglei@huawei.com>

arch/arm/include/asm/tlb.h+9−2

@@ -148,7 +148,8 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) · 147 unmodified lines
148	148	}
149	149
150	150	static inline void
151		−tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start, unsigned long end)
	151	+arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	152	+ unsigned long start, unsigned long end)
152	153	{
153	154	tlb->mm = mm;
154	155	tlb->fullmm = !(start \| (end+1));
@@ -166,8 +167,14 @@ tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start · 11 unmodified lines
166	167	}
167	168
168	169	static inline void
169		−tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
	170	+arch_tlb_finish_mmu(struct mmu_gather *tlb,
	171	+ unsigned long start, unsigned long end, bool force)
170	172	{
	173	+ if (force) {
	174	+ tlb->range_start = start;
	175	+ tlb->range_end = end;
	176	+ }
	177	+
171	178	tlb_flush_mmu(tlb);
172	179
173	180	/* keep the page table cache within bounds */

arch/ia64/include/asm/tlb.h+6−2

@@ -168,7 +168,8 @@ static inline void __tlb_alloc_page(struct mmu_gather *tlb) · 167 unmodified lines
168	168
169	169
170	170	static inline void
171		−tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start, unsigned long end)
	171	+arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	172	+ unsigned long start, unsigned long end)
172	173	{
173	174	tlb->mm = mm;
174	175	tlb->max = ARRAY_SIZE(tlb->local);
@@ -185,8 +186,11 @@ tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start · 10 unmodified lines
185	186	* collected.
186	187	*/
187	188	static inline void
188		−tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
	189	+arch_tlb_finish_mmu(struct mmu_gather *tlb,
	190	+ unsigned long start, unsigned long end, bool force)
189	191	{
	192	+ if (force)
	193	+ tlb->need_flush = 1;
190	194	/*
191	195	* Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and
192	196	* tlb->end_addr.

arch/s390/include/asm/tlb.h+11−6

@@ -47,10 +47,9 @@ struct mmu_table_batch { · 46 unmodified lines
47	47	extern void tlb_table_flush(struct mmu_gather *tlb);
48	48	extern void tlb_remove_table(struct mmu_gather tlb, void table);
49	49
50		−static inline void tlb_gather_mmu(struct mmu_gather *tlb,
51		− struct mm_struct *mm,
52		− unsigned long start,
53		− unsigned long end)
	50	+static inline void
	51	+arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	52	+ unsigned long start, unsigned long end)
54	53	{
55	54	tlb->mm = mm;
56	55	tlb->start = start;
@@ -76,9 +75,15 @@ static inline void tlb_flush_mmu(struct mmu_gather *tlb) · 19 unmodified lines
76	75	tlb_flush_mmu_free(tlb);
77	76	}
78	77
79		−static inline void tlb_finish_mmu(struct mmu_gather *tlb,
80		− unsigned long start, unsigned long end)
	78	+static inline void
	79	+arch_tlb_finish_mmu(struct mmu_gather *tlb,
	80	+ unsigned long start, unsigned long end, bool force)
81	81	{
	82	+ if (force) {
	83	+ tlb->start = start;
	84	+ tlb->end = end;
	85	+ }
	86	+
82	87	tlb_flush_mmu(tlb);
83	88	}
84	89

arch/sh/include/asm/tlb.h+5−3

@@ -36,7 +36,8 @@ static inline void init_tlb_gather(struct mmu_gather *tlb) · 35 unmodified lines
36	36	}
37	37
38	38	static inline void
39		−tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start, unsigned long end)
	39	+arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	40	+ unsigned long start, unsigned long end)
40	41	{
41	42	tlb->mm = mm;
42	43	tlb->start = start;
@@ -47,9 +48,10 @@ tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start · 4 unmodified lines
47	48	}
48	49
49	50	static inline void
50		−tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
	51	+arch_tlb_finish_mmu(struct mmu_gather *tlb,
	52	+ unsigned long start, unsigned long end, bool force)
51	53	{
52		− if (tlb->fullmm)
	54	+ if (tlb->fullmm \|\| force)
53	55	flush_tlb_mm(tlb->mm);
54	56
55	57	/* keep the page table cache within bounds */

arch/um/include/asm/tlb.h+10−3

@@ -45,7 +45,8 @@ static inline void init_tlb_gather(struct mmu_gather *tlb) · 44 unmodified lines
45	45	}
46	46
47	47	static inline void
48		−tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start, unsigned long end)
	48	+arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	49	+ unsigned long start, unsigned long end)
49	50	{
50	51	tlb->mm = mm;
51	52	tlb->start = start;
@@ -80,13 +81,19 @@ tlb_flush_mmu(struct mmu_gather *tlb) · 28 unmodified lines
80	81	tlb_flush_mmu_free(tlb);
81	82	}
82	83
83		−/* tlb_finish_mmu
	84	+/* arch_tlb_finish_mmu
84	85	* Called at the end of the shootdown operation to free up any resources
85	86	* that were required.
86	87	*/
87	88	static inline void
88		−tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
	89	+arch_tlb_finish_mmu(struct mmu_gather *tlb,
	90	+ unsigned long start, unsigned long end, bool force)
89	91	{
	92	+ if (force) {
	93	+ tlb->start = start;
	94	+ tlb->end = end;
	95	+ tlb->need_flush = 1;
	96	+ }
90	97	tlb_flush_mmu(tlb);
91	98
92	99	/* keep the page table cache within bounds */

drivers/block/zram/zram_drv.c+2−2

@@ -308,7 +308,7 @@ static ssize_t comp_algorithm_store(struct device *dev, · 307 unmodified lines
308	308	struct device_attribute attr, const char buf, size_t len)
309	309	{
310	310	struct zram *zram = dev_to_zram(dev);
311		− char compressor[CRYPTO_MAX_ALG_NAME];
	311	+ char compressor[ARRAY_SIZE(zram->compressor)];
312	312	size_t sz;
313	313
314	314	strlcpy(compressor, buf, sizeof(compressor));
@@ -327,7 +327,7 @@ static ssize_t comp_algorithm_store(struct device *dev, · 12 unmodified lines
327	327	return -EBUSY;
328	328	}
329	329
330		− strlcpy(zram->compressor, compressor, sizeof(compressor));
	330	+ strcpy(zram->compressor, compressor);
331	331	up_write(&zram->init_lock);
332	332	return len;
333	333	}

fs/proc/meminfo.c+4−4

@@ -106,13 +106,13 @@ static int meminfo_proc_show(struct seq_file m, void v) · 105 unmodified lines
106	106	global_node_page_state(NR_FILE_MAPPED));
107	107	show_val_kb(m, "Shmem: ", i.sharedram);
108	108	show_val_kb(m, "Slab: ",
109		− global_page_state(NR_SLAB_RECLAIMABLE) +
110		− global_page_state(NR_SLAB_UNRECLAIMABLE));
	109	+ global_node_page_state(NR_SLAB_RECLAIMABLE) +
	110	+ global_node_page_state(NR_SLAB_UNRECLAIMABLE));
111	111
112	112	show_val_kb(m, "SReclaimable: ",
113		− global_page_state(NR_SLAB_RECLAIMABLE));
	113	+ global_node_page_state(NR_SLAB_RECLAIMABLE));
114	114	show_val_kb(m, "SUnreclaim: ",
115		− global_page_state(NR_SLAB_UNRECLAIMABLE));
	115	+ global_node_page_state(NR_SLAB_UNRECLAIMABLE));
116	116	seq_printf(m, "KernelStack: %8lu kB\n",
117	117	global_page_state(NR_KERNEL_STACK_KB));
118	118	show_val_kb(m, "PageTables: ",

fs/proc/task_mmu.c+5−2

@@ -16,9 +16,10 @@ · 15 unmodified lines
16	16	#include <linux/mmu_notifier.h>
17	17	#include <linux/page_idle.h>
18	18	#include <linux/shmem_fs.h>
	19	+#include <linux/uaccess.h>
19	20
20	21	#include <asm/elf.h>
21		−#include <linux/uaccess.h>
	22	+#include <asm/tlb.h>
22	23	#include <asm/tlbflush.h>
23	24	#include "internal.h"
24	25
@@ -1008,6 +1009,7 @@ static ssize_t clear_refs_write(struct file file, const char __user buf, · 983 unmodified lines
1008	1009	struct mm_struct *mm;
1009	1010	struct vm_area_struct *vma;
1010	1011	enum clear_refs_types type;
	1012	+ struct mmu_gather tlb;
1011	1013	int itype;
1012	1014	int rv;
1013	1015
@@ -1054,6 +1056,7 @@ static ssize_t clear_refs_write(struct file file, const char __user buf, · 40 unmodified lines
1054	1056	}
1055	1057
1056	1058	down_read(&mm->mmap_sem);
	1059	+ tlb_gather_mmu(&tlb, mm, 0, -1);
1057	1060	if (type == CLEAR_REFS_SOFT_DIRTY) {
1058	1061	for (vma = mm->mmap; vma; vma = vma->vm_next) {
1059	1062	if (!(vma->vm_flags & VM_SOFTDIRTY))
@@ -1075,7 +1078,7 @@ static ssize_t clear_refs_write(struct file file, const char __user buf, · 15 unmodified lines
1075	1078	walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
1076	1079	if (type == CLEAR_REFS_SOFT_DIRTY)
1077	1080	mmu_notifier_invalidate_range_end(mm, 0, -1);
1078		− flush_tlb_mm(mm);
	1081	+ tlb_finish_mmu(&tlb, 0, -1);
1079	1082	up_read(&mm->mmap_sem);
1080	1083	out_mm:
1081	1084	mmput(mm);

fs/userfaultfd.c+2−2

@@ -1600,7 +1600,7 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, · 1599 unmodified lines
1600	1600	uffdio_copy.len);
1601	1601	mmput(ctx->mm);
1602	1602	} else {
1603		− return -ENOSPC;
	1603	+ return -ESRCH;
1604	1604	}
1605	1605	if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1606	1606	return -EFAULT;
@@ -1647,7 +1647,7 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, · 40 unmodified lines
1647	1647	uffdio_zeropage.range.len);
1648	1648	mmput(ctx->mm);
1649	1649	} else {
1650		− return -ENOSPC;
	1650	+ return -ESRCH;
1651	1651	}
1652	1652	if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1653	1653	return -EFAULT;

include/asm-generic/tlb.h+4−3

@@ -112,10 +112,11 @@ struct mmu_gather { · 111 unmodified lines
112	112
113	113	#define HAVE_GENERIC_MMU_GATHER
114	114
115		−void tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start, unsigned long end);
	115	+void arch_tlb_gather_mmu(struct mmu_gather *tlb,
	116	+ struct mm_struct *mm, unsigned long start, unsigned long end);
116	117	void tlb_flush_mmu(struct mmu_gather *tlb);
117		−void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
118		− unsigned long end);
	118	+void arch_tlb_finish_mmu(struct mmu_gather *tlb,
	119	+ unsigned long start, unsigned long end, bool force);
119	120	extern bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page,
120	121	int page_size);
121	122

include/linux/mm_types.h+38−26

@@ -487,14 +487,12 @@ struct mm_struct { · 486 unmodified lines
487	487	/* numa_scan_seq prevents two threads setting pte_numa */
488	488	int numa_scan_seq;
489	489	#endif
490		−#if defined(CONFIG_NUMA_BALANCING) \|\| defined(CONFIG_COMPACTION)
491	490	/*
492	491	* An operation with batched TLB flushing is going on. Anything that
493	492	* can move process memory needs to flush the TLB when moving a
494	493	* PROT_NONE or PROT_NUMA mapped page.
495	494	*/
496		− bool tlb_flush_pending;
497		−#endif
	495	+ atomic_t tlb_flush_pending;
498	496	#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
499	497	/* See flush_tlb_batched_pending() */
500	498	bool tlb_flush_batched;
@@ -522,46 +520,60 @@ static inline cpumask_t mm_cpumask(struct mm_struct mm) · 21 unmodified lines
522	520	return mm->cpu_vm_mask_var;
523	521	}
524	522
525		−#if defined(CONFIG_NUMA_BALANCING) \|\| defined(CONFIG_COMPACTION)
	523	+struct mmu_gather;
	524	+extern void tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	525	+ unsigned long start, unsigned long end);
	526	+extern void tlb_finish_mmu(struct mmu_gather *tlb,
	527	+ unsigned long start, unsigned long end);
	528	+
526	529	/*
527	530	* Memory barriers to keep this state in sync are graciously provided by
528	531	* the page table locks, outside of which no page table modifications happen.
529		− * The barriers below prevent the compiler from re-ordering the instructions
530		− * around the memory barriers that are already present in the code.
	532	+ * The barriers are used to ensure the order between tlb_flush_pending updates,
	533	+ * which happen while the lock is not taken, and the PTE updates, which happen
	534	+ * while the lock is taken, are serialized.
531	535	*/
532	536	static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
533	537	{
534		− barrier();
535		− return mm->tlb_flush_pending;
	538	+ return atomic_read(&mm->tlb_flush_pending) > 0;
	539	+}
	540	+
	541	+/*
	542	+ * Returns true if there are two above TLB batching threads in parallel.
	543	+ */
	544	+static inline bool mm_tlb_flush_nested(struct mm_struct *mm)
	545	+{
	546	+ return atomic_read(&mm->tlb_flush_pending) > 1;
	547	+}
	548	+
	549	+static inline void init_tlb_flush_pending(struct mm_struct *mm)
	550	+{
	551	+ atomic_set(&mm->tlb_flush_pending, 0);
536	552	}
537		−static inline void set_tlb_flush_pending(struct mm_struct *mm)
	553	+
	554	+static inline void inc_tlb_flush_pending(struct mm_struct *mm)
538	555	{
539		− mm->tlb_flush_pending = true;
	556	+ atomic_inc(&mm->tlb_flush_pending);
540	557
541	558	/*
542		− * Guarantee that the tlb_flush_pending store does not leak into the
	559	+ * Guarantee that the tlb_flush_pending increase does not leak into the
543	560	* critical section updating the page tables
544	561	*/
545	562	smp_mb__before_spinlock();
546	563	}
	564	+
547	565	/* Clearing is done after a TLB flush, which also provides a barrier. */
548		−static inline void clear_tlb_flush_pending(struct mm_struct *mm)
549		−{
550		− barrier();
551		− mm->tlb_flush_pending = false;
552		−}
553		−#else
554		−static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
555		−{
556		− return false;
557		−}
558		−static inline void set_tlb_flush_pending(struct mm_struct *mm)
559		−{
560		−}
561		−static inline void clear_tlb_flush_pending(struct mm_struct *mm)
	566	+static inline void dec_tlb_flush_pending(struct mm_struct *mm)
562	567	{
	568	+ /*
	569	+ * Guarantee that the tlb_flush_pending does not not leak into the
	570	+ * critical section, since we must order the PTE change and changes to
	571	+ * the pending TLB flush indication. We could have relied on TLB flush
	572	+ * as a memory barrier, but this behavior is not clearly documented.
	573	+ */
	574	+ smp_mb__before_atomic();
	575	+ atomic_dec(&mm->tlb_flush_pending);
563	576	}
564		−#endif
565	577
566	578	struct vm_fault;
567	579

kernel/fork.c+1−1

@@ -807,7 +807,7 @@ static struct mm_struct mm_init(struct mm_struct mm, struct task_struct *p, · 806 unmodified lines
807	807	mm_init_aio(mm);
808	808	mm_init_owner(mm, p);
809	809	mmu_notifier_mm_init(mm);
810		− clear_tlb_flush_pending(mm);
	810	+ init_tlb_flush_pending(mm);
811	811	#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
812	812	mm->pmd_huge_pte = NULL;
813	813	#endif

kernel/power/snapshot.c+1−1

@@ -1650,7 +1650,7 @@ static unsigned long minimum_image_size(unsigned long saveable) · 1649 unmodified lines
1650	1650	{
1651	1651	unsigned long size;
1652	1652
1653		− size = global_page_state(NR_SLAB_RECLAIMABLE)
	1653	+ size = global_node_page_state(NR_SLAB_RECLAIMABLE)
1654	1654	+ global_node_page_state(NR_ACTIVE_ANON)
1655	1655	+ global_node_page_state(NR_INACTIVE_ANON)
1656	1656	+ global_node_page_state(NR_ACTIVE_FILE)

lib/fault-inject.c+5−3

@@ -110,10 +110,12 @@ bool should_fail(struct fault_attr *attr, ssize_t size) · 109 unmodified lines
110	110	if (in_task()) {
111	111	unsigned int fail_nth = READ_ONCE(current->fail_nth);
112	112
113		− if (fail_nth && !WRITE_ONCE(current->fail_nth, fail_nth - 1))
114		− goto fail;
	113	+ if (fail_nth) {
	114	+ if (!WRITE_ONCE(current->fail_nth, fail_nth - 1))
	115	+ goto fail;
115	116
116		− return false;
	117	+ return false;
	118	+ }
117	119	}
118	120
119	121	/* No need to check any other properties if the probability is 0 */

lib/test_kmod.c+8−8

@@ -485,7 +485,7 @@ static ssize_t config_show(struct device *dev, · 484 unmodified lines
485	485	config->test_driver);
486	486	else
487	487	len += snprintf(buf+len, PAGE_SIZE - len,
488		− "driver:\tEMTPY\n");
	488	+ "driver:\tEMPTY\n");
489	489
490	490	if (config->test_fs)
491	491	len += snprintf(buf+len, PAGE_SIZE - len,
@@ -493,7 +493,7 @@ static ssize_t config_show(struct device *dev, · 1 unmodified line
493	493	config->test_fs);
494	494	else
495	495	len += snprintf(buf+len, PAGE_SIZE - len,
496		− "fs:\tEMTPY\n");
	496	+ "fs:\tEMPTY\n");
497	497
498	498	mutex_unlock(&test_dev->config_mutex);
499	499
@@ -746,11 +746,11 @@ static int trigger_config_run_type(struct kmod_test_device *test_dev, · 246 unmodified lines
746	746	strlen(test_str));
747	747	break;
748	748	case TEST_KMOD_FS_TYPE:
749		− break;
750	749	kfree_const(config->test_fs);
751	750	config->test_driver = NULL;
752	751	copied = config_copy_test_fs(config, test_str,
753	752	strlen(test_str));
	753	+ break;
754	754	default:
755	755	mutex_unlock(&test_dev->config_mutex);
756	756	return -EINVAL;
@@ -880,10 +880,10 @@ static int test_dev_config_update_uint_sync(struct kmod_test_device *test_dev, · 123 unmodified lines
880	880	int (test_sync)(struct kmod_test_device test_dev))
881	881	{
882	882	int ret;
883		− long new;
	883	+ unsigned long new;
884	884	unsigned int old_val;
885	885
886		− ret = kstrtol(buf, 10, &new);
	886	+ ret = kstrtoul(buf, 10, &new);
887	887	if (ret)
888	888	return ret;
889	889
@@ -918,9 +918,9 @@ static int test_dev_config_update_uint_range(struct kmod_test_device *test_dev, · 28 unmodified lines
918	918	unsigned int max)
919	919	{
920	920	int ret;
921		− long new;
	921	+ unsigned long new;
922	922
923		− ret = kstrtol(buf, 10, &new);
	923	+ ret = kstrtoul(buf, 10, &new);
924	924	if (ret)
925	925	return ret;
926	926
@@ -1146,7 +1146,7 @@ static struct kmod_test_device *register_test_dev_kmod(void) · 219 unmodified lines
1146	1146	struct kmod_test_device *test_dev = NULL;
1147	1147	int ret;
1148	1148
1149		− mutex_unlock(&reg_dev_mutex);
	1149	+ mutex_lock(&reg_dev_mutex);
1150	1150
1151	1151	/* int should suffice for number of devices, test for wrap */
1152	1152	if (unlikely(num_test_devs + 1) < 0) {

mm/balloon_compaction.c+1−1

@@ -24,7 +24,7 @@ struct page balloon_page_enqueue(struct balloon_dev_info b_dev_info) · 23 unmodified lines
24	24	{
25	25	unsigned long flags;
26	26	struct page *page = alloc_page(balloon_mapping_gfp_mask() \|
27		− __GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_ZERO);
	27	+ __GFP_NOMEMALLOC \| __GFP_NORETRY);
28	28	if (!page)
29	29	return NULL;
30	30

mm/debug.c+1−5

@@ -124,9 +124,7 @@ void dump_mm(const struct mm_struct *mm) · 123 unmodified lines
124	124	#ifdef CONFIG_NUMA_BALANCING
125	125	"numa_next_scan %lu numa_scan_offset %lu numa_scan_seq %d\n"
126	126	#endif
127		−#if defined(CONFIG_NUMA_BALANCING) \|\| defined(CONFIG_COMPACTION)
128	127	"tlb_flush_pending %d\n"
129		−#endif
130	128	"def_flags: %#lx(%pGv)\n",
131	129
132	130	mm, mm->mmap, mm->vmacache_seqnum, mm->task_size,
@@ -158,9 +156,7 @@ void dump_mm(const struct mm_struct *mm) · 25 unmodified lines
158	156	#ifdef CONFIG_NUMA_BALANCING
159	157	mm->numa_next_scan, mm->numa_scan_offset, mm->numa_scan_seq,
160	158	#endif
161		−#if defined(CONFIG_NUMA_BALANCING) \|\| defined(CONFIG_COMPACTION)
162		− mm->tlb_flush_pending,
163		−#endif
	159	+ atomic_read(&mm->tlb_flush_pending),
164	160	mm->def_flags, &mm->def_flags
165	161	);
166	162	}

mm/huge_memory.c+7−0

@@ -1495,6 +1495,13 @@ int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) · 1494 unmodified lines
1495	1495	goto clear_pmdnuma;
1496	1496	}
1497	1497
	1498	+ /*
	1499	+ * The page_table_lock above provides a memory barrier
	1500	+ * with change_protection_range.
	1501	+ */
	1502	+ if (mm_tlb_flush_pending(vma->vm_mm))
	1503	+ flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
	1504	+
1498	1505	/*
1499	1506	* Migrate the THP to the requested node, returns with page unlocked
1500	1507	* and access rights restored.

mm/hugetlb.c+1−1

@@ -4062,9 +4062,9 @@ out: · 4061 unmodified lines
4062	4062	return ret;
4063	4063	out_release_unlock:
4064	4064	spin_unlock(ptl);
4065		−out_release_nounlock:
4066	4065	if (vm_shared)
4067	4066	unlock_page(page);
	4067	+out_release_nounlock:
4068	4068	put_page(page);
4069	4069	goto out;
4070	4070	}

mm/ksm.c+2−1

@@ -1038,7 +1038,8 @@ static int write_protect_page(struct vm_area_struct vma, struct page page, · 1037 unmodified lines
1038	1038	goto out_unlock;
1039	1039
1040	1040	if (pte_write(pvmw.pte) \|\| pte_dirty(pvmw.pte) \|\|
1041		− (pte_protnone(pvmw.pte) && pte_savedwrite(pvmw.pte))) {
	1041	+ (pte_protnone(pvmw.pte) && pte_savedwrite(pvmw.pte)) \|\|
	1042	+ mm_tlb_flush_pending(mm)) {
1042	1043	pte_t entry;
1043	1044
1044	1045	swapped = PageSwapCache(page);

mm/memory.c+35−7

@@ -215,12 +215,8 @@ static bool tlb_next_batch(struct mmu_gather *tlb) · 214 unmodified lines
215	215	return true;
216	216	}
217	217
218		−/* tlb_gather_mmu
219		− * Called to initialize an (on-stack) mmu_gather structure for page-table
220		− * tear-down from @mm. The @fullmm argument is used when @mm is without
221		− * users and we're going to destroy the full address space (exit/execve).
222		− */
223		−void tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm, unsigned long start, unsigned long end)
	218	+void arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	219	+ unsigned long start, unsigned long end)
224	220	{
225	221	tlb->mm = mm;
226	222
@@ -275,10 +271,14 @@ void tlb_flush_mmu(struct mmu_gather *tlb) · 48 unmodified lines
275	271	* Called at the end of the shootdown operation to free up any resources
276	272	* that were required.
277	273	*/
278		−void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
	274	+void arch_tlb_finish_mmu(struct mmu_gather *tlb,
	275	+ unsigned long start, unsigned long end, bool force)
279	276	{
280	277	struct mmu_gather_batch batch, next;
281	278
	279	+ if (force)
	280	+ __tlb_adjust_range(tlb, start, end - start);
	281	+
282	282	tlb_flush_mmu(tlb);
283	283
284	284	/* keep the page table cache within bounds */
@@ -398,6 +398,34 @@ void tlb_remove_table(struct mmu_gather tlb, void table) · 113 unmodified lines
398	398
399	399	#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
400	400
	401	+/* tlb_gather_mmu
	402	+ * Called to initialize an (on-stack) mmu_gather structure for page-table
	403	+ * tear-down from @mm. The @fullmm argument is used when @mm is without
	404	+ * users and we're going to destroy the full address space (exit/execve).
	405	+ */
	406	+void tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
	407	+ unsigned long start, unsigned long end)
	408	+{
	409	+ arch_tlb_gather_mmu(tlb, mm, start, end);
	410	+ inc_tlb_flush_pending(tlb->mm);
	411	+}
	412	+
	413	+void tlb_finish_mmu(struct mmu_gather *tlb,
	414	+ unsigned long start, unsigned long end)
	415	+{
	416	+ /*
	417	+ * If there are parallel threads are doing PTE changes on same range
	418	+ * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB
	419	+ * flush by batching, a thread has stable TLB entry can fail to flush
	420	+ * the TLB by observing pte_none\|!pte_dirty, for example so flush TLB
	421	+ * forcefully if we detect parallel PTE batching threads.
	422	+ */
	423	+ bool force = mm_tlb_flush_nested(tlb->mm);
	424	+
	425	+ arch_tlb_finish_mmu(tlb, start, end, force);
	426	+ dec_tlb_flush_pending(tlb->mm);
	427	+}
	428	+
401	429	/*
402	430	* Note: this doesn't free the actual pages themselves. That
403	431	* has been handled earlier when unmapping all the memory regions.

mm/migrate.c+0−6

@@ -1937,12 +1937,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, · 1936 unmodified lines
1937	1937	put_page(new_page);
1938	1938	goto out_fail;
1939	1939	}
1940		− /*
1941		− * We are not sure a pending tlb flush here is for a huge page
1942		− * mapping or not. Hence use the tlb range variant
1943		− */
1944		− if (mm_tlb_flush_pending(mm))
1945		− flush_tlb_range(vma, mmun_start, mmun_end);
1946	1940
1947	1941	/* Prepare a page as a migration target */
1948	1942	__SetPageLocked(new_page);

mm/mprotect.c+2−2

@@ -244,7 +244,7 @@ static unsigned long change_protection_range(struct vm_area_struct *vma, · 243 unmodified lines
244	244	BUG_ON(addr >= end);
245	245	pgd = pgd_offset(mm, addr);
246	246	flush_cache_range(vma, addr, end);
247		− set_tlb_flush_pending(mm);
	247	+ inc_tlb_flush_pending(mm);
248	248	do {
249	249	next = pgd_addr_end(addr, end);
250	250	if (pgd_none_or_clear_bad(pgd))
@@ -256,7 +256,7 @@ static unsigned long change_protection_range(struct vm_area_struct *vma, · 5 unmodified lines
256	256	/* Only flush the TLB if we actually modified any entries: */
257	257	if (pages)
258	258	flush_tlb_range(vma, start, end);
259		− clear_tlb_flush_pending(mm);
	259	+ dec_tlb_flush_pending(mm);
260	260
261	261	return pages;
262	262	}

mm/page_alloc.c+6−5

@@ -4458,8 +4458,9 @@ long si_mem_available(void) · 4457 unmodified lines
4458	4458	* Part of the reclaimable slab consists of items that are in use,
4459	4459	* and cannot be freed. Cap this estimate at the low watermark.
4460	4460	*/
4461		− available += global_page_state(NR_SLAB_RECLAIMABLE) -
4462		− min(global_page_state(NR_SLAB_RECLAIMABLE) / 2, wmark_low);
	4461	+ available += global_node_page_state(NR_SLAB_RECLAIMABLE) -
	4462	+ min(global_node_page_state(NR_SLAB_RECLAIMABLE) / 2,
	4463	+ wmark_low);
4463	4464
4464	4465	if (available < 0)
4465	4466	available = 0;
@@ -4602,8 +4603,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) · 136 unmodified lines
4602	4603	global_node_page_state(NR_FILE_DIRTY),
4603	4604	global_node_page_state(NR_WRITEBACK),
4604	4605	global_node_page_state(NR_UNSTABLE_NFS),
4605		− global_page_state(NR_SLAB_RECLAIMABLE),
4606		− global_page_state(NR_SLAB_UNRECLAIMABLE),
	4606	+ global_node_page_state(NR_SLAB_RECLAIMABLE),
	4607	+ global_node_page_state(NR_SLAB_UNRECLAIMABLE),
4607	4608	global_node_page_state(NR_FILE_MAPPED),
4608	4609	global_node_page_state(NR_SHMEM),
4609	4610	global_page_state(NR_PAGETABLE),
@@ -7668,7 +7669,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, · 3058 unmodified lines
7668	7669
7669	7670	/* Make sure the range is really isolated. */
7670	7671	if (test_pages_isolated(outer_start, end, false)) {
7671		− pr_info("%s: [%lx, %lx) PFNs busy\n",
	7672	+ pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
7672	7673	__func__, outer_start, end);
7673	7674	ret = -EBUSY;
7674	7675	goto done;

mm/rmap.c+30−22

@@ -888,10 +888,10 @@ static bool page_mkclean_one(struct page page, struct vm_area_struct vma, · 887 unmodified lines
888	888	.flags = PVMW_SYNC,
889	889	};
890	890	int *cleaned = arg;
	891	+ bool invalidation_needed = false;
891	892
892	893	while (page_vma_mapped_walk(&pvmw)) {
893	894	int ret = 0;
894		− address = pvmw.address;
895	895	if (pvmw.pte) {
896	896	pte_t entry;
897	897	pte_t *pte = pvmw.pte;
@@ -899,11 +899,11 @@ static bool page_mkclean_one(struct page page, struct vm_area_struct vma, · 1 unmodified line
899	899	if (!pte_dirty(pte) && !pte_write(pte))
900	900	continue;
901	901
902		− flush_cache_page(vma, address, pte_pfn(*pte));
903		− entry = ptep_clear_flush(vma, address, pte);
	902	+ flush_cache_page(vma, pvmw.address, pte_pfn(*pte));
	903	+ entry = ptep_clear_flush(vma, pvmw.address, pte);
904	904	entry = pte_wrprotect(entry);
905	905	entry = pte_mkclean(entry);
906		− set_pte_at(vma->vm_mm, address, pte, entry);
	906	+ set_pte_at(vma->vm_mm, pvmw.address, pte, entry);
907	907	ret = 1;
908	908	} else {
909	909	#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
@@ -913,11 +913,11 @@ static bool page_mkclean_one(struct page page, struct vm_area_struct vma, · 3 unmodified lines
913	913	if (!pmd_dirty(pmd) && !pmd_write(pmd))
914	914	continue;
915	915
916		− flush_cache_page(vma, address, page_to_pfn(page));
917		− entry = pmdp_huge_clear_flush(vma, address, pmd);
	916	+ flush_cache_page(vma, pvmw.address, page_to_pfn(page));
	917	+ entry = pmdp_huge_clear_flush(vma, pvmw.address, pmd);
918	918	entry = pmd_wrprotect(entry);
919	919	entry = pmd_mkclean(entry);
920		− set_pmd_at(vma->vm_mm, address, pmd, entry);
	920	+ set_pmd_at(vma->vm_mm, pvmw.address, pmd, entry);
921	921	ret = 1;
922	922	#else
923	923	/* unexpected pmd-mapped page? */
@@ -926,11 +926,16 @@ static bool page_mkclean_one(struct page page, struct vm_area_struct vma, · 2 unmodified lines
926	926	}
927	927
928	928	if (ret) {
929		− mmu_notifier_invalidate_page(vma->vm_mm, address);
930	929	(*cleaned)++;
	930	+ invalidation_needed = true;
931	931	}
932	932	}
933	933
	934	+ if (invalidation_needed) {
	935	+ mmu_notifier_invalidate_range(vma->vm_mm, address,
	936	+ address + (1UL << compound_order(page)));
	937	+ }
	938	+
934	939	return true;
935	940	}
936	941
@@ -1323,7 +1328,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 386 unmodified lines
1323	1328	};
1324	1329	pte_t pteval;
1325	1330	struct page *subpage;
1326		− bool ret = true;
	1331	+ bool ret = true, invalidation_needed = false;
1327	1332	enum ttu_flags flags = (enum ttu_flags)arg;
1328	1333
1329	1334	/* munlock has nothing to gain from examining un-locked vmas */
@@ -1363,11 +1368,9 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 33 unmodified lines
1363	1368	VM_BUG_ON_PAGE(!pvmw.pte, page);
1364	1369
1365	1370	subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1366		− address = pvmw.address;
1367		−
1368	1371
1369	1372	if (!(flags & TTU_IGNORE_ACCESS)) {
1370		− if (ptep_clear_flush_young_notify(vma, address,
	1373	+ if (ptep_clear_flush_young_notify(vma, pvmw.address,
1371	1374	pvmw.pte)) {
1372	1375	ret = false;
1373	1376	page_vma_mapped_walk_done(&pvmw);
@@ -1376,7 +1379,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 2 unmodified lines
1376	1379	}
1377	1380
1378	1381	/* Nuke the page table entry. */
1379		− flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
	1382	+ flush_cache_page(vma, pvmw.address, pte_pfn(*pvmw.pte));
1380	1383	if (should_defer_flush(mm, flags)) {
1381	1384	/*
1382	1385	* We clear the PTE but do not flush so potentially
@@ -1386,11 +1389,12 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 3 unmodified lines
1386	1389	* transition on a cached TLB entry is written through
1387	1390	* and traps if the PTE is unmapped.
1388	1391	*/
1389		− pteval = ptep_get_and_clear(mm, address, pvmw.pte);
	1392	+ pteval = ptep_get_and_clear(mm, pvmw.address,
	1393	+ pvmw.pte);
1390	1394
1391	1395	set_tlb_ubc_flush_pending(mm, pte_dirty(pteval));
1392	1396	} else {
1393		− pteval = ptep_clear_flush(vma, address, pvmw.pte);
	1397	+ pteval = ptep_clear_flush(vma, pvmw.address, pvmw.pte);
1394	1398	}
1395	1399
1396	1400	/* Move the dirty bit to the page. Now the pte is gone. */
@@ -1405,12 +1409,12 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 8 unmodified lines
1405	1409	if (PageHuge(page)) {
1406	1410	int nr = 1 << compound_order(page);
1407	1411	hugetlb_count_sub(nr, mm);
1408		− set_huge_swap_pte_at(mm, address,
	1412	+ set_huge_swap_pte_at(mm, pvmw.address,
1409	1413	pvmw.pte, pteval,
1410	1414	vma_mmu_pagesize(vma));
1411	1415	} else {
1412	1416	dec_mm_counter(mm, mm_counter(page));
1413		− set_pte_at(mm, address, pvmw.pte, pteval);
	1417	+ set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
1414	1418	}
1415	1419
1416	1420	} else if (pte_unused(pteval)) {
@@ -1434,7 +1438,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 17 unmodified lines
1434	1438	swp_pte = swp_entry_to_pte(entry);
1435	1439	if (pte_soft_dirty(pteval))
1436	1440	swp_pte = pte_swp_mksoft_dirty(swp_pte);
1437		− set_pte_at(mm, address, pvmw.pte, swp_pte);
	1441	+ set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
1438	1442	} else if (PageAnon(page)) {
1439	1443	swp_entry_t entry = { .val = page_private(subpage) };
1440	1444	pte_t swp_pte;
@@ -1460,7 +1464,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 19 unmodified lines
1460	1464	* If the page was redirtied, it cannot be
1461	1465	* discarded. Remap the page to page table.
1462	1466	*/
1463		− set_pte_at(mm, address, pvmw.pte, pteval);
	1467	+ set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
1464	1468	SetPageSwapBacked(page);
1465	1469	ret = false;
1466	1470	page_vma_mapped_walk_done(&pvmw);
@@ -1468,7 +1472,7 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 1 unmodified line
1468	1472	}
1469	1473
1470	1474	if (swap_duplicate(entry) < 0) {
1471		− set_pte_at(mm, address, pvmw.pte, pteval);
	1475	+ set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
1472	1476	ret = false;
1473	1477	page_vma_mapped_walk_done(&pvmw);
1474	1478	break;
@@ -1484,14 +1488,18 @@ static bool try_to_unmap_one(struct page page, struct vm_area_struct vma, · 9 unmodified lines
1484	1488	swp_pte = swp_entry_to_pte(entry);
1485	1489	if (pte_soft_dirty(pteval))
1486	1490	swp_pte = pte_swp_mksoft_dirty(swp_pte);
1487		− set_pte_at(mm, address, pvmw.pte, swp_pte);
	1491	+ set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
1488	1492	} else
1489	1493	dec_mm_counter(mm, mm_counter_file(page));
1490	1494	discard:
1491	1495	page_remove_rmap(subpage, PageHuge(page));
1492	1496	put_page(page);
1493		− mmu_notifier_invalidate_page(mm, address);
	1497	+ invalidation_needed = true;
1494	1498	}
	1499	+
	1500	+ if (invalidation_needed)
	1501	+ mmu_notifier_invalidate_range(mm, address,
	1502	+ address + (1UL << compound_order(page)));
1495	1503	return ret;
1496	1504	}
1497	1505

mm/shmem.c+10−2

@@ -1022,7 +1022,11 @@ static int shmem_setattr(struct dentry dentry, struct iattr attr) · 1021 unmodified lines
1022	1022	*/
1023	1023	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
1024	1024	spin_lock(&sbinfo->shrinklist_lock);
1025		− if (list_empty(&info->shrinklist)) {
	1025	+ /*
	1026	+ * _careful to defend against unlocked access to
	1027	+ * ->shrink_list in shmem_unused_huge_shrink()
	1028	+ */
	1029	+ if (list_empty_careful(&info->shrinklist)) {
1026	1030	list_add_tail(&info->shrinklist,
1027	1031	&sbinfo->shrinklist);
1028	1032	sbinfo->shrinklist_len++;
@@ -1817,7 +1821,11 @@ alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, info, sbinfo, · 788 unmodified lines
1817	1821	* to shrink under memory pressure.
1818	1822	*/
1819	1823	spin_lock(&sbinfo->shrinklist_lock);
1820		− if (list_empty(&info->shrinklist)) {
	1824	+ /*
	1825	+ * _careful to defend against unlocked access to
	1826	+ * ->shrink_list in shmem_unused_huge_shrink()
	1827	+ */
	1828	+ if (list_empty_careful(&info->shrinklist)) {
1821	1829	list_add_tail(&info->shrinklist,
1822	1830	&sbinfo->shrinklist);
1823	1831	sbinfo->shrinklist_len++;

mm/util.c+1−1

@@ -633,7 +633,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) · 632 unmodified lines
633	633	* which are reclaimable, under pressure. The dentry
634	634	* cache and most inode caches should fall into this
635	635	*/
636		− free += global_page_state(NR_SLAB_RECLAIMABLE);
	636	+ free += global_node_page_state(NR_SLAB_RECLAIMABLE);
637	637
638	638	/*
639	639	* Leave reserved pages. The pages are not for anonymous pages.