Merge branch 'core-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull unified TLB flushing from Ingo Molnar: "This contains the generic mmu_gather feature from Peter Zijlstra, which is an all-arch unification of TLB flushing APIs, via the following (broad) steps: - enhance the <asm-generic/tlb.h> APIs to cover more arch details - convert most TLB flushing arch implementations to the generic <asm-generic/tlb.h> APIs. - remove leftovers of per arch implementations After this series every single architecture makes use of the unified TLB flushing APIs" * 'core-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: mm/resource: Use resource_overlaps() to simplify region_intersects() ia64/tlb: Eradicate tlb_migrate_finish() callback asm-generic/tlb: Remove tlb_table_flush() asm-generic/tlb: Remove tlb_flush_mmu_free() asm-generic/tlb: Remove CONFIG_HAVE_GENERIC_MMU_GATHER asm-generic/tlb: Remove arch_tlb*_mmu() s390/tlb: Convert to generic mmu_gather asm-generic/tlb: Introduce CONFIG_HAVE_MMU_GATHER_NO_GATHER=y arch/tlb: Clean up simple architectures um/tlb: Convert to generic mmu_gather sh/tlb: Convert SH to generic mmu_gather ia64/tlb: Convert to generic mmu_gather arm/tlb: Convert to generic mmu_gather asm-generic/tlb, arch: Invert CONFIG_HAVE_RCU_TABLE_INVALIDATE asm-generic/tlb, ia64: Conditionally provide tlb_migrate_finish() asm-generic/tlb: Provide generic tlb_flush() based on flush_tlb_mm() asm-generic/tlb, arch: Provide generic tlb_flush() based on flush_tlb_range() asm-generic/tlb, arch: Provide generic VIPT cache flush asm-generic/tlb, arch: Provide CONFIG_HAVE_MMU_GATHER_PAGE_SIZE asm-generic/tlb: Provide a comment

Linus Torvalds committed 7 years ago

Commit 171c2bc2 parents 423ea32 f6c601055 files · +482 −1296Browse files

Unified · Split

55 files

Documentation/core-api/cachetlb.rst +0 −10
arch/Kconfig +7 −1
arch/alpha/Kconfig +1 −0
arch/alpha/include/asm/tlb.h +0 −6
arch/arc/include/asm/tlb.h +0 −32
arch/arm/include/asm/tlb.h +13 −242
arch/arm64/Kconfig +0 −1
arch/arm64/include/asm/tlb.h +1 −0
arch/c6x/Kconfig +1 −0
arch/c6x/include/asm/tlb.h +0 −2
arch/h8300/include/asm/tlb.h +0 −2
arch/hexagon/include/asm/tlb.h +0 −12
arch/ia64/include/asm/machvec.h +0 −13
arch/ia64/include/asm/machvec_sn2.h +0 −2
arch/ia64/include/asm/tlb.h +1 −258
arch/ia64/include/asm/tlbflush.h +25 −0
arch/ia64/mm/tlb.c +21 −2
arch/ia64/sn/kernel/sn2/sn2_smp.c +0 −7
arch/m68k/Kconfig +1 −0
arch/m68k/include/asm/tlb.h +0 −14
arch/microblaze/Kconfig +1 −0
arch/microblaze/include/asm/tlb.h +0 −9
arch/mips/include/asm/tlb.h +0 −17
arch/nds32/include/asm/tlb.h +0 −16
arch/nds32/include/asm/tlbflush.h +0 −1
arch/nios2/Kconfig +1 −0
arch/nios2/include/asm/tlb.h +2 −12
arch/openrisc/Kconfig +1 −0
arch/openrisc/include/asm/tlb.h +2 −6
arch/parisc/include/asm/tlb.h +0 −18
arch/powerpc/Kconfig +2 −0
arch/powerpc/include/asm/tlb.h +1 −17
arch/riscv/include/asm/tlb.h +1 −0
arch/s390/Kconfig +2 −0
arch/s390/include/asm/tlb.h +39 −91
arch/s390/mm/pgalloc.c +1 −62
arch/sh/include/asm/pgalloc.h +9 −0
arch/sh/include/asm/tlb.h +1 −131
arch/sparc/Kconfig +1 −0
arch/sparc/include/asm/tlb_32.h +0 −18
arch/um/include/asm/tlb.h +2 −156
arch/unicore32/Kconfig +1 −0
arch/unicore32/include/asm/tlb.h +3 −4
arch/x86/Kconfig +0 −1
arch/x86/include/asm/tlb.h +1 −0
arch/xtensa/include/asm/tlb.h +0 −26
include/asm-generic/tlb.h +259 −29
kernel/iomem.c +2 −2
kernel/resource.c +5 −6
kernel/sched/core.c +0 −1
mm/huge_memory.c +2 −2
mm/hugetlb.c +1 −1
mm/madvise.c +1 −1
mm/memory.c +3 −3
mm/mmu_gather.c +67 −62

Documentation/core-api/cachetlb.rst+0−10

@@ -101,16 +101,6 @@ changes occur: · 100 unmodified lines
101	101	translations for software managed TLB configurations.
102	102	The sparc64 port currently does this.
103	103
104		−6) ``void tlb_migrate_finish(struct mm_struct *mm)``
105		−
106		− This interface is called at the end of an explicit
107		− process migration. This interface provides a hook
108		− to allow a platform to update TLB or context-specific
109		− information for the address space.
110		−
111		− The ia64 sn2 platform is one example of a platform
112		− that uses this interface.
113		−
114	104	Next, we have the cache flushing interfaces. In general, when Linux
115	105	is changing an existing virtual-->physical mapping to a new value,
116	106	the sequence will be in one of the following forms::

arch/Kconfig+7−1

@@ -383,7 +383,13 @@ config HAVE_ARCH_JUMP_LABEL_RELATIVE · 382 unmodified lines
383	383	config HAVE_RCU_TABLE_FREE
384	384	bool
385	385
386		−config HAVE_RCU_TABLE_INVALIDATE
	386	+config HAVE_RCU_TABLE_NO_INVALIDATE
	387	+ bool
	388	+
	389	+config HAVE_MMU_GATHER_PAGE_SIZE
	390	+ bool
	391	+
	392	+config HAVE_MMU_GATHER_NO_GATHER
387	393	bool
388	394
389	395	config ARCH_HAVE_NMI_SAFE_CMPXCHG

arch/alpha/Kconfig+1−0

@@ -36,6 +36,7 @@ config ALPHA · 35 unmodified lines
36	36	select ODD_RT_SIGACTION
37	37	select OLD_SIGSUSPEND
38	38	select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67
	39	+ select MMU_GATHER_NO_RANGE
39	40	help
40	41	The Alpha is a 64-bit general-purpose processor designed and
41	42	marketed by the Digital Equipment Corporation of blessed memory,

arch/alpha/include/asm/tlb.h+0−6

@@ -2,12 +2,6 @@ · 1 unmodified line
2	2	#ifndef _ALPHA_TLB_H
3	3	#define _ALPHA_TLB_H
4	4
5		−#define tlb_start_vma(tlb, vma) do { } while (0)
6		−#define tlb_end_vma(tlb, vma) do { } while (0)
7		−#define __tlb_remove_tlb_entry(tlb, pte, addr) do { } while (0)
8		−
9		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
10		−
11	5	#include <asm-generic/tlb.h>
12	6
13	7	#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte)

arch/arc/include/asm/tlb.h+0−32

@@ -9,38 +9,6 @@ · 8 unmodified lines
9	9	#ifndef _ASM_ARC_TLB_H
10	10	#define _ASM_ARC_TLB_H
11	11
12		−#define tlb_flush(tlb) \
13		−do { \
14		− if (tlb->fullmm) \
15		− flush_tlb_mm((tlb)->mm); \
16		−} while (0)
17		−
18		−/*
19		− * This pair is called at time of munmap/exit to flush cache and TLB entries
20		− * for mappings being torn down.
21		− * 1) cache-flush part -implemented via tlb_start_vma( ) for VIPT aliasing D$
22		− * 2) tlb-flush part - implemted via tlb_end_vma( ) flushes the TLB range
23		− *
24		− * Note, read http://lkml.org/lkml/2004/1/15/6
25		− */
26		−#ifndef CONFIG_ARC_CACHE_VIPT_ALIASING
27		−#define tlb_start_vma(tlb, vma)
28		−#else
29		−#define tlb_start_vma(tlb, vma) \
30		−do { \
31		− if (!tlb->fullmm) \
32		− flush_cache_range(vma, vma->vm_start, vma->vm_end); \
33		−} while(0)
34		−#endif
35		−
36		−#define tlb_end_vma(tlb, vma) \
37		−do { \
38		− if (!tlb->fullmm) \
39		− flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
40		−} while (0)
41		−
42		−#define __tlb_remove_tlb_entry(tlb, ptep, address)
43		−
44	12	#include <linux/pagemap.h>
45	13	#include <asm-generic/tlb.h>
46	14

arch/arm/include/asm/tlb.h+13−242

@@ -33,271 +33,42 @@ · 32 unmodified lines
33	33	#include <asm/pgalloc.h>
34	34	#include <asm/tlbflush.h>
35	35
36		−#define MMU_GATHER_BUNDLE 8
37		−
38		−#ifdef CONFIG_HAVE_RCU_TABLE_FREE
39	36	static inline void __tlb_remove_table(void *_table)
40	37	{
41	38	free_page_and_swap_cache((struct page *)_table);
42	39	}
43	40
44		−struct mmu_table_batch {
45		− struct rcu_head rcu;
46		− unsigned int nr;
47		− void *tables[0];
48		−};
49		−
50		−#define MAX_TABLE_BATCH \
51		− ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
52		−
53		−extern void tlb_table_flush(struct mmu_gather *tlb);
54		−extern void tlb_remove_table(struct mmu_gather tlb, void table);
55		−
56		−#define tlb_remove_entry(tlb, entry) tlb_remove_table(tlb, entry)
57		−#else
58		−#define tlb_remove_entry(tlb, entry) tlb_remove_page(tlb, entry)
59		−#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
60		−
61		−/*
62		− * TLB handling. This allows us to remove pages from the page
63		− * tables, and efficiently handle the TLB issues.
64		− */
65		−struct mmu_gather {
66		− struct mm_struct *mm;
67		−#ifdef CONFIG_HAVE_RCU_TABLE_FREE
68		− struct mmu_table_batch *batch;
69		− unsigned int need_flush;
70		−#endif
71		− unsigned int fullmm;
72		− struct vm_area_struct *vma;
73		− unsigned long start, end;
74		− unsigned long range_start;
75		− unsigned long range_end;
76		− unsigned int nr;
77		− unsigned int max;
78		− struct page **pages;
79		− struct page *local[MMU_GATHER_BUNDLE];
80		−};
81		−
82		−DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
83		−
84		−/*
85		− * This is unnecessarily complex. There's three ways the TLB shootdown
86		− * code is used:
87		− * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region().
88		− * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.
89		− * tlb->vma will be non-NULL.
90		− * 2. Unmapping all vmas. See exit_mmap().
91		− * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.
92		− * tlb->vma will be non-NULL. Additionally, page tables will be freed.
93		− * 3. Unmapping argument pages. See shift_arg_pages().
94		− * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.
95		− * tlb->vma will be NULL.
96		− */
97		−static inline void tlb_flush(struct mmu_gather *tlb)
98		−{
99		− if (tlb->fullmm \|\| !tlb->vma)
100		− flush_tlb_mm(tlb->mm);
101		− else if (tlb->range_end > 0) {
102		− flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
103		− tlb->range_start = TASK_SIZE;
104		− tlb->range_end = 0;
105		− }
106		−}
107		−
108		−static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
109		−{
110		− if (!tlb->fullmm) {
111		− if (addr < tlb->range_start)
112		− tlb->range_start = addr;
113		− if (addr + PAGE_SIZE > tlb->range_end)
114		− tlb->range_end = addr + PAGE_SIZE;
115		− }
116		−}
117		−
118		−static inline void __tlb_alloc_page(struct mmu_gather *tlb)
119		−{
120		− unsigned long addr = __get_free_pages(GFP_NOWAIT \| __GFP_NOWARN, 0);
121		−
122		− if (addr) {
123		− tlb->pages = (void *)addr;
124		− tlb->max = PAGE_SIZE / sizeof(struct page *);
125		− }
126		−}
127		−
128		−static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
129		−{
130		− tlb_flush(tlb);
131		−#ifdef CONFIG_HAVE_RCU_TABLE_FREE
132		− tlb_table_flush(tlb);
133		−#endif
134		−}
135		−
136		−static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
137		−{
138		− free_pages_and_swap_cache(tlb->pages, tlb->nr);
139		− tlb->nr = 0;
140		− if (tlb->pages == tlb->local)
141		− __tlb_alloc_page(tlb);
142		−}
143		−
144		−static inline void tlb_flush_mmu(struct mmu_gather *tlb)
145		−{
146		− tlb_flush_mmu_tlbonly(tlb);
147		− tlb_flush_mmu_free(tlb);
148		−}
149		−
150		−static inline void
151		−arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
152		− unsigned long start, unsigned long end)
153		−{
154		− tlb->mm = mm;
155		− tlb->fullmm = !(start \| (end+1));
156		− tlb->start = start;
157		− tlb->end = end;
158		− tlb->vma = NULL;
159		− tlb->max = ARRAY_SIZE(tlb->local);
160		− tlb->pages = tlb->local;
161		− tlb->nr = 0;
162		− __tlb_alloc_page(tlb);
	41	+#include <asm-generic/tlb.h>
163	42
164		−#ifdef CONFIG_HAVE_RCU_TABLE_FREE
165		− tlb->batch = NULL;
	43	+#ifndef CONFIG_HAVE_RCU_TABLE_FREE
	44	+#define tlb_remove_table(tlb, entry) tlb_remove_page(tlb, entry)
166	45	#endif
167		−}
168		−
169		−static inline void
170		−arch_tlb_finish_mmu(struct mmu_gather *tlb,
171		− unsigned long start, unsigned long end, bool force)
172		−{
173		− if (force) {
174		− tlb->range_start = start;
175		− tlb->range_end = end;
176		− }
177		−
178		− tlb_flush_mmu(tlb);
179		−
180		− /* keep the page table cache within bounds */
181		− check_pgt_cache();
182		−
183		− if (tlb->pages != tlb->local)
184		− free_pages((unsigned long)tlb->pages, 0);
185		−}
186		−
187		−/*
188		− * Memorize the range for the TLB flush.
189		− */
190		−static inline void
191		−tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep, unsigned long addr)
192		−{
193		− tlb_add_flush(tlb, addr);
194		−}
195		−
196		−#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
197		− tlb_remove_tlb_entry(tlb, ptep, address)
198		−/*
199		− * In the case of tlb vma handling, we can optimise these away in the
200		− * case where we're doing a full MM flush. When we're doing a munmap,
201		− * the vmas are adjusted to only cover the region to be torn down.
202		− */
203		−static inline void
204		−tlb_start_vma(struct mmu_gather tlb, struct vm_area_struct vma)
205		−{
206		− if (!tlb->fullmm) {
207		− flush_cache_range(vma, vma->vm_start, vma->vm_end);
208		− tlb->vma = vma;
209		− tlb->range_start = TASK_SIZE;
210		− tlb->range_end = 0;
211		− }
212		−}
213	46
214	47	static inline void
215		−tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma)
216		−{
217		− if (!tlb->fullmm)
218		− tlb_flush(tlb);
219		−}
220		−
221		−static inline bool __tlb_remove_page(struct mmu_gather tlb, struct page page)
222		−{
223		− tlb->pages[tlb->nr++] = page;
224		− VM_WARN_ON(tlb->nr > tlb->max);
225		− if (tlb->nr == tlb->max)
226		− return true;
227		− return false;
228		−}
229		−
230		−static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
231		−{
232		− if (__tlb_remove_page(tlb, page))
233		− tlb_flush_mmu(tlb);
234		−}
235		−
236		−static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
237		− struct page *page, int page_size)
238		−{
239		− return __tlb_remove_page(tlb, page);
240		−}
241		−
242		−static inline void tlb_remove_page_size(struct mmu_gather *tlb,
243		− struct page *page, int page_size)
244		−{
245		− return tlb_remove_page(tlb, page);
246		−}
247		−
248		−static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
249		− unsigned long addr)
	48	+__pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr)
250	49	{
251	50	pgtable_page_dtor(pte);
252	51
253		−#ifdef CONFIG_ARM_LPAE
254		− tlb_add_flush(tlb, addr);
255		−#else
	52	+#ifndef CONFIG_ARM_LPAE
256	53	/*
257	54	* With the classic ARM MMU, a pte page has two corresponding pmd
258	55	* entries, each covering 1MB.
259	56	*/
260		− addr &= PMD_MASK;
261		− tlb_add_flush(tlb, addr + SZ_1M - PAGE_SIZE);
262		− tlb_add_flush(tlb, addr + SZ_1M);
	57	+ addr = (addr & PMD_MASK) + SZ_1M;
	58	+ __tlb_adjust_range(tlb, addr - PAGE_SIZE, 2 * PAGE_SIZE);
263	59	#endif
264	60
265		− tlb_remove_entry(tlb, pte);
266		−}
267		−
268		−static inline void __pmd_free_tlb(struct mmu_gather tlb, pmd_t pmdp,
269		− unsigned long addr)
270		−{
271		−#ifdef CONFIG_ARM_LPAE
272		− tlb_add_flush(tlb, addr);
273		− tlb_remove_entry(tlb, virt_to_page(pmdp));
274		−#endif
	61	+ tlb_remove_table(tlb, pte);
275	62	}
276	63
277	64	static inline void
278		−tlb_remove_pmd_tlb_entry(struct mmu_gather tlb, pmd_t pmdp, unsigned long addr)
279		−{
280		− tlb_add_flush(tlb, addr);
281		−}
282		−
283		−#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
284		−#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
285		−#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
286		−
287		−#define tlb_migrate_finish(mm) do { } while (0)
288		−
289		−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
290		−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
291		− unsigned int page_size)
	65	+__pmd_free_tlb(struct mmu_gather tlb, pmd_t pmdp, unsigned long addr)
292	66	{
293		−}
294		−
295		−static inline void tlb_flush_remove_tables(struct mm_struct *mm)
296		−{
297		−}
	67	+#ifdef CONFIG_ARM_LPAE
	68	+ struct page *page = virt_to_page(pmdp);
298	69
299		−static inline void tlb_flush_remove_tables_local(void *arg)
300		−{
	70	+ tlb_remove_table(tlb, page);
	71	+#endif
301	72	}
302	73
303	74	#endif /* CONFIG_MMU */

arch/arm64/Kconfig+0−1

@@ -149,7 +149,6 @@ config ARM64 · 148 unmodified lines
149	149	select HAVE_PERF_USER_STACK_DUMP
150	150	select HAVE_REGS_AND_STACK_ACCESS_API
151	151	select HAVE_RCU_TABLE_FREE
152		− select HAVE_RCU_TABLE_INVALIDATE
153	152	select HAVE_RSEQ
154	153	select HAVE_STACKPROTECTOR
155	154	select HAVE_SYSCALL_TRACEPOINTS

arch/arm64/include/asm/tlb.h+1−0

@@ -27,6 +27,7 @@ static inline void __tlb_remove_table(void *_table) · 26 unmodified lines
27	27	free_page_and_swap_cache((struct page *)_table);
28	28	}
29	29
	30	+#define tlb_flush tlb_flush
30	31	static void tlb_flush(struct mmu_gather *tlb);
31	32
32	33	#include <asm-generic/tlb.h>

arch/c6x/Kconfig+1−0

@@ -20,6 +20,7 @@ config C6X · 19 unmodified lines
20	20	select GENERIC_CLOCKEVENTS
21	21	select MODULES_USE_ELF_RELA
22	22	select ARCH_NO_COHERENT_DMA_MMAP
	23	+ select MMU_GATHER_NO_RANGE if MMU
23	24
24	25	config MMU
25	26	def_bool n

arch/c6x/include/asm/tlb.h+0−2

@@ -2,8 +2,6 @@ · 1 unmodified line
2	2	#ifndef _ASM_C6X_TLB_H
3	3	#define _ASM_C6X_TLB_H
4	4
5		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
6		−
7	5	#include <asm-generic/tlb.h>
8	6
9	7	#endif /* _ASM_C6X_TLB_H */

arch/h8300/include/asm/tlb.h+0−2

@@ -2,8 +2,6 @@ · 1 unmodified line
2	2	#ifndef __H8300_TLB_H__
3	3	#define __H8300_TLB_H__
4	4
5		−#define tlb_flush(tlb) do { } while (0)
6		−
7	5	#include <asm-generic/tlb.h>
8	6
9	7	#endif

arch/hexagon/include/asm/tlb.h+0−12

@@ -22,18 +22,6 @@ · 21 unmodified lines
22	22	#include <linux/pagemap.h>
23	23	#include <asm/tlbflush.h>
24	24
25		−/*
26		− * We don't need any special per-pte or per-vma handling...
27		− */
28		−#define tlb_start_vma(tlb, vma) do { } while (0)
29		−#define tlb_end_vma(tlb, vma) do { } while (0)
30		−#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
31		−
32		−/*
33		− * .. because we flush the whole mm when it fills up
34		− */
35		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
36		−
37	25	#include <asm-generic/tlb.h>
38	26
39	27	#endif

arch/ia64/include/asm/machvec.h+0−13

@@ -30,7 +30,6 @@ typedef void ia64_mv_irq_init_t (void); · 29 unmodified lines
30	30	typedef void ia64_mv_send_ipi_t (int, int, int, int);
31	31	typedef void ia64_mv_timer_interrupt_t (int, void *);
32	32	typedef void ia64_mv_global_tlb_purge_t (struct mm_struct *, unsigned long, unsigned long, unsigned long);
33		−typedef void ia64_mv_tlb_migrate_finish_t (struct mm_struct *);
34	33	typedef u8 ia64_mv_irq_to_vector (int);
35	34	typedef unsigned int ia64_mv_local_vector_to_irq (u8);
36	35	typedef char ia64_mv_pci_get_legacy_mem_t (struct pci_bus );
@@ -79,11 +78,6 @@ machvec_noop (void) · 42 unmodified lines
79	78	{
80	79	}
81	80
82		−static inline void
83		−machvec_noop_mm (struct mm_struct *mm)
84		−{
85		−}
86		−
87	81	static inline void
88	82	machvec_noop_task (struct task_struct *task)
89	83	{
@@ -96,7 +90,6 @@ machvec_noop_bus (struct pci_bus *bus) · 6 unmodified lines
96	90
97	91	extern void machvec_setup (char **);
98	92	extern void machvec_timer_interrupt (int, void *);
99		−extern void machvec_tlb_migrate_finish (struct mm_struct *);
100	93
101	94	# if defined (CONFIG_IA64_HP_SIM)
102	95	# include <asm/machvec_hpsim.h>
@@ -124,7 +117,6 @@ extern void machvec_tlb_migrate_finish (struct mm_struct *); · 21 unmodified lines
124	117	# define platform_send_ipi ia64_mv.send_ipi
125	118	# define platform_timer_interrupt ia64_mv.timer_interrupt
126	119	# define platform_global_tlb_purge ia64_mv.global_tlb_purge
127		−# define platform_tlb_migrate_finish ia64_mv.tlb_migrate_finish
128	120	# define platform_dma_init ia64_mv.dma_init
129	121	# define platform_dma_get_ops ia64_mv.dma_get_ops
130	122	# define platform_irq_to_vector ia64_mv.irq_to_vector
@@ -167,7 +159,6 @@ struct ia64_machine_vector { · 36 unmodified lines
167	159	ia64_mv_send_ipi_t *send_ipi;
168	160	ia64_mv_timer_interrupt_t *timer_interrupt;
169	161	ia64_mv_global_tlb_purge_t *global_tlb_purge;
170		− ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish;
171	162	ia64_mv_dma_init *dma_init;
172	163	ia64_mv_dma_get_ops *dma_get_ops;
173	164	ia64_mv_irq_to_vector *irq_to_vector;
@@ -206,7 +197,6 @@ struct ia64_machine_vector { · 32 unmodified lines
206	197	platform_send_ipi, \
207	198	platform_timer_interrupt, \
208	199	platform_global_tlb_purge, \
209		− platform_tlb_migrate_finish, \
210	200	platform_dma_init, \
211	201	platform_dma_get_ops, \
212	202	platform_irq_to_vector, \
@@ -270,9 +260,6 @@ extern const struct dma_map_ops dma_get_ops(struct device ); · 57 unmodified lines
270	260	#ifndef platform_global_tlb_purge
271	261	# define platform_global_tlb_purge ia64_global_tlb_purge /* default to architected version */
272	262	#endif
273		−#ifndef platform_tlb_migrate_finish
274		−# define platform_tlb_migrate_finish machvec_noop_mm
275		−#endif
276	263	#ifndef platform_kernel_launch_event
277	264	# define platform_kernel_launch_event machvec_noop
278	265	#endif

arch/ia64/include/asm/machvec_sn2.h+0−2

@@ -34,7 +34,6 @@ extern ia64_mv_irq_init_t sn_irq_init; · 33 unmodified lines
34	34	extern ia64_mv_send_ipi_t sn2_send_IPI;
35	35	extern ia64_mv_timer_interrupt_t sn_timer_interrupt;
36	36	extern ia64_mv_global_tlb_purge_t sn2_global_tlb_purge;
37		−extern ia64_mv_tlb_migrate_finish_t sn_tlb_migrate_finish;
38	37	extern ia64_mv_irq_to_vector sn_irq_to_vector;
39	38	extern ia64_mv_local_vector_to_irq sn_local_vector_to_irq;
40	39	extern ia64_mv_pci_get_legacy_mem_t sn_pci_get_legacy_mem;
@@ -77,7 +76,6 @@ extern ia64_mv_pci_fixup_bus_t sn_pci_fixup_bus; · 36 unmodified lines
77	76	#define platform_send_ipi sn2_send_IPI
78	77	#define platform_timer_interrupt sn_timer_interrupt
79	78	#define platform_global_tlb_purge sn2_global_tlb_purge
80		−#define platform_tlb_migrate_finish sn_tlb_migrate_finish
81	79	#define platform_pci_fixup sn_pci_fixup
82	80	#define platform_inb __sn_inb
83	81	#define platform_inw __sn_inw

arch/ia64/include/asm/tlb.h+1−258

@@ -47,263 +47,6 @@ · 46 unmodified lines
47	47	#include <asm/tlbflush.h>
48	48	#include <asm/machvec.h>
49	49
50		−/*
51		− * If we can't allocate a page to make a big batch of page pointers
52		− * to work on, then just handle a few from the on-stack structure.
53		− */
54		−#define IA64_GATHER_BUNDLE 8
55		−
56		−struct mmu_gather {
57		− struct mm_struct *mm;
58		− unsigned int nr;
59		− unsigned int max;
60		− unsigned char fullmm; /* non-zero means full mm flush */
61		− unsigned char need_flush; /* really unmapped some PTEs? */
62		− unsigned long start, end;
63		− unsigned long start_addr;
64		− unsigned long end_addr;
65		− struct page **pages;
66		− struct page *local[IA64_GATHER_BUNDLE];
67		−};
68		−
69		−struct ia64_tr_entry {
70		− u64 ifa;
71		− u64 itir;
72		− u64 pte;
73		− u64 rr;
74		−}; /Record for tr entry!/
75		−
76		−extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
77		−extern void ia64_ptr_entry(u64 target_mask, int slot);
78		−
79		−extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
80		−
81		−/*
82		− region register macros
83		−*/
84		−#define RR_TO_VE(val) (((val) >> 0) & 0x0000000000000001)
85		−#define RR_VE(val) (((val) & 0x0000000000000001) << 0)
86		−#define RR_VE_MASK 0x0000000000000001L
87		−#define RR_VE_SHIFT 0
88		−#define RR_TO_PS(val) (((val) >> 2) & 0x000000000000003f)
89		−#define RR_PS(val) (((val) & 0x000000000000003f) << 2)
90		−#define RR_PS_MASK 0x00000000000000fcL
91		−#define RR_PS_SHIFT 2
92		−#define RR_RID_MASK 0x00000000ffffff00L
93		−#define RR_TO_RID(val) ((val >> 8) & 0xffffff)
94		−
95		−static inline void
96		−ia64_tlb_flush_mmu_tlbonly(struct mmu_gather *tlb, unsigned long start, unsigned long end)
97		−{
98		− tlb->need_flush = 0;
99		−
100		− if (tlb->fullmm) {
101		− /*
102		− * Tearing down the entire address space. This happens both as a result
103		− * of exit() and execve(). The latter case necessitates the call to
104		− * flush_tlb_mm() here.
105		− */
106		− flush_tlb_mm(tlb->mm);
107		− } else if (unlikely (end - start >= 102410241024*1024UL
108		− \|\| REGION_NUMBER(start) != REGION_NUMBER(end - 1)))
109		− {
110		− /*
111		− * If we flush more than a tera-byte or across regions, we're probably
112		− * better off just flushing the entire TLB(s). This should be very rare
113		− * and is not worth optimizing for.
114		− */
115		− flush_tlb_all();
116		− } else {
117		− /*
118		− * flush_tlb_range() takes a vma instead of a mm pointer because
119		− * some architectures want the vm_flags for ITLB/DTLB flush.
120		− */
121		− struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
122		−
123		− /* flush the address range from the tlb: */
124		− flush_tlb_range(&vma, start, end);
125		− /* now flush the virt. page-table area mapping the address range: */
126		− flush_tlb_range(&vma, ia64_thash(start), ia64_thash(end));
127		− }
128		−
129		−}
130		−
131		−static inline void
132		−ia64_tlb_flush_mmu_free(struct mmu_gather *tlb)
133		−{
134		− unsigned long i;
135		− unsigned int nr;
136		−
137		− /* lastly, release the freed pages */
138		− nr = tlb->nr;
139		−
140		− tlb->nr = 0;
141		− tlb->start_addr = ~0UL;
142		− for (i = 0; i < nr; ++i)
143		− free_page_and_swap_cache(tlb->pages[i]);
144		−}
145		−
146		−/*
147		− * Flush the TLB for address range START to END and, if not in fast mode, release the
148		− * freed pages that where gathered up to this point.
149		− */
150		−static inline void
151		−ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
152		−{
153		− if (!tlb->need_flush)
154		− return;
155		− ia64_tlb_flush_mmu_tlbonly(tlb, start, end);
156		− ia64_tlb_flush_mmu_free(tlb);
157		−}
158		−
159		−static inline void __tlb_alloc_page(struct mmu_gather *tlb)
160		−{
161		− unsigned long addr = __get_free_pages(GFP_NOWAIT \| __GFP_NOWARN, 0);
162		−
163		− if (addr) {
164		− tlb->pages = (void *)addr;
165		− tlb->max = PAGE_SIZE / sizeof(void *);
166		− }
167		−}
168		−
169		−
170		−static inline void
171		−arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
172		− unsigned long start, unsigned long end)
173		−{
174		− tlb->mm = mm;
175		− tlb->max = ARRAY_SIZE(tlb->local);
176		− tlb->pages = tlb->local;
177		− tlb->nr = 0;
178		− tlb->fullmm = !(start \| (end+1));
179		− tlb->start = start;
180		− tlb->end = end;
181		− tlb->start_addr = ~0UL;
182		−}
183		−
184		−/*
185		− * Called at the end of the shootdown operation to free up any resources that were
186		− * collected.
187		− */
188		−static inline void
189		−arch_tlb_finish_mmu(struct mmu_gather *tlb,
190		− unsigned long start, unsigned long end, bool force)
191		−{
192		− if (force)
193		− tlb->need_flush = 1;
194		− /*
195		− * Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and
196		− * tlb->end_addr.
197		− */
198		− ia64_tlb_flush_mmu(tlb, start, end);
199		−
200		− /* keep the page table cache within bounds */
201		− check_pgt_cache();
202		−
203		− if (tlb->pages != tlb->local)
204		− free_pages((unsigned long)tlb->pages, 0);
205		−}
206		−
207		−/*
208		− * Logically, this routine frees PAGE. On MP machines, the actual freeing of the page
209		− * must be delayed until after the TLB has been flushed (see comments at the beginning of
210		− * this file).
211		− */
212		−static inline bool __tlb_remove_page(struct mmu_gather tlb, struct page page)
213		−{
214		− tlb->need_flush = 1;
215		−
216		− if (!tlb->nr && tlb->pages == tlb->local)
217		− __tlb_alloc_page(tlb);
218		−
219		− tlb->pages[tlb->nr++] = page;
220		− VM_WARN_ON(tlb->nr > tlb->max);
221		− if (tlb->nr == tlb->max)
222		− return true;
223		− return false;
224		−}
225		−
226		−static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
227		−{
228		− ia64_tlb_flush_mmu_tlbonly(tlb, tlb->start_addr, tlb->end_addr);
229		−}
230		−
231		−static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
232		−{
233		− ia64_tlb_flush_mmu_free(tlb);
234		−}
235		−
236		−static inline void tlb_flush_mmu(struct mmu_gather *tlb)
237		−{
238		− ia64_tlb_flush_mmu(tlb, tlb->start_addr, tlb->end_addr);
239		−}
240		−
241		−static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
242		−{
243		− if (__tlb_remove_page(tlb, page))
244		− tlb_flush_mmu(tlb);
245		−}
246		−
247		−static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
248		− struct page *page, int page_size)
249		−{
250		− return __tlb_remove_page(tlb, page);
251		−}
252		−
253		−static inline void tlb_remove_page_size(struct mmu_gather *tlb,
254		− struct page *page, int page_size)
255		−{
256		− return tlb_remove_page(tlb, page);
257		−}
258		−
259		−/*
260		− * Remove TLB entry for PTE mapped at virtual address ADDRESS. This is called for any
261		− * PTE, not just those pointing to (normal) physical memory.
262		− */
263		−static inline void
264		−__tlb_remove_tlb_entry (struct mmu_gather tlb, pte_t ptep, unsigned long address)
265		−{
266		− if (tlb->start_addr == ~0UL)
267		− tlb->start_addr = address;
268		− tlb->end_addr = address + PAGE_SIZE;
269		−}
270		−
271		−#define tlb_migrate_finish(mm) platform_tlb_migrate_finish(mm)
272		−
273		−#define tlb_start_vma(tlb, vma) do { } while (0)
274		−#define tlb_end_vma(tlb, vma) do { } while (0)
275		−
276		−#define tlb_remove_tlb_entry(tlb, ptep, addr) \
277		−do { \
278		− tlb->need_flush = 1; \
279		− __tlb_remove_tlb_entry(tlb, ptep, addr); \
280		−} while (0)
281		−
282		−#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
283		− tlb_remove_tlb_entry(tlb, ptep, address)
284		−
285		−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
286		−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
287		− unsigned int page_size)
288		−{
289		−}
290		−
291		−#define pte_free_tlb(tlb, ptep, address) \
292		−do { \
293		− tlb->need_flush = 1; \
294		− __pte_free_tlb(tlb, ptep, address); \
295		−} while (0)
296		−
297		−#define pmd_free_tlb(tlb, ptep, address) \
298		−do { \
299		− tlb->need_flush = 1; \
300		− __pmd_free_tlb(tlb, ptep, address); \
301		−} while (0)
302		−
303		−#define pud_free_tlb(tlb, pudp, address) \
304		−do { \
305		− tlb->need_flush = 1; \
306		− __pud_free_tlb(tlb, pudp, address); \
307		−} while (0)
	50	+#include <asm-generic/tlb.h>
308	51
309	52	#endif /* _ASM_IA64_TLB_H */

arch/ia64/include/asm/tlbflush.h+25−0

@@ -14,6 +14,31 @@ · 13 unmodified lines
14	14	#include <asm/mmu_context.h>
15	15	#include <asm/page.h>
16	16
	17	+struct ia64_tr_entry {
	18	+ u64 ifa;
	19	+ u64 itir;
	20	+ u64 pte;
	21	+ u64 rr;
	22	+}; /Record for tr entry!/
	23	+
	24	+extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
	25	+extern void ia64_ptr_entry(u64 target_mask, int slot);
	26	+extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
	27	+
	28	+/*
	29	+ region register macros
	30	+*/
	31	+#define RR_TO_VE(val) (((val) >> 0) & 0x0000000000000001)
	32	+#define RR_VE(val) (((val) & 0x0000000000000001) << 0)
	33	+#define RR_VE_MASK 0x0000000000000001L
	34	+#define RR_VE_SHIFT 0
	35	+#define RR_TO_PS(val) (((val) >> 2) & 0x000000000000003f)
	36	+#define RR_PS(val) (((val) & 0x000000000000003f) << 2)
	37	+#define RR_PS_MASK 0x00000000000000fcL
	38	+#define RR_PS_SHIFT 2
	39	+#define RR_RID_MASK 0x00000000ffffff00L
	40	+#define RR_TO_RID(val) ((val >> 8) & 0xffffff)
	41	+
17	42	/*
18	43	* Now for some TLB flushing routines. This is the kind of stuff that
19	44	* can be very expensive, so try to avoid them whenever possible.

arch/ia64/mm/tlb.c+21−2

@@ -305,8 +305,8 @@ local_flush_tlb_all (void) · 304 unmodified lines
305	305	ia64_srlz_i(); /* srlz.i implies srlz.d */
306	306	}
307	307
308		−void
309		−flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
	308	+static void
	309	+__flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
310	310	unsigned long end)
311	311	{
312	312	struct mm_struct *mm = vma->vm_mm;
@@ -343,6 +343,25 @@ flush_tlb_range (struct vm_area_struct *vma, unsigned long start, · 30 unmodified lines
343	343	preempt_enable();
344	344	ia64_srlz_i(); /* srlz.i implies srlz.d */
345	345	}
	346	+
	347	+void flush_tlb_range(struct vm_area_struct *vma,
	348	+ unsigned long start, unsigned long end)
	349	+{
	350	+ if (unlikely(end - start >= 102410241024*1024UL
	351	+ \|\| REGION_NUMBER(start) != REGION_NUMBER(end - 1))) {
	352	+ /*
	353	+ * If we flush more than a tera-byte or across regions, we're
	354	+ * probably better off just flushing the entire TLB(s). This
	355	+ * should be very rare and is not worth optimizing for.
	356	+ */
	357	+ flush_tlb_all();
	358	+ } else {
	359	+ /* flush the address range from the tlb */
	360	+ __flush_tlb_range(vma, start, end);
	361	+ /* flush the virt. page-table area mapping the addr range */
	362	+ __flush_tlb_range(vma, ia64_thash(start), ia64_thash(end));
	363	+ }
	364	+}
346	365	EXPORT_SYMBOL(flush_tlb_range);
347	366
348	367	void ia64_tlb_init(void)

arch/ia64/sn/kernel/sn2/sn2_smp.c+0−7

@@ -120,13 +120,6 @@ void sn_migrate(struct task_struct *task) · 119 unmodified lines
120	120	cpu_relax();
121	121	}
122	122
123		−void sn_tlb_migrate_finish(struct mm_struct *mm)
124		−{
125		− /* flush_tlb_mm is inefficient if more than 1 users of mm */
126		− if (mm == current->mm && mm && atomic_read(&mm->mm_users) == 1)
127		− flush_tlb_mm(mm);
128		−}
129		−
130	123	static void
131	124	sn2_ipi_flush_all_tlb(struct mm_struct *mm)
132	125	{

arch/m68k/Kconfig+1−0

@@ -28,6 +28,7 @@ config M68K · 27 unmodified lines
28	28	select OLD_SIGSUSPEND3
29	29	select OLD_SIGACTION
30	30	select ARCH_DISCARD_MEMBLOCK
	31	+ select MMU_GATHER_NO_RANGE if MMU
31	32
32	33	config CPU_BIG_ENDIAN
33	34	def_bool y

arch/m68k/include/asm/tlb.h+0−14

@@ -2,20 +2,6 @@ · 1 unmodified line
2	2	#ifndef _M68K_TLB_H
3	3	#define _M68K_TLB_H
4	4
5		−/*
6		− * m68k doesn't need any special per-pte or
7		− * per-vma handling..
8		− */
9		−#define tlb_start_vma(tlb, vma) do { } while (0)
10		−#define tlb_end_vma(tlb, vma) do { } while (0)
11		−#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
12		−
13		−/*
14		− * .. because we flush the whole mm when it
15		− * fills up.
16		− */
17		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
18		−
19	5	#include <asm-generic/tlb.h>
20	6
21	7	#endif /* _M68K_TLB_H */

arch/microblaze/Kconfig+1−0

@@ -41,6 +41,7 @@ config MICROBLAZE · 40 unmodified lines
41	41	select TRACING_SUPPORT
42	42	select VIRT_TO_BUS
43	43	select CPU_NO_EFFICIENT_FFS
	44	+ select MMU_GATHER_NO_RANGE if MMU
44	45
45	46	# Endianness selection
46	47	choice

arch/microblaze/include/asm/tlb.h+0−9

@@ -11,16 +11,7 @@ · 10 unmodified lines
11	11	#ifndef _ASM_MICROBLAZE_TLB_H
12	12	#define _ASM_MICROBLAZE_TLB_H
13	13
14		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
15		−
16	14	#include <linux/pagemap.h>
17		−
18		−#ifdef CONFIG_MMU
19		−#define tlb_start_vma(tlb, vma) do { } while (0)
20		−#define tlb_end_vma(tlb, vma) do { } while (0)
21		−#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
22		−#endif
23		−
24	15	#include <asm-generic/tlb.h>
25	16
26	17	#endif /* _ASM_MICROBLAZE_TLB_H */

arch/mips/include/asm/tlb.h+0−17

@@ -5,23 +5,6 @@ · 4 unmodified lines
5	5	#include <asm/cpu-features.h>
6	6	#include <asm/mipsregs.h>
7	7
8		−/*
9		− * MIPS doesn't need any special per-pte or per-vma handling, except
10		− * we need to flush cache for area to be unmapped.
11		− */
12		−#define tlb_start_vma(tlb, vma) \
13		− do { \
14		− if (!tlb->fullmm) \
15		− flush_cache_range(vma, vma->vm_start, vma->vm_end); \
16		− } while (0)
17		−#define tlb_end_vma(tlb, vma) do { } while (0)
18		−#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
19		−
20		−/*
21		− * .. because we flush the whole mm when it fills up.
22		− */
23		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
24		−
25	8	#define _UNIQUE_ENTRYHI(base, idx) \
26	9	(((base) + ((idx) << (PAGE_SHIFT + 1))) \| \
27	10	(cpu_has_tlbinv ? MIPS_ENTRYHI_EHINV : 0))

arch/nds32/include/asm/tlb.h+0−16

@@ -4,22 +4,6 @@ · 3 unmodified lines
4	4	#ifndef __ASMNDS32_TLB_H
5	5	#define __ASMNDS32_TLB_H
6	6
7		−#define tlb_start_vma(tlb,vma) \
8		− do { \
9		− if (!tlb->fullmm) \
10		− flush_cache_range(vma, vma->vm_start, vma->vm_end); \
11		− } while (0)
12		−
13		−#define tlb_end_vma(tlb,vma) \
14		− do { \
15		− if(!tlb->fullmm) \
16		− flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
17		− } while (0)
18		−
19		−#define __tlb_remove_tlb_entry(tlb, pte, addr) do { } while (0)
20		−
21		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
22		−
23	7	#include <asm-generic/tlb.h>
24	8
25	9	#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)

arch/nds32/include/asm/tlbflush.h+0−1

@@ -42,6 +42,5 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); · 41 unmodified lines
42	42
43	43	void update_mmu_cache(struct vm_area_struct *vma,
44	44	unsigned long address, pte_t * pte);
45		−void tlb_migrate_finish(struct mm_struct *mm);
46	45
47	46	#endif

arch/nios2/Kconfig+1−0

@@ -24,6 +24,7 @@ config NIOS2 · 23 unmodified lines
24	24	select USB_ARCH_HAS_HCD if USB_SUPPORT
25	25	select CPU_NO_EFFICIENT_FFS
26	26	select ARCH_DISCARD_MEMBLOCK
	27	+ select MMU_GATHER_NO_RANGE if MMU
27	28
28	29	config GENERIC_CSUM
29	30	def_bool y

arch/nios2/include/asm/tlb.h+2−12

@@ -11,22 +11,12 @@ · 10 unmodified lines
11	11	#ifndef _ASM_NIOS2_TLB_H
12	12	#define _ASM_NIOS2_TLB_H
13	13
14		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
15		−
16	14	extern void set_mmu_pid(unsigned long pid);
17	15
18	16	/*
19		− * NiosII doesn't need any special per-pte or per-vma handling, except
20		− * we need to flush cache for the area to be unmapped.
	17	+ * NIOS32 does have flush_tlb_range(), but it lacks a limit and fallback to
	18	+ * full mm invalidation. So use flush_tlb_mm() for everything.
21	19	*/
22		−#define tlb_start_vma(tlb, vma) \
23		− do { \
24		− if (!tlb->fullmm) \
25		− flush_cache_range(vma, vma->vm_start, vma->vm_end); \
26		− } while (0)
27		−
28		−#define tlb_end_vma(tlb, vma) do { } while (0)
29		−#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
30	20
31	21	#include <linux/pagemap.h>
32	22	#include <asm-generic/tlb.h>

arch/openrisc/Kconfig+1−0

@@ -36,6 +36,7 @@ config OPENRISC · 35 unmodified lines
36	36	select OMPIC if SMP
37	37	select ARCH_WANT_FRAME_POINTERS
38	38	select GENERIC_IRQ_MULTI_HANDLER
	39	+ select MMU_GATHER_NO_RANGE if MMU
39	40
40	41	config CPU_BIG_ENDIAN
41	42	def_bool y

arch/openrisc/include/asm/tlb.h+2−6

@@ -20,14 +20,10 @@ · 19 unmodified lines
20	20	#define __ASM_OPENRISC_TLB_H__
21	21
22	22	/*
23		− * or32 doesn't need any special per-pte or
24		− * per-vma handling..
	23	+ * OpenRISC doesn't have an efficient flush_tlb_range() so use flush_tlb_mm()
	24	+ * for everything.
25	25	*/
26		−#define tlb_start_vma(tlb, vma) do { } while (0)
27		−#define tlb_end_vma(tlb, vma) do { } while (0)
28		−#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
29	26
30		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
31	27	#include <linux/pagemap.h>
32	28	#include <asm-generic/tlb.h>
33	29

arch/parisc/include/asm/tlb.h+0−18

@@ -2,24 +2,6 @@ · 1 unmodified line
2	2	#ifndef _PARISC_TLB_H
3	3	#define _PARISC_TLB_H
4	4
5		−#define tlb_flush(tlb) \
6		−do { if ((tlb)->fullmm) \
7		− flush_tlb_mm((tlb)->mm);\
8		−} while (0)
9		−
10		−#define tlb_start_vma(tlb, vma) \
11		−do { if (!(tlb)->fullmm) \
12		− flush_cache_range(vma, vma->vm_start, vma->vm_end); \
13		−} while (0)
14		−
15		−#define tlb_end_vma(tlb, vma) \
16		−do { if (!(tlb)->fullmm) \
17		− flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
18		−} while (0)
19		−
20		−#define __tlb_remove_tlb_entry(tlb, pte, address) \
21		− do { } while (0)
22		−
23	5	#include <asm-generic/tlb.h>
24	6
25	7	#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)

arch/powerpc/Kconfig+2−0

@@ -218,6 +218,8 @@ config PPC · 217 unmodified lines
218	218	select HAVE_PERF_REGS
219	219	select HAVE_PERF_USER_STACK_DUMP
220	220	select HAVE_RCU_TABLE_FREE if SMP
	221	+ select HAVE_RCU_TABLE_NO_INVALIDATE if HAVE_RCU_TABLE_FREE
	222	+ select HAVE_MMU_GATHER_PAGE_SIZE
221	223	select HAVE_REGS_AND_STACK_ACCESS_API
222	224	select HAVE_RELIABLE_STACKTRACE if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN
223	225	select HAVE_SYSCALL_TRACEPOINTS

arch/powerpc/include/asm/tlb.h+1−17

@@ -27,8 +27,8 @@ · 26 unmodified lines
27	27	#define tlb_start_vma(tlb, vma) do { } while (0)
28	28	#define tlb_end_vma(tlb, vma) do { } while (0)
29	29	#define __tlb_remove_tlb_entry __tlb_remove_tlb_entry
30		−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
31	30
	31	+#define tlb_flush tlb_flush
32	32	extern void tlb_flush(struct mmu_gather *tlb);
33	33
34	34	/* Get the generic bits... */
@@ -46,22 +46,6 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep, · 11 unmodified lines
46	46	#endif
47	47	}
48	48
49		−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
50		− unsigned int page_size)
51		−{
52		− if (!tlb->page_size)
53		− tlb->page_size = page_size;
54		− else if (tlb->page_size != page_size) {
55		− if (!tlb->fullmm)
56		− tlb_flush_mmu(tlb);
57		− /*
58		− * update the page size after flush for the new
59		− * mmu_gather.
60		− */
61		− tlb->page_size = page_size;
62		− }
63		−}
64		−
65	49	#ifdef CONFIG_SMP
66	50	static inline int mm_is_core_local(struct mm_struct *mm)
67	51	{

arch/riscv/include/asm/tlb.h+1−0

@@ -18,6 +18,7 @@ struct mmu_gather; · 17 unmodified lines
18	18
19	19	static void tlb_flush(struct mmu_gather *tlb);
20	20
	21	+#define tlb_flush tlb_flush
21	22	#include <asm-generic/tlb.h>
22	23
23	24	static inline void tlb_flush(struct mmu_gather *tlb)

arch/s390/Kconfig+2−0

@@ -164,11 +164,13 @@ config S390 · 163 unmodified lines
164	164	select HAVE_PERF_USER_STACK_DUMP
165	165	select HAVE_MEMBLOCK_NODE_MAP
166	166	select HAVE_MEMBLOCK_PHYS_MAP
	167	+ select HAVE_MMU_GATHER_NO_GATHER
167	168	select HAVE_MOD_ARCH_SPECIFIC
168	169	select HAVE_NOP_MCOUNT
169	170	select HAVE_OPROFILE
170	171	select HAVE_PCI
171	172	select HAVE_PERF_EVENTS
	173	+ select HAVE_RCU_TABLE_FREE
172	174	select HAVE_REGS_AND_STACK_ACCESS_API
173	175	select HAVE_RSEQ
174	176	select HAVE_SYSCALL_TRACEPOINTS

arch/s390/include/asm/tlb.h+39−91

@@ -22,98 +22,39 @@ · 21 unmodified lines
22	22	* Pages used for the page tables is a different story. FIXME: more
23	23	*/
24	24
25		−#include <linux/mm.h>
26		−#include <linux/pagemap.h>
27		−#include <linux/swap.h>
28		−#include <asm/processor.h>
29		−#include <asm/pgalloc.h>
30		−#include <asm/tlbflush.h>
31		−
32		−struct mmu_gather {
33		− struct mm_struct *mm;
34		− struct mmu_table_batch *batch;
35		− unsigned int fullmm;
36		− unsigned long start, end;
37		−};
38		−
39		−struct mmu_table_batch {
40		− struct rcu_head rcu;
41		− unsigned int nr;
42		− void *tables[0];
43		−};
44		−
45		−#define MAX_TABLE_BATCH \
46		− ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
47		−
48		−extern void tlb_table_flush(struct mmu_gather *tlb);
49		−extern void tlb_remove_table(struct mmu_gather tlb, void table);
50		−
51		−static inline void
52		−arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
53		− unsigned long start, unsigned long end)
54		−{
55		− tlb->mm = mm;
56		− tlb->start = start;
57		− tlb->end = end;
58		− tlb->fullmm = !(start \| (end+1));
59		− tlb->batch = NULL;
60		−}
61		−
62		−static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
63		−{
64		− __tlb_flush_mm_lazy(tlb->mm);
65		−}
66		−
67		−static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
68		−{
69		− tlb_table_flush(tlb);
70		−}
71		−
	25	+void __tlb_remove_table(void *_table);
	26	+static inline void tlb_flush(struct mmu_gather *tlb);
	27	+static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
	28	+ struct page *page, int page_size);
72	29
73		−static inline void tlb_flush_mmu(struct mmu_gather *tlb)
74		−{
75		− tlb_flush_mmu_tlbonly(tlb);
76		− tlb_flush_mmu_free(tlb);
77		−}
	30	+#define tlb_start_vma(tlb, vma) do { } while (0)
	31	+#define tlb_end_vma(tlb, vma) do { } while (0)
78	32
79		−static inline void
80		−arch_tlb_finish_mmu(struct mmu_gather *tlb,
81		− unsigned long start, unsigned long end, bool force)
82		−{
83		− if (force) {
84		− tlb->start = start;
85		− tlb->end = end;
86		− }
	33	+#define tlb_flush tlb_flush
	34	+#define pte_free_tlb pte_free_tlb
	35	+#define pmd_free_tlb pmd_free_tlb
	36	+#define p4d_free_tlb p4d_free_tlb
	37	+#define pud_free_tlb pud_free_tlb
87	38
88		− tlb_flush_mmu(tlb);
89		−}
	39	+#include <asm/pgalloc.h>
	40	+#include <asm/tlbflush.h>
	41	+#include <asm-generic/tlb.h>
90	42
91	43	/*
92	44	* Release the page cache reference for a pte removed by
93	45	* tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page
94	46	* has already been freed, so just do free_page_and_swap_cache.
95	47	*/
96		−static inline bool __tlb_remove_page(struct mmu_gather tlb, struct page page)
97		−{
98		− free_page_and_swap_cache(page);
99		− return false; /* avoid calling tlb_flush_mmu */
100		−}
101		−
102		−static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
103		−{
104		− free_page_and_swap_cache(page);
105		−}
106		−
107	48	static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
108	49	struct page *page, int page_size)
109	50	{
110		− return __tlb_remove_page(tlb, page);
	51	+ free_page_and_swap_cache(page);
	52	+ return false;
111	53	}
112	54
113		−static inline void tlb_remove_page_size(struct mmu_gather *tlb,
114		− struct page *page, int page_size)
	55	+static inline void tlb_flush(struct mmu_gather *tlb)
115	56	{
116		− return tlb_remove_page(tlb, page);
	57	+ __tlb_flush_mm_lazy(tlb->mm);
117	58	}
118	59
119	60	/*
@@ -121,8 +62,17 @@ static inline void tlb_remove_page_size(struct mmu_gather *tlb, · 1 unmodified line
121	62	* page table from the tlb.
122	63	*/
123	64	static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
124		− unsigned long address)
	65	+ unsigned long address)
125	66	{
	67	+ __tlb_adjust_range(tlb, address, PAGE_SIZE);
	68	+ tlb->mm->context.flush_mm = 1;
	69	+ tlb->freed_tables = 1;
	70	+ tlb->cleared_ptes = 1;
	71	+ /*
	72	+ * page_table_free_rcu takes care of the allocation bit masks
	73	+ * of the 2K table fragments in the 4K page table page,
	74	+ * then calls tlb_remove_table.
	75	+ */
126	76	page_table_free_rcu(tlb, (unsigned long *) pte, address);
127	77	}
128	78
@@ -139,6 +89,10 @@ static inline void pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd, · 10 unmodified lines
139	89	if (mm_pmd_folded(tlb->mm))
140	90	return;
141	91	pgtable_pmd_page_dtor(virt_to_page(pmd));
	92	+ __tlb_adjust_range(tlb, address, PAGE_SIZE);
	93	+ tlb->mm->context.flush_mm = 1;
	94	+ tlb->freed_tables = 1;
	95	+ tlb->cleared_puds = 1;
142	96	tlb_remove_table(tlb, pmd);
143	97	}
144	98
@@ -154,6 +108,10 @@ static inline void p4d_free_tlb(struct mmu_gather tlb, p4d_t p4d, · 9 unmodified lines
154	108	{
155	109	if (mm_p4d_folded(tlb->mm))
156	110	return;
	111	+ __tlb_adjust_range(tlb, address, PAGE_SIZE);
	112	+ tlb->mm->context.flush_mm = 1;
	113	+ tlb->freed_tables = 1;
	114	+ tlb->cleared_p4ds = 1;
157	115	tlb_remove_table(tlb, p4d);
158	116	}
159	117
@@ -169,21 +127,11 @@ static inline void pud_free_tlb(struct mmu_gather tlb, pud_t pud, · 9 unmodified lines
169	127	{
170	128	if (mm_pud_folded(tlb->mm))
171	129	return;
	130	+ tlb->mm->context.flush_mm = 1;
	131	+ tlb->freed_tables = 1;
	132	+ tlb->cleared_puds = 1;
172	133	tlb_remove_table(tlb, pud);
173	134	}
174	135
175		−#define tlb_start_vma(tlb, vma) do { } while (0)
176		−#define tlb_end_vma(tlb, vma) do { } while (0)
177		−#define tlb_remove_tlb_entry(tlb, ptep, addr) do { } while (0)
178		−#define tlb_remove_pmd_tlb_entry(tlb, pmdp, addr) do { } while (0)
179		−#define tlb_migrate_finish(mm) do { } while (0)
180		−#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
181		− tlb_remove_tlb_entry(tlb, ptep, address)
182		−
183		−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
184		−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
185		− unsigned int page_size)
186		−{
187		−}
188	136
189	137	#endif /* _S390_TLB_H */

arch/s390/mm/pgalloc.c+1−62

@@ -290,7 +290,7 @@ void page_table_free_rcu(struct mmu_gather tlb, unsigned long table, · 289 unmodified lines
290	290	tlb_remove_table(tlb, table);
291	291	}
292	292
293		−static void __tlb_remove_table(void *_table)
	293	+void __tlb_remove_table(void *_table)
294	294	{
295	295	unsigned int mask = (unsigned long) _table & 3;
296	296	void table = (void )((unsigned long) _table ^ mask);
@@ -316,67 +316,6 @@ static void __tlb_remove_table(void *_table) · 19 unmodified lines
316	316	}
317	317	}
318	318
319		−static void tlb_remove_table_smp_sync(void *arg)
320		−{
321		− /* Simply deliver the interrupt */
322		−}
323		−
324		−static void tlb_remove_table_one(void *table)
325		−{
326		− /*
327		− * This isn't an RCU grace period and hence the page-tables cannot be
328		− * assumed to be actually RCU-freed.
329		− *
330		− * It is however sufficient for software page-table walkers that rely
331		− * on IRQ disabling. See the comment near struct mmu_table_batch.
332		− */
333		− smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
334		− __tlb_remove_table(table);
335		−}
336		−
337		−static void tlb_remove_table_rcu(struct rcu_head *head)
338		−{
339		− struct mmu_table_batch *batch;
340		− int i;
341		−
342		− batch = container_of(head, struct mmu_table_batch, rcu);
343		−
344		− for (i = 0; i < batch->nr; i++)
345		− __tlb_remove_table(batch->tables[i]);
346		−
347		− free_page((unsigned long)batch);
348		−}
349		−
350		−void tlb_table_flush(struct mmu_gather *tlb)
351		−{
352		− struct mmu_table_batch **batch = &tlb->batch;
353		−
354		− if (*batch) {
355		− call_rcu(&(*batch)->rcu, tlb_remove_table_rcu);
356		− *batch = NULL;
357		− }
358		−}
359		−
360		−void tlb_remove_table(struct mmu_gather tlb, void table)
361		−{
362		− struct mmu_table_batch **batch = &tlb->batch;
363		−
364		− tlb->mm->context.flush_mm = 1;
365		− if (*batch == NULL) {
366		− batch = (struct mmu_table_batch )
367		− __get_free_page(GFP_NOWAIT \| __GFP_NOWARN);
368		− if (*batch == NULL) {
369		− __tlb_flush_mm_lazy(tlb->mm);
370		− tlb_remove_table_one(table);
371		− return;
372		− }
373		− (*batch)->nr = 0;
374		− }
375		− (batch)->tables[(batch)->nr++] = table;
376		− if ((*batch)->nr == MAX_TABLE_BATCH)
377		− tlb_flush_mmu(tlb);
378		−}
379		−
380	319	/*
381	320	* Base infrastructure required to generate basic asces, region, segment,
382	321	* and page tables that do not make use of enhanced features like EDAT1.

arch/sh/include/asm/pgalloc.h+9−0

@@ -70,6 +70,15 @@ do { \ · 69 unmodified lines
70	70	tlb_remove_page((tlb), (pte)); \
71	71	} while (0)
72	72
	73	+#if CONFIG_PGTABLE_LEVELS > 2
	74	+#define __pmd_free_tlb(tlb, pmdp, addr) \
	75	+do { \
	76	+ struct page *page = virt_to_page(pmdp); \
	77	+ pgtable_pmd_page_dtor(page); \
	78	+ tlb_remove_page((tlb), page); \
	79	+} while (0);
	80	+#endif
	81	+
73	82	static inline void check_pgt_cache(void)
74	83	{
75	84	quicklist_trim(QUICK_PT, NULL, 25, 16);

arch/sh/include/asm/tlb.h+1−131

@@ -11,133 +11,8 @@ · 10 unmodified lines
11	11
12	12	#ifdef CONFIG_MMU
13	13	#include <linux/swap.h>
14		−#include <asm/pgalloc.h>
15		−#include <asm/tlbflush.h>
16		−#include <asm/mmu_context.h>
17	14
18		−/*
19		− * TLB handling. This allows us to remove pages from the page
20		− * tables, and efficiently handle the TLB issues.
21		− */
22		−struct mmu_gather {
23		− struct mm_struct *mm;
24		− unsigned int fullmm;
25		− unsigned long start, end;
26		−};
27		−
28		−static inline void init_tlb_gather(struct mmu_gather *tlb)
29		−{
30		− tlb->start = TASK_SIZE;
31		− tlb->end = 0;
32		−
33		− if (tlb->fullmm) {
34		− tlb->start = 0;
35		− tlb->end = TASK_SIZE;
36		− }
37		−}
38		−
39		−static inline void
40		−arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
41		− unsigned long start, unsigned long end)
42		−{
43		− tlb->mm = mm;
44		− tlb->start = start;
45		− tlb->end = end;
46		− tlb->fullmm = !(start \| (end+1));
47		−
48		− init_tlb_gather(tlb);
49		−}
50		−
51		−static inline void
52		−arch_tlb_finish_mmu(struct mmu_gather *tlb,
53		− unsigned long start, unsigned long end, bool force)
54		−{
55		− if (tlb->fullmm \|\| force)
56		− flush_tlb_mm(tlb->mm);
57		−
58		− /* keep the page table cache within bounds */
59		− check_pgt_cache();
60		−}
61		−
62		−static inline void
63		−tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep, unsigned long address)
64		−{
65		− if (tlb->start > address)
66		− tlb->start = address;
67		− if (tlb->end < address + PAGE_SIZE)
68		− tlb->end = address + PAGE_SIZE;
69		−}
70		−
71		−#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
72		− tlb_remove_tlb_entry(tlb, ptep, address)
73		−
74		−/*
75		− * In the case of tlb vma handling, we can optimise these away in the
76		− * case where we're doing a full MM flush. When we're doing a munmap,
77		− * the vmas are adjusted to only cover the region to be torn down.
78		− */
79		−static inline void
80		−tlb_start_vma(struct mmu_gather tlb, struct vm_area_struct vma)
81		−{
82		− if (!tlb->fullmm)
83		− flush_cache_range(vma, vma->vm_start, vma->vm_end);
84		−}
85		−
86		−static inline void
87		−tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma)
88		−{
89		− if (!tlb->fullmm && tlb->end) {
90		− flush_tlb_range(vma, tlb->start, tlb->end);
91		− init_tlb_gather(tlb);
92		− }
93		−}
94		−
95		−static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
96		−{
97		−}
98		−
99		−static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
100		−{
101		−}
102		−
103		−static inline void tlb_flush_mmu(struct mmu_gather *tlb)
104		−{
105		−}
106		−
107		−static inline int __tlb_remove_page(struct mmu_gather tlb, struct page page)
108		−{
109		− free_page_and_swap_cache(page);
110		− return false; /* avoid calling tlb_flush_mmu */
111		−}
112		−
113		−static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
114		−{
115		− __tlb_remove_page(tlb, page);
116		−}
117		−
118		−static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
119		− struct page *page, int page_size)
120		−{
121		− return __tlb_remove_page(tlb, page);
122		−}
123		−
124		−static inline void tlb_remove_page_size(struct mmu_gather *tlb,
125		− struct page *page, int page_size)
126		−{
127		− return tlb_remove_page(tlb, page);
128		−}
129		−
130		−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
131		−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
132		− unsigned int page_size)
133		−{
134		−}
135		−
136		−#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep)
137		−#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
138		−#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
139		−
140		−#define tlb_migrate_finish(mm) do { } while (0)
	15	+#include <asm-generic/tlb.h>
141	16
142	17	#if defined(CONFIG_CPU_SH4) \|\| defined(CONFIG_SUPERH64)
143	18	extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);
@@ -157,11 +32,6 @@ static inline void tlb_unwire_entry(void) · 13 unmodified lines
157	32
158	33	#else /* CONFIG_MMU */
159	34
160		−#define tlb_start_vma(tlb, vma) do { } while (0)
161		−#define tlb_end_vma(tlb, vma) do { } while (0)
162		−#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
163		−#define tlb_flush(tlb) do { } while (0)
164		−
165	35	#include <asm-generic/tlb.h>
166	36
167	37	#endif /* CONFIG_MMU */

arch/sparc/Kconfig+1−0

@@ -63,6 +63,7 @@ config SPARC64 · 62 unmodified lines
63	63	select HAVE_KRETPROBES
64	64	select HAVE_KPROBES
65	65	select HAVE_RCU_TABLE_FREE if SMP
	66	+ select HAVE_RCU_TABLE_NO_INVALIDATE if HAVE_RCU_TABLE_FREE
66	67	select HAVE_MEMBLOCK_NODE_MAP
67	68	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
68	69	select HAVE_DYNAMIC_FTRACE

arch/sparc/include/asm/tlb_32.h+0−18

@@ -2,24 +2,6 @@ · 1 unmodified line
2	2	#ifndef _SPARC_TLB_H
3	3	#define _SPARC_TLB_H
4	4
5		−#define tlb_start_vma(tlb, vma) \
6		−do { \
7		− flush_cache_range(vma, vma->vm_start, vma->vm_end); \
8		−} while (0)
9		−
10		−#define tlb_end_vma(tlb, vma) \
11		−do { \
12		− flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
13		−} while (0)
14		−
15		−#define __tlb_remove_tlb_entry(tlb, pte, address) \
16		− do { } while (0)
17		−
18		−#define tlb_flush(tlb) \
19		−do { \
20		− flush_tlb_mm((tlb)->mm); \
21		−} while (0)
22		−
23	5	#include <asm-generic/tlb.h>
24	6
25	7	#endif /* _SPARC_TLB_H */

arch/um/include/asm/tlb.h+2−156

@@ -2,162 +2,8 @@ · 1 unmodified line
2	2	#ifndef __UM_TLB_H
3	3	#define __UM_TLB_H
4	4
5		−#include <linux/pagemap.h>
6		−#include <linux/swap.h>
7		−#include <asm/percpu.h>
8		−#include <asm/pgalloc.h>
9	5	#include <asm/tlbflush.h>
10		−
11		−#define tlb_start_vma(tlb, vma) do { } while (0)
12		−#define tlb_end_vma(tlb, vma) do { } while (0)
13		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
14		−
15		−/* struct mmu_gather is an opaque type used by the mm code for passing around
16		− * any data needed by arch specific code for tlb_remove_page.
17		− */
18		−struct mmu_gather {
19		− struct mm_struct *mm;
20		− unsigned int need_flush; /* Really unmapped some ptes? */
21		− unsigned long start;
22		− unsigned long end;
23		− unsigned int fullmm; /* non-zero means full mm flush */
24		−};
25		−
26		−static inline void __tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep,
27		− unsigned long address)
28		−{
29		− if (tlb->start > address)
30		− tlb->start = address;
31		− if (tlb->end < address + PAGE_SIZE)
32		− tlb->end = address + PAGE_SIZE;
33		−}
34		−
35		−static inline void init_tlb_gather(struct mmu_gather *tlb)
36		−{
37		− tlb->need_flush = 0;
38		−
39		− tlb->start = TASK_SIZE;
40		− tlb->end = 0;
41		−
42		− if (tlb->fullmm) {
43		− tlb->start = 0;
44		− tlb->end = TASK_SIZE;
45		− }
46		−}
47		−
48		−static inline void
49		−arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
50		− unsigned long start, unsigned long end)
51		−{
52		− tlb->mm = mm;
53		− tlb->start = start;
54		− tlb->end = end;
55		− tlb->fullmm = !(start \| (end+1));
56		−
57		− init_tlb_gather(tlb);
58		−}
59		−
60		−extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
61		− unsigned long end);
62		−
63		−static inline void
64		−tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
65		−{
66		− flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end);
67		−}
68		−
69		−static inline void
70		−tlb_flush_mmu_free(struct mmu_gather *tlb)
71		−{
72		− init_tlb_gather(tlb);
73		−}
74		−
75		−static inline void
76		−tlb_flush_mmu(struct mmu_gather *tlb)
77		−{
78		− if (!tlb->need_flush)
79		− return;
80		−
81		− tlb_flush_mmu_tlbonly(tlb);
82		− tlb_flush_mmu_free(tlb);
83		−}
84		−
85		−/* arch_tlb_finish_mmu
86		− * Called at the end of the shootdown operation to free up any resources
87		− * that were required.
88		− */
89		−static inline void
90		−arch_tlb_finish_mmu(struct mmu_gather *tlb,
91		− unsigned long start, unsigned long end, bool force)
92		−{
93		− if (force) {
94		− tlb->start = start;
95		− tlb->end = end;
96		− tlb->need_flush = 1;
97		− }
98		− tlb_flush_mmu(tlb);
99		−
100		− /* keep the page table cache within bounds */
101		− check_pgt_cache();
102		−}
103		−
104		−/* tlb_remove_page
105		− * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)),
106		− * while handling the additional races in SMP caused by other CPUs
107		− * caching valid mappings in their TLBs.
108		− */
109		−static inline int __tlb_remove_page(struct mmu_gather tlb, struct page page)
110		−{
111		− tlb->need_flush = 1;
112		− free_page_and_swap_cache(page);
113		− return false; /* avoid calling tlb_flush_mmu */
114		−}
115		−
116		−static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
117		−{
118		− __tlb_remove_page(tlb, page);
119		−}
120		−
121		−static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
122		− struct page *page, int page_size)
123		−{
124		− return __tlb_remove_page(tlb, page);
125		−}
126		−
127		−static inline void tlb_remove_page_size(struct mmu_gather *tlb,
128		− struct page *page, int page_size)
129		−{
130		− return tlb_remove_page(tlb, page);
131		−}
132		−
133		−/**
134		− * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
135		− *
136		− * Record the fact that pte's were really umapped in ->need_flush, so we can
137		− * later optimise away the tlb invalidate. This helps when userspace is
138		− * unmapping already-unmapped pages, which happens quite a lot.
139		− */
140		−#define tlb_remove_tlb_entry(tlb, ptep, address) \
141		− do { \
142		− tlb->need_flush = 1; \
143		− __tlb_remove_tlb_entry(tlb, ptep, address); \
144		− } while (0)
145		−
146		−#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
147		− tlb_remove_tlb_entry(tlb, ptep, address)
148		−
149		−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
150		−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
151		− unsigned int page_size)
152		−{
153		−}
154		−
155		−#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
156		−
157		−#define pud_free_tlb(tlb, pudp, addr) __pud_free_tlb(tlb, pudp, addr)
158		−
159		−#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
160		−
161		−#define tlb_migrate_finish(mm) do {} while (0)
	6	+#include <asm-generic/cacheflush.h>
	7	+#include <asm-generic/tlb.h>
162	8
163	9	#endif

arch/unicore32/Kconfig+1−0

@@ -20,6 +20,7 @@ config UNICORE32 · 19 unmodified lines
20	20	select GENERIC_IOMAP
21	21	select MODULES_USE_ELF_REL
22	22	select NEED_DMA_MAP_STATE
	23	+ select MMU_GATHER_NO_RANGE if MMU
23	24	help
24	25	UniCore-32 is 32-bit Instruction Set Architecture,
25	26	including a series of low-power-consumption RISC chip

arch/unicore32/include/asm/tlb.h+3−4

@@ -12,10 +12,9 @@ · 11 unmodified lines
12	12	#ifndef __UNICORE_TLB_H__
13	13	#define __UNICORE_TLB_H__
14	14
15		−#define tlb_start_vma(tlb, vma) do { } while (0)
16		−#define tlb_end_vma(tlb, vma) do { } while (0)
17		−#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
18		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
	15	+/*
	16	+ * unicore32 lacks an efficient flush_tlb_range(), use flush_tlb_mm().
	17	+ */
19	18
20	19	#define __pte_free_tlb(tlb, pte, addr) \
21	20	do { \

arch/x86/Kconfig+0−1

@@ -183,7 +183,6 @@ config X86 · 182 unmodified lines
183	183	select HAVE_PERF_REGS
184	184	select HAVE_PERF_USER_STACK_DUMP
185	185	select HAVE_RCU_TABLE_FREE if PARAVIRT
186		− select HAVE_RCU_TABLE_INVALIDATE if HAVE_RCU_TABLE_FREE
187	186	select HAVE_REGS_AND_STACK_ACCESS_API
188	187	select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER \|\| UNWINDER_ORC) && STACK_VALIDATION
189	188	select HAVE_FUNCTION_ARG_ACCESS_API

arch/x86/include/asm/tlb.h+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#define tlb_end_vma(tlb, vma) do { } while (0)
7	7	#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
8	8
	9	+#define tlb_flush tlb_flush
9	10	static inline void tlb_flush(struct mmu_gather *tlb);
10	11
11	12	#include <asm-generic/tlb.h>

arch/xtensa/include/asm/tlb.h+0−26

@@ -14,32 +14,6 @@ · 13 unmodified lines
14	14	#include <asm/cache.h>
15	15	#include <asm/page.h>
16	16
17		−#if (DCACHE_WAY_SIZE <= PAGE_SIZE)
18		−
19		−/* Note, read http://lkml.org/lkml/2004/1/15/6 */
20		−
21		−# define tlb_start_vma(tlb,vma) do { } while (0)
22		−# define tlb_end_vma(tlb,vma) do { } while (0)
23		−
24		−#else
25		−
26		−# define tlb_start_vma(tlb, vma) \
27		− do { \
28		− if (!tlb->fullmm) \
29		− flush_cache_range(vma, vma->vm_start, vma->vm_end); \
30		− } while(0)
31		−
32		−# define tlb_end_vma(tlb, vma) \
33		− do { \
34		− if (!tlb->fullmm) \
35		− flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
36		− } while(0)
37		−
38		−#endif
39		−
40		−#define __tlb_remove_tlb_entry(tlb,pte,addr) do { } while (0)
41		−#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
42		−
43	17	#include <asm-generic/tlb.h>
44	18
45	19	#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte)

include/asm-generic/tlb.h+259−29

@@ -19,9 +19,131 @@ · 18 unmodified lines
19	19	#include <linux/swap.h>
20	20	#include <asm/pgalloc.h>
21	21	#include <asm/tlbflush.h>
	22	+#include <asm/cacheflush.h>
22	23
23	24	#ifdef CONFIG_MMU
24	25
	26	+/*
	27	+ * Generic MMU-gather implementation.
	28	+ *
	29	+ * The mmu_gather data structure is used by the mm code to implement the
	30	+ * correct and efficient ordering of freeing pages and TLB invalidations.
	31	+ *
	32	+ * This correct ordering is:
	33	+ *
	34	+ * 1) unhook page
	35	+ * 2) TLB invalidate page
	36	+ * 3) free page
	37	+ *
	38	+ * That is, we must never free a page before we have ensured there are no live
	39	+ * translations left to it. Otherwise it might be possible to observe (or
	40	+ * worse, change) the page content after it has been reused.
	41	+ *
	42	+ * The mmu_gather API consists of:
	43	+ *
	44	+ * - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather
	45	+ *
	46	+ * Finish in particular will issue a (final) TLB invalidate and free
	47	+ * all (remaining) queued pages.
	48	+ *
	49	+ * - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA
	50	+ *
	51	+ * Defaults to flushing at tlb_end_vma() to reset the range; helps when
	52	+ * there's large holes between the VMAs.
	53	+ *
	54	+ * - tlb_remove_page() / __tlb_remove_page()
	55	+ * - tlb_remove_page_size() / __tlb_remove_page_size()
	56	+ *
	57	+ * __tlb_remove_page_size() is the basic primitive that queues a page for
	58	+ * freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a
	59	+ * boolean indicating if the queue is (now) full and a call to
	60	+ * tlb_flush_mmu() is required.
	61	+ *
	62	+ * tlb_remove_page() and tlb_remove_page_size() imply the call to
	63	+ * tlb_flush_mmu() when required and has no return value.
	64	+ *
	65	+ * - tlb_change_page_size()
	66	+ *
	67	+ * call before __tlb_remove_page*() to set the current page-size; implies a
	68	+ * possible tlb_flush_mmu() call.
	69	+ *
	70	+ * - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()
	71	+ *
	72	+ * tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
	73	+ * related state, like the range)
	74	+ *
	75	+ * tlb_flush_mmu() - in addition to the above TLB invalidate, also frees
	76	+ * whatever pages are still batched.
	77	+ *
	78	+ * - mmu_gather::fullmm
	79	+ *
	80	+ * A flag set by tlb_gather_mmu() to indicate we're going to free
	81	+ * the entire mm; this allows a number of optimizations.
	82	+ *
	83	+ * - We can ignore tlb_{start,end}_vma(); because we don't
	84	+ * care about ranges. Everything will be shot down.
	85	+ *
	86	+ * - (RISC) architectures that use ASIDs can cycle to a new ASID
	87	+ * and delay the invalidation until ASID space runs out.
	88	+ *
	89	+ * - mmu_gather::need_flush_all
	90	+ *
	91	+ * A flag that can be set by the arch code if it wants to force
	92	+ * flush the entire TLB irrespective of the range. For instance
	93	+ * x86-PAE needs this when changing top-level entries.
	94	+ *
	95	+ * And allows the architecture to provide and implement tlb_flush():
	96	+ *
	97	+ * tlb_flush() may, in addition to the above mentioned mmu_gather fields, make
	98	+ * use of:
	99	+ *
	100	+ * - mmu_gather::start / mmu_gather::end
	101	+ *
	102	+ * which provides the range that needs to be flushed to cover the pages to
	103	+ * be freed.
	104	+ *
	105	+ * - mmu_gather::freed_tables
	106	+ *
	107	+ * set when we freed page table pages
	108	+ *
	109	+ * - tlb_get_unmap_shift() / tlb_get_unmap_size()
	110	+ *
	111	+ * returns the smallest TLB entry size unmapped in this range.
	112	+ *
	113	+ * If an architecture does not provide tlb_flush() a default implementation
	114	+ * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is
	115	+ * specified, in which case we'll default to flush_tlb_mm().
	116	+ *
	117	+ * Additionally there are a few opt-in features:
	118	+ *
	119	+ * HAVE_MMU_GATHER_PAGE_SIZE
	120	+ *
	121	+ * This ensures we call tlb_flush() every time tlb_change_page_size() actually
	122	+ * changes the size and provides mmu_gather::page_size to tlb_flush().
	123	+ *
	124	+ * HAVE_RCU_TABLE_FREE
	125	+ *
	126	+ * This provides tlb_remove_table(), to be used instead of tlb_remove_page()
	127	+ * for page directores (__p*_free_tlb()). This provides separate freeing of
	128	+ * the page-table pages themselves in a semi-RCU fashion (see comment below).
	129	+ * Useful if your architecture doesn't use IPIs for remote TLB invalidates
	130	+ * and therefore doesn't naturally serialize with software page-table walkers.
	131	+ *
	132	+ * When used, an architecture is expected to provide __tlb_remove_table()
	133	+ * which does the actual freeing of these pages.
	134	+ *
	135	+ * HAVE_RCU_TABLE_NO_INVALIDATE
	136	+ *
	137	+ * This makes HAVE_RCU_TABLE_FREE avoid calling tlb_flush_mmu_tlbonly() before
	138	+ * freeing the page-table pages. This can be avoided if you use
	139	+ * HAVE_RCU_TABLE_FREE and your architecture does _NOT_ use the Linux
	140	+ * page-tables natively.
	141	+ *
	142	+ * MMU_GATHER_NO_RANGE
	143	+ *
	144	+ * Use this if your architecture lacks an efficient flush_tlb_range().
	145	+ */
	146	+
25	147	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
26	148	/*
27	149	* Semi RCU freeing of the page directories.
@@ -60,11 +182,11 @@ struct mmu_table_batch { · 32 unmodified lines
60	182	#define MAX_TABLE_BATCH \
61	183	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
62	184
63		−extern void tlb_table_flush(struct mmu_gather *tlb);
64	185	extern void tlb_remove_table(struct mmu_gather tlb, void table);
65	186
66	187	#endif
67	188
	189	+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
68	190	/*
69	191	* If we can't allocate a page to make a big batch of page pointers
70	192	* to work on, then just handle a few from the on-stack structure.
@@ -89,14 +211,21 @@ struct mmu_gather_batch { · 18 unmodified lines
89	211	*/
90	212	#define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)
91	213
92		−/* struct mmu_gather is an opaque type used by the mm code for passing around
	214	+extern bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page,
	215	+ int page_size);
	216	+#endif
	217	+
	218	+/*
	219	+ * struct mmu_gather is an opaque type used by the mm code for passing around
93	220	* any data needed by arch specific code for tlb_remove_page.
94	221	*/
95	222	struct mmu_gather {
96	223	struct mm_struct *mm;
	224	+
97	225	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
98	226	struct mmu_table_batch *batch;
99	227	#endif
	228	+
100	229	unsigned long start;
101	230	unsigned long end;
102	231	/*
@@ -124,23 +253,30 @@ struct mmu_gather { · 21 unmodified lines
124	253	unsigned int cleared_puds : 1;
125	254	unsigned int cleared_p4ds : 1;
126	255
	256	+ /*
	257	+ * tracks VM_EXEC \| VM_HUGETLB in tlb_start_vma
	258	+ */
	259	+ unsigned int vma_exec : 1;
	260	+ unsigned int vma_huge : 1;
	261	+
	262	+ unsigned int batch_count;
	263	+
	264	+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
127	265	struct mmu_gather_batch *active;
128	266	struct mmu_gather_batch local;
129	267	struct page *__pages[MMU_GATHER_BUNDLE];
130		− unsigned int batch_count;
131		− int page_size;
132		−};
133	268
134		−#define HAVE_GENERIC_MMU_GATHER
	269	+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
	270	+ unsigned int page_size;
	271	+#endif
	272	+#endif
	273	+};
135	274
136	275	void arch_tlb_gather_mmu(struct mmu_gather *tlb,
137	276	struct mm_struct *mm, unsigned long start, unsigned long end);
138	277	void tlb_flush_mmu(struct mmu_gather *tlb);
139	278	void arch_tlb_finish_mmu(struct mmu_gather *tlb,
140	279	unsigned long start, unsigned long end, bool force);
141		−void tlb_flush_mmu_free(struct mmu_gather *tlb);
142		−extern bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page,
143		− int page_size);
144	280
145	281	static inline void __tlb_adjust_range(struct mmu_gather *tlb,
146	282	unsigned long address,
@@ -163,8 +299,94 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) · 16 unmodified lines
163	299	tlb->cleared_pmds = 0;
164	300	tlb->cleared_puds = 0;
165	301	tlb->cleared_p4ds = 0;
	302	+ /*
	303	+ * Do not reset mmu_gather::vma_* fields here, we do not
	304	+ * call into tlb_start_vma() again to set them if there is an
	305	+ * intermediate flush.
	306	+ */
	307	+}
	308	+
	309	+#ifdef CONFIG_MMU_GATHER_NO_RANGE
	310	+
	311	+#if defined(tlb_flush) \|\| defined(tlb_start_vma) \|\| defined(tlb_end_vma)
	312	+#error MMU_GATHER_NO_RANGE relies on default tlb_flush(), tlb_start_vma() and tlb_end_vma()
	313	+#endif
	314	+
	315	+/*
	316	+ * When an architecture does not have efficient means of range flushing TLBs
	317	+ * there is no point in doing intermediate flushes on tlb_end_vma() to keep the
	318	+ * range small. We equally don't have to worry about page granularity or other
	319	+ * things.
	320	+ *
	321	+ * All we need to do is issue a full flush for any !0 range.
	322	+ */
	323	+static inline void tlb_flush(struct mmu_gather *tlb)
	324	+{
	325	+ if (tlb->end)
	326	+ flush_tlb_mm(tlb->mm);
	327	+}
	328	+
	329	+static inline void
	330	+tlb_update_vma_flags(struct mmu_gather tlb, struct vm_area_struct vma) { }
	331	+
	332	+#define tlb_end_vma tlb_end_vma
	333	+static inline void tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma) { }
	334	+
	335	+#else /* CONFIG_MMU_GATHER_NO_RANGE */
	336	+
	337	+#ifndef tlb_flush
	338	+
	339	+#if defined(tlb_start_vma) \|\| defined(tlb_end_vma)
	340	+#error Default tlb_flush() relies on default tlb_start_vma() and tlb_end_vma()
	341	+#endif
	342	+
	343	+/*
	344	+ * When an architecture does not provide its own tlb_flush() implementation
	345	+ * but does have a reasonably efficient flush_vma_range() implementation
	346	+ * use that.
	347	+ */
	348	+static inline void tlb_flush(struct mmu_gather *tlb)
	349	+{
	350	+ if (tlb->fullmm \|\| tlb->need_flush_all) {
	351	+ flush_tlb_mm(tlb->mm);
	352	+ } else if (tlb->end) {
	353	+ struct vm_area_struct vma = {
	354	+ .vm_mm = tlb->mm,
	355	+ .vm_flags = (tlb->vma_exec ? VM_EXEC : 0) \|
	356	+ (tlb->vma_huge ? VM_HUGETLB : 0),
	357	+ };
	358	+
	359	+ flush_tlb_range(&vma, tlb->start, tlb->end);
	360	+ }
166	361	}
167	362
	363	+static inline void
	364	+tlb_update_vma_flags(struct mmu_gather tlb, struct vm_area_struct vma)
	365	+{
	366	+ /*
	367	+ * flush_tlb_range() implementations that look at VM_HUGETLB (tile,
	368	+ * mips-4k) flush only large pages.
	369	+ *
	370	+ * flush_tlb_range() implementations that flush I-TLB also flush D-TLB
	371	+ * (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing
	372	+ * range.
	373	+ *
	374	+ * We rely on tlb_end_vma() to issue a flush, such that when we reset
	375	+ * these values the batch is empty.
	376	+ */
	377	+ tlb->vma_huge = !!(vma->vm_flags & VM_HUGETLB);
	378	+ tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);
	379	+}
	380	+
	381	+#else
	382	+
	383	+static inline void
	384	+tlb_update_vma_flags(struct mmu_gather tlb, struct vm_area_struct vma) { }
	385	+
	386	+#endif
	387	+
	388	+#endif /* CONFIG_MMU_GATHER_NO_RANGE */
	389	+
168	390	static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
169	391	{
170	392	if (!tlb->end)
@@ -196,21 +418,18 @@ static inline void tlb_remove_page(struct mmu_gather tlb, struct page page) · 25 unmodified lines
196	418	return tlb_remove_page_size(tlb, page, PAGE_SIZE);
197	419	}
198	420
199		−#ifndef tlb_remove_check_page_size_change
200		−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
201		−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
	421	+static inline void tlb_change_page_size(struct mmu_gather *tlb,
202	422	unsigned int page_size)
203	423	{
204		− /*
205		− * We don't care about page size change, just update
206		− * mmu_gather page size here so that debug checks
207		− * doesn't throw false warning.
208		− */
209		−#ifdef CONFIG_DEBUG_VM
	424	+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
	425	+ if (tlb->page_size && tlb->page_size != page_size) {
	426	+ if (!tlb->fullmm)
	427	+ tlb_flush_mmu(tlb);
	428	+ }
	429	+
210	430	tlb->page_size = page_size;
211	431	#endif
212	432	}
213		−#endif
214	433
215	434	static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
216	435	{
@@ -237,17 +456,30 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 20 unmodified lines
237	456	* the vmas are adjusted to only cover the region to be torn down.
238	457	*/
239	458	#ifndef tlb_start_vma
240		−#define tlb_start_vma(tlb, vma) do { } while (0)
241		−#endif
	459	+static inline void tlb_start_vma(struct mmu_gather tlb, struct vm_area_struct vma)
	460	+{
	461	+ if (tlb->fullmm)
	462	+ return;
242	463
243		−#define __tlb_end_vma(tlb, vma) \
244		− do { \
245		− if (!tlb->fullmm) \
246		− tlb_flush_mmu_tlbonly(tlb); \
247		− } while (0)
	464	+ tlb_update_vma_flags(tlb, vma);
	465	+ flush_cache_range(vma, vma->vm_start, vma->vm_end);
	466	+}
	467	+#endif
248	468
249	469	#ifndef tlb_end_vma
250		−#define tlb_end_vma __tlb_end_vma
	470	+static inline void tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma)
	471	+{
	472	+ if (tlb->fullmm)
	473	+ return;
	474	+
	475	+ /*
	476	+ * Do a TLB flush and reset the range at VMA boundaries; this avoids
	477	+ * the ranges growing with the unused space between consecutive VMAs,
	478	+ * but also the mmu_gather::vma_* flags from tlb_start_vma() rely on
	479	+ * this.
	480	+ */
	481	+ tlb_flush_mmu_tlbonly(tlb);
	482	+}
251	483	#endif
252	484
253	485	#ifndef __tlb_remove_tlb_entry
@@ -372,6 +604,4 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 118 unmodified lines
372	604
373	605	#endif /* CONFIG_MMU */
374	606
375		−#define tlb_migrate_finish(mm) do {} while (0)
376		−
377	607	#endif /* _ASM_GENERIC__TLB_H */

kernel/iomem.c+2−2

@@ -55,7 +55,7 @@ static void *try_ram_remap(resource_size_t offset, size_t size, · 54 unmodified lines
55	55	*
56	56	* MEMREMAP_WB - matches the default mapping for System RAM on
57	57	* the architecture. This is usually a read-allocate write-back cache.
58		− * Morever, if MEMREMAP_WB is specified and the requested remap region is RAM
	58	+ * Moreover, if MEMREMAP_WB is specified and the requested remap region is RAM
59	59	* memremap() will bypass establishing a new mapping and instead return
60	60	* a pointer into the direct map.
61	61	*
@@ -86,7 +86,7 @@ void *memremap(resource_size_t offset, size_t size, unsigned long flags) · 24 unmodified lines
86	86	/* Try all mapping types requested until one returns non-NULL */
87	87	if (flags & MEMREMAP_WB) {
88	88	/*
89		− * MEMREMAP_WB is special in that it can be satisifed
	89	+ * MEMREMAP_WB is special in that it can be satisfied
90	90	* from the direct map. Some archs depend on the
91	91	* capability of memremap() to autodetect cases where
92	92	* the requested range is potentially in System RAM.

kernel/resource.c+5−6

@@ -520,21 +520,20 @@ EXPORT_SYMBOL_GPL(page_is_ram); · 519 unmodified lines
520	520	int region_intersects(resource_size_t start, size_t size, unsigned long flags,
521	521	unsigned long desc)
522	522	{
523		− resource_size_t end = start + size - 1;
	523	+ struct resource res;
524	524	int type = 0; int other = 0;
525	525	struct resource *p;
526	526
	527	+ res.start = start;
	528	+ res.end = start + size - 1;
	529	+
527	530	read_lock(&resource_lock);
528	531	for (p = iomem_resource.child; p ; p = p->sibling) {
529	532	bool is_type = (((p->flags & flags) == flags) &&
530	533	((desc == IORES_DESC_NONE) \|\|
531	534	(desc == p->desc)));
532	535
533		− if (start >= p->start && start <= p->end)
534		− is_type ? type++ : other++;
535		− if (end >= p->start && end <= p->end)
536		− is_type ? type++ : other++;
537		− if (p->start >= start && p->end <= end)
	536	+ if (resource_overlaps(p, &res))
538	537	is_type ? type++ : other++;
539	538	}
540	539	read_unlock(&resource_lock);

kernel/sched/core.c+0−1

@@ -1151,7 +1151,6 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 1150 unmodified lines
1151	1151	/* Need help from migration thread: drop lock and wait. */
1152	1152	task_rq_unlock(rq, p, &rf);
1153	1153	stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1154		− tlb_migrate_finish(p->mm);
1155	1154	return 0;
1156	1155	} else if (task_on_rq_queued(p)) {
1157	1156	/*

mm/huge_memory.c+2−2

@@ -1677,7 +1677,7 @@ bool madvise_free_huge_pmd(struct mmu_gather tlb, struct vm_area_struct vma, · 1676 unmodified lines
1677	1677	struct mm_struct *mm = tlb->mm;
1678	1678	bool ret = false;
1679	1679
1680		− tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
	1680	+ tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1681	1681
1682	1682	ptl = pmd_trans_huge_lock(pmd, vma);
1683	1683	if (!ptl)
@@ -1753,7 +1753,7 @@ int zap_huge_pmd(struct mmu_gather tlb, struct vm_area_struct vma, · 69 unmodified lines
1753	1753	pmd_t orig_pmd;
1754	1754	spinlock_t *ptl;
1755	1755
1756		− tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
	1756	+ tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1757	1757
1758	1758	ptl = __pmd_trans_huge_lock(pmd, vma);
1759	1759	if (!ptl)

mm/hugetlb.c+1−1

@@ -3353,7 +3353,7 @@ void __unmap_hugepage_range(struct mmu_gather tlb, struct vm_area_struct vma, · 3352 unmodified lines
3353	3353	* This is a hugetlb vma, all the pte entries should point
3354	3354	* to huge page.
3355	3355	*/
3356		− tlb_remove_check_page_size_change(tlb, sz);
	3356	+ tlb_change_page_size(tlb, sz);
3357	3357	tlb_start_vma(tlb, vma);
3358	3358
3359	3359	/*

mm/madvise.c+1−1

@@ -328,7 +328,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, · 327 unmodified lines
328	328	if (pmd_trans_unstable(pmd))
329	329	return 0;
330	330
331		− tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
	331	+ tlb_change_page_size(tlb, PAGE_SIZE);
332	332	orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
333	333	flush_tlb_batched_pending(mm);
334	334	arch_enter_lazy_mmu_mode();

mm/memory.c+3−3

@@ -356,7 +356,7 @@ void free_pgd_range(struct mmu_gather *tlb, · 355 unmodified lines
356	356	* We add page table cache pages with PAGE_SIZE,
357	357	* (see pte_free_tlb()), flush the tlb if we need
358	358	*/
359		− tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
	359	+ tlb_change_page_size(tlb, PAGE_SIZE);
360	360	pgd = pgd_offset(tlb->mm, addr);
361	361	do {
362	362	next = pgd_addr_end(addr, end);
@@ -1046,7 +1046,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, · 683 unmodified lines
1046	1046	pte_t *pte;
1047	1047	swp_entry_t entry;
1048	1048
1049		− tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
	1049	+ tlb_change_page_size(tlb, PAGE_SIZE);
1050	1050	again:
1051	1051	init_rss_vec(rss);
1052	1052	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
@@ -1155,7 +1155,7 @@ again: · 102 unmodified lines
1155	1155	*/
1156	1156	if (force_flush) {
1157	1157	force_flush = 0;
1158		− tlb_flush_mmu_free(tlb);
	1158	+ tlb_flush_mmu(tlb);
1159	1159	if (addr != end)
1160	1160	goto again;
1161	1161	}

mm/mmu_gather.c+67−62

@@ -11,7 +11,7 @@ · 10 unmodified lines
11	11	#include <asm/pgalloc.h>
12	12	#include <asm/tlb.h>
13	13
14		−#ifdef HAVE_GENERIC_MMU_GATHER
	14	+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
15	15
16	16	static bool tlb_next_batch(struct mmu_gather *tlb)
17	17	{
@@ -41,35 +41,10 @@ static bool tlb_next_batch(struct mmu_gather *tlb) · 23 unmodified lines
41	41	return true;
42	42	}
43	43
44		−void arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
45		− unsigned long start, unsigned long end)
46		−{
47		− tlb->mm = mm;
48		−
49		− /* Is it from 0 to ~0? */
50		− tlb->fullmm = !(start \| (end+1));
51		− tlb->need_flush_all = 0;
52		− tlb->local.next = NULL;
53		− tlb->local.nr = 0;
54		− tlb->local.max = ARRAY_SIZE(tlb->__pages);
55		− tlb->active = &tlb->local;
56		− tlb->batch_count = 0;
57		−
58		−#ifdef CONFIG_HAVE_RCU_TABLE_FREE
59		− tlb->batch = NULL;
60		−#endif
61		− tlb->page_size = 0;
62		−
63		− __tlb_reset_range(tlb);
64		−}
65		−
66		−void tlb_flush_mmu_free(struct mmu_gather *tlb)
	44	+static void tlb_batch_pages_flush(struct mmu_gather *tlb)
67	45	{
68	46	struct mmu_gather_batch *batch;
69	47
70		−#ifdef CONFIG_HAVE_RCU_TABLE_FREE
71		− tlb_table_flush(tlb);
72		−#endif
73	48	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
74	49	free_pages_and_swap_cache(batch->pages, batch->nr);
75	50	batch->nr = 0;
@@ -77,31 +52,10 @@ void tlb_flush_mmu_free(struct mmu_gather *tlb) · 1 unmodified line
77	52	tlb->active = &tlb->local;
78	53	}
79	54
80		−void tlb_flush_mmu(struct mmu_gather *tlb)
81		−{
82		− tlb_flush_mmu_tlbonly(tlb);
83		− tlb_flush_mmu_free(tlb);
84		−}
85		−
86		−/* tlb_finish_mmu
87		− * Called at the end of the shootdown operation to free up any resources
88		− * that were required.
89		− */
90		−void arch_tlb_finish_mmu(struct mmu_gather *tlb,
91		− unsigned long start, unsigned long end, bool force)
	55	+static void tlb_batch_list_free(struct mmu_gather *tlb)
92	56	{
93	57	struct mmu_gather_batch batch, next;
94	58
95		− if (force) {
96		− __tlb_reset_range(tlb);
97		− __tlb_adjust_range(tlb, start, end - start);
98		− }
99		−
100		− tlb_flush_mmu(tlb);
101		−
102		− /* keep the page table cache within bounds */
103		− check_pgt_cache();
104		−
105	59	for (batch = tlb->local.next; batch; batch = next) {
106	60	next = batch->next;
107	61	free_pages((unsigned long)batch, 0);
@@ -109,19 +63,15 @@ void arch_tlb_finish_mmu(struct mmu_gather *tlb, · 1 unmodified line
109	63	tlb->local.next = NULL;
110	64	}
111	65
112		−/* __tlb_remove_page
113		− * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
114		− * handling the additional races in SMP caused by other CPUs caching valid
115		− * mappings in their TLBs. Returns the number of free page slots left.
116		− * When out of page slots we must call tlb_flush_mmu().
117		− *returns true if the caller should flush.
118		− */
119	66	bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page, int page_size)
120	67	{
121	68	struct mmu_gather_batch *batch;
122	69
123	70	VM_BUG_ON(!tlb->end);
	71	+
	72	+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
124	73	VM_WARN_ON(tlb->page_size != page_size);
	74	+#endif
125	75
126	76	batch = tlb->active;
127	77	/*
@@ -139,7 +89,7 @@ bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page, int page_ · 11 unmodified lines
139	89	return false;
140	90	}
141	91
142		−#endif /* HAVE_GENERIC_MMU_GATHER */
	92	+#endif /* HAVE_MMU_GATHER_NO_GATHER */
143	93
144	94	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
145	95
@@ -152,7 +102,7 @@ bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page, int page_ · 6 unmodified lines
152	102	*/
153	103	static inline void tlb_table_invalidate(struct mmu_gather *tlb)
154	104	{
155		−#ifdef CONFIG_HAVE_RCU_TABLE_INVALIDATE
	105	+#ifndef CONFIG_HAVE_RCU_TABLE_NO_INVALIDATE
156	106	/*
157	107	* Invalidate page-table caches used by hardware walkers. Then we still
158	108	* need to RCU-sched wait while freeing the pages because software
@@ -193,7 +143,7 @@ static void tlb_remove_table_rcu(struct rcu_head *head) · 34 unmodified lines
193	143	free_page((unsigned long)batch);
194	144	}
195	145
196		−void tlb_table_flush(struct mmu_gather *tlb)
	146	+static void tlb_table_flush(struct mmu_gather *tlb)
197	147	{
198	148	struct mmu_table_batch **batch = &tlb->batch;
199	149
@@ -225,6 +175,22 @@ void tlb_remove_table(struct mmu_gather tlb, void table) · 25 unmodified lines
225	175
226	176	#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
227	177
	178	+static void tlb_flush_mmu_free(struct mmu_gather *tlb)
	179	+{
	180	+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
	181	+ tlb_table_flush(tlb);
	182	+#endif
	183	+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
	184	+ tlb_batch_pages_flush(tlb);
	185	+#endif
	186	+}
	187	+
	188	+void tlb_flush_mmu(struct mmu_gather *tlb)
	189	+{
	190	+ tlb_flush_mmu_tlbonly(tlb);
	191	+ tlb_flush_mmu_free(tlb);
	192	+}
	193	+
228	194	/**
229	195	* tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
230	196	* @tlb: the mmu_gather structure to initialize
@@ -240,10 +206,40 @@ void tlb_remove_table(struct mmu_gather tlb, void table) · 9 unmodified lines
240	206	void tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
241	207	unsigned long start, unsigned long end)
242	208	{
243		− arch_tlb_gather_mmu(tlb, mm, start, end);
	209	+ tlb->mm = mm;
	210	+
	211	+ /* Is it from 0 to ~0? */
	212	+ tlb->fullmm = !(start \| (end+1));
	213	+
	214	+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
	215	+ tlb->need_flush_all = 0;
	216	+ tlb->local.next = NULL;
	217	+ tlb->local.nr = 0;
	218	+ tlb->local.max = ARRAY_SIZE(tlb->__pages);
	219	+ tlb->active = &tlb->local;
	220	+ tlb->batch_count = 0;
	221	+#endif
	222	+
	223	+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
	224	+ tlb->batch = NULL;
	225	+#endif
	226	+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
	227	+ tlb->page_size = 0;
	228	+#endif
	229	+
	230	+ __tlb_reset_range(tlb);
244	231	inc_tlb_flush_pending(tlb->mm);
245	232	}
246	233
	234	+/**
	235	+ * tlb_finish_mmu - finish an mmu_gather structure
	236	+ * @tlb: the mmu_gather structure to finish
	237	+ * @start: start of the region that will be removed from the page-table
	238	+ * @end: end of the region that will be removed from the page-table
	239	+ *
	240	+ * Called at the end of the shootdown operation to free up any resources that
	241	+ * were required.
	242	+ */
247	243	void tlb_finish_mmu(struct mmu_gather *tlb,
248	244	unsigned long start, unsigned long end)
249	245	{
@@ -254,8 +250,17 @@ void tlb_finish_mmu(struct mmu_gather *tlb, · 4 unmodified lines
254	250	* the TLB by observing pte_none\|!pte_dirty, for example so flush TLB
255	251	* forcefully if we detect parallel PTE batching threads.
256	252	*/
257		− bool force = mm_tlb_flush_nested(tlb->mm);
	253	+ if (mm_tlb_flush_nested(tlb->mm)) {
	254	+ __tlb_reset_range(tlb);
	255	+ __tlb_adjust_range(tlb, start, end - start);
	256	+ }
258	257
259		− arch_tlb_finish_mmu(tlb, start, end, force);
	258	+ tlb_flush_mmu(tlb);
	259	+
	260	+ /* keep the page table cache within bounds */
	261	+ check_pgt_cache();
	262	+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
	263	+ tlb_batch_list_free(tlb);
	264	+#endif
260	265	dec_tlb_flush_pending(tlb->mm);
261	266	}