lael-labs/linux

Documentation/core-api/cachetlb.rst+0−10

@@ -101,16 +101,6 @@ changes occur: · 100 unmodified lines
101	translations for software managed TLB configurations.	101	translations for software managed TLB configurations.
102	The sparc64 port currently does this.	102	The sparc64 port currently does this.
103		103
104	6) ``void tlb_migrate_finish(struct mm_struct *mm)``
105
106	This interface is called at the end of an explicit
107	process migration. This interface provides a hook
108	to allow a platform to update TLB or context-specific
109	information for the address space.
110
111	The ia64 sn2 platform is one example of a platform
112	that uses this interface.
113
114	Next, we have the cache flushing interfaces. In general, when Linux	104	Next, we have the cache flushing interfaces. In general, when Linux
115	is changing an existing virtual-->physical mapping to a new value,	105	is changing an existing virtual-->physical mapping to a new value,
116	the sequence will be in one of the following forms::	106	the sequence will be in one of the following forms::

arch/Kconfig+7−1

 config HAVE_RCU_TABLE_FREE
 	bool
-config HAVE_RCU_TABLE_INVALIDATE
+config HAVE_RCU_TABLE_NO_INVALIDATE
+	bool
+config HAVE_MMU_GATHER_PAGE_SIZE
+	bool
+config HAVE_MMU_GATHER_NO_GATHER
 	bool
 config ARCH_HAVE_NMI_SAFE_CMPXCHG

arch/alpha/Kconfig+1−0

@@ -36,6 +36,7 @@ config ALPHA · 35 unmodified lines
36	select ODD_RT_SIGACTION	36	select ODD_RT_SIGACTION
37	select OLD_SIGSUSPEND	37	select OLD_SIGSUSPEND
38	select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67	38	select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67
		39	select MMU_GATHER_NO_RANGE
39	help	40	help
40	The Alpha is a 64-bit general-purpose processor designed and	41	The Alpha is a 64-bit general-purpose processor designed and
41	marketed by the Digital Equipment Corporation of blessed memory,	42	marketed by the Digital Equipment Corporation of blessed memory,

arch/alpha/include/asm/tlb.h+0−6

 #ifndef _ALPHA_TLB_H
 #define _ALPHA_TLB_H
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, pte, addr)	do { } while (0)
-#define tlb_flush(tlb)				flush_tlb_mm((tlb)->mm)
 #include <asm-generic/tlb.h>
 #define __pte_free_tlb(tlb, pte, address)		pte_free((tlb)->mm, pte)

arch/arc/include/asm/tlb.h+0−32

@@ -9,38 +9,6 @@ · 8 unmodified lines
9	#ifndef _ASM_ARC_TLB_H	9	#ifndef _ASM_ARC_TLB_H
10	#define _ASM_ARC_TLB_H	10	#define _ASM_ARC_TLB_H
11		11
12	#define tlb_flush(tlb) \
13	do { \
14	if (tlb->fullmm) \
15	flush_tlb_mm((tlb)->mm); \
16	} while (0)
17
18	/*
19	* This pair is called at time of munmap/exit to flush cache and TLB entries
20	* for mappings being torn down.
21	* 1) cache-flush part -implemented via tlb_start_vma( ) for VIPT aliasing D$
22	* 2) tlb-flush part - implemted via tlb_end_vma( ) flushes the TLB range
23	*
24	* Note, read http://lkml.org/lkml/2004/1/15/6
25	*/
26	#ifndef CONFIG_ARC_CACHE_VIPT_ALIASING
27	#define tlb_start_vma(tlb, vma)
28	#else
29	#define tlb_start_vma(tlb, vma) \
30	do { \
31	if (!tlb->fullmm) \
32	flush_cache_range(vma, vma->vm_start, vma->vm_end); \
33	} while(0)
34	#endif
35
36	#define tlb_end_vma(tlb, vma) \
37	do { \
38	if (!tlb->fullmm) \
39	flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
40	} while (0)
41
42	#define __tlb_remove_tlb_entry(tlb, ptep, address)
43
44	#include <linux/pagemap.h>	12	#include <linux/pagemap.h>
45	#include <asm-generic/tlb.h>	13	#include <asm-generic/tlb.h>
46		14

arch/arm/include/asm/tlb.h+13−242

@@ -33,271 +33,42 @@ · 32 unmodified lines
33	#include <asm/pgalloc.h>	33	#include <asm/pgalloc.h>
34	#include <asm/tlbflush.h>	34	#include <asm/tlbflush.h>
35		35
36	#define MMU_GATHER_BUNDLE 8
37
38	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
39	static inline void __tlb_remove_table(void *_table)	36	static inline void __tlb_remove_table(void *_table)
40	{	37	{
41	free_page_and_swap_cache((struct page *)_table);	38	free_page_and_swap_cache((struct page *)_table);
42	}	39	}
43		40
44	struct mmu_table_batch {	41	#include <asm-generic/tlb.h>
45	struct rcu_head rcu;
46	unsigned int nr;
47	void *tables[0];
48	};
49
50	#define MAX_TABLE_BATCH \
51	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
52
53	extern void tlb_table_flush(struct mmu_gather *tlb);
54	extern void tlb_remove_table(struct mmu_gather tlb, void table);
55
56	#define tlb_remove_entry(tlb, entry) tlb_remove_table(tlb, entry)
57	#else
58	#define tlb_remove_entry(tlb, entry) tlb_remove_page(tlb, entry)
59	#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
60
61	/*
62	* TLB handling. This allows us to remove pages from the page
63	* tables, and efficiently handle the TLB issues.
64	*/
65	struct mmu_gather {
66	struct mm_struct *mm;
67	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
68	struct mmu_table_batch *batch;
69	unsigned int need_flush;
70	#endif
71	unsigned int fullmm;
72	struct vm_area_struct *vma;
73	unsigned long start, end;
74	unsigned long range_start;
75	unsigned long range_end;
76	unsigned int nr;
77	unsigned int max;
78	struct page **pages;
79	struct page *local[MMU_GATHER_BUNDLE];
80	};
81
82	DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
83
84	/*
85	* This is unnecessarily complex. There's three ways the TLB shootdown
86	* code is used:
87	* 1. Unmapping a range of vmas. See zap_page_range(), unmap_region().
88	* tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.
89	* tlb->vma will be non-NULL.
90	* 2. Unmapping all vmas. See exit_mmap().
91	* tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.
92	* tlb->vma will be non-NULL. Additionally, page tables will be freed.
93	* 3. Unmapping argument pages. See shift_arg_pages().
94	* tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.
95	* tlb->vma will be NULL.
96	*/
97	static inline void tlb_flush(struct mmu_gather *tlb)
98	{
99	if (tlb->fullmm \|\| !tlb->vma)
100	flush_tlb_mm(tlb->mm);
101	else if (tlb->range_end > 0) {
102	flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
103	tlb->range_start = TASK_SIZE;
104	tlb->range_end = 0;
105	}
106	}
107
108	static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
109	{
110	if (!tlb->fullmm) {
111	if (addr < tlb->range_start)
112	tlb->range_start = addr;
113	if (addr + PAGE_SIZE > tlb->range_end)
114	tlb->range_end = addr + PAGE_SIZE;
115	}
116	}
117
118	static inline void __tlb_alloc_page(struct mmu_gather *tlb)
119	{
120	unsigned long addr = __get_free_pages(GFP_NOWAIT \| __GFP_NOWARN, 0);
121
122	if (addr) {
123	tlb->pages = (void *)addr;
124	tlb->max = PAGE_SIZE / sizeof(struct page *);
125	}
126	}
127
128	static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
129	{
130	tlb_flush(tlb);
131	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
132	tlb_table_flush(tlb);
133	#endif
134	}
135
136	static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
137	{
138	free_pages_and_swap_cache(tlb->pages, tlb->nr);
139	tlb->nr = 0;
140	if (tlb->pages == tlb->local)
141	__tlb_alloc_page(tlb);
142	}
143
144	static inline void tlb_flush_mmu(struct mmu_gather *tlb)
145	{
146	tlb_flush_mmu_tlbonly(tlb);
147	tlb_flush_mmu_free(tlb);
148	}
149
150	static inline void
151	arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
152	unsigned long start, unsigned long end)
153	{
154	tlb->mm = mm;
155	tlb->fullmm = !(start \| (end+1));
156	tlb->start = start;
157	tlb->end = end;
158	tlb->vma = NULL;
159	tlb->max = ARRAY_SIZE(tlb->local);
160	tlb->pages = tlb->local;
161	tlb->nr = 0;
162	__tlb_alloc_page(tlb);
163		42
164	#ifdef CONFIG_HAVE_RCU_TABLE_FREE	43	#ifndef CONFIG_HAVE_RCU_TABLE_FREE
165	tlb->batch = NULL;	44	#define tlb_remove_table(tlb, entry) tlb_remove_page(tlb, entry)
166	#endif	45	#endif
167	}
168
169	static inline void
170	arch_tlb_finish_mmu(struct mmu_gather *tlb,
171	unsigned long start, unsigned long end, bool force)
172	{
173	if (force) {
174	tlb->range_start = start;
175	tlb->range_end = end;
176	}
177
178	tlb_flush_mmu(tlb);
179
180	/* keep the page table cache within bounds */
181	check_pgt_cache();
182
183	if (tlb->pages != tlb->local)
184	free_pages((unsigned long)tlb->pages, 0);
185	}
186
187	/*
188	* Memorize the range for the TLB flush.
189	*/
190	static inline void
191	tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep, unsigned long addr)
192	{
193	tlb_add_flush(tlb, addr);
194	}
195
196	#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
197	tlb_remove_tlb_entry(tlb, ptep, address)
198	/*
199	* In the case of tlb vma handling, we can optimise these away in the
200	* case where we're doing a full MM flush. When we're doing a munmap,
201	* the vmas are adjusted to only cover the region to be torn down.
202	*/
203	static inline void
204	tlb_start_vma(struct mmu_gather tlb, struct vm_area_struct vma)
205	{
206	if (!tlb->fullmm) {
207	flush_cache_range(vma, vma->vm_start, vma->vm_end);
208	tlb->vma = vma;
209	tlb->range_start = TASK_SIZE;
210	tlb->range_end = 0;
211	}
212	}
213		46
214	static inline void	47	static inline void
215	tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma)	48	__pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr)
216	{
217	if (!tlb->fullmm)
218	tlb_flush(tlb);
219	}
220
221	static inline bool __tlb_remove_page(struct mmu_gather tlb, struct page page)
222	{
223	tlb->pages[tlb->nr++] = page;
224	VM_WARN_ON(tlb->nr > tlb->max);
225	if (tlb->nr == tlb->max)
226	return true;
227	return false;
228	}
229
230	static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
231	{
232	if (__tlb_remove_page(tlb, page))
233	tlb_flush_mmu(tlb);
234	}
235
236	static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
237	struct page *page, int page_size)
238	{
239	return __tlb_remove_page(tlb, page);
240	}
241
242	static inline void tlb_remove_page_size(struct mmu_gather *tlb,
243	struct page *page, int page_size)
244	{
245	return tlb_remove_page(tlb, page);
246	}
247
248	static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
249	unsigned long addr)
250	{	49	{
251	pgtable_page_dtor(pte);	50	pgtable_page_dtor(pte);
252		51
253	#ifdef CONFIG_ARM_LPAE	52	#ifndef CONFIG_ARM_LPAE
254	tlb_add_flush(tlb, addr);
255	#else
256	/*	53	/*
257	* With the classic ARM MMU, a pte page has two corresponding pmd	54	* With the classic ARM MMU, a pte page has two corresponding pmd
258	* entries, each covering 1MB.	55	* entries, each covering 1MB.
259	*/	56	*/
260	addr &= PMD_MASK;	57	addr = (addr & PMD_MASK) + SZ_1M;
261	tlb_add_flush(tlb, addr + SZ_1M - PAGE_SIZE);	58	__tlb_adjust_range(tlb, addr - PAGE_SIZE, 2 * PAGE_SIZE);
262	tlb_add_flush(tlb, addr + SZ_1M);
263	#endif	59	#endif
264		60
265	tlb_remove_entry(tlb, pte);	61	tlb_remove_table(tlb, pte);
266	}
267
268	static inline void __pmd_free_tlb(struct mmu_gather tlb, pmd_t pmdp,
269	unsigned long addr)
270	{
271	#ifdef CONFIG_ARM_LPAE
272	tlb_add_flush(tlb, addr);
273	tlb_remove_entry(tlb, virt_to_page(pmdp));
274	#endif
275	}	62	}
276		63
277	static inline void	64	static inline void
278	tlb_remove_pmd_tlb_entry(struct mmu_gather tlb, pmd_t pmdp, unsigned long addr)	65	__pmd_free_tlb(struct mmu_gather tlb, pmd_t pmdp, unsigned long addr)
279	{
280	tlb_add_flush(tlb, addr);
281	}
282
283	#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
284	#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
285	#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
286
287	#define tlb_migrate_finish(mm) do { } while (0)
288
289	#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
290	static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
291	unsigned int page_size)
292	{	66	{
293	}	67	#ifdef CONFIG_ARM_LPAE
294		68	struct page *page = virt_to_page(pmdp);
295	static inline void tlb_flush_remove_tables(struct mm_struct *mm)
296	{
297	}
298		69
299	static inline void tlb_flush_remove_tables_local(void *arg)	70	tlb_remove_table(tlb, page);
300	{	71	#endif
301	}	72	}
302		73
303	#endif /* CONFIG_MMU */	74	#endif /* CONFIG_MMU */

arch/arm64/Kconfig+0−1

@@ -149,7 +149,6 @@ config ARM64 · 148 unmodified lines
149	select HAVE_PERF_USER_STACK_DUMP	149	select HAVE_PERF_USER_STACK_DUMP
150	select HAVE_REGS_AND_STACK_ACCESS_API	150	select HAVE_REGS_AND_STACK_ACCESS_API
151	select HAVE_RCU_TABLE_FREE	151	select HAVE_RCU_TABLE_FREE
152	select HAVE_RCU_TABLE_INVALIDATE
153	select HAVE_RSEQ	152	select HAVE_RSEQ
154	select HAVE_STACKPROTECTOR	153	select HAVE_STACKPROTECTOR
155	select HAVE_SYSCALL_TRACEPOINTS	154	select HAVE_SYSCALL_TRACEPOINTS

arch/arm64/include/asm/tlb.h+1−0

@@ -27,6 +27,7 @@ static inline void __tlb_remove_table(void *_table) · 26 unmodified lines
27	free_page_and_swap_cache((struct page *)_table);	27	free_page_and_swap_cache((struct page *)_table);
28	}	28	}
29		29
		30	#define tlb_flush tlb_flush
30	static void tlb_flush(struct mmu_gather *tlb);	31	static void tlb_flush(struct mmu_gather *tlb);
31		32
32	#include <asm-generic/tlb.h>	33	#include <asm-generic/tlb.h>

arch/c6x/Kconfig+1−0

@@ -20,6 +20,7 @@ config C6X · 19 unmodified lines
20	select GENERIC_CLOCKEVENTS	20	select GENERIC_CLOCKEVENTS
21	select MODULES_USE_ELF_RELA	21	select MODULES_USE_ELF_RELA
22	select ARCH_NO_COHERENT_DMA_MMAP	22	select ARCH_NO_COHERENT_DMA_MMAP
		23	select MMU_GATHER_NO_RANGE if MMU
23		24
24	config MMU	25	config MMU
25	def_bool n	26	def_bool n

arch/c6x/include/asm/tlb.h+0−2

@@ -2,8 +2,6 @@ · 1 unmodified line
2	#ifndef _ASM_C6X_TLB_H	2	#ifndef _ASM_C6X_TLB_H
3	#define _ASM_C6X_TLB_H	3	#define _ASM_C6X_TLB_H
4		4
5	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
6
7	#include <asm-generic/tlb.h>	5	#include <asm-generic/tlb.h>
8		6
9	#endif /* _ASM_C6X_TLB_H */	7	#endif /* _ASM_C6X_TLB_H */

arch/h8300/include/asm/tlb.h+0−2

@@ -2,8 +2,6 @@ · 1 unmodified line
2	#ifndef __H8300_TLB_H__	2	#ifndef __H8300_TLB_H__
3	#define __H8300_TLB_H__	3	#define __H8300_TLB_H__
4		4
5	#define tlb_flush(tlb) do { } while (0)
6
7	#include <asm-generic/tlb.h>	5	#include <asm-generic/tlb.h>
8		6
9	#endif	7	#endif

arch/hexagon/include/asm/tlb.h+0−12

@@ -22,18 +22,6 @@ · 21 unmodified lines
22	#include <linux/pagemap.h>	22	#include <linux/pagemap.h>
23	#include <asm/tlbflush.h>	23	#include <asm/tlbflush.h>
24		24
25	/*
26	* We don't need any special per-pte or per-vma handling...
27	*/
28	#define tlb_start_vma(tlb, vma) do { } while (0)
29	#define tlb_end_vma(tlb, vma) do { } while (0)
30	#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
31
32	/*
33	* .. because we flush the whole mm when it fills up
34	*/
35	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
36
37	#include <asm-generic/tlb.h>	25	#include <asm-generic/tlb.h>
38		26
39	#endif	27	#endif

arch/ia64/include/asm/machvec.h+0−13

 typedef void ia64_mv_send_ipi_t (int, int, int, int);
 typedef void ia64_mv_timer_interrupt_t (int, void *);
 typedef void ia64_mv_global_tlb_purge_t (struct mm_struct *, unsigned long, unsigned long, unsigned long);
-typedef void ia64_mv_tlb_migrate_finish_t (struct mm_struct *);
 typedef u8 ia64_mv_irq_to_vector (int);
 typedef unsigned int ia64_mv_local_vector_to_irq (u8);
 typedef char *ia64_mv_pci_get_legacy_mem_t (struct pci_bus *);
 {
 }
-static inline void
-machvec_noop_mm (struct mm_struct *mm)
-{
-}
 static inline void
 machvec_noop_task (struct task_struct *task)
 {
 extern void machvec_setup (char **);
 extern void machvec_timer_interrupt (int, void *);
-extern void machvec_tlb_migrate_finish (struct mm_struct *);
 # if defined (CONFIG_IA64_HP_SIM)
 #  include <asm/machvec_hpsim.h>
 #  define platform_send_ipi	ia64_mv.send_ipi
 #  define platform_timer_interrupt	ia64_mv.timer_interrupt
 #  define platform_global_tlb_purge	ia64_mv.global_tlb_purge
-#  define platform_tlb_migrate_finish	ia64_mv.tlb_migrate_finish
 #  define platform_dma_init		ia64_mv.dma_init
 #  define platform_dma_get_ops		ia64_mv.dma_get_ops
 #  define platform_irq_to_vector	ia64_mv.irq_to_vector
 	ia64_mv_send_ipi_t *send_ipi;
 	ia64_mv_timer_interrupt_t *timer_interrupt;
 	ia64_mv_global_tlb_purge_t *global_tlb_purge;
-	ia64_mv_tlb_migrate_finish_t *tlb_migrate_finish;
 	ia64_mv_dma_init *dma_init;
 	ia64_mv_dma_get_ops *dma_get_ops;
 	ia64_mv_irq_to_vector *irq_to_vector;
 	platform_send_ipi,			\
 	platform_timer_interrupt,		\
 	platform_global_tlb_purge,		\
-	platform_tlb_migrate_finish,		\
 	platform_dma_init,			\
 	platform_dma_get_ops,			\
 	platform_irq_to_vector,			\
 #ifndef platform_global_tlb_purge
 # define platform_global_tlb_purge	ia64_global_tlb_purge /* default to architected version */
 #endif
-#ifndef platform_tlb_migrate_finish
-# define platform_tlb_migrate_finish	machvec_noop_mm
-#endif
 #ifndef platform_kernel_launch_event
 # define platform_kernel_launch_event	machvec_noop
 #endif

arch/ia64/include/asm/machvec_sn2.h+0−2

 extern ia64_mv_send_ipi_t sn2_send_IPI;
 extern ia64_mv_timer_interrupt_t sn_timer_interrupt;
 extern ia64_mv_global_tlb_purge_t sn2_global_tlb_purge;
-extern ia64_mv_tlb_migrate_finish_t	sn_tlb_migrate_finish;
 extern ia64_mv_irq_to_vector sn_irq_to_vector;
 extern ia64_mv_local_vector_to_irq sn_local_vector_to_irq;
 extern ia64_mv_pci_get_legacy_mem_t sn_pci_get_legacy_mem;
 #define platform_send_ipi		sn2_send_IPI
 #define platform_timer_interrupt	sn_timer_interrupt
 #define platform_global_tlb_purge       sn2_global_tlb_purge
-#define platform_tlb_migrate_finish	sn_tlb_migrate_finish
 #define platform_pci_fixup		sn_pci_fixup
 #define platform_inb			__sn_inb
 #define platform_inw			__sn_inw

arch/ia64/include/asm/tlb.h+1−258

@@ -47,263 +47,6 @@ · 46 unmodified lines
47	#include <asm/tlbflush.h>	47	#include <asm/tlbflush.h>
48	#include <asm/machvec.h>	48	#include <asm/machvec.h>
49		49
50	/*	50	#include <asm-generic/tlb.h>
51	* If we can't allocate a page to make a big batch of page pointers
52	* to work on, then just handle a few from the on-stack structure.
53	*/
54	#define IA64_GATHER_BUNDLE 8
55
56	struct mmu_gather {
57	struct mm_struct *mm;
58	unsigned int nr;
59	unsigned int max;
60	unsigned char fullmm; /* non-zero means full mm flush */
61	unsigned char need_flush; /* really unmapped some PTEs? */
62	unsigned long start, end;
63	unsigned long start_addr;
64	unsigned long end_addr;
65	struct page **pages;
66	struct page *local[IA64_GATHER_BUNDLE];
67	};
68
69	struct ia64_tr_entry {
70	u64 ifa;
71	u64 itir;
72	u64 pte;
73	u64 rr;
74	}; /Record for tr entry!/
75
76	extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
77	extern void ia64_ptr_entry(u64 target_mask, int slot);
78
79	extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
80
81	/*
82	region register macros
83	*/
84	#define RR_TO_VE(val) (((val) >> 0) & 0x0000000000000001)
85	#define RR_VE(val) (((val) & 0x0000000000000001) << 0)
86	#define RR_VE_MASK 0x0000000000000001L
87	#define RR_VE_SHIFT 0
88	#define RR_TO_PS(val) (((val) >> 2) & 0x000000000000003f)
89	#define RR_PS(val) (((val) & 0x000000000000003f) << 2)
90	#define RR_PS_MASK 0x00000000000000fcL
91	#define RR_PS_SHIFT 2
92	#define RR_RID_MASK 0x00000000ffffff00L
93	#define RR_TO_RID(val) ((val >> 8) & 0xffffff)
94
95	static inline void
96	ia64_tlb_flush_mmu_tlbonly(struct mmu_gather *tlb, unsigned long start, unsigned long end)
97	{
98	tlb->need_flush = 0;
99
100	if (tlb->fullmm) {
101	/*
102	* Tearing down the entire address space. This happens both as a result
103	* of exit() and execve(). The latter case necessitates the call to
104	* flush_tlb_mm() here.
105	*/
106	flush_tlb_mm(tlb->mm);
107	} else if (unlikely (end - start >= 102410241024*1024UL
108	\|\| REGION_NUMBER(start) != REGION_NUMBER(end - 1)))
109	{
110	/*
111	* If we flush more than a tera-byte or across regions, we're probably
112	* better off just flushing the entire TLB(s). This should be very rare
113	* and is not worth optimizing for.
114	*/
115	flush_tlb_all();
116	} else {
117	/*
118	* flush_tlb_range() takes a vma instead of a mm pointer because
119	* some architectures want the vm_flags for ITLB/DTLB flush.
120	*/
121	struct vm_area_struct vma = TLB_FLUSH_VMA(tlb->mm, 0);
122
123	/* flush the address range from the tlb: */
124	flush_tlb_range(&vma, start, end);
125	/* now flush the virt. page-table area mapping the address range: */
126	flush_tlb_range(&vma, ia64_thash(start), ia64_thash(end));
127	}
128
129	}
130
131	static inline void
132	ia64_tlb_flush_mmu_free(struct mmu_gather *tlb)
133	{
134	unsigned long i;
135	unsigned int nr;
136
137	/* lastly, release the freed pages */
138	nr = tlb->nr;
139
140	tlb->nr = 0;
141	tlb->start_addr = ~0UL;
142	for (i = 0; i < nr; ++i)
143	free_page_and_swap_cache(tlb->pages[i]);
144	}
145
146	/*
147	* Flush the TLB for address range START to END and, if not in fast mode, release the
148	* freed pages that where gathered up to this point.
149	*/
150	static inline void
151	ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
152	{
153	if (!tlb->need_flush)
154	return;
155	ia64_tlb_flush_mmu_tlbonly(tlb, start, end);
156	ia64_tlb_flush_mmu_free(tlb);
157	}
158
159	static inline void __tlb_alloc_page(struct mmu_gather *tlb)
160	{
161	unsigned long addr = __get_free_pages(GFP_NOWAIT \| __GFP_NOWARN, 0);
162
163	if (addr) {
164	tlb->pages = (void *)addr;
165	tlb->max = PAGE_SIZE / sizeof(void *);
166	}
167	}
168
169
170	static inline void
171	arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
172	unsigned long start, unsigned long end)
173	{
174	tlb->mm = mm;
175	tlb->max = ARRAY_SIZE(tlb->local);
176	tlb->pages = tlb->local;
177	tlb->nr = 0;
178	tlb->fullmm = !(start \| (end+1));
179	tlb->start = start;
180	tlb->end = end;
181	tlb->start_addr = ~0UL;
182	}
183
184	/*
185	* Called at the end of the shootdown operation to free up any resources that were
186	* collected.
187	*/
188	static inline void
189	arch_tlb_finish_mmu(struct mmu_gather *tlb,
190	unsigned long start, unsigned long end, bool force)
191	{
192	if (force)
193	tlb->need_flush = 1;
194	/*
195	* Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and
196	* tlb->end_addr.
197	*/
198	ia64_tlb_flush_mmu(tlb, start, end);
199
200	/* keep the page table cache within bounds */
201	check_pgt_cache();
202
203	if (tlb->pages != tlb->local)
204	free_pages((unsigned long)tlb->pages, 0);
205	}
206
207	/*
208	* Logically, this routine frees PAGE. On MP machines, the actual freeing of the page
209	* must be delayed until after the TLB has been flushed (see comments at the beginning of
210	* this file).
211	*/
212	static inline bool __tlb_remove_page(struct mmu_gather tlb, struct page page)
213	{
214	tlb->need_flush = 1;
215
216	if (!tlb->nr && tlb->pages == tlb->local)
217	__tlb_alloc_page(tlb);
218
219	tlb->pages[tlb->nr++] = page;
220	VM_WARN_ON(tlb->nr > tlb->max);
221	if (tlb->nr == tlb->max)
222	return true;
223	return false;
224	}
225
226	static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
227	{
228	ia64_tlb_flush_mmu_tlbonly(tlb, tlb->start_addr, tlb->end_addr);
229	}
230
231	static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
232	{
233	ia64_tlb_flush_mmu_free(tlb);
234	}
235
236	static inline void tlb_flush_mmu(struct mmu_gather *tlb)
237	{
238	ia64_tlb_flush_mmu(tlb, tlb->start_addr, tlb->end_addr);
239	}
240
241	static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
242	{
243	if (__tlb_remove_page(tlb, page))
244	tlb_flush_mmu(tlb);
245	}
246
247	static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
248	struct page *page, int page_size)
249	{
250	return __tlb_remove_page(tlb, page);
251	}
252
253	static inline void tlb_remove_page_size(struct mmu_gather *tlb,
254	struct page *page, int page_size)
255	{
256	return tlb_remove_page(tlb, page);
257	}
258
259	/*
260	* Remove TLB entry for PTE mapped at virtual address ADDRESS. This is called for any
261	* PTE, not just those pointing to (normal) physical memory.
262	*/
263	static inline void
264	__tlb_remove_tlb_entry (struct mmu_gather tlb, pte_t ptep, unsigned long address)
265	{
266	if (tlb->start_addr == ~0UL)
267	tlb->start_addr = address;
268	tlb->end_addr = address + PAGE_SIZE;
269	}
270
271	#define tlb_migrate_finish(mm) platform_tlb_migrate_finish(mm)
272
273	#define tlb_start_vma(tlb, vma) do { } while (0)
274	#define tlb_end_vma(tlb, vma) do { } while (0)
275
276	#define tlb_remove_tlb_entry(tlb, ptep, addr) \
277	do { \
278	tlb->need_flush = 1; \
279	__tlb_remove_tlb_entry(tlb, ptep, addr); \
280	} while (0)
281
282	#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
283	tlb_remove_tlb_entry(tlb, ptep, address)
284
285	#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
286	static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
287	unsigned int page_size)
288	{
289	}
290
291	#define pte_free_tlb(tlb, ptep, address) \
292	do { \
293	tlb->need_flush = 1; \
294	__pte_free_tlb(tlb, ptep, address); \
295	} while (0)
296
297	#define pmd_free_tlb(tlb, ptep, address) \
298	do { \
299	tlb->need_flush = 1; \
300	__pmd_free_tlb(tlb, ptep, address); \
301	} while (0)
302
303	#define pud_free_tlb(tlb, pudp, address) \
304	do { \
305	tlb->need_flush = 1; \
306	__pud_free_tlb(tlb, pudp, address); \
307	} while (0)
308		51
309	#endif /* _ASM_IA64_TLB_H */	52	#endif /* _ASM_IA64_TLB_H */

arch/ia64/include/asm/tlbflush.h+25−0

@@ -14,6 +14,31 @@ · 13 unmodified lines
14	#include <asm/mmu_context.h>	14	#include <asm/mmu_context.h>
15	#include <asm/page.h>	15	#include <asm/page.h>
16		16
		17	struct ia64_tr_entry {
		18	u64 ifa;
		19	u64 itir;
		20	u64 pte;
		21	u64 rr;
		22	}; /Record for tr entry!/
		23
		24	extern int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size);
		25	extern void ia64_ptr_entry(u64 target_mask, int slot);
		26	extern struct ia64_tr_entry *ia64_idtrs[NR_CPUS];
		27
		28	/*
		29	region register macros
		30	*/
		31	#define RR_TO_VE(val) (((val) >> 0) & 0x0000000000000001)
		32	#define RR_VE(val) (((val) & 0x0000000000000001) << 0)
		33	#define RR_VE_MASK 0x0000000000000001L
		34	#define RR_VE_SHIFT 0
		35	#define RR_TO_PS(val) (((val) >> 2) & 0x000000000000003f)
		36	#define RR_PS(val) (((val) & 0x000000000000003f) << 2)
		37	#define RR_PS_MASK 0x00000000000000fcL
		38	#define RR_PS_SHIFT 2
		39	#define RR_RID_MASK 0x00000000ffffff00L
		40	#define RR_TO_RID(val) ((val >> 8) & 0xffffff)
		41
17	/*	42	/*
18	* Now for some TLB flushing routines. This is the kind of stuff that	43	* Now for some TLB flushing routines. This is the kind of stuff that
19	* can be very expensive, so try to avoid them whenever possible.	44	* can be very expensive, so try to avoid them whenever possible.

arch/ia64/mm/tlb.c+21−2

@@ -305,8 +305,8 @@ local_flush_tlb_all (void) · 304 unmodified lines
305	ia64_srlz_i(); /* srlz.i implies srlz.d */	305	ia64_srlz_i(); /* srlz.i implies srlz.d */
306	}	306	}
307		307
308	void	308	static void
309	flush_tlb_range (struct vm_area_struct *vma, unsigned long start,	309	__flush_tlb_range (struct vm_area_struct *vma, unsigned long start,
310	unsigned long end)	310	unsigned long end)
311	{	311	{
312	struct mm_struct *mm = vma->vm_mm;	312	struct mm_struct *mm = vma->vm_mm;
@@ -343,6 +343,25 @@ flush_tlb_range (struct vm_area_struct *vma, unsigned long start, · 30 unmodified lines
343	preempt_enable();	343	preempt_enable();
344	ia64_srlz_i(); /* srlz.i implies srlz.d */	344	ia64_srlz_i(); /* srlz.i implies srlz.d */
345	}	345	}
		346
		347	void flush_tlb_range(struct vm_area_struct *vma,
		348	unsigned long start, unsigned long end)
		349	{
		350	if (unlikely(end - start >= 102410241024*1024UL
		351	\|\| REGION_NUMBER(start) != REGION_NUMBER(end - 1))) {
		352	/*
		353	* If we flush more than a tera-byte or across regions, we're
		354	* probably better off just flushing the entire TLB(s). This
		355	* should be very rare and is not worth optimizing for.
		356	*/
		357	flush_tlb_all();
		358	} else {
		359	/* flush the address range from the tlb */
		360	__flush_tlb_range(vma, start, end);
		361	/* flush the virt. page-table area mapping the addr range */
		362	__flush_tlb_range(vma, ia64_thash(start), ia64_thash(end));
		363	}
		364	}
346	EXPORT_SYMBOL(flush_tlb_range);	365	EXPORT_SYMBOL(flush_tlb_range);
347		366
348	void ia64_tlb_init(void)	367	void ia64_tlb_init(void)

arch/ia64/sn/kernel/sn2/sn2_smp.c+0−7

@@ -120,13 +120,6 @@ void sn_migrate(struct task_struct *task) · 119 unmodified lines
120	cpu_relax();	120	cpu_relax();
121	}	121	}
122		122
123	void sn_tlb_migrate_finish(struct mm_struct *mm)
124	{
125	/* flush_tlb_mm is inefficient if more than 1 users of mm */
126	if (mm == current->mm && mm && atomic_read(&mm->mm_users) == 1)
127	flush_tlb_mm(mm);
128	}
129
130	static void	123	static void
131	sn2_ipi_flush_all_tlb(struct mm_struct *mm)	124	sn2_ipi_flush_all_tlb(struct mm_struct *mm)
132	{	125	{

arch/m68k/Kconfig+1−0

@@ -28,6 +28,7 @@ config M68K · 27 unmodified lines
28	select OLD_SIGSUSPEND3	28	select OLD_SIGSUSPEND3
29	select OLD_SIGACTION	29	select OLD_SIGACTION
30	select ARCH_DISCARD_MEMBLOCK	30	select ARCH_DISCARD_MEMBLOCK
		31	select MMU_GATHER_NO_RANGE if MMU
31		32
32	config CPU_BIG_ENDIAN	33	config CPU_BIG_ENDIAN
33	def_bool y	34	def_bool y

arch/m68k/include/asm/tlb.h+0−14

@@ -2,20 +2,6 @@ · 1 unmodified line
2	#ifndef _M68K_TLB_H	2	#ifndef _M68K_TLB_H
3	#define _M68K_TLB_H	3	#define _M68K_TLB_H
4		4
5	/*
6	* m68k doesn't need any special per-pte or
7	* per-vma handling..
8	*/
9	#define tlb_start_vma(tlb, vma) do { } while (0)
10	#define tlb_end_vma(tlb, vma) do { } while (0)
11	#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
12
13	/*
14	* .. because we flush the whole mm when it
15	* fills up.
16	*/
17	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
18
19	#include <asm-generic/tlb.h>	5	#include <asm-generic/tlb.h>
20		6
21	#endif /* _M68K_TLB_H */	7	#endif /* _M68K_TLB_H */

arch/microblaze/Kconfig+1−0

@@ -41,6 +41,7 @@ config MICROBLAZE · 40 unmodified lines
41	select TRACING_SUPPORT	41	select TRACING_SUPPORT
42	select VIRT_TO_BUS	42	select VIRT_TO_BUS
43	select CPU_NO_EFFICIENT_FFS	43	select CPU_NO_EFFICIENT_FFS
		44	select MMU_GATHER_NO_RANGE if MMU
44		45
45	# Endianness selection	46	# Endianness selection
46	choice	47	choice

arch/microblaze/include/asm/tlb.h+0−9

@@ -11,16 +11,7 @@ · 10 unmodified lines
11	#ifndef _ASM_MICROBLAZE_TLB_H	11	#ifndef _ASM_MICROBLAZE_TLB_H
12	#define _ASM_MICROBLAZE_TLB_H	12	#define _ASM_MICROBLAZE_TLB_H
13		13
14	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
15
16	#include <linux/pagemap.h>	14	#include <linux/pagemap.h>
17
18	#ifdef CONFIG_MMU
19	#define tlb_start_vma(tlb, vma) do { } while (0)
20	#define tlb_end_vma(tlb, vma) do { } while (0)
21	#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
22	#endif
23
24	#include <asm-generic/tlb.h>	15	#include <asm-generic/tlb.h>
25		16
26	#endif /* _ASM_MICROBLAZE_TLB_H */	17	#endif /* _ASM_MICROBLAZE_TLB_H */

arch/mips/include/asm/tlb.h+0−17

@@ -5,23 +5,6 @@ · 4 unmodified lines
5	#include <asm/cpu-features.h>	5	#include <asm/cpu-features.h>
6	#include <asm/mipsregs.h>	6	#include <asm/mipsregs.h>
7		7
8	/*
9	* MIPS doesn't need any special per-pte or per-vma handling, except
10	* we need to flush cache for area to be unmapped.
11	*/
12	#define tlb_start_vma(tlb, vma) \
13	do { \
14	if (!tlb->fullmm) \
15	flush_cache_range(vma, vma->vm_start, vma->vm_end); \
16	} while (0)
17	#define tlb_end_vma(tlb, vma) do { } while (0)
18	#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
19
20	/*
21	* .. because we flush the whole mm when it fills up.
22	*/
23	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
24
25	#define _UNIQUE_ENTRYHI(base, idx) \	8	#define _UNIQUE_ENTRYHI(base, idx) \
26	(((base) + ((idx) << (PAGE_SHIFT + 1))) \| \	9	(((base) + ((idx) << (PAGE_SHIFT + 1))) \| \
27	(cpu_has_tlbinv ? MIPS_ENTRYHI_EHINV : 0))	10	(cpu_has_tlbinv ? MIPS_ENTRYHI_EHINV : 0))

arch/nds32/include/asm/tlb.h+0−16

@@ -4,22 +4,6 @@ · 3 unmodified lines
4	#ifndef __ASMNDS32_TLB_H	4	#ifndef __ASMNDS32_TLB_H
5	#define __ASMNDS32_TLB_H	5	#define __ASMNDS32_TLB_H
6		6
7	#define tlb_start_vma(tlb,vma) \
8	do { \
9	if (!tlb->fullmm) \
10	flush_cache_range(vma, vma->vm_start, vma->vm_end); \
11	} while (0)
12
13	#define tlb_end_vma(tlb,vma) \
14	do { \
15	if(!tlb->fullmm) \
16	flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
17	} while (0)
18
19	#define __tlb_remove_tlb_entry(tlb, pte, addr) do { } while (0)
20
21	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
22
23	#include <asm-generic/tlb.h>	7	#include <asm-generic/tlb.h>
24		8
25	#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)	9	#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)

arch/nds32/include/asm/tlbflush.h+0−1

@@ -42,6 +42,5 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); · 41 unmodified lines
42		42
43	void update_mmu_cache(struct vm_area_struct *vma,	43	void update_mmu_cache(struct vm_area_struct *vma,
44	unsigned long address, pte_t * pte);	44	unsigned long address, pte_t * pte);
45	void tlb_migrate_finish(struct mm_struct *mm);
46		45
47	#endif	46	#endif

arch/nios2/Kconfig+1−0

@@ -24,6 +24,7 @@ config NIOS2 · 23 unmodified lines
24	select USB_ARCH_HAS_HCD if USB_SUPPORT	24	select USB_ARCH_HAS_HCD if USB_SUPPORT
25	select CPU_NO_EFFICIENT_FFS	25	select CPU_NO_EFFICIENT_FFS
26	select ARCH_DISCARD_MEMBLOCK	26	select ARCH_DISCARD_MEMBLOCK
		27	select MMU_GATHER_NO_RANGE if MMU
27		28
28	config GENERIC_CSUM	29	config GENERIC_CSUM
29	def_bool y	30	def_bool y

arch/nios2/include/asm/tlb.h+2−12

 #ifndef _ASM_NIOS2_TLB_H
 #define _ASM_NIOS2_TLB_H
-#define tlb_flush(tlb)	flush_tlb_mm((tlb)->mm)
 extern void set_mmu_pid(unsigned long pid);
 /*
- * NiosII doesn't need any special per-pte or per-vma handling, except
+ * NIOS32 does have flush_tlb_range(), but it lacks a limit and fallback to
- * we need to flush cache for the area to be unmapped.
+ * full mm invalidation. So use flush_tlb_mm() for everything.
  */
-#define tlb_start_vma(tlb, vma)					\
-	do {							\
-		if (!tlb->fullmm)				\
-			flush_cache_range(vma, vma->vm_start, vma->vm_end); \
-	}  while (0)
-#define tlb_end_vma(tlb, vma)	do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>

arch/openrisc/Kconfig+1−0

@@ -36,6 +36,7 @@ config OPENRISC · 35 unmodified lines
36	select OMPIC if SMP	36	select OMPIC if SMP
37	select ARCH_WANT_FRAME_POINTERS	37	select ARCH_WANT_FRAME_POINTERS
38	select GENERIC_IRQ_MULTI_HANDLER	38	select GENERIC_IRQ_MULTI_HANDLER
		39	select MMU_GATHER_NO_RANGE if MMU
39		40
40	config CPU_BIG_ENDIAN	41	config CPU_BIG_ENDIAN
41	def_bool y	42	def_bool y

arch/openrisc/include/asm/tlb.h+2−6

 #define __ASM_OPENRISC_TLB_H__
 /*
- * or32 doesn't need any special per-pte or
+ * OpenRISC doesn't have an efficient flush_tlb_range() so use flush_tlb_mm()
- * per-vma handling..
+ * for everything.
  */
-#define tlb_start_vma(tlb, vma) do { } while (0)
-#define tlb_end_vma(tlb, vma) do { } while (0)
-#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>

arch/parisc/include/asm/tlb.h+0−18

@@ -2,24 +2,6 @@ · 1 unmodified line
2	#ifndef _PARISC_TLB_H	2	#ifndef _PARISC_TLB_H
3	#define _PARISC_TLB_H	3	#define _PARISC_TLB_H
4		4
5	#define tlb_flush(tlb) \
6	do { if ((tlb)->fullmm) \
7	flush_tlb_mm((tlb)->mm);\
8	} while (0)
9
10	#define tlb_start_vma(tlb, vma) \
11	do { if (!(tlb)->fullmm) \
12	flush_cache_range(vma, vma->vm_start, vma->vm_end); \
13	} while (0)
14
15	#define tlb_end_vma(tlb, vma) \
16	do { if (!(tlb)->fullmm) \
17	flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
18	} while (0)
19
20	#define __tlb_remove_tlb_entry(tlb, pte, address) \
21	do { } while (0)
22
23	#include <asm-generic/tlb.h>	5	#include <asm-generic/tlb.h>
24		6
25	#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)	7	#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)

arch/powerpc/Kconfig+2−0

 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_RCU_TABLE_FREE		if SMP
+	select HAVE_RCU_TABLE_NO_INVALIDATE	if HAVE_RCU_TABLE_FREE
+	select HAVE_MMU_GATHER_PAGE_SIZE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE		if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN
 	select HAVE_SYSCALL_TRACEPOINTS

arch/powerpc/include/asm/tlb.h+1−17

@@ -27,8 +27,8 @@ · 26 unmodified lines
27	#define tlb_start_vma(tlb, vma) do { } while (0)	27	#define tlb_start_vma(tlb, vma) do { } while (0)
28	#define tlb_end_vma(tlb, vma) do { } while (0)	28	#define tlb_end_vma(tlb, vma) do { } while (0)
29	#define __tlb_remove_tlb_entry __tlb_remove_tlb_entry	29	#define __tlb_remove_tlb_entry __tlb_remove_tlb_entry
30	#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
31		30
		31	#define tlb_flush tlb_flush
32	extern void tlb_flush(struct mmu_gather *tlb);	32	extern void tlb_flush(struct mmu_gather *tlb);
33		33
34	/* Get the generic bits... */	34	/* Get the generic bits... */
@@ -46,22 +46,6 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep, · 11 unmodified lines
46	#endif	46	#endif
47	}	47	}
48		48
49	static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
50	unsigned int page_size)
51	{
52	if (!tlb->page_size)
53	tlb->page_size = page_size;
54	else if (tlb->page_size != page_size) {
55	if (!tlb->fullmm)
56	tlb_flush_mmu(tlb);
57	/*
58	* update the page size after flush for the new
59	* mmu_gather.
60	*/
61	tlb->page_size = page_size;
62	}
63	}
64
65	#ifdef CONFIG_SMP	49	#ifdef CONFIG_SMP
66	static inline int mm_is_core_local(struct mm_struct *mm)	50	static inline int mm_is_core_local(struct mm_struct *mm)
67	{	51	{

arch/riscv/include/asm/tlb.h+1−0

@@ -18,6 +18,7 @@ struct mmu_gather; · 17 unmodified lines
18		18
19	static void tlb_flush(struct mmu_gather *tlb);	19	static void tlb_flush(struct mmu_gather *tlb);
20		20
		21	#define tlb_flush tlb_flush
21	#include <asm-generic/tlb.h>	22	#include <asm-generic/tlb.h>
22		23
23	static inline void tlb_flush(struct mmu_gather *tlb)	24	static inline void tlb_flush(struct mmu_gather *tlb)

arch/s390/Kconfig+2−0

 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_MEMBLOCK_PHYS_MAP
+	select HAVE_MMU_GATHER_NO_GATHER
 	select HAVE_MOD_ARCH_SPECIFIC
 	select HAVE_NOP_MCOUNT
 	select HAVE_OPROFILE
 	select HAVE_PCI
 	select HAVE_PERF_EVENTS
+	select HAVE_RCU_TABLE_FREE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RSEQ
 	select HAVE_SYSCALL_TRACEPOINTS

arch/s390/include/asm/tlb.h+39−91

  * Pages used for the page tables is a different story. FIXME: more
  */
-#include <linux/mm.h>
+void __tlb_remove_table(void *_table);
-#include <linux/pagemap.h>
+static inline void tlb_flush(struct mmu_gather *tlb);
-#include <linux/swap.h>
+static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
-#include <asm/processor.h>
+					  struct page *page, int page_size);
-#include <asm/pgalloc.h>
-#include <asm/tlbflush.h>
-struct mmu_gather {
-	struct mm_struct *mm;
-	struct mmu_table_batch *batch;
-	unsigned int fullmm;
-	unsigned long start, end;
-};
-struct mmu_table_batch {
-	struct rcu_head		rcu;
-	unsigned int		nr;
-	void			*tables[0];
-};
-#define MAX_TABLE_BATCH		\
-	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
-extern void tlb_table_flush(struct mmu_gather *tlb);
-extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
-static inline void
-arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
-			unsigned long start, unsigned long end)
-{
-	tlb->mm = mm;
-	tlb->start = start;
-	tlb->end = end;
-	tlb->fullmm = !(start | (end+1));
-	tlb->batch = NULL;
-}
-static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
-{
-	__tlb_flush_mm_lazy(tlb->mm);
-}
-static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
-{
-	tlb_table_flush(tlb);
-}
-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
+#define tlb_start_vma(tlb, vma)			do { } while (0)
-{
+#define tlb_end_vma(tlb, vma)			do { } while (0)
-	tlb_flush_mmu_tlbonly(tlb);
-	tlb_flush_mmu_free(tlb);
-}
-static inline void
+#define tlb_flush tlb_flush
-arch_tlb_finish_mmu(struct mmu_gather *tlb,
+#define pte_free_tlb pte_free_tlb
-		unsigned long start, unsigned long end, bool force)
+#define pmd_free_tlb pmd_free_tlb
-{
+#define p4d_free_tlb p4d_free_tlb
-	if (force) {
+#define pud_free_tlb pud_free_tlb
-		tlb->start = start;
-		tlb->end = end;
-	}
-	tlb_flush_mmu(tlb);
+#include <asm/pgalloc.h>
-}
+#include <asm/tlbflush.h>
+#include <asm-generic/tlb.h>
 /*
  * Release the page cache reference for a pte removed by
  * tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page
  * has already been freed, so just do free_page_and_swap_cache.
  */
-static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	free_page_and_swap_cache(page);
-	return false; /* avoid calling tlb_flush_mmu */
-}
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
-{
-	free_page_and_swap_cache(page);
-}
 static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
 					  struct page *page, int page_size)
 {
-	return __tlb_remove_page(tlb, page);
+	free_page_and_swap_cache(page);
+	return false;
 }
-static inline void tlb_remove_page_size(struct mmu_gather *tlb,
+static inline void tlb_flush(struct mmu_gather *tlb)
-					struct page *page, int page_size)
 {
-	return tlb_remove_page(tlb, page);
+	__tlb_flush_mm_lazy(tlb->mm);
 }
 /*
  * page table from the tlb.
  */
 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
-				unsigned long address)
+                                unsigned long address)
 {
+	__tlb_adjust_range(tlb, address, PAGE_SIZE);
+	tlb->mm->context.flush_mm = 1;
+	tlb->freed_tables = 1;
+	tlb->cleared_ptes = 1;
+	/*
+	 * page_table_free_rcu takes care of the allocation bit masks
+	 * of the 2K table fragments in the 4K page table page,
+	 * then calls tlb_remove_table.
+	 */
 	page_table_free_rcu(tlb, (unsigned long *) pte, address);
 }
 	if (mm_pmd_folded(tlb->mm))
 		return;
 	pgtable_pmd_page_dtor(virt_to_page(pmd));
+	__tlb_adjust_range(tlb, address, PAGE_SIZE);
+	tlb->mm->context.flush_mm = 1;
+	tlb->freed_tables = 1;
+	tlb->cleared_puds = 1;
 	tlb_remove_table(tlb, pmd);
 }
 {
 	if (mm_p4d_folded(tlb->mm))
 		return;
+	__tlb_adjust_range(tlb, address, PAGE_SIZE);
+	tlb->mm->context.flush_mm = 1;
+	tlb->freed_tables = 1;
+	tlb->cleared_p4ds = 1;
 	tlb_remove_table(tlb, p4d);
 }
 {
 	if (mm_pud_folded(tlb->mm))
 		return;
+	tlb->mm->context.flush_mm = 1;
+	tlb->freed_tables = 1;
+	tlb->cleared_puds = 1;
 	tlb_remove_table(tlb, pud);
 }
-#define tlb_start_vma(tlb, vma)			do { } while (0)
-#define tlb_end_vma(tlb, vma)			do { } while (0)
-#define tlb_remove_tlb_entry(tlb, ptep, addr)	do { } while (0)
-#define tlb_remove_pmd_tlb_entry(tlb, pmdp, addr)	do { } while (0)
-#define tlb_migrate_finish(mm)			do { } while (0)
-#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\
-	tlb_remove_tlb_entry(tlb, ptep, address)
-#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
-static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
-						     unsigned int page_size)
-{
-}
 #endif /* _S390_TLB_H */

arch/s390/mm/pgalloc.c+1−62

@@ -290,7 +290,7 @@ void page_table_free_rcu(struct mmu_gather tlb, unsigned long table, · 289 unmodified lines
290	tlb_remove_table(tlb, table);	290	tlb_remove_table(tlb, table);
291	}	291	}
292		292
293	static void __tlb_remove_table(void *_table)	293	void __tlb_remove_table(void *_table)
294	{	294	{
295	unsigned int mask = (unsigned long) _table & 3;	295	unsigned int mask = (unsigned long) _table & 3;
296	void table = (void )((unsigned long) _table ^ mask);	296	void table = (void )((unsigned long) _table ^ mask);
@@ -316,67 +316,6 @@ static void __tlb_remove_table(void *_table) · 19 unmodified lines
316	}	316	}
317	}	317	}
318		318
319	static void tlb_remove_table_smp_sync(void *arg)
320	{
321	/* Simply deliver the interrupt */
322	}
323
324	static void tlb_remove_table_one(void *table)
325	{
326	/*
327	* This isn't an RCU grace period and hence the page-tables cannot be
328	* assumed to be actually RCU-freed.
329	*
330	* It is however sufficient for software page-table walkers that rely
331	* on IRQ disabling. See the comment near struct mmu_table_batch.
332	*/
333	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
334	__tlb_remove_table(table);
335	}
336
337	static void tlb_remove_table_rcu(struct rcu_head *head)
338	{
339	struct mmu_table_batch *batch;
340	int i;
341
342	batch = container_of(head, struct mmu_table_batch, rcu);
343
344	for (i = 0; i < batch->nr; i++)
345	__tlb_remove_table(batch->tables[i]);
346
347	free_page((unsigned long)batch);
348	}
349
350	void tlb_table_flush(struct mmu_gather *tlb)
351	{
352	struct mmu_table_batch **batch = &tlb->batch;
353
354	if (*batch) {
355	call_rcu(&(*batch)->rcu, tlb_remove_table_rcu);
356	*batch = NULL;
357	}
358	}
359
360	void tlb_remove_table(struct mmu_gather tlb, void table)
361	{
362	struct mmu_table_batch **batch = &tlb->batch;
363
364	tlb->mm->context.flush_mm = 1;
365	if (*batch == NULL) {
366	batch = (struct mmu_table_batch )
367	__get_free_page(GFP_NOWAIT \| __GFP_NOWARN);
368	if (*batch == NULL) {
369	__tlb_flush_mm_lazy(tlb->mm);
370	tlb_remove_table_one(table);
371	return;
372	}
373	(*batch)->nr = 0;
374	}
375	(batch)->tables[(batch)->nr++] = table;
376	if ((*batch)->nr == MAX_TABLE_BATCH)
377	tlb_flush_mmu(tlb);
378	}
379
380	/*	319	/*
381	* Base infrastructure required to generate basic asces, region, segment,	320	* Base infrastructure required to generate basic asces, region, segment,
382	* and page tables that do not make use of enhanced features like EDAT1.	321	* and page tables that do not make use of enhanced features like EDAT1.

arch/sh/include/asm/pgalloc.h+9−0

@@ -70,6 +70,15 @@ do { \ · 69 unmodified lines
70	tlb_remove_page((tlb), (pte)); \	70	tlb_remove_page((tlb), (pte)); \
71	} while (0)	71	} while (0)
72		72
		73	#if CONFIG_PGTABLE_LEVELS > 2
		74	#define __pmd_free_tlb(tlb, pmdp, addr) \
		75	do { \
		76	struct page *page = virt_to_page(pmdp); \
		77	pgtable_pmd_page_dtor(page); \
		78	tlb_remove_page((tlb), page); \
		79	} while (0);
		80	#endif
		81
73	static inline void check_pgt_cache(void)	82	static inline void check_pgt_cache(void)
74	{	83	{
75	quicklist_trim(QUICK_PT, NULL, 25, 16);	84	quicklist_trim(QUICK_PT, NULL, 25, 16);

arch/sh/include/asm/tlb.h+1−131

@@ -11,133 +11,8 @@ · 10 unmodified lines
11		11
12	#ifdef CONFIG_MMU	12	#ifdef CONFIG_MMU
13	#include <linux/swap.h>	13	#include <linux/swap.h>
14	#include <asm/pgalloc.h>
15	#include <asm/tlbflush.h>
16	#include <asm/mmu_context.h>
17		14
18	/*	15	#include <asm-generic/tlb.h>
19	* TLB handling. This allows us to remove pages from the page
20	* tables, and efficiently handle the TLB issues.
21	*/
22	struct mmu_gather {
23	struct mm_struct *mm;
24	unsigned int fullmm;
25	unsigned long start, end;
26	};
27
28	static inline void init_tlb_gather(struct mmu_gather *tlb)
29	{
30	tlb->start = TASK_SIZE;
31	tlb->end = 0;
32
33	if (tlb->fullmm) {
34	tlb->start = 0;
35	tlb->end = TASK_SIZE;
36	}
37	}
38
39	static inline void
40	arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
41	unsigned long start, unsigned long end)
42	{
43	tlb->mm = mm;
44	tlb->start = start;
45	tlb->end = end;
46	tlb->fullmm = !(start \| (end+1));
47
48	init_tlb_gather(tlb);
49	}
50
51	static inline void
52	arch_tlb_finish_mmu(struct mmu_gather *tlb,
53	unsigned long start, unsigned long end, bool force)
54	{
55	if (tlb->fullmm \|\| force)
56	flush_tlb_mm(tlb->mm);
57
58	/* keep the page table cache within bounds */
59	check_pgt_cache();
60	}
61
62	static inline void
63	tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep, unsigned long address)
64	{
65	if (tlb->start > address)
66	tlb->start = address;
67	if (tlb->end < address + PAGE_SIZE)
68	tlb->end = address + PAGE_SIZE;
69	}
70
71	#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
72	tlb_remove_tlb_entry(tlb, ptep, address)
73
74	/*
75	* In the case of tlb vma handling, we can optimise these away in the
76	* case where we're doing a full MM flush. When we're doing a munmap,
77	* the vmas are adjusted to only cover the region to be torn down.
78	*/
79	static inline void
80	tlb_start_vma(struct mmu_gather tlb, struct vm_area_struct vma)
81	{
82	if (!tlb->fullmm)
83	flush_cache_range(vma, vma->vm_start, vma->vm_end);
84	}
85
86	static inline void
87	tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma)
88	{
89	if (!tlb->fullmm && tlb->end) {
90	flush_tlb_range(vma, tlb->start, tlb->end);
91	init_tlb_gather(tlb);
92	}
93	}
94
95	static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
96	{
97	}
98
99	static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
100	{
101	}
102
103	static inline void tlb_flush_mmu(struct mmu_gather *tlb)
104	{
105	}
106
107	static inline int __tlb_remove_page(struct mmu_gather tlb, struct page page)
108	{
109	free_page_and_swap_cache(page);
110	return false; /* avoid calling tlb_flush_mmu */
111	}
112
113	static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
114	{
115	__tlb_remove_page(tlb, page);
116	}
117
118	static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
119	struct page *page, int page_size)
120	{
121	return __tlb_remove_page(tlb, page);
122	}
123
124	static inline void tlb_remove_page_size(struct mmu_gather *tlb,
125	struct page *page, int page_size)
126	{
127	return tlb_remove_page(tlb, page);
128	}
129
130	#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
131	static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
132	unsigned int page_size)
133	{
134	}
135
136	#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep)
137	#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
138	#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
139
140	#define tlb_migrate_finish(mm) do { } while (0)
141		16
142	#if defined(CONFIG_CPU_SH4) \|\| defined(CONFIG_SUPERH64)	17	#if defined(CONFIG_CPU_SH4) \|\| defined(CONFIG_SUPERH64)
143	extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);	18	extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);
@@ -157,11 +32,6 @@ static inline void tlb_unwire_entry(void) · 13 unmodified lines
157		32
158	#else /* CONFIG_MMU */	33	#else /* CONFIG_MMU */
159		34
160	#define tlb_start_vma(tlb, vma) do { } while (0)
161	#define tlb_end_vma(tlb, vma) do { } while (0)
162	#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
163	#define tlb_flush(tlb) do { } while (0)
164
165	#include <asm-generic/tlb.h>	35	#include <asm-generic/tlb.h>
166		36
167	#endif /* CONFIG_MMU */	37	#endif /* CONFIG_MMU */

arch/sparc/Kconfig+1−0

@@ -63,6 +63,7 @@ config SPARC64 · 62 unmodified lines
63	select HAVE_KRETPROBES	63	select HAVE_KRETPROBES
64	select HAVE_KPROBES	64	select HAVE_KPROBES
65	select HAVE_RCU_TABLE_FREE if SMP	65	select HAVE_RCU_TABLE_FREE if SMP
		66	select HAVE_RCU_TABLE_NO_INVALIDATE if HAVE_RCU_TABLE_FREE
66	select HAVE_MEMBLOCK_NODE_MAP	67	select HAVE_MEMBLOCK_NODE_MAP
67	select HAVE_ARCH_TRANSPARENT_HUGEPAGE	68	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
68	select HAVE_DYNAMIC_FTRACE	69	select HAVE_DYNAMIC_FTRACE

arch/sparc/include/asm/tlb_32.h+0−18

@@ -2,24 +2,6 @@ · 1 unmodified line
2	#ifndef _SPARC_TLB_H	2	#ifndef _SPARC_TLB_H
3	#define _SPARC_TLB_H	3	#define _SPARC_TLB_H
4		4
5	#define tlb_start_vma(tlb, vma) \
6	do { \
7	flush_cache_range(vma, vma->vm_start, vma->vm_end); \
8	} while (0)
9
10	#define tlb_end_vma(tlb, vma) \
11	do { \
12	flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
13	} while (0)
14
15	#define __tlb_remove_tlb_entry(tlb, pte, address) \
16	do { } while (0)
17
18	#define tlb_flush(tlb) \
19	do { \
20	flush_tlb_mm((tlb)->mm); \
21	} while (0)
22
23	#include <asm-generic/tlb.h>	5	#include <asm-generic/tlb.h>
24		6
25	#endif /* _SPARC_TLB_H */	7	#endif /* _SPARC_TLB_H */

arch/um/include/asm/tlb.h+2−156

@@ -2,162 +2,8 @@ · 1 unmodified line
2	#ifndef __UM_TLB_H	2	#ifndef __UM_TLB_H
3	#define __UM_TLB_H	3	#define __UM_TLB_H
4		4
5	#include <linux/pagemap.h>
6	#include <linux/swap.h>
7	#include <asm/percpu.h>
8	#include <asm/pgalloc.h>
9	#include <asm/tlbflush.h>	5	#include <asm/tlbflush.h>
10		6	#include <asm-generic/cacheflush.h>
11	#define tlb_start_vma(tlb, vma) do { } while (0)	7	#include <asm-generic/tlb.h>
12	#define tlb_end_vma(tlb, vma) do { } while (0)
13	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
14
15	/* struct mmu_gather is an opaque type used by the mm code for passing around
16	* any data needed by arch specific code for tlb_remove_page.
17	*/
18	struct mmu_gather {
19	struct mm_struct *mm;
20	unsigned int need_flush; /* Really unmapped some ptes? */
21	unsigned long start;
22	unsigned long end;
23	unsigned int fullmm; /* non-zero means full mm flush */
24	};
25
26	static inline void __tlb_remove_tlb_entry(struct mmu_gather tlb, pte_t ptep,
27	unsigned long address)
28	{
29	if (tlb->start > address)
30	tlb->start = address;
31	if (tlb->end < address + PAGE_SIZE)
32	tlb->end = address + PAGE_SIZE;
33	}
34
35	static inline void init_tlb_gather(struct mmu_gather *tlb)
36	{
37	tlb->need_flush = 0;
38
39	tlb->start = TASK_SIZE;
40	tlb->end = 0;
41
42	if (tlb->fullmm) {
43	tlb->start = 0;
44	tlb->end = TASK_SIZE;
45	}
46	}
47
48	static inline void
49	arch_tlb_gather_mmu(struct mmu_gather tlb, struct mm_struct mm,
50	unsigned long start, unsigned long end)
51	{
52	tlb->mm = mm;
53	tlb->start = start;
54	tlb->end = end;
55	tlb->fullmm = !(start \| (end+1));
56
57	init_tlb_gather(tlb);
58	}
59
60	extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
61	unsigned long end);
62
63	static inline void
64	tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
65	{
66	flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end);
67	}
68
69	static inline void
70	tlb_flush_mmu_free(struct mmu_gather *tlb)
71	{
72	init_tlb_gather(tlb);
73	}
74
75	static inline void
76	tlb_flush_mmu(struct mmu_gather *tlb)
77	{
78	if (!tlb->need_flush)
79	return;
80
81	tlb_flush_mmu_tlbonly(tlb);
82	tlb_flush_mmu_free(tlb);
83	}
84
85	/* arch_tlb_finish_mmu
86	* Called at the end of the shootdown operation to free up any resources
87	* that were required.
88	*/
89	static inline void
90	arch_tlb_finish_mmu(struct mmu_gather *tlb,
91	unsigned long start, unsigned long end, bool force)
92	{
93	if (force) {
94	tlb->start = start;
95	tlb->end = end;
96	tlb->need_flush = 1;
97	}
98	tlb_flush_mmu(tlb);
99
100	/* keep the page table cache within bounds */
101	check_pgt_cache();
102	}
103
104	/* tlb_remove_page
105	* Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)),
106	* while handling the additional races in SMP caused by other CPUs
107	* caching valid mappings in their TLBs.
108	*/
109	static inline int __tlb_remove_page(struct mmu_gather tlb, struct page page)
110	{
111	tlb->need_flush = 1;
112	free_page_and_swap_cache(page);
113	return false; /* avoid calling tlb_flush_mmu */
114	}
115
116	static inline void tlb_remove_page(struct mmu_gather tlb, struct page page)
117	{
118	__tlb_remove_page(tlb, page);
119	}
120
121	static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
122	struct page *page, int page_size)
123	{
124	return __tlb_remove_page(tlb, page);
125	}
126
127	static inline void tlb_remove_page_size(struct mmu_gather *tlb,
128	struct page *page, int page_size)
129	{
130	return tlb_remove_page(tlb, page);
131	}
132
133	/**
134	* tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
135	*
136	* Record the fact that pte's were really umapped in ->need_flush, so we can
137	* later optimise away the tlb invalidate. This helps when userspace is
138	* unmapping already-unmapped pages, which happens quite a lot.
139	*/
140	#define tlb_remove_tlb_entry(tlb, ptep, address) \
141	do { \
142	tlb->need_flush = 1; \
143	__tlb_remove_tlb_entry(tlb, ptep, address); \
144	} while (0)
145
146	#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
147	tlb_remove_tlb_entry(tlb, ptep, address)
148
149	#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
150	static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
151	unsigned int page_size)
152	{
153	}
154
155	#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
156
157	#define pud_free_tlb(tlb, pudp, addr) __pud_free_tlb(tlb, pudp, addr)
158
159	#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
160
161	#define tlb_migrate_finish(mm) do {} while (0)
162		8
163	#endif	9	#endif

arch/unicore32/Kconfig+1−0

@@ -20,6 +20,7 @@ config UNICORE32 · 19 unmodified lines
20	select GENERIC_IOMAP	20	select GENERIC_IOMAP
21	select MODULES_USE_ELF_REL	21	select MODULES_USE_ELF_REL
22	select NEED_DMA_MAP_STATE	22	select NEED_DMA_MAP_STATE
		23	select MMU_GATHER_NO_RANGE if MMU
23	help	24	help
24	UniCore-32 is 32-bit Instruction Set Architecture,	25	UniCore-32 is 32-bit Instruction Set Architecture,
25	including a series of low-power-consumption RISC chip	26	including a series of low-power-consumption RISC chip

arch/unicore32/include/asm/tlb.h+3−4

 #ifndef __UNICORE_TLB_H__
 #define __UNICORE_TLB_H__
-#define tlb_start_vma(tlb, vma)				do { } while (0)
+/*
-#define tlb_end_vma(tlb, vma)				do { } while (0)
+ * unicore32 lacks an efficient flush_tlb_range(), use flush_tlb_mm().
-#define __tlb_remove_tlb_entry(tlb, ptep, address)	do { } while (0)
+ */
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
 #define __pte_free_tlb(tlb, pte, addr)				\
 	do {							\

arch/x86/Kconfig+0−1

@@ -183,7 +183,6 @@ config X86 · 182 unmodified lines
183	select HAVE_PERF_REGS	183	select HAVE_PERF_REGS
184	select HAVE_PERF_USER_STACK_DUMP	184	select HAVE_PERF_USER_STACK_DUMP
185	select HAVE_RCU_TABLE_FREE if PARAVIRT	185	select HAVE_RCU_TABLE_FREE if PARAVIRT
186	select HAVE_RCU_TABLE_INVALIDATE if HAVE_RCU_TABLE_FREE
187	select HAVE_REGS_AND_STACK_ACCESS_API	186	select HAVE_REGS_AND_STACK_ACCESS_API
188	select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER \|\| UNWINDER_ORC) && STACK_VALIDATION	187	select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER \|\| UNWINDER_ORC) && STACK_VALIDATION
189	select HAVE_FUNCTION_ARG_ACCESS_API	188	select HAVE_FUNCTION_ARG_ACCESS_API

arch/x86/include/asm/tlb.h+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	#define tlb_end_vma(tlb, vma) do { } while (0)	6	#define tlb_end_vma(tlb, vma) do { } while (0)
7	#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)	7	#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
8		8
		9	#define tlb_flush tlb_flush
9	static inline void tlb_flush(struct mmu_gather *tlb);	10	static inline void tlb_flush(struct mmu_gather *tlb);
10		11
11	#include <asm-generic/tlb.h>	12	#include <asm-generic/tlb.h>

arch/xtensa/include/asm/tlb.h+0−26

@@ -14,32 +14,6 @@ · 13 unmodified lines
14	#include <asm/cache.h>	14	#include <asm/cache.h>
15	#include <asm/page.h>	15	#include <asm/page.h>
16		16
17	#if (DCACHE_WAY_SIZE <= PAGE_SIZE)
18
19	/* Note, read http://lkml.org/lkml/2004/1/15/6 */
20
21	# define tlb_start_vma(tlb,vma) do { } while (0)
22	# define tlb_end_vma(tlb,vma) do { } while (0)
23
24	#else
25
26	# define tlb_start_vma(tlb, vma) \
27	do { \
28	if (!tlb->fullmm) \
29	flush_cache_range(vma, vma->vm_start, vma->vm_end); \
30	} while(0)
31
32	# define tlb_end_vma(tlb, vma) \
33	do { \
34	if (!tlb->fullmm) \
35	flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
36	} while(0)
37
38	#endif
39
40	#define __tlb_remove_tlb_entry(tlb,pte,addr) do { } while (0)
41	#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
42
43	#include <asm-generic/tlb.h>	17	#include <asm-generic/tlb.h>
44		18
45	#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte)	19	#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte)

include/asm-generic/tlb.h+259−29

@@ -19,9 +19,131 @@ · 18 unmodified lines
19	#include <linux/swap.h>	19	#include <linux/swap.h>
20	#include <asm/pgalloc.h>	20	#include <asm/pgalloc.h>
21	#include <asm/tlbflush.h>	21	#include <asm/tlbflush.h>
		22	#include <asm/cacheflush.h>
22		23
23	#ifdef CONFIG_MMU	24	#ifdef CONFIG_MMU
24		25
		26	/*
		27	* Generic MMU-gather implementation.
		28	*
		29	* The mmu_gather data structure is used by the mm code to implement the
		30	* correct and efficient ordering of freeing pages and TLB invalidations.
		31	*
		32	* This correct ordering is:
		33	*
		34	* 1) unhook page
		35	* 2) TLB invalidate page
		36	* 3) free page
		37	*
		38	* That is, we must never free a page before we have ensured there are no live
		39	* translations left to it. Otherwise it might be possible to observe (or
		40	* worse, change) the page content after it has been reused.
		41	*
		42	* The mmu_gather API consists of:
		43	*
		44	* - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather
		45	*
		46	* Finish in particular will issue a (final) TLB invalidate and free
		47	* all (remaining) queued pages.
		48	*
		49	* - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA
		50	*
		51	* Defaults to flushing at tlb_end_vma() to reset the range; helps when
		52	* there's large holes between the VMAs.
		53	*
		54	* - tlb_remove_page() / __tlb_remove_page()
		55	* - tlb_remove_page_size() / __tlb_remove_page_size()
		56	*
		57	* __tlb_remove_page_size() is the basic primitive that queues a page for
		58	* freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a
		59	* boolean indicating if the queue is (now) full and a call to
		60	* tlb_flush_mmu() is required.
		61	*
		62	* tlb_remove_page() and tlb_remove_page_size() imply the call to
		63	* tlb_flush_mmu() when required and has no return value.
		64	*
		65	* - tlb_change_page_size()
		66	*
		67	* call before __tlb_remove_page*() to set the current page-size; implies a
		68	* possible tlb_flush_mmu() call.
		69	*
		70	* - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()
		71	*
		72	* tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
		73	* related state, like the range)
		74	*
		75	* tlb_flush_mmu() - in addition to the above TLB invalidate, also frees
		76	* whatever pages are still batched.
		77	*
		78	* - mmu_gather::fullmm
		79	*
		80	* A flag set by tlb_gather_mmu() to indicate we're going to free
		81	* the entire mm; this allows a number of optimizations.
		82	*
		83	* - We can ignore tlb_{start,end}_vma(); because we don't
		84	* care about ranges. Everything will be shot down.
		85	*
		86	* - (RISC) architectures that use ASIDs can cycle to a new ASID
		87	* and delay the invalidation until ASID space runs out.
		88	*
		89	* - mmu_gather::need_flush_all
		90	*
		91	* A flag that can be set by the arch code if it wants to force
		92	* flush the entire TLB irrespective of the range. For instance
		93	* x86-PAE needs this when changing top-level entries.
		94	*
		95	* And allows the architecture to provide and implement tlb_flush():
		96	*
		97	* tlb_flush() may, in addition to the above mentioned mmu_gather fields, make
		98	* use of:
		99	*
		100	* - mmu_gather::start / mmu_gather::end
		101	*
		102	* which provides the range that needs to be flushed to cover the pages to
		103	* be freed.
		104	*
		105	* - mmu_gather::freed_tables
		106	*
		107	* set when we freed page table pages
		108	*
		109	* - tlb_get_unmap_shift() / tlb_get_unmap_size()
		110	*
		111	* returns the smallest TLB entry size unmapped in this range.
		112	*
		113	* If an architecture does not provide tlb_flush() a default implementation
		114	* based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is
		115	* specified, in which case we'll default to flush_tlb_mm().
		116	*
		117	* Additionally there are a few opt-in features:
		118	*
		119	* HAVE_MMU_GATHER_PAGE_SIZE
		120	*
		121	* This ensures we call tlb_flush() every time tlb_change_page_size() actually
		122	* changes the size and provides mmu_gather::page_size to tlb_flush().
		123	*
		124	* HAVE_RCU_TABLE_FREE
		125	*
		126	* This provides tlb_remove_table(), to be used instead of tlb_remove_page()
		127	* for page directores (__p*_free_tlb()). This provides separate freeing of
		128	* the page-table pages themselves in a semi-RCU fashion (see comment below).
		129	* Useful if your architecture doesn't use IPIs for remote TLB invalidates
		130	* and therefore doesn't naturally serialize with software page-table walkers.
		131	*
		132	* When used, an architecture is expected to provide __tlb_remove_table()
		133	* which does the actual freeing of these pages.
		134	*
		135	* HAVE_RCU_TABLE_NO_INVALIDATE
		136	*
		137	* This makes HAVE_RCU_TABLE_FREE avoid calling tlb_flush_mmu_tlbonly() before
		138	* freeing the page-table pages. This can be avoided if you use
		139	* HAVE_RCU_TABLE_FREE and your architecture does _NOT_ use the Linux
		140	* page-tables natively.
		141	*
		142	* MMU_GATHER_NO_RANGE
		143	*
		144	* Use this if your architecture lacks an efficient flush_tlb_range().
		145	*/
		146
25	#ifdef CONFIG_HAVE_RCU_TABLE_FREE	147	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
26	/*	148	/*
27	* Semi RCU freeing of the page directories.	149	* Semi RCU freeing of the page directories.
@@ -60,11 +182,11 @@ struct mmu_table_batch { · 32 unmodified lines
60	#define MAX_TABLE_BATCH \	182	#define MAX_TABLE_BATCH \
61	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))	183	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
62		184
63	extern void tlb_table_flush(struct mmu_gather *tlb);
64	extern void tlb_remove_table(struct mmu_gather tlb, void table);	185	extern void tlb_remove_table(struct mmu_gather tlb, void table);
65		186
66	#endif	187	#endif
67		188
		189	#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
68	/*	190	/*
69	* If we can't allocate a page to make a big batch of page pointers	191	* If we can't allocate a page to make a big batch of page pointers
70	* to work on, then just handle a few from the on-stack structure.	192	* to work on, then just handle a few from the on-stack structure.
@@ -89,14 +211,21 @@ struct mmu_gather_batch { · 18 unmodified lines
89	*/	211	*/
90	#define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)	212	#define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)
91		213
92	/* struct mmu_gather is an opaque type used by the mm code for passing around	214	extern bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page,
		215	int page_size);
		216	#endif
		217
		218	/*
		219	* struct mmu_gather is an opaque type used by the mm code for passing around
93	* any data needed by arch specific code for tlb_remove_page.	220	* any data needed by arch specific code for tlb_remove_page.
94	*/	221	*/
95	struct mmu_gather {	222	struct mmu_gather {
96	struct mm_struct *mm;	223	struct mm_struct *mm;
		224
97	#ifdef CONFIG_HAVE_RCU_TABLE_FREE	225	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
98	struct mmu_table_batch *batch;	226	struct mmu_table_batch *batch;
99	#endif	227	#endif
		228
100	unsigned long start;	229	unsigned long start;
101	unsigned long end;	230	unsigned long end;
102	/*	231	/*
@@ -124,23 +253,30 @@ struct mmu_gather { · 21 unmodified lines
124	unsigned int cleared_puds : 1;	253	unsigned int cleared_puds : 1;
125	unsigned int cleared_p4ds : 1;	254	unsigned int cleared_p4ds : 1;
126		255
		256	/*
		257	* tracks VM_EXEC \| VM_HUGETLB in tlb_start_vma
		258	*/
		259	unsigned int vma_exec : 1;
		260	unsigned int vma_huge : 1;
		261
		262	unsigned int batch_count;
		263
		264	#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
127	struct mmu_gather_batch *active;	265	struct mmu_gather_batch *active;
128	struct mmu_gather_batch local;	266	struct mmu_gather_batch local;
129	struct page *__pages[MMU_GATHER_BUNDLE];	267	struct page *__pages[MMU_GATHER_BUNDLE];
130	unsigned int batch_count;
131	int page_size;
132	};
133		268
134	#define HAVE_GENERIC_MMU_GATHER	269	#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
		270	unsigned int page_size;
		271	#endif
		272	#endif
		273	};
135		274
136	void arch_tlb_gather_mmu(struct mmu_gather *tlb,	275	void arch_tlb_gather_mmu(struct mmu_gather *tlb,
137	struct mm_struct *mm, unsigned long start, unsigned long end);	276	struct mm_struct *mm, unsigned long start, unsigned long end);
138	void tlb_flush_mmu(struct mmu_gather *tlb);	277	void tlb_flush_mmu(struct mmu_gather *tlb);
139	void arch_tlb_finish_mmu(struct mmu_gather *tlb,	278	void arch_tlb_finish_mmu(struct mmu_gather *tlb,
140	unsigned long start, unsigned long end, bool force);	279	unsigned long start, unsigned long end, bool force);
141	void tlb_flush_mmu_free(struct mmu_gather *tlb);
142	extern bool __tlb_remove_page_size(struct mmu_gather tlb, struct page page,
143	int page_size);
144		280
145	static inline void __tlb_adjust_range(struct mmu_gather *tlb,	281	static inline void __tlb_adjust_range(struct mmu_gather *tlb,
146	unsigned long address,	282	unsigned long address,
@@ -163,8 +299,94 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) · 16 unmodified lines
163	tlb->cleared_pmds = 0;	299	tlb->cleared_pmds = 0;
164	tlb->cleared_puds = 0;	300	tlb->cleared_puds = 0;
165	tlb->cleared_p4ds = 0;	301	tlb->cleared_p4ds = 0;
		302	/*
		303	* Do not reset mmu_gather::vma_* fields here, we do not
		304	* call into tlb_start_vma() again to set them if there is an
		305	* intermediate flush.
		306	*/
		307	}
		308
		309	#ifdef CONFIG_MMU_GATHER_NO_RANGE
		310
		311	#if defined(tlb_flush) \|\| defined(tlb_start_vma) \|\| defined(tlb_end_vma)
		312	#error MMU_GATHER_NO_RANGE relies on default tlb_flush(), tlb_start_vma() and tlb_end_vma()
		313	#endif
		314
		315	/*
		316	* When an architecture does not have efficient means of range flushing TLBs
		317	* there is no point in doing intermediate flushes on tlb_end_vma() to keep the
		318	* range small. We equally don't have to worry about page granularity or other
		319	* things.
		320	*
		321	* All we need to do is issue a full flush for any !0 range.
		322	*/
		323	static inline void tlb_flush(struct mmu_gather *tlb)
		324	{
		325	if (tlb->end)
		326	flush_tlb_mm(tlb->mm);
		327	}
		328
		329	static inline void
		330	tlb_update_vma_flags(struct mmu_gather tlb, struct vm_area_struct vma) { }
		331
		332	#define tlb_end_vma tlb_end_vma
		333	static inline void tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma) { }
		334
		335	#else /* CONFIG_MMU_GATHER_NO_RANGE */
		336
		337	#ifndef tlb_flush
		338
		339	#if defined(tlb_start_vma) \|\| defined(tlb_end_vma)
		340	#error Default tlb_flush() relies on default tlb_start_vma() and tlb_end_vma()
		341	#endif
		342
		343	/*
		344	* When an architecture does not provide its own tlb_flush() implementation
		345	* but does have a reasonably efficient flush_vma_range() implementation
		346	* use that.
		347	*/
		348	static inline void tlb_flush(struct mmu_gather *tlb)
		349	{
		350	if (tlb->fullmm \|\| tlb->need_flush_all) {
		351	flush_tlb_mm(tlb->mm);
		352	} else if (tlb->end) {
		353	struct vm_area_struct vma = {
		354	.vm_mm = tlb->mm,
		355	.vm_flags = (tlb->vma_exec ? VM_EXEC : 0) \|
		356	(tlb->vma_huge ? VM_HUGETLB : 0),
		357	};
		358
		359	flush_tlb_range(&vma, tlb->start, tlb->end);
		360	}
166	}	361	}
167		362
		363	static inline void
		364	tlb_update_vma_flags(struct mmu_gather tlb, struct vm_area_struct vma)
		365	{
		366	/*
		367	* flush_tlb_range() implementations that look at VM_HUGETLB (tile,
		368	* mips-4k) flush only large pages.
		369	*
		370	* flush_tlb_range() implementations that flush I-TLB also flush D-TLB
		371	* (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing
		372	* range.
		373	*
		374	* We rely on tlb_end_vma() to issue a flush, such that when we reset
		375	* these values the batch is empty.
		376	*/
		377	tlb->vma_huge = !!(vma->vm_flags & VM_HUGETLB);
		378	tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);
		379	}
		380
		381	#else
		382
		383	static inline void
		384	tlb_update_vma_flags(struct mmu_gather tlb, struct vm_area_struct vma) { }
		385
		386	#endif
		387
		388	#endif /* CONFIG_MMU_GATHER_NO_RANGE */
		389
168	static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)	390	static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
169	{	391	{
170	if (!tlb->end)	392	if (!tlb->end)
@@ -196,21 +418,18 @@ static inline void tlb_remove_page(struct mmu_gather tlb, struct page page) · 25 unmodified lines
196	return tlb_remove_page_size(tlb, page, PAGE_SIZE);	418	return tlb_remove_page_size(tlb, page, PAGE_SIZE);
197	}	419	}
198		420
199	#ifndef tlb_remove_check_page_size_change	421	static inline void tlb_change_page_size(struct mmu_gather *tlb,
200	#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
201	static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
202	unsigned int page_size)	422	unsigned int page_size)
203	{	423	{
204	/*	424	#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
205	* We don't care about page size change, just update	425	if (tlb->page_size && tlb->page_size != page_size) {
206	* mmu_gather page size here so that debug checks	426	if (!tlb->fullmm)
207	* doesn't throw false warning.	427	tlb_flush_mmu(tlb);
208	*/	428	}
209	#ifdef CONFIG_DEBUG_VM	429
210	tlb->page_size = page_size;	430	tlb->page_size = page_size;
211	#endif	431	#endif
212	}	432	}
213	#endif
214		433
215	static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)	434	static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
216	{	435	{
@@ -237,17 +456,30 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 20 unmodified lines
237	* the vmas are adjusted to only cover the region to be torn down.	456	* the vmas are adjusted to only cover the region to be torn down.
238	*/	457	*/
239	#ifndef tlb_start_vma	458	#ifndef tlb_start_vma
240	#define tlb_start_vma(tlb, vma) do { } while (0)	459	static inline void tlb_start_vma(struct mmu_gather tlb, struct vm_area_struct vma)
241	#endif	460	{
		461	if (tlb->fullmm)
		462	return;
242		463
243	#define __tlb_end_vma(tlb, vma) \	464	tlb_update_vma_flags(tlb, vma);
244	do { \	465	flush_cache_range(vma, vma->vm_start, vma->vm_end);
245	if (!tlb->fullmm) \	466	}
246	tlb_flush_mmu_tlbonly(tlb); \	467	#endif
247	} while (0)
248		468
249	#ifndef tlb_end_vma	469	#ifndef tlb_end_vma
250	#define tlb_end_vma __tlb_end_vma	470	static inline void tlb_end_vma(struct mmu_gather tlb, struct vm_area_struct vma)
		471	{
		472	if (tlb->fullmm)
		473	return;
		474
		475	/*
		476	* Do a TLB flush and reset the range at VMA boundaries; this avoids
		477	* the ranges growing with the unused space between consecutive VMAs,
		478	* but also the mmu_gather::vma_* flags from tlb_start_vma() rely on
		479	* this.
		480	*/
		481	tlb_flush_mmu_tlbonly(tlb);
		482	}
251	#endif	483	#endif
252		484
253	#ifndef __tlb_remove_tlb_entry	485	#ifndef __tlb_remove_tlb_entry
@@ -372,6 +604,4 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 118 unmodified lines
372		604
373	#endif /* CONFIG_MMU */	605	#endif /* CONFIG_MMU */
374		606
375	#define tlb_migrate_finish(mm) do {} while (0)
376
377	#endif /* _ASM_GENERIC__TLB_H */	607	#endif /* _ASM_GENERIC__TLB_H */

kernel/iomem.c+2−2

@@ -55,7 +55,7 @@ static void *try_ram_remap(resource_size_t offset, size_t size, · 54 unmodified lines
55	*	55	*
56	* MEMREMAP_WB - matches the default mapping for System RAM on	56	* MEMREMAP_WB - matches the default mapping for System RAM on
57	* the architecture. This is usually a read-allocate write-back cache.	57	* the architecture. This is usually a read-allocate write-back cache.
58	* Morever, if MEMREMAP_WB is specified and the requested remap region is RAM	58	* Moreover, if MEMREMAP_WB is specified and the requested remap region is RAM
59	* memremap() will bypass establishing a new mapping and instead return	59	* memremap() will bypass establishing a new mapping and instead return
60	* a pointer into the direct map.	60	* a pointer into the direct map.
61	*	61	*
@@ -86,7 +86,7 @@ void *memremap(resource_size_t offset, size_t size, unsigned long flags) · 24 unmodified lines
86	/* Try all mapping types requested until one returns non-NULL */	86	/* Try all mapping types requested until one returns non-NULL */
87	if (flags & MEMREMAP_WB) {	87	if (flags & MEMREMAP_WB) {
88	/*	88	/*
89	* MEMREMAP_WB is special in that it can be satisifed	89	* MEMREMAP_WB is special in that it can be satisfied
90	* from the direct map. Some archs depend on the	90	* from the direct map. Some archs depend on the
91	* capability of memremap() to autodetect cases where	91	* capability of memremap() to autodetect cases where
92	* the requested range is potentially in System RAM.	92	* the requested range is potentially in System RAM.

kernel/resource.c+5−6

 int region_intersects(resource_size_t start, size_t size, unsigned long flags,
 		      unsigned long desc)
 {
-	resource_size_t end = start + size - 1;
+	struct resource res;
 	int type = 0; int other = 0;
 	struct resource *p;
+	res.start = start;
+	res.end = start + size - 1;
 	read_lock(&resource_lock);
 	for (p = iomem_resource.child; p ; p = p->sibling) {
 		bool is_type = (((p->flags & flags) == flags) &&
 				((desc == IORES_DESC_NONE) ||
 				 (desc == p->desc)));
-		if (start >= p->start && start <= p->end)
+		if (resource_overlaps(p, &res))
-			is_type ? type++ : other++;
-		if (end >= p->start && end <= p->end)
-			is_type ? type++ : other++;
-		if (p->start >= start && p->end <= end)
 			is_type ? type++ : other++;
 	}
 	read_unlock(&resource_lock);

kernel/sched/core.c+0−1

@@ -1151,7 +1151,6 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, · 1150 unmodified lines
1151	/* Need help from migration thread: drop lock and wait. */	1151	/* Need help from migration thread: drop lock and wait. */
1152	task_rq_unlock(rq, p, &rf);	1152	task_rq_unlock(rq, p, &rf);
1153	stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);	1153	stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1154	tlb_migrate_finish(p->mm);
1155	return 0;	1154	return 0;
1156	} else if (task_on_rq_queued(p)) {	1155	} else if (task_on_rq_queued(p)) {
1157	/*	1156	/*

mm/huge_memory.c+2−2

@@ -1677,7 +1677,7 @@ bool madvise_free_huge_pmd(struct mmu_gather tlb, struct vm_area_struct vma, · 1676 unmodified lines
1677	struct mm_struct *mm = tlb->mm;	1677	struct mm_struct *mm = tlb->mm;
1678	bool ret = false;	1678	bool ret = false;
1679		1679
1680	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);	1680	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1681		1681
1682	ptl = pmd_trans_huge_lock(pmd, vma);	1682	ptl = pmd_trans_huge_lock(pmd, vma);
1683	if (!ptl)	1683	if (!ptl)
@@ -1753,7 +1753,7 @@ int zap_huge_pmd(struct mmu_gather tlb, struct vm_area_struct vma, · 69 unmodified lines
1753	pmd_t orig_pmd;	1753	pmd_t orig_pmd;
1754	spinlock_t *ptl;	1754	spinlock_t *ptl;
1755		1755
1756	tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);	1756	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
1757		1757
1758	ptl = __pmd_trans_huge_lock(pmd, vma);	1758	ptl = __pmd_trans_huge_lock(pmd, vma);
1759	if (!ptl)	1759	if (!ptl)

mm/hugetlb.c+1−1

@@ -3353,7 +3353,7 @@ void __unmap_hugepage_range(struct mmu_gather tlb, struct vm_area_struct vma, · 3352 unmodified lines
3353	* This is a hugetlb vma, all the pte entries should point	3353	* This is a hugetlb vma, all the pte entries should point
3354	* to huge page.	3354	* to huge page.
3355	*/	3355	*/
3356	tlb_remove_check_page_size_change(tlb, sz);	3356	tlb_change_page_size(tlb, sz);
3357	tlb_start_vma(tlb, vma);	3357	tlb_start_vma(tlb, vma);
3358		3358
3359	/*	3359	/*

mm/madvise.c+1−1

@@ -328,7 +328,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, · 327 unmodified lines
328	if (pmd_trans_unstable(pmd))	328	if (pmd_trans_unstable(pmd))
329	return 0;	329	return 0;
330		330
331	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);	331	tlb_change_page_size(tlb, PAGE_SIZE);
332	orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);	332	orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
333	flush_tlb_batched_pending(mm);	333	flush_tlb_batched_pending(mm);
334	arch_enter_lazy_mmu_mode();	334	arch_enter_lazy_mmu_mode();

mm/memory.c+3−3

@@ -356,7 +356,7 @@ void free_pgd_range(struct mmu_gather *tlb, · 355 unmodified lines
356	* We add page table cache pages with PAGE_SIZE,	356	* We add page table cache pages with PAGE_SIZE,
357	* (see pte_free_tlb()), flush the tlb if we need	357	* (see pte_free_tlb()), flush the tlb if we need
358	*/	358	*/
359	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);	359	tlb_change_page_size(tlb, PAGE_SIZE);
360	pgd = pgd_offset(tlb->mm, addr);	360	pgd = pgd_offset(tlb->mm, addr);
361	do {	361	do {
362	next = pgd_addr_end(addr, end);	362	next = pgd_addr_end(addr, end);
@@ -1046,7 +1046,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, · 683 unmodified lines
1046	pte_t *pte;	1046	pte_t *pte;
1047	swp_entry_t entry;	1047	swp_entry_t entry;
1048		1048
1049	tlb_remove_check_page_size_change(tlb, PAGE_SIZE);	1049	tlb_change_page_size(tlb, PAGE_SIZE);
1050	again:	1050	again:
1051	init_rss_vec(rss);	1051	init_rss_vec(rss);
1052	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);	1052	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
@@ -1155,7 +1155,7 @@ again: · 102 unmodified lines
1155	*/	1155	*/
1156	if (force_flush) {	1156	if (force_flush) {
1157	force_flush = 0;	1157	force_flush = 0;
1158	tlb_flush_mmu_free(tlb);	1158	tlb_flush_mmu(tlb);
1159	if (addr != end)	1159	if (addr != end)
1160	goto again;	1160	goto again;
1161	}	1161	}

mm/mmu_gather.c+67−62

 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
-#ifdef HAVE_GENERIC_MMU_GATHER
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
 static bool tlb_next_batch(struct mmu_gather *tlb)
 {
 	return true;
 }
-void arch_tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
+static void tlb_batch_pages_flush(struct mmu_gather *tlb)
-				unsigned long start, unsigned long end)
-{
-	tlb->mm = mm;
-	/* Is it from 0 to ~0? */
-	tlb->fullmm     = !(start | (end+1));
-	tlb->need_flush_all = 0;
-	tlb->local.next = NULL;
-	tlb->local.nr   = 0;
-	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
-	tlb->active     = &tlb->local;
-	tlb->batch_count = 0;
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	tlb->batch = NULL;
-#endif
-	tlb->page_size = 0;
-	__tlb_reset_range(tlb);
-}
-void tlb_flush_mmu_free(struct mmu_gather *tlb)
 {
 	struct mmu_gather_batch *batch;
-#ifdef CONFIG_HAVE_RCU_TABLE_FREE
-	tlb_table_flush(tlb);
-#endif
 	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
 		free_pages_and_swap_cache(batch->pages, batch->nr);
 		batch->nr = 0;
 	tlb->active = &tlb->local;
 }
-void tlb_flush_mmu(struct mmu_gather *tlb)
+static void tlb_batch_list_free(struct mmu_gather *tlb)
-{
-	tlb_flush_mmu_tlbonly(tlb);
-	tlb_flush_mmu_free(tlb);
-}
-/* tlb_finish_mmu
- *	Called at the end of the shootdown operation to free up any resources
- *	that were required.
- */
-void arch_tlb_finish_mmu(struct mmu_gather *tlb,
-		unsigned long start, unsigned long end, bool force)
 {
 	struct mmu_gather_batch *batch, *next;
-	if (force) {
-		__tlb_reset_range(tlb);
-		__tlb_adjust_range(tlb, start, end - start);
-	}
-	tlb_flush_mmu(tlb);
-	/* keep the page table cache within bounds */
-	check_pgt_cache();
 	for (batch = tlb->local.next; batch; batch = next) {
 		next = batch->next;
 		free_pages((unsigned long)batch, 0);
 	tlb->local.next = NULL;
 }
-/* __tlb_remove_page
- *	Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
- *	handling the additional races in SMP caused by other CPUs caching valid
- *	mappings in their TLBs. Returns the number of free page slots left.
- *	When out of page slots we must call tlb_flush_mmu().
- *returns true if the caller should flush.
- */
 bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
 {
 	struct mmu_gather_batch *batch;
 	VM_BUG_ON(!tlb->end);
+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
 	VM_WARN_ON(tlb->page_size != page_size);
+#endif
 	batch = tlb->active;
 	/*
 	return false;
 }
-#endif /* HAVE_GENERIC_MMU_GATHER */
+#endif /* HAVE_MMU_GATHER_NO_GATHER */
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
  */
 static inline void tlb_table_invalidate(struct mmu_gather *tlb)
 {
-#ifdef CONFIG_HAVE_RCU_TABLE_INVALIDATE
+#ifndef CONFIG_HAVE_RCU_TABLE_NO_INVALIDATE
 	/*
 	 * Invalidate page-table caches used by hardware walkers. Then we still
 	 * need to RCU-sched wait while freeing the pages because software
 	free_page((unsigned long)batch);
 }
-void tlb_table_flush(struct mmu_gather *tlb)
+static void tlb_table_flush(struct mmu_gather *tlb)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 #endif /* CONFIG_HAVE_RCU_TABLE_FREE */
+static void tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb_table_flush(tlb);
+#endif
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
+	tlb_batch_pages_flush(tlb);
+#endif
+}
+void tlb_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu_tlbonly(tlb);
+	tlb_flush_mmu_free(tlb);
+}
 /**
  * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
  * @tlb: the mmu_gather structure to initialize
 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
 			unsigned long start, unsigned long end)
 {
-	arch_tlb_gather_mmu(tlb, mm, start, end);
+	tlb->mm = mm;
+	/* Is it from 0 to ~0? */
+	tlb->fullmm     = !(start | (end+1));
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
+	tlb->need_flush_all = 0;
+	tlb->local.next = NULL;
+	tlb->local.nr   = 0;
+	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
+	tlb->active     = &tlb->local;
+	tlb->batch_count = 0;
+#endif
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+	tlb->batch = NULL;
+#endif
+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
+	tlb->page_size = 0;
+#endif
+	__tlb_reset_range(tlb);
 	inc_tlb_flush_pending(tlb->mm);
 }
+/**
+ * tlb_finish_mmu - finish an mmu_gather structure
+ * @tlb: the mmu_gather structure to finish
+ * @start: start of the region that will be removed from the page-table
+ * @end: end of the region that will be removed from the page-table
+ *
+ * Called at the end of the shootdown operation to free up any resources that
+ * were required.
+ */
 void tlb_finish_mmu(struct mmu_gather *tlb,
 		unsigned long start, unsigned long end)
 {
 	 * the TLB by observing pte_none|!pte_dirty, for example so flush TLB
 	 * forcefully if we detect parallel PTE batching threads.
 	 */
-	bool force = mm_tlb_flush_nested(tlb->mm);
+	if (mm_tlb_flush_nested(tlb->mm)) {
+		__tlb_reset_range(tlb);
+		__tlb_adjust_range(tlb, start, end - start);
+	}
-	arch_tlb_finish_mmu(tlb, start, end, force);
+	tlb_flush_mmu(tlb);
+	/* keep the page table cache within bounds */
+	check_pgt_cache();
+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
+	tlb_batch_list_free(tlb);
+#endif
 	dec_tlb_flush_pending(tlb->mm);
 }