History

Merge branch 'core-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull unified TLB flushing from Ingo Molnar: "This contains the generic mmu_gather feature from Peter Zijlstra, which is an all-arch unification of TLB flushing APIs, via the following (broad) steps: - enhance the <asm-generic/tlb.h> APIs to cover more arch details - convert most TLB flushing arch implementations to the generic <asm-generic/tlb.h> APIs. - remove leftovers of per arch implementations After this series every single architecture makes use of the unified TLB flushing APIs" * 'core-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: mm/resource: Use resource_overlaps() to simplify region_intersects() ia64/tlb: Eradicate tlb_migrate_finish() callback asm-generic/tlb: Remove tlb_table_flush() asm-generic/tlb: Remove tlb_flush_mmu_free() asm-generic/tlb: Remove CONFIG_HAVE_GENERIC_MMU_GATHER asm-generic/tlb: Remove arch_tlb*_mmu() s390/tlb: Convert to generic mmu_gather asm-generic/tlb: Introduce CONFIG_HAVE_MMU_GATHER_NO_GATHER=y arch/tlb: Clean up simple architectures um/tlb: Convert to generic mmu_gather sh/tlb: Convert SH to generic mmu_gather ia64/tlb: Convert to generic mmu_gather arm/tlb: Convert to generic mmu_gather asm-generic/tlb, arch: Invert CONFIG_HAVE_RCU_TABLE_INVALIDATE asm-generic/tlb, ia64: Conditionally provide tlb_migrate_finish() asm-generic/tlb: Provide generic tlb_flush() based on flush_tlb_mm() asm-generic/tlb, arch: Provide generic tlb_flush() based on flush_tlb_range() asm-generic/tlb, arch: Provide generic VIPT cache flush asm-generic/tlb, arch: Provide CONFIG_HAVE_MMU_GATHER_PAGE_SIZE asm-generic/tlb: Provide a comment

Linus Torvalds committed 7 years ago

Commit 171c2bc2 parents 423ea32 f6c601055 files · +482 −1296Browse files

Showing the changes to include/asm-generic (1 of 55 files). Show all

Unified · Split

include/asm-generic/tlb.h+259−29
@@ -19,9 +19,131 @@ · 18 unmodified lines
19#include <linux/swap.h>19#include <linux/swap.h>
20#include <asm/pgalloc.h>20#include <asm/pgalloc.h>
21#include <asm/tlbflush.h>21#include <asm/tlbflush.h>
22#include <asm/cacheflush.h>
22 23
23#ifdef CONFIG_MMU24#ifdef CONFIG_MMU
24 25
26/*
27 * Generic MMU-gather implementation.
28 *
29 * The mmu_gather data structure is used by the mm code to implement the
30 * correct and efficient ordering of freeing pages and TLB invalidations.
31 *
32 * This correct ordering is:
33 *
34 * 1) unhook page
35 * 2) TLB invalidate page
36 * 3) free page
37 *
38 * That is, we must never free a page before we have ensured there are no live
39 * translations left to it. Otherwise it might be possible to observe (or
40 * worse, change) the page content after it has been reused.
41 *
42 * The mmu_gather API consists of:
43 *
44 * - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather
45 *
46 * Finish in particular will issue a (final) TLB invalidate and free
47 * all (remaining) queued pages.
48 *
49 * - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA
50 *
51 * Defaults to flushing at tlb_end_vma() to reset the range; helps when
52 * there's large holes between the VMAs.
53 *
54 * - tlb_remove_page() / __tlb_remove_page()
55 * - tlb_remove_page_size() / __tlb_remove_page_size()
56 *
57 * __tlb_remove_page_size() is the basic primitive that queues a page for
58 * freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a
59 * boolean indicating if the queue is (now) full and a call to
60 * tlb_flush_mmu() is required.
61 *
62 * tlb_remove_page() and tlb_remove_page_size() imply the call to
63 * tlb_flush_mmu() when required and has no return value.
64 *
65 * - tlb_change_page_size()
66 *
67 * call before __tlb_remove_page*() to set the current page-size; implies a
68 * possible tlb_flush_mmu() call.
69 *
70 * - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()
71 *
72 * tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
73 * related state, like the range)
74 *
75 * tlb_flush_mmu() - in addition to the above TLB invalidate, also frees
76 * whatever pages are still batched.
77 *
78 * - mmu_gather::fullmm
79 *
80 * A flag set by tlb_gather_mmu() to indicate we're going to free
81 * the entire mm; this allows a number of optimizations.
82 *
83 * - We can ignore tlb_{start,end}_vma(); because we don't
84 * care about ranges. Everything will be shot down.
85 *
86 * - (RISC) architectures that use ASIDs can cycle to a new ASID
87 * and delay the invalidation until ASID space runs out.
88 *
89 * - mmu_gather::need_flush_all
90 *
91 * A flag that can be set by the arch code if it wants to force
92 * flush the entire TLB irrespective of the range. For instance
93 * x86-PAE needs this when changing top-level entries.
94 *
95 * And allows the architecture to provide and implement tlb_flush():
96 *
97 * tlb_flush() may, in addition to the above mentioned mmu_gather fields, make
98 * use of:
99 *
100 * - mmu_gather::start / mmu_gather::end
101 *
102 * which provides the range that needs to be flushed to cover the pages to
103 * be freed.
104 *
105 * - mmu_gather::freed_tables
106 *
107 * set when we freed page table pages
108 *
109 * - tlb_get_unmap_shift() / tlb_get_unmap_size()
110 *
111 * returns the smallest TLB entry size unmapped in this range.
112 *
113 * If an architecture does not provide tlb_flush() a default implementation
114 * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is
115 * specified, in which case we'll default to flush_tlb_mm().
116 *
117 * Additionally there are a few opt-in features:
118 *
119 * HAVE_MMU_GATHER_PAGE_SIZE
120 *
121 * This ensures we call tlb_flush() every time tlb_change_page_size() actually
122 * changes the size and provides mmu_gather::page_size to tlb_flush().
123 *
124 * HAVE_RCU_TABLE_FREE
125 *
126 * This provides tlb_remove_table(), to be used instead of tlb_remove_page()
127 * for page directores (__p*_free_tlb()). This provides separate freeing of
128 * the page-table pages themselves in a semi-RCU fashion (see comment below).
129 * Useful if your architecture doesn't use IPIs for remote TLB invalidates
130 * and therefore doesn't naturally serialize with software page-table walkers.
131 *
132 * When used, an architecture is expected to provide __tlb_remove_table()
133 * which does the actual freeing of these pages.
134 *
135 * HAVE_RCU_TABLE_NO_INVALIDATE
136 *
137 * This makes HAVE_RCU_TABLE_FREE avoid calling tlb_flush_mmu_tlbonly() before
138 * freeing the page-table pages. This can be avoided if you use
139 * HAVE_RCU_TABLE_FREE and your architecture does _NOT_ use the Linux
140 * page-tables natively.
141 *
142 * MMU_GATHER_NO_RANGE
143 *
144 * Use this if your architecture lacks an efficient flush_tlb_range().
145 */
146
25#ifdef CONFIG_HAVE_RCU_TABLE_FREE147#ifdef CONFIG_HAVE_RCU_TABLE_FREE
26/*148/*
27 * Semi RCU freeing of the page directories.149 * Semi RCU freeing of the page directories.
@@ -60,11 +182,11 @@ struct mmu_table_batch { · 32 unmodified lines
60#define MAX_TABLE_BATCH \182#define MAX_TABLE_BATCH \
61 ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))183 ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
62 184
63extern void tlb_table_flush(struct mmu_gather *tlb);
64extern void tlb_remove_table(struct mmu_gather *tlb, void *table);185extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
65 186
66#endif187#endif
67 188
189#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
68/*190/*
69 * If we can't allocate a page to make a big batch of page pointers191 * If we can't allocate a page to make a big batch of page pointers
70 * to work on, then just handle a few from the on-stack structure.192 * to work on, then just handle a few from the on-stack structure.
@@ -89,14 +211,21 @@ struct mmu_gather_batch { · 18 unmodified lines
89 */211 */
90#define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)212#define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)
91 213
92/* struct mmu_gather is an opaque type used by the mm code for passing around214extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
215 int page_size);
216#endif
217
218/*
219 * struct mmu_gather is an opaque type used by the mm code for passing around
93 * any data needed by arch specific code for tlb_remove_page.220 * any data needed by arch specific code for tlb_remove_page.
94 */221 */
95struct mmu_gather {222struct mmu_gather {
96 struct mm_struct *mm;223 struct mm_struct *mm;
224
97#ifdef CONFIG_HAVE_RCU_TABLE_FREE225#ifdef CONFIG_HAVE_RCU_TABLE_FREE
98 struct mmu_table_batch *batch;226 struct mmu_table_batch *batch;
99#endif227#endif
228
100 unsigned long start;229 unsigned long start;
101 unsigned long end;230 unsigned long end;
102 /*231 /*
@@ -124,23 +253,30 @@ struct mmu_gather { · 21 unmodified lines
124 unsigned int cleared_puds : 1;253 unsigned int cleared_puds : 1;
125 unsigned int cleared_p4ds : 1;254 unsigned int cleared_p4ds : 1;
126 255
256 /*
257 * tracks VM_EXEC | VM_HUGETLB in tlb_start_vma
258 */
259 unsigned int vma_exec : 1;
260 unsigned int vma_huge : 1;
261
262 unsigned int batch_count;
263
264#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
127 struct mmu_gather_batch *active;265 struct mmu_gather_batch *active;
128 struct mmu_gather_batch local;266 struct mmu_gather_batch local;
129 struct page *__pages[MMU_GATHER_BUNDLE];267 struct page *__pages[MMU_GATHER_BUNDLE];
130 unsigned int batch_count;
131 int page_size;
132};
133 268
134#define HAVE_GENERIC_MMU_GATHER269#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
270 unsigned int page_size;
271#endif
272#endif
273};
135 274
136void arch_tlb_gather_mmu(struct mmu_gather *tlb,275void arch_tlb_gather_mmu(struct mmu_gather *tlb,
137 struct mm_struct *mm, unsigned long start, unsigned long end);276 struct mm_struct *mm, unsigned long start, unsigned long end);
138void tlb_flush_mmu(struct mmu_gather *tlb);277void tlb_flush_mmu(struct mmu_gather *tlb);
139void arch_tlb_finish_mmu(struct mmu_gather *tlb,278void arch_tlb_finish_mmu(struct mmu_gather *tlb,
140 unsigned long start, unsigned long end, bool force);279 unsigned long start, unsigned long end, bool force);
141void tlb_flush_mmu_free(struct mmu_gather *tlb);
142extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
143 int page_size);
144 280
145static inline void __tlb_adjust_range(struct mmu_gather *tlb,281static inline void __tlb_adjust_range(struct mmu_gather *tlb,
146 unsigned long address,282 unsigned long address,
@@ -163,8 +299,94 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) · 16 unmodified lines
163 tlb->cleared_pmds = 0;299 tlb->cleared_pmds = 0;
164 tlb->cleared_puds = 0;300 tlb->cleared_puds = 0;
165 tlb->cleared_p4ds = 0;301 tlb->cleared_p4ds = 0;
302 /*
303 * Do not reset mmu_gather::vma_* fields here, we do not
304 * call into tlb_start_vma() again to set them if there is an
305 * intermediate flush.
306 */
307}
308
309#ifdef CONFIG_MMU_GATHER_NO_RANGE
310
311#if defined(tlb_flush) || defined(tlb_start_vma) || defined(tlb_end_vma)
312#error MMU_GATHER_NO_RANGE relies on default tlb_flush(), tlb_start_vma() and tlb_end_vma()
313#endif
314
315/*
316 * When an architecture does not have efficient means of range flushing TLBs
317 * there is no point in doing intermediate flushes on tlb_end_vma() to keep the
318 * range small. We equally don't have to worry about page granularity or other
319 * things.
320 *
321 * All we need to do is issue a full flush for any !0 range.
322 */
323static inline void tlb_flush(struct mmu_gather *tlb)
324{
325 if (tlb->end)
326 flush_tlb_mm(tlb->mm);
327}
328
329static inline void
330tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
331
332#define tlb_end_vma tlb_end_vma
333static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
334
335#else /* CONFIG_MMU_GATHER_NO_RANGE */
336
337#ifndef tlb_flush
338
339#if defined(tlb_start_vma) || defined(tlb_end_vma)
340#error Default tlb_flush() relies on default tlb_start_vma() and tlb_end_vma()
341#endif
342
343/*
344 * When an architecture does not provide its own tlb_flush() implementation
345 * but does have a reasonably efficient flush_vma_range() implementation
346 * use that.
347 */
348static inline void tlb_flush(struct mmu_gather *tlb)
349{
350 if (tlb->fullmm || tlb->need_flush_all) {
351 flush_tlb_mm(tlb->mm);
352 } else if (tlb->end) {
353 struct vm_area_struct vma = {
354 .vm_mm = tlb->mm,
355 .vm_flags = (tlb->vma_exec ? VM_EXEC : 0) |
356 (tlb->vma_huge ? VM_HUGETLB : 0),
357 };
358
359 flush_tlb_range(&vma, tlb->start, tlb->end);
360 }
166}361}
167 362
363static inline void
364tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma)
365{
366 /*
367 * flush_tlb_range() implementations that look at VM_HUGETLB (tile,
368 * mips-4k) flush only large pages.
369 *
370 * flush_tlb_range() implementations that flush I-TLB also flush D-TLB
371 * (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing
372 * range.
373 *
374 * We rely on tlb_end_vma() to issue a flush, such that when we reset
375 * these values the batch is empty.
376 */
377 tlb->vma_huge = !!(vma->vm_flags & VM_HUGETLB);
378 tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);
379}
380
381#else
382
383static inline void
384tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
385
386#endif
387
388#endif /* CONFIG_MMU_GATHER_NO_RANGE */
389
168static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)390static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
169{391{
170 if (!tlb->end)392 if (!tlb->end)
@@ -196,21 +418,18 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) · 25 unmodified lines
196 return tlb_remove_page_size(tlb, page, PAGE_SIZE);418 return tlb_remove_page_size(tlb, page, PAGE_SIZE);
197}419}
198 420
199#ifndef tlb_remove_check_page_size_change421static inline void tlb_change_page_size(struct mmu_gather *tlb,
200#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
201static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
202 unsigned int page_size)422 unsigned int page_size)
203{423{
204 /*424#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
205 * We don't care about page size change, just update425 if (tlb->page_size && tlb->page_size != page_size) {
206 * mmu_gather page size here so that debug checks426 if (!tlb->fullmm)
207 * doesn't throw false warning.427 tlb_flush_mmu(tlb);
208 */428 }
209#ifdef CONFIG_DEBUG_VM429
210 tlb->page_size = page_size;430 tlb->page_size = page_size;
211#endif431#endif
212}432}
213#endif
214 433
215static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)434static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
216{435{
@@ -237,17 +456,30 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 20 unmodified lines
237 * the vmas are adjusted to only cover the region to be torn down.456 * the vmas are adjusted to only cover the region to be torn down.
238 */457 */
239#ifndef tlb_start_vma458#ifndef tlb_start_vma
240#define tlb_start_vma(tlb, vma) do { } while (0)459static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
241#endif460{
461 if (tlb->fullmm)
462 return;
242 463
243#define __tlb_end_vma(tlb, vma) \464 tlb_update_vma_flags(tlb, vma);
244 do { \465 flush_cache_range(vma, vma->vm_start, vma->vm_end);
245 if (!tlb->fullmm) \466}
246 tlb_flush_mmu_tlbonly(tlb); \467#endif
247 } while (0)
248 468
249#ifndef tlb_end_vma469#ifndef tlb_end_vma
250#define tlb_end_vma __tlb_end_vma470static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
471{
472 if (tlb->fullmm)
473 return;
474
475 /*
476 * Do a TLB flush and reset the range at VMA boundaries; this avoids
477 * the ranges growing with the unused space between consecutive VMAs,
478 * but also the mmu_gather::vma_* flags from tlb_start_vma() rely on
479 * this.
480 */
481 tlb_flush_mmu_tlbonly(tlb);
482}
251#endif483#endif
252 484
253#ifndef __tlb_remove_tlb_entry485#ifndef __tlb_remove_tlb_entry
@@ -372,6 +604,4 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 118 unmodified lines
372 604
373#endif /* CONFIG_MMU */605#endif /* CONFIG_MMU */
374 606
375#define tlb_migrate_finish(mm) do {} while (0)
376
377#endif /* _ASM_GENERIC__TLB_H */607#endif /* _ASM_GENERIC__TLB_H */