History

Merge branch 'core-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull unified TLB flushing from Ingo Molnar: "This contains the generic mmu_gather feature from Peter Zijlstra, which is an all-arch unification of TLB flushing APIs, via the following (broad) steps: - enhance the <asm-generic/tlb.h> APIs to cover more arch details - convert most TLB flushing arch implementations to the generic <asm-generic/tlb.h> APIs. - remove leftovers of per arch implementations After this series every single architecture makes use of the unified TLB flushing APIs" * 'core-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: mm/resource: Use resource_overlaps() to simplify region_intersects() ia64/tlb: Eradicate tlb_migrate_finish() callback asm-generic/tlb: Remove tlb_table_flush() asm-generic/tlb: Remove tlb_flush_mmu_free() asm-generic/tlb: Remove CONFIG_HAVE_GENERIC_MMU_GATHER asm-generic/tlb: Remove arch_tlb*_mmu() s390/tlb: Convert to generic mmu_gather asm-generic/tlb: Introduce CONFIG_HAVE_MMU_GATHER_NO_GATHER=y arch/tlb: Clean up simple architectures um/tlb: Convert to generic mmu_gather sh/tlb: Convert SH to generic mmu_gather ia64/tlb: Convert to generic mmu_gather arm/tlb: Convert to generic mmu_gather asm-generic/tlb, arch: Invert CONFIG_HAVE_RCU_TABLE_INVALIDATE asm-generic/tlb, ia64: Conditionally provide tlb_migrate_finish() asm-generic/tlb: Provide generic tlb_flush() based on flush_tlb_mm() asm-generic/tlb, arch: Provide generic tlb_flush() based on flush_tlb_range() asm-generic/tlb, arch: Provide generic VIPT cache flush asm-generic/tlb, arch: Provide CONFIG_HAVE_MMU_GATHER_PAGE_SIZE asm-generic/tlb: Provide a comment

Linus Torvalds committed 7 years ago

Commit 171c2bc2 parents 423ea32 f6c601055 files · +482 −1296Browse files

Showing the changes to include/asm-generic (1 of 55 files). Show all

Unified · Split

include/asm-generic/tlb.h+259−29
@@ -19,9 +19,131 @@ · 18 unmodified lines
1919 #include <linux/swap.h>
2020 #include <asm/pgalloc.h>
2121 #include <asm/tlbflush.h>
22+#include <asm/cacheflush.h>
2223
2324 #ifdef CONFIG_MMU
2425
26+/*
27+ * Generic MMU-gather implementation.
28+ *
29+ * The mmu_gather data structure is used by the mm code to implement the
30+ * correct and efficient ordering of freeing pages and TLB invalidations.
31+ *
32+ * This correct ordering is:
33+ *
34+ * 1) unhook page
35+ * 2) TLB invalidate page
36+ * 3) free page
37+ *
38+ * That is, we must never free a page before we have ensured there are no live
39+ * translations left to it. Otherwise it might be possible to observe (or
40+ * worse, change) the page content after it has been reused.
41+ *
42+ * The mmu_gather API consists of:
43+ *
44+ * - tlb_gather_mmu() / tlb_finish_mmu(); start and finish a mmu_gather
45+ *
46+ * Finish in particular will issue a (final) TLB invalidate and free
47+ * all (remaining) queued pages.
48+ *
49+ * - tlb_start_vma() / tlb_end_vma(); marks the start / end of a VMA
50+ *
51+ * Defaults to flushing at tlb_end_vma() to reset the range; helps when
52+ * there's large holes between the VMAs.
53+ *
54+ * - tlb_remove_page() / __tlb_remove_page()
55+ * - tlb_remove_page_size() / __tlb_remove_page_size()
56+ *
57+ * __tlb_remove_page_size() is the basic primitive that queues a page for
58+ * freeing. __tlb_remove_page() assumes PAGE_SIZE. Both will return a
59+ * boolean indicating if the queue is (now) full and a call to
60+ * tlb_flush_mmu() is required.
61+ *
62+ * tlb_remove_page() and tlb_remove_page_size() imply the call to
63+ * tlb_flush_mmu() when required and has no return value.
64+ *
65+ * - tlb_change_page_size()
66+ *
67+ * call before __tlb_remove_page*() to set the current page-size; implies a
68+ * possible tlb_flush_mmu() call.
69+ *
70+ * - tlb_flush_mmu() / tlb_flush_mmu_tlbonly()
71+ *
72+ * tlb_flush_mmu_tlbonly() - does the TLB invalidate (and resets
73+ * related state, like the range)
74+ *
75+ * tlb_flush_mmu() - in addition to the above TLB invalidate, also frees
76+ * whatever pages are still batched.
77+ *
78+ * - mmu_gather::fullmm
79+ *
80+ * A flag set by tlb_gather_mmu() to indicate we're going to free
81+ * the entire mm; this allows a number of optimizations.
82+ *
83+ * - We can ignore tlb_{start,end}_vma(); because we don't
84+ * care about ranges. Everything will be shot down.
85+ *
86+ * - (RISC) architectures that use ASIDs can cycle to a new ASID
87+ * and delay the invalidation until ASID space runs out.
88+ *
89+ * - mmu_gather::need_flush_all
90+ *
91+ * A flag that can be set by the arch code if it wants to force
92+ * flush the entire TLB irrespective of the range. For instance
93+ * x86-PAE needs this when changing top-level entries.
94+ *
95+ * And allows the architecture to provide and implement tlb_flush():
96+ *
97+ * tlb_flush() may, in addition to the above mentioned mmu_gather fields, make
98+ * use of:
99+ *
100+ * - mmu_gather::start / mmu_gather::end
101+ *
102+ * which provides the range that needs to be flushed to cover the pages to
103+ * be freed.
104+ *
105+ * - mmu_gather::freed_tables
106+ *
107+ * set when we freed page table pages
108+ *
109+ * - tlb_get_unmap_shift() / tlb_get_unmap_size()
110+ *
111+ * returns the smallest TLB entry size unmapped in this range.
112+ *
113+ * If an architecture does not provide tlb_flush() a default implementation
114+ * based on flush_tlb_range() will be used, unless MMU_GATHER_NO_RANGE is
115+ * specified, in which case we'll default to flush_tlb_mm().
116+ *
117+ * Additionally there are a few opt-in features:
118+ *
119+ * HAVE_MMU_GATHER_PAGE_SIZE
120+ *
121+ * This ensures we call tlb_flush() every time tlb_change_page_size() actually
122+ * changes the size and provides mmu_gather::page_size to tlb_flush().
123+ *
124+ * HAVE_RCU_TABLE_FREE
125+ *
126+ * This provides tlb_remove_table(), to be used instead of tlb_remove_page()
127+ * for page directores (__p*_free_tlb()). This provides separate freeing of
128+ * the page-table pages themselves in a semi-RCU fashion (see comment below).
129+ * Useful if your architecture doesn't use IPIs for remote TLB invalidates
130+ * and therefore doesn't naturally serialize with software page-table walkers.
131+ *
132+ * When used, an architecture is expected to provide __tlb_remove_table()
133+ * which does the actual freeing of these pages.
134+ *
135+ * HAVE_RCU_TABLE_NO_INVALIDATE
136+ *
137+ * This makes HAVE_RCU_TABLE_FREE avoid calling tlb_flush_mmu_tlbonly() before
138+ * freeing the page-table pages. This can be avoided if you use
139+ * HAVE_RCU_TABLE_FREE and your architecture does _NOT_ use the Linux
140+ * page-tables natively.
141+ *
142+ * MMU_GATHER_NO_RANGE
143+ *
144+ * Use this if your architecture lacks an efficient flush_tlb_range().
145+ */
146+
25147 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
26148 /*
27149 * Semi RCU freeing of the page directories.
@@ -60,11 +182,11 @@ struct mmu_table_batch { · 32 unmodified lines
60182 #define MAX_TABLE_BATCH \
61183 ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
62184
63−extern void tlb_table_flush(struct mmu_gather *tlb);
64185 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
65186
66187 #endif
67188
189+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
68190 /*
69191 * If we can't allocate a page to make a big batch of page pointers
70192 * to work on, then just handle a few from the on-stack structure.
@@ -89,14 +211,21 @@ struct mmu_gather_batch { · 18 unmodified lines
89211 */
90212 #define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH)
91213
92−/* struct mmu_gather is an opaque type used by the mm code for passing around
214+extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
215+ int page_size);
216+#endif
217+
218+/*
219+ * struct mmu_gather is an opaque type used by the mm code for passing around
93220 * any data needed by arch specific code for tlb_remove_page.
94221 */
95222 struct mmu_gather {
96223 struct mm_struct *mm;
224+
97225 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
98226 struct mmu_table_batch *batch;
99227 #endif
228+
100229 unsigned long start;
101230 unsigned long end;
102231 /*
@@ -124,23 +253,30 @@ struct mmu_gather { · 21 unmodified lines
124253 unsigned int cleared_puds : 1;
125254 unsigned int cleared_p4ds : 1;
126255
256+ /*
257+ * tracks VM_EXEC | VM_HUGETLB in tlb_start_vma
258+ */
259+ unsigned int vma_exec : 1;
260+ unsigned int vma_huge : 1;
261+
262+ unsigned int batch_count;
263+
264+#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
127265 struct mmu_gather_batch *active;
128266 struct mmu_gather_batch local;
129267 struct page *__pages[MMU_GATHER_BUNDLE];
130− unsigned int batch_count;
131− int page_size;
132−};
133268
134−#define HAVE_GENERIC_MMU_GATHER
269+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
270+ unsigned int page_size;
271+#endif
272+#endif
273+};
135274
136275 void arch_tlb_gather_mmu(struct mmu_gather *tlb,
137276 struct mm_struct *mm, unsigned long start, unsigned long end);
138277 void tlb_flush_mmu(struct mmu_gather *tlb);
139278 void arch_tlb_finish_mmu(struct mmu_gather *tlb,
140279 unsigned long start, unsigned long end, bool force);
141−void tlb_flush_mmu_free(struct mmu_gather *tlb);
142−extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
143− int page_size);
144280
145281 static inline void __tlb_adjust_range(struct mmu_gather *tlb,
146282 unsigned long address,
@@ -163,8 +299,94 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) · 16 unmodified lines
163299 tlb->cleared_pmds = 0;
164300 tlb->cleared_puds = 0;
165301 tlb->cleared_p4ds = 0;
302+ /*
303+ * Do not reset mmu_gather::vma_* fields here, we do not
304+ * call into tlb_start_vma() again to set them if there is an
305+ * intermediate flush.
306+ */
307+}
308+
309+#ifdef CONFIG_MMU_GATHER_NO_RANGE
310+
311+#if defined(tlb_flush) || defined(tlb_start_vma) || defined(tlb_end_vma)
312+#error MMU_GATHER_NO_RANGE relies on default tlb_flush(), tlb_start_vma() and tlb_end_vma()
313+#endif
314+
315+/*
316+ * When an architecture does not have efficient means of range flushing TLBs
317+ * there is no point in doing intermediate flushes on tlb_end_vma() to keep the
318+ * range small. We equally don't have to worry about page granularity or other
319+ * things.
320+ *
321+ * All we need to do is issue a full flush for any !0 range.
322+ */
323+static inline void tlb_flush(struct mmu_gather *tlb)
324+{
325+ if (tlb->end)
326+ flush_tlb_mm(tlb->mm);
327+}
328+
329+static inline void
330+tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
331+
332+#define tlb_end_vma tlb_end_vma
333+static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
334+
335+#else /* CONFIG_MMU_GATHER_NO_RANGE */
336+
337+#ifndef tlb_flush
338+
339+#if defined(tlb_start_vma) || defined(tlb_end_vma)
340+#error Default tlb_flush() relies on default tlb_start_vma() and tlb_end_vma()
341+#endif
342+
343+/*
344+ * When an architecture does not provide its own tlb_flush() implementation
345+ * but does have a reasonably efficient flush_vma_range() implementation
346+ * use that.
347+ */
348+static inline void tlb_flush(struct mmu_gather *tlb)
349+{
350+ if (tlb->fullmm || tlb->need_flush_all) {
351+ flush_tlb_mm(tlb->mm);
352+ } else if (tlb->end) {
353+ struct vm_area_struct vma = {
354+ .vm_mm = tlb->mm,
355+ .vm_flags = (tlb->vma_exec ? VM_EXEC : 0) |
356+ (tlb->vma_huge ? VM_HUGETLB : 0),
357+ };
358+
359+ flush_tlb_range(&vma, tlb->start, tlb->end);
360+ }
166361 }
167362
363+static inline void
364+tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma)
365+{
366+ /*
367+ * flush_tlb_range() implementations that look at VM_HUGETLB (tile,
368+ * mips-4k) flush only large pages.
369+ *
370+ * flush_tlb_range() implementations that flush I-TLB also flush D-TLB
371+ * (tile, xtensa, arm), so it's ok to just add VM_EXEC to an existing
372+ * range.
373+ *
374+ * We rely on tlb_end_vma() to issue a flush, such that when we reset
375+ * these values the batch is empty.
376+ */
377+ tlb->vma_huge = !!(vma->vm_flags & VM_HUGETLB);
378+ tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);
379+}
380+
381+#else
382+
383+static inline void
384+tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { }
385+
386+#endif
387+
388+#endif /* CONFIG_MMU_GATHER_NO_RANGE */
389+
168390 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
169391 {
170392 if (!tlb->end)
@@ -196,21 +418,18 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) · 25 unmodified lines
196418 return tlb_remove_page_size(tlb, page, PAGE_SIZE);
197419 }
198420
199−#ifndef tlb_remove_check_page_size_change
200−#define tlb_remove_check_page_size_change tlb_remove_check_page_size_change
201−static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
421+static inline void tlb_change_page_size(struct mmu_gather *tlb,
202422 unsigned int page_size)
203423 {
204− /*
205− * We don't care about page size change, just update
206− * mmu_gather page size here so that debug checks
207− * doesn't throw false warning.
208− */
209−#ifdef CONFIG_DEBUG_VM
424+#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE
425+ if (tlb->page_size && tlb->page_size != page_size) {
426+ if (!tlb->fullmm)
427+ tlb_flush_mmu(tlb);
428+ }
429+
210430 tlb->page_size = page_size;
211431 #endif
212432 }
213−#endif
214433
215434 static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)
216435 {
@@ -237,17 +456,30 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 20 unmodified lines
237456 * the vmas are adjusted to only cover the region to be torn down.
238457 */
239458 #ifndef tlb_start_vma
240−#define tlb_start_vma(tlb, vma) do { } while (0)
241−#endif
459+static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
460+{
461+ if (tlb->fullmm)
462+ return;
242463
243−#define __tlb_end_vma(tlb, vma) \
244− do { \
245− if (!tlb->fullmm) \
246− tlb_flush_mmu_tlbonly(tlb); \
247− } while (0)
464+ tlb_update_vma_flags(tlb, vma);
465+ flush_cache_range(vma, vma->vm_start, vma->vm_end);
466+}
467+#endif
248468
249469 #ifndef tlb_end_vma
250−#define tlb_end_vma __tlb_end_vma
470+static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
471+{
472+ if (tlb->fullmm)
473+ return;
474+
475+ /*
476+ * Do a TLB flush and reset the range at VMA boundaries; this avoids
477+ * the ranges growing with the unused space between consecutive VMAs,
478+ * but also the mmu_gather::vma_* flags from tlb_start_vma() rely on
479+ * this.
480+ */
481+ tlb_flush_mmu_tlbonly(tlb);
482+}
251483 #endif
252484
253485 #ifndef __tlb_remove_tlb_entry
@@ -372,6 +604,4 @@ static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) · 118 unmodified lines
372604
373605 #endif /* CONFIG_MMU */
374606
375−#define tlb_migrate_finish(mm) do {} while (0)
376−
377607 #endif /* _ASM_GENERIC__TLB_H */