Every arch has the same code to ensure atomic operations and a check for !HIGHMEM page. Remove the duplicate code by defining a core kmap_atomic() which only calls the arch specific kmap_atomic_high() when the page is high memory. [akpm@linux-foundation.org: coding style fixes] Signed-off-by: Ira Weiny <ira.weiny@intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Reviewed-by: Christoph Hellwig <hch@lst.de> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Andy Lutomirski <luto@kernel.org> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Christian König <christian.koenig@amd.com> Cc: Chris Zankel <chris@zankel.net> Cc: Daniel Vetter <daniel.vetter@ffwll.ch> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: "David S. Miller" <davem@davemloft.net> Cc: Helge Deller <deller@gmx.de> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com> Cc: Max Filippov <jcmvbkbc@gmail.com> Cc: Paul Mackerras <paulus@samba.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/20200507150004.1423069-7-ira.weiny@intel.com Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Unified · Split
| @@ -30,7 +30,6 @@ · 29 unmodified lines | |||
| 30 | 30 | ||
| 31 | #include <asm/cacheflush.h> | 31 | #include <asm/cacheflush.h> |
| 32 | 32 | ||
| 33 | extern void *kmap_atomic(struct page *page); | ||
| 34 | extern void __kunmap_atomic(void *kvaddr); | 33 | extern void __kunmap_atomic(void *kvaddr); |
| 35 | 34 | ||
| 36 | extern void kmap_init(void); | 35 | extern void kmap_init(void); |
| @@ -49,16 +49,11 @@ · 48 unmodified lines | |||
| 49 | extern pte_t * pkmap_page_table; | 49 | extern pte_t * pkmap_page_table; |
| 50 | static pte_t * fixmap_page_table; | 50 | static pte_t * fixmap_page_table; |
| 51 | 51 | ||
| 52 | void *kmap_atomic(struct page *page) | 52 | void *kmap_atomic_high(struct page *page) |
| 53 | { | 53 | { |
| 54 | int idx, cpu_idx; | 54 | int idx, cpu_idx; |
| 55 | unsigned long vaddr; | 55 | unsigned long vaddr; |
| 56 | 56 | ||
| 57 | preempt_disable(); | ||
| 58 | pagefault_disable(); | ||
| 59 | if (!PageHighMem(page)) | ||
| 60 | return page_address(page); | ||
| 61 | |||
| 62 | cpu_idx = kmap_atomic_idx_push(); | 57 | cpu_idx = kmap_atomic_idx_push(); |
| 63 | idx = cpu_idx + KM_TYPE_NR * smp_processor_id(); | 58 | idx = cpu_idx + KM_TYPE_NR * smp_processor_id(); |
| 64 | vaddr = FIXMAP_ADDR(idx); | 59 | vaddr = FIXMAP_ADDR(idx); |
| @@ -68,7 +63,7 @@ void *kmap_atomic(struct page *page) · 3 unmodified lines | |||
| 68 | 63 | ||
| 69 | return (void *)vaddr; | 64 | return (void *)vaddr; |
| 70 | } | 65 | } |
| 71 | EXPORT_SYMBOL(kmap_atomic); | 66 | EXPORT_SYMBOL(kmap_atomic_high); |
| 72 | 67 | ||
| 73 | void __kunmap_atomic(void *kv) | 68 | void __kunmap_atomic(void *kv) |
| 74 | { | 69 | { |
| @@ -60,7 +60,6 @@ static inline void *kmap_high_get(struct page *page) · 59 unmodified lines | |||
| 60 | * when CONFIG_HIGHMEM is not set. | 60 | * when CONFIG_HIGHMEM is not set. |
| 61 | */ | 61 | */ |
| 62 | #ifdef CONFIG_HIGHMEM | 62 | #ifdef CONFIG_HIGHMEM |
| 63 | extern void *kmap_atomic(struct page *page); | ||
| 64 | extern void __kunmap_atomic(void *kvaddr); | 63 | extern void __kunmap_atomic(void *kvaddr); |
| 65 | extern void *kmap_atomic_pfn(unsigned long pfn); | 64 | extern void *kmap_atomic_pfn(unsigned long pfn); |
| 66 | #endif | 65 | #endif |
| @@ -31,18 +31,13 @@ static inline pte_t get_fixmap_pte(unsigned long vaddr) · 30 unmodified lines | |||
| 31 | return *ptep; | 31 | return *ptep; |
| 32 | } | 32 | } |
| 33 | 33 | ||
| 34 | void *kmap_atomic(struct page *page) | 34 | void *kmap_atomic_high(struct page *page) |
| 35 | { | 35 | { |
| 36 | unsigned int idx; | 36 | unsigned int idx; |
| 37 | unsigned long vaddr; | 37 | unsigned long vaddr; |
| 38 | void *kmap; | 38 | void *kmap; |
| 39 | int type; | 39 | int type; |
| 40 | 40 | ||
| 41 | preempt_disable(); | ||
| 42 | pagefault_disable(); | ||
| 43 | if (!PageHighMem(page)) | ||
| 44 | return page_address(page); | ||
| 45 | |||
| 46 | #ifdef CONFIG_DEBUG_HIGHMEM | 41 | #ifdef CONFIG_DEBUG_HIGHMEM |
| 47 | /* | 42 | /* |
| 48 | * There is no cache coherency issue when non VIVT, so force the | 43 | * There is no cache coherency issue when non VIVT, so force the |
| @@ -76,7 +71,7 @@ void *kmap_atomic(struct page *page) · 27 unmodified lines | |||
| 76 | 71 | ||
| 77 | return (void *)vaddr; | 72 | return (void *)vaddr; |
| 78 | } | 73 | } |
| 79 | EXPORT_SYMBOL(kmap_atomic); | 74 | EXPORT_SYMBOL(kmap_atomic_high); |
| 80 | 75 | ||
| 81 | void __kunmap_atomic(void *kvaddr) | 76 | void __kunmap_atomic(void *kvaddr) |
| 82 | { | 77 | { |
| @@ -32,7 +32,6 @@ extern pte_t *pkmap_page_table; · 31 unmodified lines | |||
| 32 | 32 | ||
| 33 | #define ARCH_HAS_KMAP_FLUSH_TLB | 33 | #define ARCH_HAS_KMAP_FLUSH_TLB |
| 34 | extern void kmap_flush_tlb(unsigned long addr); | 34 | extern void kmap_flush_tlb(unsigned long addr); |
| 35 | extern void *kmap_atomic(struct page *page); | ||
| 36 | extern void __kunmap_atomic(void *kvaddr); | 35 | extern void __kunmap_atomic(void *kvaddr); |
| 37 | extern void *kmap_atomic_pfn(unsigned long pfn); | 36 | extern void *kmap_atomic_pfn(unsigned long pfn); |
| 38 | extern struct page *kmap_atomic_to_page(void *ptr); | 37 | extern struct page *kmap_atomic_to_page(void *ptr); |
| @@ -21,16 +21,11 @@ EXPORT_SYMBOL(kmap_flush_tlb); · 20 unmodified lines | |||
| 21 | 21 | ||
| 22 | EXPORT_SYMBOL(kmap); | 22 | EXPORT_SYMBOL(kmap); |
| 23 | 23 | ||
| 24 | void *kmap_atomic(struct page *page) | 24 | void *kmap_atomic_high(struct page *page) |
| 25 | { | 25 | { |
| 26 | unsigned long vaddr; | 26 | unsigned long vaddr; |
| 27 | int idx, type; | 27 | int idx, type; |
| 28 | 28 | ||
| 29 | preempt_disable(); | ||
| 30 | pagefault_disable(); | ||
| 31 | if (!PageHighMem(page)) | ||
| 32 | return page_address(page); | ||
| 33 | |||
| 34 | type = kmap_atomic_idx_push(); | 29 | type = kmap_atomic_idx_push(); |
| 35 | idx = type + KM_TYPE_NR*smp_processor_id(); | 30 | idx = type + KM_TYPE_NR*smp_processor_id(); |
| 36 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); | 31 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); |
| @@ -42,7 +37,7 @@ void *kmap_atomic(struct page *page) · 5 unmodified lines | |||
| 42 | 37 | ||
| 43 | return (void *)vaddr; | 38 | return (void *)vaddr; |
| 44 | } | 39 | } |
| 45 | EXPORT_SYMBOL(kmap_atomic); | 40 | EXPORT_SYMBOL(kmap_atomic_high); |
| 46 | 41 | ||
| 47 | void __kunmap_atomic(void *kvaddr) | 42 | void __kunmap_atomic(void *kvaddr) |
| 48 | { | 43 | { |
| @@ -63,9 +63,9 @@ static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot) · 62 unmodified lines | |||
| 63 | } | 63 | } |
| 64 | extern void __kunmap_atomic(void *kvaddr); | 64 | extern void __kunmap_atomic(void *kvaddr); |
| 65 | 65 | ||
| 66 | static inline void *kmap_atomic(struct page *page) | 66 | static inline void *kmap_atomic_high(struct page *page) |
| 67 | { | 67 | { |
| 68 | return kmap_atomic_prot(page, kmap_prot); | 68 | return kmap_atomic_high_prot(page, kmap_prot); |
| 69 | } | 69 | } |
| 70 | 70 | ||
| 71 | #define flush_cache_kmaps() { flush_icache(); flush_dcache(); } | 71 | #define flush_cache_kmaps() { flush_icache(); flush_dcache(); } |
| @@ -48,7 +48,6 @@ extern pte_t *pkmap_page_table; · 47 unmodified lines | |||
| 48 | 48 | ||
| 49 | #define ARCH_HAS_KMAP_FLUSH_TLB | 49 | #define ARCH_HAS_KMAP_FLUSH_TLB |
| 50 | extern void kmap_flush_tlb(unsigned long addr); | 50 | extern void kmap_flush_tlb(unsigned long addr); |
| 51 | extern void *kmap_atomic(struct page *page); | ||
| 52 | extern void __kunmap_atomic(void *kvaddr); | 51 | extern void __kunmap_atomic(void *kvaddr); |
| 53 | extern void *kmap_atomic_pfn(unsigned long pfn); | 52 | extern void *kmap_atomic_pfn(unsigned long pfn); |
| 54 | 53 | ||
| @@ -14,9 +14,9 @@ · 13 unmodified lines | |||
| 14 | #include <linux/sched.h> | 14 | #include <linux/sched.h> |
| 15 | #include <linux/syscalls.h> | 15 | #include <linux/syscalls.h> |
| 16 | #include <linux/mm.h> | 16 | #include <linux/mm.h> |
| 17 | #include <linux/highmem.h> | ||
| 17 | 18 | ||
| 18 | #include <asm/cacheflush.h> | 19 | #include <asm/cacheflush.h> |
| 19 | #include <asm/highmem.h> | ||
| 20 | #include <asm/processor.h> | 20 | #include <asm/processor.h> |
| 21 | #include <asm/cpu.h> | 21 | #include <asm/cpu.h> |
| 22 | #include <asm/cpu-features.h> | 22 | #include <asm/cpu-features.h> |
| @@ -18,25 +18,11 @@ void kmap_flush_tlb(unsigned long addr) · 17 unmodified lines | |||
| 18 | } | 18 | } |
| 19 | EXPORT_SYMBOL(kmap_flush_tlb); | 19 | EXPORT_SYMBOL(kmap_flush_tlb); |
| 20 | 20 | ||
| 21 | /* | 21 | void *kmap_atomic_high(struct page *page) |
| 22 | * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because | ||
| 23 | * no global lock is needed and because the kmap code must perform a global TLB | ||
| 24 | * invalidation when the kmap pool wraps. | ||
| 25 | * | ||
| 26 | * However when holding an atomic kmap is is not legal to sleep, so atomic | ||
| 27 | * kmaps are appropriate for short, tight code paths only. | ||
| 28 | */ | ||
| 29 | |||
| 30 | void *kmap_atomic(struct page *page) | ||
| 31 | { | 22 | { |
| 32 | unsigned long vaddr; | 23 | unsigned long vaddr; |
| 33 | int idx, type; | 24 | int idx, type; |
| 34 | 25 | ||
| 35 | preempt_disable(); | ||
| 36 | pagefault_disable(); | ||
| 37 | if (!PageHighMem(page)) | ||
| 38 | return page_address(page); | ||
| 39 | |||
| 40 | type = kmap_atomic_idx_push(); | 26 | type = kmap_atomic_idx_push(); |
| 41 | idx = type + KM_TYPE_NR*smp_processor_id(); | 27 | idx = type + KM_TYPE_NR*smp_processor_id(); |
| 42 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); | 28 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); |
| @@ -48,7 +34,7 @@ void *kmap_atomic(struct page *page) · 5 unmodified lines | |||
| 48 | 34 | ||
| 49 | return (void*) vaddr; | 35 | return (void*) vaddr; |
| 50 | } | 36 | } |
| 51 | EXPORT_SYMBOL(kmap_atomic); | 37 | EXPORT_SYMBOL(kmap_atomic_high); |
| 52 | 38 | ||
| 53 | void __kunmap_atomic(void *kvaddr) | 39 | void __kunmap_atomic(void *kvaddr) |
| 54 | { | 40 | { |
| @@ -51,7 +51,6 @@ extern void kmap_init(void); · 50 unmodified lines | |||
| 51 | * when CONFIG_HIGHMEM is not set. | 51 | * when CONFIG_HIGHMEM is not set. |
| 52 | */ | 52 | */ |
| 53 | #ifdef CONFIG_HIGHMEM | 53 | #ifdef CONFIG_HIGHMEM |
| 54 | extern void *kmap_atomic(struct page *page); | ||
| 55 | extern void __kunmap_atomic(void *kvaddr); | 54 | extern void __kunmap_atomic(void *kvaddr); |
| 56 | extern void *kmap_atomic_pfn(unsigned long pfn); | 55 | extern void *kmap_atomic_pfn(unsigned long pfn); |
| 57 | extern struct page *kmap_atomic_to_page(void *ptr); | 56 | extern struct page *kmap_atomic_to_page(void *ptr); |
| @@ -10,18 +10,13 @@ · 9 unmodified lines | |||
| 10 | #include <asm/fixmap.h> | 10 | #include <asm/fixmap.h> |
| 11 | #include <asm/tlbflush.h> | 11 | #include <asm/tlbflush.h> |
| 12 | 12 | ||
| 13 | void *kmap_atomic(struct page *page) | 13 | void *kmap_atomic_high(struct page *page) |
| 14 | { | 14 | { |
| 15 | unsigned int idx; | 15 | unsigned int idx; |
| 16 | unsigned long vaddr, pte; | 16 | unsigned long vaddr, pte; |
| 17 | int type; | 17 | int type; |
| 18 | pte_t *ptep; | 18 | pte_t *ptep; |
| 19 | 19 | ||
| 20 | preempt_disable(); | ||
| 21 | pagefault_disable(); | ||
| 22 | if (!PageHighMem(page)) | ||
| 23 | return page_address(page); | ||
| 24 | |||
| 25 | type = kmap_atomic_idx_push(); | 20 | type = kmap_atomic_idx_push(); |
| 26 | 21 | ||
| 27 | idx = type + KM_TYPE_NR * smp_processor_id(); | 22 | idx = type + KM_TYPE_NR * smp_processor_id(); |
| @@ -36,8 +31,7 @@ void *kmap_atomic(struct page *page) · 8 unmodified lines | |||
| 36 | __nds32__isb(); | 31 | __nds32__isb(); |
| 37 | return (void *)vaddr; | 32 | return (void *)vaddr; |
| 38 | } | 33 | } |
| 39 | 34 | EXPORT_SYMBOL(kmap_atomic_high); | |
| 40 | EXPORT_SYMBOL(kmap_atomic); | ||
| 41 | 35 | ||
| 42 | void __kunmap_atomic(void *kvaddr) | 36 | void __kunmap_atomic(void *kvaddr) |
| 43 | { | 37 | { |
| @@ -53,5 +47,4 @@ void __kunmap_atomic(void *kvaddr) · 9 unmodified lines | |||
| 53 | pagefault_enable(); | 47 | pagefault_enable(); |
| 54 | preempt_enable(); | 48 | preempt_enable(); |
| 55 | } | 49 | } |
| 56 | |||
| 57 | EXPORT_SYMBOL(__kunmap_atomic); | 50 | EXPORT_SYMBOL(__kunmap_atomic); |
| @@ -71,9 +71,9 @@ static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot) · 70 unmodified lines | |||
| 71 | } | 71 | } |
| 72 | extern void __kunmap_atomic(void *kvaddr); | 72 | extern void __kunmap_atomic(void *kvaddr); |
| 73 | 73 | ||
| 74 | static inline void *kmap_atomic(struct page *page) | 74 | static inline void *kmap_atomic_high(struct page *page) |
| 75 | { | 75 | { |
| 76 | return kmap_atomic_prot(page, kmap_prot); | 76 | return kmap_atomic_high_prot(page, kmap_prot); |
| 77 | } | 77 | } |
| 78 | 78 | ||
| 79 | 79 | ||
| @@ -24,12 +24,6 @@ · 23 unmodified lines | |||
| 24 | #include <linux/highmem.h> | 24 | #include <linux/highmem.h> |
| 25 | #include <linux/module.h> | 25 | #include <linux/module.h> |
| 26 | 26 | ||
| 27 | /* | ||
| 28 | * The use of kmap_atomic/kunmap_atomic is discouraged - kmap/kunmap | ||
| 29 | * gives a more generic (and caching) interface. But kmap_atomic can | ||
| 30 | * be used in IRQ contexts, so in some (very limited) cases we need | ||
| 31 | * it. | ||
| 32 | */ | ||
| 33 | void *kmap_atomic_high_prot(struct page *page, pgprot_t prot) | 27 | void *kmap_atomic_high_prot(struct page *page, pgprot_t prot) |
| 34 | { | 28 | { |
| 35 | unsigned long vaddr; | 29 | unsigned long vaddr; |
| @@ -50,7 +50,6 @@ void kmap_init(void) __init; · 49 unmodified lines | |||
| 50 | 50 | ||
| 51 | #define PKMAP_END (PKMAP_ADDR(LAST_PKMAP)) | 51 | #define PKMAP_END (PKMAP_ADDR(LAST_PKMAP)) |
| 52 | 52 | ||
| 53 | void *kmap_atomic(struct page *page); | ||
| 54 | void __kunmap_atomic(void *kvaddr); | 53 | void __kunmap_atomic(void *kvaddr); |
| 55 | 54 | ||
| 56 | #define flush_cache_kmaps() flush_cache_all() | 55 | #define flush_cache_kmaps() flush_cache_all() |
| @@ -53,16 +53,11 @@ void __init kmap_init(void) · 52 unmodified lines | |||
| 53 | kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); | 53 | kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); |
| 54 | } | 54 | } |
| 55 | 55 | ||
| 56 | void *kmap_atomic(struct page *page) | 56 | void *kmap_atomic_high(struct page *page) |
| 57 | { | 57 | { |
| 58 | unsigned long vaddr; | 58 | unsigned long vaddr; |
| 59 | long idx, type; | 59 | long idx, type; |
| 60 | 60 | ||
| 61 | preempt_disable(); | ||
| 62 | pagefault_disable(); | ||
| 63 | if (!PageHighMem(page)) | ||
| 64 | return page_address(page); | ||
| 65 | |||
| 66 | type = kmap_atomic_idx_push(); | 61 | type = kmap_atomic_idx_push(); |
| 67 | idx = type + KM_TYPE_NR*smp_processor_id(); | 62 | idx = type + KM_TYPE_NR*smp_processor_id(); |
| 68 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); | 63 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); |
| @@ -87,7 +82,7 @@ void *kmap_atomic(struct page *page) · 18 unmodified lines | |||
| 87 | 82 | ||
| 88 | return (void*) vaddr; | 83 | return (void*) vaddr; |
| 89 | } | 84 | } |
| 90 | EXPORT_SYMBOL(kmap_atomic); | 85 | EXPORT_SYMBOL(kmap_atomic_high); |
| 91 | 86 | ||
| 92 | void __kunmap_atomic(void *kvaddr) | 87 | void __kunmap_atomic(void *kvaddr) |
| 93 | { | 88 | { |
| @@ -68,7 +68,10 @@ static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot) · 67 unmodified lines | |||
| 68 | 68 | ||
| 69 | return kmap_atomic_high_prot(page, prot); | 69 | return kmap_atomic_high_prot(page, prot); |
| 70 | } | 70 | } |
| 71 | void *kmap_atomic(struct page *page); | 71 | static inline void *kmap_atomic_high(struct page *page) |
| 72 | { | ||
| 73 | return kmap_atomic_high_prot(page, kmap_prot); | ||
| 74 | } | ||
| 72 | void __kunmap_atomic(void *kvaddr); | 75 | void __kunmap_atomic(void *kvaddr); |
| 73 | void *kmap_atomic_pfn(unsigned long pfn); | 76 | void *kmap_atomic_pfn(unsigned long pfn); |
| 74 | void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot); | 77 | void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot); |
| @@ -4,14 +4,6 @@ · 3 unmodified lines | |||
| 4 | #include <linux/swap.h> /* for totalram_pages */ | 4 | #include <linux/swap.h> /* for totalram_pages */ |
| 5 | #include <linux/memblock.h> | 5 | #include <linux/memblock.h> |
| 6 | 6 | ||
| 7 | /* | ||
| 8 | * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because | ||
| 9 | * no global lock is needed and because the kmap code must perform a global TLB | ||
| 10 | * invalidation when the kmap pool wraps. | ||
| 11 | * | ||
| 12 | * However when holding an atomic kmap it is not legal to sleep, so atomic | ||
| 13 | * kmaps are appropriate for short, tight code paths only. | ||
| 14 | */ | ||
| 15 | void *kmap_atomic_high_prot(struct page *page, pgprot_t prot) | 7 | void *kmap_atomic_high_prot(struct page *page, pgprot_t prot) |
| 16 | { | 8 | { |
| 17 | unsigned long vaddr; | 9 | unsigned long vaddr; |
| @@ -28,12 +20,6 @@ void *kmap_atomic_high_prot(struct page *page, pgprot_t prot) · 10 unmodified lines | |||
| 28 | } | 20 | } |
| 29 | EXPORT_SYMBOL(kmap_atomic_high_prot); | 21 | EXPORT_SYMBOL(kmap_atomic_high_prot); |
| 30 | 22 | ||
| 31 | void *kmap_atomic(struct page *page) | ||
| 32 | { | ||
| 33 | return kmap_atomic_prot(page, kmap_prot); | ||
| 34 | } | ||
| 35 | EXPORT_SYMBOL(kmap_atomic); | ||
| 36 | |||
| 37 | /* | 23 | /* |
| 38 | * This is the same as kmap_atomic() but can map memory that doesn't | 24 | * This is the same as kmap_atomic() but can map memory that doesn't |
| 39 | * have a struct page associated with it. | 25 | * have a struct page associated with it. |
| @@ -68,7 +68,6 @@ static inline void flush_cache_kmaps(void) · 67 unmodified lines | |||
| 68 | flush_cache_all(); | 68 | flush_cache_all(); |
| 69 | } | 69 | } |
| 70 | 70 | ||
| 71 | void *kmap_atomic(struct page *page); | ||
| 72 | void __kunmap_atomic(void *kvaddr); | 71 | void __kunmap_atomic(void *kvaddr); |
| 73 | 72 | ||
| 74 | void kmap_init(void); | 73 | void kmap_init(void); |
| @@ -37,16 +37,11 @@ static inline enum fixed_addresses kmap_idx(int type, unsigned long color) · 36 unmodified lines | |||
| 37 | color; | 37 | color; |
| 38 | } | 38 | } |
| 39 | 39 | ||
| 40 | void *kmap_atomic(struct page *page) | 40 | void *kmap_atomic_high(struct page *page) |
| 41 | { | 41 | { |
| 42 | enum fixed_addresses idx; | 42 | enum fixed_addresses idx; |
| 43 | unsigned long vaddr; | 43 | unsigned long vaddr; |
| 44 | 44 | ||
| 45 | preempt_disable(); | ||
| 46 | pagefault_disable(); | ||
| 47 | if (!PageHighMem(page)) | ||
| 48 | return page_address(page); | ||
| 49 | |||
| 50 | idx = kmap_idx(kmap_atomic_idx_push(), | 45 | idx = kmap_idx(kmap_atomic_idx_push(), |
| 51 | DCACHE_ALIAS(page_to_phys(page))); | 46 | DCACHE_ALIAS(page_to_phys(page))); |
| 52 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); | 47 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); |
| @@ -57,7 +52,7 @@ void *kmap_atomic(struct page *page) · 4 unmodified lines | |||
| 57 | 52 | ||
| 58 | return (void *)vaddr; | 53 | return (void *)vaddr; |
| 59 | } | 54 | } |
| 60 | EXPORT_SYMBOL(kmap_atomic); | 55 | EXPORT_SYMBOL(kmap_atomic_high); |
| 61 | 56 | ||
| 62 | void __kunmap_atomic(void *kvaddr) | 57 | void __kunmap_atomic(void *kvaddr) |
| 63 | { | 58 | { |
| @@ -32,6 +32,7 @@ static inline void invalidate_kernel_vmap_range(void *vaddr, int size) · 31 unmodified lines | |||
| 32 | #include <asm/kmap_types.h> | 32 | #include <asm/kmap_types.h> |
| 33 | 33 | ||
| 34 | #ifdef CONFIG_HIGHMEM | 34 | #ifdef CONFIG_HIGHMEM |
| 35 | extern void *kmap_atomic_high(struct page *page); | ||
| 35 | #include <asm/highmem.h> | 36 | #include <asm/highmem.h> |
| 36 | 37 | ||
| 37 | #ifndef ARCH_HAS_KMAP_FLUSH_TLB | 38 | #ifndef ARCH_HAS_KMAP_FLUSH_TLB |
| @@ -62,6 +63,28 @@ static inline void kunmap(struct page *page) · 24 unmodified lines | |||
| 62 | kunmap_high(page); | 63 | kunmap_high(page); |
| 63 | } | 64 | } |
| 64 | 65 | ||
| 66 | /* | ||
| 67 | * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because | ||
| 68 | * no global lock is needed and because the kmap code must perform a global TLB | ||
| 69 | * invalidation when the kmap pool wraps. | ||
| 70 | * | ||
| 71 | * However when holding an atomic kmap is is not legal to sleep, so atomic | ||
| 72 | * kmaps are appropriate for short, tight code paths only. | ||
| 73 | * | ||
| 74 | * The use of kmap_atomic/kunmap_atomic is discouraged - kmap/kunmap | ||
| 75 | * gives a more generic (and caching) interface. But kmap_atomic can | ||
| 76 | * be used in IRQ contexts, so in some (very limited) cases we need | ||
| 77 | * it. | ||
| 78 | */ | ||
| 79 | static inline void *kmap_atomic(struct page *page) | ||
| 80 | { | ||
| 81 | preempt_disable(); | ||
| 82 | pagefault_disable(); | ||
| 83 | if (!PageHighMem(page)) | ||
| 84 | return page_address(page); | ||
| 85 | return kmap_atomic_high(page); | ||
| 86 | } | ||
| 87 | |||
| 65 | /* declarations for linux/mm/highmem.c */ | 88 | /* declarations for linux/mm/highmem.c */ |
| 66 | unsigned int nr_free_highpages(void); | 89 | unsigned int nr_free_highpages(void); |
| 67 | extern atomic_long_t _totalhigh_pages; | 90 | extern atomic_long_t _totalhigh_pages; |