History

dma-direct: provide mmap and get_sgtable method overrides

For dma-direct we know that the DMA address is an encoding of the physical address that we can trivially decode. Use that fact to provide implementations that do not need the arch_dma_coherent_to_pfn architecture hook. Note that we still can only support mmap of non-coherent memory only if the architecture provides a way to set an uncached bit in the page tables. This must be true for architectures that use the generic remap helpers, but other architectures can also manually select it.

Christoph Hellwig committed 7 years ago

Commit 34dc0ea1 parent 4e1003a16 files · +85 −75Browse files

Unified · Split

16 files
  1. arch/arc/Kconfig +0 −1
  2. arch/arm/Kconfig +0 −1
  3. arch/arm/mm/dma-mapping.c +0 −6
  4. arch/arm64/Kconfig +0 −1
  5. arch/ia64/Kconfig +1 −1
  6. arch/ia64/kernel/dma-mapping.c +0 −6
  7. arch/microblaze/Kconfig +0 −1
  8. arch/mips/Kconfig +2 −2
  9. arch/mips/mm/dma-noncoherent.c +0 −6
  10. arch/powerpc/platforms/Kconfig.cputype +0 −1
  11. include/linux/dma-direct.h +7 −0
  12. include/linux/dma-noncoherent.h +0 −2
  13. kernel/dma/Kconfig +9 −3
  14. kernel/dma/direct.c +59 −0
  15. kernel/dma/mapping.c +7 −38
  16. kernel/dma/remap.c +0 −6
arch/arc/Kconfig+0−1
@@ -6,7 +6,6 @@ · 5 unmodified lines
6config ARC6config ARC
7 def_bool y7 def_bool y
8 select ARC_TIMERS8 select ARC_TIMERS
9 select ARCH_HAS_DMA_COHERENT_TO_PFN
10 select ARCH_HAS_DMA_PREP_COHERENT9 select ARCH_HAS_DMA_PREP_COHERENT
11 select ARCH_HAS_PTE_SPECIAL10 select ARCH_HAS_PTE_SPECIAL
12 select ARCH_HAS_SETUP_DMA_OPS11 select ARCH_HAS_SETUP_DMA_OPS
arch/arm/Kconfig+0−1
@@ -7,7 +7,6 @@ config ARM · 6 unmodified lines
7 select ARCH_HAS_BINFMT_FLAT7 select ARCH_HAS_BINFMT_FLAT
8 select ARCH_HAS_DEBUG_VIRTUAL if MMU8 select ARCH_HAS_DEBUG_VIRTUAL if MMU
9 select ARCH_HAS_DEVMEM_IS_ALLOWED9 select ARCH_HAS_DEVMEM_IS_ALLOWED
10 select ARCH_HAS_DMA_COHERENT_TO_PFN if SWIOTLB
11 select ARCH_HAS_DMA_WRITE_COMBINE if !ARM_DMA_MEM_BUFFERABLE10 select ARCH_HAS_DMA_WRITE_COMBINE if !ARM_DMA_MEM_BUFFERABLE
12 select ARCH_HAS_ELF_RANDOMIZE11 select ARCH_HAS_ELF_RANDOMIZE
13 select ARCH_HAS_FORTIFY_SOURCE12 select ARCH_HAS_FORTIFY_SOURCE
arch/arm/mm/dma-mapping.c+0−6
@@ -2346,12 +2346,6 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, · 2345 unmodified lines
2346 size, dir);2346 size, dir);
2347}2347}
2348 2348
2349long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
2350 dma_addr_t dma_addr)
2351{
2352 return dma_to_pfn(dev, dma_addr);
2353}
2354
2355void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,2349void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
2356 gfp_t gfp, unsigned long attrs)2350 gfp_t gfp, unsigned long attrs)
2357{2351{
arch/arm64/Kconfig+0−1
@@ -12,7 +12,6 @@ config ARM64 · 11 unmodified lines
12 select ARCH_CLOCKSOURCE_DATA12 select ARCH_CLOCKSOURCE_DATA
13 select ARCH_HAS_DEBUG_VIRTUAL13 select ARCH_HAS_DEBUG_VIRTUAL
14 select ARCH_HAS_DEVMEM_IS_ALLOWED14 select ARCH_HAS_DEVMEM_IS_ALLOWED
15 select ARCH_HAS_DMA_COHERENT_TO_PFN
16 select ARCH_HAS_DMA_PREP_COHERENT15 select ARCH_HAS_DMA_PREP_COHERENT
17 select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI16 select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI
18 select ARCH_HAS_FAST_MULTIPLIER17 select ARCH_HAS_FAST_MULTIPLIER
arch/ia64/Kconfig+1−1
@@ -33,7 +33,7 @@ config IA64 · 32 unmodified lines
33 select HAVE_ARCH_TRACEHOOK33 select HAVE_ARCH_TRACEHOOK
34 select HAVE_MEMBLOCK_NODE_MAP34 select HAVE_MEMBLOCK_NODE_MAP
35 select HAVE_VIRT_CPU_ACCOUNTING35 select HAVE_VIRT_CPU_ACCOUNTING
36 select ARCH_HAS_DMA_COHERENT_TO_PFN36 select DMA_NONCOHERENT_MMAP
37 select ARCH_HAS_SYNC_DMA_FOR_CPU37 select ARCH_HAS_SYNC_DMA_FOR_CPU
38 select VIRT_TO_BUS38 select VIRT_TO_BUS
39 select GENERIC_IRQ_PROBE39 select GENERIC_IRQ_PROBE
arch/ia64/kernel/dma-mapping.c+0−6
@@ -19,9 +19,3 @@ void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, · 18 unmodified lines
19{19{
20 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);20 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
21}21}
22
23long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
24 dma_addr_t dma_addr)
25{
26 return page_to_pfn(virt_to_page(cpu_addr));
27}
arch/microblaze/Kconfig+0−1
@@ -4,7 +4,6 @@ config MICROBLAZE · 3 unmodified lines
4 select ARCH_32BIT_OFF_T4 select ARCH_32BIT_OFF_T
5 select ARCH_NO_SWAP5 select ARCH_NO_SWAP
6 select ARCH_HAS_BINFMT_FLAT if !MMU6 select ARCH_HAS_BINFMT_FLAT if !MMU
7 select ARCH_HAS_DMA_COHERENT_TO_PFN if MMU
8 select ARCH_HAS_DMA_PREP_COHERENT7 select ARCH_HAS_DMA_PREP_COHERENT
9 select ARCH_HAS_GCOV_PROFILE_ALL8 select ARCH_HAS_GCOV_PROFILE_ALL
10 select ARCH_HAS_SYNC_DMA_FOR_CPU9 select ARCH_HAS_SYNC_DMA_FOR_CPU
arch/mips/Kconfig+2−2
@@ -1134,9 +1134,9 @@ config DMA_NONCOHERENT · 1133 unmodified lines
1134 select ARCH_HAS_DMA_WRITE_COMBINE1134 select ARCH_HAS_DMA_WRITE_COMBINE
1135 select ARCH_HAS_SYNC_DMA_FOR_DEVICE1135 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
1136 select ARCH_HAS_UNCACHED_SEGMENT1136 select ARCH_HAS_UNCACHED_SEGMENT
1137 select NEED_DMA_MAP_STATE1137 select DMA_NONCOHERENT_MMAP
1138 select ARCH_HAS_DMA_COHERENT_TO_PFN
1139 select DMA_NONCOHERENT_CACHE_SYNC1138 select DMA_NONCOHERENT_CACHE_SYNC
1139 select NEED_DMA_MAP_STATE
1140 1140
1141config SYS_HAS_EARLY_PRINTK1141config SYS_HAS_EARLY_PRINTK
1142 bool1142 bool
arch/mips/mm/dma-noncoherent.c+0−6
@@ -59,12 +59,6 @@ void *cached_kernel_address(void *addr) · 58 unmodified lines
59 return __va(addr) - UNCAC_BASE;59 return __va(addr) - UNCAC_BASE;
60}60}
61 61
62long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
63 dma_addr_t dma_addr)
64{
65 return page_to_pfn(virt_to_page(cached_kernel_address(cpu_addr)));
66}
67
68static inline void dma_sync_virt(void *addr, size_t size,62static inline void dma_sync_virt(void *addr, size_t size,
69 enum dma_data_direction dir)63 enum dma_data_direction dir)
70{64{
arch/powerpc/platforms/Kconfig.cputype+0−1
@@ -459,7 +459,6 @@ config NOT_COHERENT_CACHE · 458 unmodified lines
459 bool459 bool
460 depends on 4xx || PPC_8xx || E200 || PPC_MPC512x || \460 depends on 4xx || PPC_8xx || E200 || PPC_MPC512x || \
461 GAMECUBE_COMMON || AMIGAONE461 GAMECUBE_COMMON || AMIGAONE
462 select ARCH_HAS_DMA_COHERENT_TO_PFN
463 select ARCH_HAS_DMA_PREP_COHERENT462 select ARCH_HAS_DMA_PREP_COHERENT
464 select ARCH_HAS_SYNC_DMA_FOR_DEVICE463 select ARCH_HAS_SYNC_DMA_FOR_DEVICE
465 select ARCH_HAS_SYNC_DMA_FOR_CPU464 select ARCH_HAS_SYNC_DMA_FOR_CPU
include/linux/dma-direct.h+7−0
@@ -68,5 +68,12 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, · 67 unmodified lines
68 dma_addr_t dma_addr, unsigned long attrs);68 dma_addr_t dma_addr, unsigned long attrs);
69struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,69struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
70 gfp_t gfp, unsigned long attrs);70 gfp_t gfp, unsigned long attrs);
71int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,
72 void *cpu_addr, dma_addr_t dma_addr, size_t size,
73 unsigned long attrs);
74bool dma_direct_can_mmap(struct device *dev);
75int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
76 void *cpu_addr, dma_addr_t dma_addr, size_t size,
77 unsigned long attrs);
71int dma_direct_supported(struct device *dev, u64 mask);78int dma_direct_supported(struct device *dev, u64 mask);
72#endif /* _LINUX_DMA_DIRECT_H */79#endif /* _LINUX_DMA_DIRECT_H */
include/linux/dma-noncoherent.h+0−2
@@ -41,8 +41,6 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, · 40 unmodified lines
41 gfp_t gfp, unsigned long attrs);41 gfp_t gfp, unsigned long attrs);
42void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,42void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
43 dma_addr_t dma_addr, unsigned long attrs);43 dma_addr_t dma_addr, unsigned long attrs);
44long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
45 dma_addr_t dma_addr);
46 44
47#ifdef CONFIG_MMU45#ifdef CONFIG_MMU
48/*46/*
kernel/dma/Kconfig+9−3
@@ -51,9 +51,6 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL · 50 unmodified lines
51config ARCH_HAS_DMA_PREP_COHERENT51config ARCH_HAS_DMA_PREP_COHERENT
52 bool52 bool
53 53
54config ARCH_HAS_DMA_COHERENT_TO_PFN
55 bool
56
57config ARCH_HAS_FORCE_DMA_UNENCRYPTED54config ARCH_HAS_FORCE_DMA_UNENCRYPTED
58 bool55 bool
59 56
@@ -68,9 +65,18 @@ config SWIOTLB · 8 unmodified lines
68 bool65 bool
69 select NEED_DMA_MAP_STATE66 select NEED_DMA_MAP_STATE
70 67
68#
69# Should be selected if we can mmap non-coherent mappings to userspace.
70# The only thing that is really required is a way to set an uncached bit
71# in the pagetables
72#
73config DMA_NONCOHERENT_MMAP
74 bool
75
71config DMA_REMAP76config DMA_REMAP
72 depends on MMU77 depends on MMU
73 select GENERIC_ALLOCATOR78 select GENERIC_ALLOCATOR
79 select DMA_NONCOHERENT_MMAP
74 bool80 bool
75 81
76config DMA_DIRECT_REMAP82config DMA_DIRECT_REMAP
kernel/dma/direct.c+59−0
@@ -43,6 +43,12 @@ static inline dma_addr_t phys_to_dma_direct(struct device *dev, · 42 unmodified lines
43 return phys_to_dma(dev, phys);43 return phys_to_dma(dev, phys);
44}44}
45 45
46static inline struct page *dma_direct_to_page(struct device *dev,
47 dma_addr_t dma_addr)
48{
49 return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr)));
50}
51
46u64 dma_direct_get_required_mask(struct device *dev)52u64 dma_direct_get_required_mask(struct device *dev)
47{53{
48 u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT);54 u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT);
@@ -379,6 +385,59 @@ dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, · 330 unmodified lines
379}385}
380EXPORT_SYMBOL(dma_direct_map_resource);386EXPORT_SYMBOL(dma_direct_map_resource);
381 387
388int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,
389 void *cpu_addr, dma_addr_t dma_addr, size_t size,
390 unsigned long attrs)
391{
392 struct page *page = dma_direct_to_page(dev, dma_addr);
393 int ret;
394
395 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
396 if (!ret)
397 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
398 return ret;
399}
400
401#ifdef CONFIG_MMU
402bool dma_direct_can_mmap(struct device *dev)
403{
404 return dev_is_dma_coherent(dev) ||
405 IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP);
406}
407
408int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
409 void *cpu_addr, dma_addr_t dma_addr, size_t size,
410 unsigned long attrs)
411{
412 unsigned long user_count = vma_pages(vma);
413 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
414 unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr));
415 int ret = -ENXIO;
416
417 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
418
419 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
420 return ret;
421
422 if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff)
423 return -ENXIO;
424 return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
425 user_count << PAGE_SHIFT, vma->vm_page_prot);
426}
427#else /* CONFIG_MMU */
428bool dma_direct_can_mmap(struct device *dev)
429{
430 return false;
431}
432
433int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
434 void *cpu_addr, dma_addr_t dma_addr, size_t size,
435 unsigned long attrs)
436{
437 return -ENXIO;
438}
439#endif /* CONFIG_MMU */
440
382/*441/*
383 * Because 32-bit DMA masks are so common we expect every architecture to be442 * Because 32-bit DMA masks are so common we expect every architecture to be
384 * able to satisfy them - either by not supporting more physical memory, or by443 * able to satisfy them - either by not supporting more physical memory, or by
kernel/dma/mapping.c+7−38
@@ -112,24 +112,9 @@ int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, · 111 unmodified lines
112 void *cpu_addr, dma_addr_t dma_addr, size_t size,112 void *cpu_addr, dma_addr_t dma_addr, size_t size,
113 unsigned long attrs)113 unsigned long attrs)
114{114{
115 struct page *page;115 struct page *page = virt_to_page(cpu_addr);
116 int ret;116 int ret;
117 117
118 if (!dev_is_dma_coherent(dev)) {
119 unsigned long pfn;
120
121 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
122 return -ENXIO;
123
124 /* If the PFN is not valid, we do not have a struct page */
125 pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
126 if (!pfn_valid(pfn))
127 return -ENXIO;
128 page = pfn_to_page(pfn);
129 } else {
130 page = virt_to_page(cpu_addr);
131 }
132
133 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);118 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
134 if (!ret)119 if (!ret)
135 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);120 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
@@ -154,7 +139,7 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, · 18 unmodified lines
154 const struct dma_map_ops *ops = get_dma_ops(dev);139 const struct dma_map_ops *ops = get_dma_ops(dev);
155 140
156 if (dma_is_direct(ops))141 if (dma_is_direct(ops))
157 return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr,142 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
158 size, attrs);143 size, attrs);
159 if (!ops->get_sgtable)144 if (!ops->get_sgtable)
160 return -ENXIO;145 return -ENXIO;
@@ -192,7 +177,6 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, · 31 unmodified lines
192 unsigned long user_count = vma_pages(vma);177 unsigned long user_count = vma_pages(vma);
193 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;178 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
194 unsigned long off = vma->vm_pgoff;179 unsigned long off = vma->vm_pgoff;
195 unsigned long pfn;
196 int ret = -ENXIO;180 int ret = -ENXIO;
197 181
198 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);182 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
@@ -203,19 +187,8 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, · 4 unmodified lines
203 if (off >= count || user_count > count - off)187 if (off >= count || user_count > count - off)
204 return -ENXIO;188 return -ENXIO;
205 189
206 if (!dev_is_dma_coherent(dev)) {190 return remap_pfn_range(vma, vma->vm_start,
207 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))191 page_to_pfn(virt_to_page(cpu_addr)) + vma->vm_pgoff,
208 return -ENXIO;
209
210 /* If the PFN is not valid, we do not have a struct page */
211 pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
212 if (!pfn_valid(pfn))
213 return -ENXIO;
214 } else {
215 pfn = page_to_pfn(virt_to_page(cpu_addr));
216 }
217
218 return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
219 user_count << PAGE_SHIFT, vma->vm_page_prot);192 user_count << PAGE_SHIFT, vma->vm_page_prot);
220#else193#else
221 return -ENXIO;194 return -ENXIO;
@@ -233,12 +206,8 @@ bool dma_can_mmap(struct device *dev) · 11 unmodified lines
233{206{
234 const struct dma_map_ops *ops = get_dma_ops(dev);207 const struct dma_map_ops *ops = get_dma_ops(dev);
235 208
236 if (dma_is_direct(ops)) {209 if (dma_is_direct(ops))
237 return IS_ENABLED(CONFIG_MMU) &&210 return dma_direct_can_mmap(dev);
238 (dev_is_dma_coherent(dev) ||
239 IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN));
240 }
241
242 return ops->mmap != NULL;211 return ops->mmap != NULL;
243}212}
244EXPORT_SYMBOL_GPL(dma_can_mmap);213EXPORT_SYMBOL_GPL(dma_can_mmap);
@@ -263,7 +232,7 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, · 18 unmodified lines
263 const struct dma_map_ops *ops = get_dma_ops(dev);232 const struct dma_map_ops *ops = get_dma_ops(dev);
264 233
265 if (dma_is_direct(ops))234 if (dma_is_direct(ops))
266 return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size,235 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
267 attrs);236 attrs);
268 if (!ops->mmap)237 if (!ops->mmap)
269 return -ENXIO;238 return -ENXIO;
kernel/dma/remap.c+0−6
@@ -259,10 +259,4 @@ void arch_dma_free(struct device *dev, size_t size, void *vaddr, · 258 unmodified lines
259 dma_free_contiguous(dev, page, size);259 dma_free_contiguous(dev, page, size);
260 }260 }
261}261}
262
263long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
264 dma_addr_t dma_addr)
265{
266 return __phys_to_pfn(dma_to_phys(dev, dma_addr));
267}
268#endif /* CONFIG_DMA_DIRECT_REMAP */262#endif /* CONFIG_DMA_DIRECT_REMAP */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t