History

Merge tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping updates from Christoph Hellwig: - make support for dma_ops optional - move more code out of line - add generic support for a dma_ops bypass mode - misc cleanups * tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping: dma-contiguous: cleanup dma_alloc_contiguous dma-debug: use named initializers for dir2name powerpc: use the generic dma_ops_bypass mode dma-mapping: add a dma_ops_bypass flag to struct device dma-mapping: make support for dma ops optional dma-mapping: inline the fast path dma-direct calls dma-mapping: move the remaining DMA API calls out of line

Linus Torvalds committed 6 years ago

Commit 2ed90db2 parents 9fa867d 274b3f726 files · +415 −416Browse files

Unified · Split

26 files
  1. arch/alpha/Kconfig +1 −0
  2. arch/arm/Kconfig +1 −0
  3. arch/ia64/Kconfig +1 −0
  4. arch/mips/Kconfig +1 −0
  5. arch/parisc/Kconfig +1 −0
  6. arch/powerpc/Kconfig +2 −0
  7. arch/powerpc/include/asm/device.h +0 −5
  8. arch/powerpc/kernel/dma-iommu.c +9 −81
  9. arch/s390/Kconfig +1 −0
  10. arch/sparc/Kconfig +1 −0
  11. arch/x86/Kconfig +1 −0
  12. drivers/infiniband/core/device.c +5 −1
  13. drivers/iommu/Kconfig +2 −0
  14. drivers/macintosh/macio_asic.c +2 −2
  15. drivers/misc/mic/Kconfig +4 −0
  16. drivers/vdpa/Kconfig +1 −0
  17. drivers/xen/Kconfig +1 −0
  18. include/linux/device.h +10 −1
  19. include/linux/dma-direct.h +104 −0
  20. include/linux/dma-mapping.h +29 −222
  21. kernel/dma/Kconfig +12 −0
  22. kernel/dma/Makefile +2 −1
  23. kernel/dma/contiguous.c +14 −17
  24. kernel/dma/debug.c +6 −2
  25. kernel/dma/direct.c +0 −74
  26. kernel/dma/mapping.c +204 −10
arch/alpha/Kconfig+1−0
@@ -7,6 +7,7 @@ config ALPHA · 6 unmodified lines
7 select ARCH_NO_PREEMPT7 select ARCH_NO_PREEMPT
8 select ARCH_NO_SG_CHAIN8 select ARCH_NO_SG_CHAIN
9 select ARCH_USE_CMPXCHG_LOCKREF9 select ARCH_USE_CMPXCHG_LOCKREF
10 select DMA_OPS if PCI
10 select FORCE_PCI if !ALPHA_JENSEN11 select FORCE_PCI if !ALPHA_JENSEN
11 select PCI_DOMAINS if PCI12 select PCI_DOMAINS if PCI
12 select PCI_SYSCALL if PCI13 select PCI_SYSCALL if PCI
arch/arm/Kconfig+1−0
@@ -41,6 +41,7 @@ config ARM · 40 unmodified lines
41 select CPU_PM if SUSPEND || CPU_IDLE41 select CPU_PM if SUSPEND || CPU_IDLE
42 select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS42 select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS
43 select DMA_DECLARE_COHERENT43 select DMA_DECLARE_COHERENT
44 select DMA_OPS
44 select DMA_REMAP if MMU45 select DMA_REMAP if MMU
45 select EDAC_SUPPORT46 select EDAC_SUPPORT
46 select EDAC_ATOMIC_SCRUB47 select EDAC_ATOMIC_SCRUB
arch/ia64/Kconfig+1−0
@@ -192,6 +192,7 @@ config IA64_SGI_UV · 191 unmodified lines
192 192
193config IA64_HP_SBA_IOMMU193config IA64_HP_SBA_IOMMU
194 bool "HP SBA IOMMU support"194 bool "HP SBA IOMMU support"
195 select DMA_OPS
195 default y196 default y
196 help197 help
197 Say Y here to add support for the SBA IOMMU found on HP zx1 and198 Say Y here to add support for the SBA IOMMU found on HP zx1 and
arch/mips/Kconfig+1−0
@@ -366,6 +366,7 @@ config MACH_JAZZ · 365 unmodified lines
366 select ARC_PROMLIB366 select ARC_PROMLIB
367 select ARCH_MIGHT_HAVE_PC_PARPORT367 select ARCH_MIGHT_HAVE_PC_PARPORT
368 select ARCH_MIGHT_HAVE_PC_SERIO368 select ARCH_MIGHT_HAVE_PC_SERIO
369 select DMA_OPS
369 select FW_ARC370 select FW_ARC
370 select FW_ARC32371 select FW_ARC32
371 select ARCH_MAY_HAVE_PC_FDC372 select ARCH_MAY_HAVE_PC_FDC
arch/parisc/Kconfig+1−0
@@ -14,6 +14,7 @@ config PARISC · 13 unmodified lines
14 select ARCH_HAS_UBSAN_SANITIZE_ALL14 select ARCH_HAS_UBSAN_SANITIZE_ALL
15 select ARCH_NO_SG_CHAIN15 select ARCH_NO_SG_CHAIN
16 select ARCH_SUPPORTS_MEMORY_FAILURE16 select ARCH_SUPPORTS_MEMORY_FAILURE
17 select DMA_OPS
17 select RTC_CLASS18 select RTC_CLASS
18 select RTC_DRV_GENERIC19 select RTC_DRV_GENERIC
19 select INIT_ALL_POSSIBLE20 select INIT_ALL_POSSIBLE
arch/powerpc/Kconfig+2−0
@@ -151,6 +151,8 @@ config PPC · 150 unmodified lines
151 select BUILDTIME_TABLE_SORT151 select BUILDTIME_TABLE_SORT
152 select CLONE_BACKWARDS152 select CLONE_BACKWARDS
153 select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN153 select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN
154 select DMA_OPS if PPC64
155 select DMA_OPS_BYPASS if PPC64
154 select DYNAMIC_FTRACE if FUNCTION_TRACER156 select DYNAMIC_FTRACE if FUNCTION_TRACER
155 select EDAC_ATOMIC_SCRUB157 select EDAC_ATOMIC_SCRUB
156 select EDAC_SUPPORT158 select EDAC_SUPPORT
arch/powerpc/include/asm/device.h+0−5
@@ -18,11 +18,6 @@ struct iommu_table; · 17 unmodified lines
18 * drivers/macintosh/macio_asic.c18 * drivers/macintosh/macio_asic.c
19 */19 */
20struct dev_archdata {20struct dev_archdata {
21 /*
22 * Set to %true if the dma_iommu_ops are requested to use a direct
23 * window instead of dynamically mapping memory.
24 */
25 bool iommu_bypass : 1;
26 /*21 /*
27 * These two used to be a union. However, with the hybrid ops we need22 * These two used to be a union. However, with the hybrid ops we need
28 * both so here we store both a DMA offset for direct mappings and23 * both so here we store both a DMA offset for direct mappings and
arch/powerpc/kernel/dma-iommu.c+9−81
@@ -14,23 +14,6 @@ · 13 unmodified lines
14 * Generic iommu implementation14 * Generic iommu implementation
15 */15 */
16 16
17/*
18 * The coherent mask may be smaller than the real mask, check if we can
19 * really use a direct window.
20 */
21static inline bool dma_iommu_alloc_bypass(struct device *dev)
22{
23 return dev->archdata.iommu_bypass && !iommu_fixed_is_weak &&
24 dma_direct_supported(dev, dev->coherent_dma_mask);
25}
26
27static inline bool dma_iommu_map_bypass(struct device *dev,
28 unsigned long attrs)
29{
30 return dev->archdata.iommu_bypass &&
31 (!iommu_fixed_is_weak || (attrs & DMA_ATTR_WEAK_ORDERING));
32}
33
34/* Allocates a contiguous real buffer and creates mappings over it.17/* Allocates a contiguous real buffer and creates mappings over it.
35 * Returns the virtual address of the buffer and sets dma_handle18 * Returns the virtual address of the buffer and sets dma_handle
36 * to the dma address (mapping) of the first page.19 * to the dma address (mapping) of the first page.
@@ -39,8 +22,6 @@ static void *dma_iommu_alloc_coherent(struct device *dev, size_t size, · 2 unmodified lines
39 dma_addr_t *dma_handle, gfp_t flag,22 dma_addr_t *dma_handle, gfp_t flag,
40 unsigned long attrs)23 unsigned long attrs)
41{24{
42 if (dma_iommu_alloc_bypass(dev))
43 return dma_direct_alloc(dev, size, dma_handle, flag, attrs);
44 return iommu_alloc_coherent(dev, get_iommu_table_base(dev), size,25 return iommu_alloc_coherent(dev, get_iommu_table_base(dev), size,
45 dma_handle, dev->coherent_dma_mask, flag,26 dma_handle, dev->coherent_dma_mask, flag,
46 dev_to_node(dev));27 dev_to_node(dev));
@@ -50,11 +31,7 @@ static void dma_iommu_free_coherent(struct device *dev, size_t size, · 3 unmodified lines
50 void *vaddr, dma_addr_t dma_handle,31 void *vaddr, dma_addr_t dma_handle,
51 unsigned long attrs)32 unsigned long attrs)
52{33{
53 if (dma_iommu_alloc_bypass(dev))34 iommu_free_coherent(get_iommu_table_base(dev), size, vaddr, dma_handle);
54 dma_direct_free(dev, size, vaddr, dma_handle, attrs);
55 else
56 iommu_free_coherent(get_iommu_table_base(dev), size, vaddr,
57 dma_handle);
58}35}
59 36
60/* Creates TCEs for a user provided buffer. The user buffer must be37/* Creates TCEs for a user provided buffer. The user buffer must be
@@ -67,9 +44,6 @@ static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page, · 6 unmodified lines
67 enum dma_data_direction direction,44 enum dma_data_direction direction,
68 unsigned long attrs)45 unsigned long attrs)
69{46{
70 if (dma_iommu_map_bypass(dev, attrs))
71 return dma_direct_map_page(dev, page, offset, size, direction,
72 attrs);
73 return iommu_map_page(dev, get_iommu_table_base(dev), page, offset,47 return iommu_map_page(dev, get_iommu_table_base(dev), page, offset,
74 size, dma_get_mask(dev), direction, attrs);48 size, dma_get_mask(dev), direction, attrs);
75}49}
@@ -79,11 +53,8 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, · 3 unmodified lines
79 size_t size, enum dma_data_direction direction,53 size_t size, enum dma_data_direction direction,
80 unsigned long attrs)54 unsigned long attrs)
81{55{
82 if (!dma_iommu_map_bypass(dev, attrs))56 iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction,
83 iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size,57 attrs);
84 direction, attrs);
85 else
86 dma_direct_unmap_page(dev, dma_handle, size, direction, attrs);
87}58}
88 59
89 60
@@ -91,8 +62,6 @@ static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, · 1 unmodified line
91 int nelems, enum dma_data_direction direction,62 int nelems, enum dma_data_direction direction,
92 unsigned long attrs)63 unsigned long attrs)
93{64{
94 if (dma_iommu_map_bypass(dev, attrs))
95 return dma_direct_map_sg(dev, sglist, nelems, direction, attrs);
96 return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems,65 return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems,
97 dma_get_mask(dev), direction, attrs);66 dma_get_mask(dev), direction, attrs);
98}67}
@@ -101,11 +70,8 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, · 2 unmodified lines
101 int nelems, enum dma_data_direction direction,70 int nelems, enum dma_data_direction direction,
102 unsigned long attrs)71 unsigned long attrs)
103{72{
104 if (!dma_iommu_map_bypass(dev, attrs))73 ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
105 ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
106 direction, attrs);74 direction, attrs);
107 else
108 dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs);
109}75}
110 76
111static bool dma_iommu_bypass_supported(struct device *dev, u64 mask)77static bool dma_iommu_bypass_supported(struct device *dev, u64 mask)
@@ -113,8 +79,9 @@ static bool dma_iommu_bypass_supported(struct device *dev, u64 mask) · 1 unmodified line
113 struct pci_dev *pdev = to_pci_dev(dev);79 struct pci_dev *pdev = to_pci_dev(dev);
114 struct pci_controller *phb = pci_bus_to_host(pdev->bus);80 struct pci_controller *phb = pci_bus_to_host(pdev->bus);
115 81
116 return phb->controller_ops.iommu_bypass_supported &&82 if (iommu_fixed_is_weak || !phb->controller_ops.iommu_bypass_supported)
117 phb->controller_ops.iommu_bypass_supported(pdev, mask);83 return false;
84 return phb->controller_ops.iommu_bypass_supported(pdev, mask);
118}85}
119 86
120/* We support DMA to/from any memory page via the iommu */87/* We support DMA to/from any memory page via the iommu */
@@ -123,7 +90,7 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask) · 2 unmodified lines
123 struct iommu_table *tbl = get_iommu_table_base(dev);90 struct iommu_table *tbl = get_iommu_table_base(dev);
124 91
125 if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {92 if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {
126 dev->archdata.iommu_bypass = true;93 dev->dma_ops_bypass = true;
127 dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");94 dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
128 return 1;95 return 1;
129 }96 }
@@ -141,7 +108,7 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask) · 11 unmodified lines
141 }108 }
142 109
143 dev_dbg(dev, "iommu: not 64-bit, using default ops\n");110 dev_dbg(dev, "iommu: not 64-bit, using default ops\n");
144 dev->archdata.iommu_bypass = false;111 dev->dma_ops_bypass = false;
145 return 1;112 return 1;
146}113}
147 114
@@ -153,47 +120,12 @@ u64 dma_iommu_get_required_mask(struct device *dev) · 5 unmodified lines
153 if (!tbl)120 if (!tbl)
154 return 0;121 return 0;
155 122
156 if (dev_is_pci(dev)) {
157 u64 bypass_mask = dma_direct_get_required_mask(dev);
158
159 if (dma_iommu_bypass_supported(dev, bypass_mask))
160 return bypass_mask;
161 }
162
163 mask = 1ULL < (fls_long(tbl->it_offset + tbl->it_size) - 1);123 mask = 1ULL < (fls_long(tbl->it_offset + tbl->it_size) - 1);
164 mask += mask - 1;124 mask += mask - 1;
165 125
166 return mask;126 return mask;
167}127}
168 128
169static void dma_iommu_sync_for_cpu(struct device *dev, dma_addr_t addr,
170 size_t size, enum dma_data_direction dir)
171{
172 if (dma_iommu_alloc_bypass(dev))
173 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
174}
175
176static void dma_iommu_sync_for_device(struct device *dev, dma_addr_t addr,
177 size_t sz, enum dma_data_direction dir)
178{
179 if (dma_iommu_alloc_bypass(dev))
180 dma_direct_sync_single_for_device(dev, addr, sz, dir);
181}
182
183extern void dma_iommu_sync_sg_for_cpu(struct device *dev,
184 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
185{
186 if (dma_iommu_alloc_bypass(dev))
187 dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir);
188}
189
190extern void dma_iommu_sync_sg_for_device(struct device *dev,
191 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
192{
193 if (dma_iommu_alloc_bypass(dev))
194 dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
195}
196
197const struct dma_map_ops dma_iommu_ops = {129const struct dma_map_ops dma_iommu_ops = {
198 .alloc = dma_iommu_alloc_coherent,130 .alloc = dma_iommu_alloc_coherent,
199 .free = dma_iommu_free_coherent,131 .free = dma_iommu_free_coherent,
@@ -203,10 +135,6 @@ const struct dma_map_ops dma_iommu_ops = { · 3 unmodified lines
203 .map_page = dma_iommu_map_page,135 .map_page = dma_iommu_map_page,
204 .unmap_page = dma_iommu_unmap_page,136 .unmap_page = dma_iommu_unmap_page,
205 .get_required_mask = dma_iommu_get_required_mask,137 .get_required_mask = dma_iommu_get_required_mask,
206 .sync_single_for_cpu = dma_iommu_sync_for_cpu,
207 .sync_single_for_device = dma_iommu_sync_for_device,
208 .sync_sg_for_cpu = dma_iommu_sync_sg_for_cpu,
209 .sync_sg_for_device = dma_iommu_sync_sg_for_device,
210 .mmap = dma_common_mmap,138 .mmap = dma_common_mmap,
211 .get_sgtable = dma_common_get_sgtable,139 .get_sgtable = dma_common_get_sgtable,
212};140};
arch/s390/Kconfig+1−0
@@ -112,6 +112,7 @@ config S390 · 111 unmodified lines
112 select ARCH_WANT_IPC_PARSE_VERSION112 select ARCH_WANT_IPC_PARSE_VERSION
113 select BUILDTIME_TABLE_SORT113 select BUILDTIME_TABLE_SORT
114 select CLONE_BACKWARDS2114 select CLONE_BACKWARDS2
115 select DMA_OPS if PCI
115 select DYNAMIC_FTRACE if FUNCTION_TRACER116 select DYNAMIC_FTRACE if FUNCTION_TRACER
116 select GENERIC_CLOCKEVENTS117 select GENERIC_CLOCKEVENTS
117 select GENERIC_CPU_AUTOPROBE118 select GENERIC_CPU_AUTOPROBE
arch/sparc/Kconfig+1−0
@@ -15,6 +15,7 @@ config SPARC · 14 unmodified lines
15 default y15 default y
16 select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI16 select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI
17 select ARCH_MIGHT_HAVE_PC_SERIO17 select ARCH_MIGHT_HAVE_PC_SERIO
18 select DMA_OPS
18 select OF19 select OF
19 select OF_PROMTREE20 select OF_PROMTREE
20 select HAVE_ASM_MODVERSIONS21 select HAVE_ASM_MODVERSIONS
arch/x86/Kconfig+1−0
@@ -909,6 +909,7 @@ config DMI · 908 unmodified lines
909 909
910config GART_IOMMU910config GART_IOMMU
911 bool "Old AMD GART IOMMU support"911 bool "Old AMD GART IOMMU support"
912 select DMA_OPS
912 select IOMMU_HELPER913 select IOMMU_HELPER
913 select SWIOTLB914 select SWIOTLB
914 depends on X86_64 && PCI && AMD_NB915 depends on X86_64 && PCI && AMD_NB
drivers/infiniband/core/device.c+5−1
@@ -1183,6 +1183,8 @@ static void setup_dma_device(struct ib_device *device) · 1182 unmodified lines
1183 struct device *parent = device->dev.parent;1183 struct device *parent = device->dev.parent;
1184 1184
1185 WARN_ON_ONCE(device->dma_device);1185 WARN_ON_ONCE(device->dma_device);
1186
1187#ifdef CONFIG_DMA_OPS
1186 if (device->dev.dma_ops) {1188 if (device->dev.dma_ops) {
1187 /*1189 /*
1188 * The caller provided custom DMA operations. Copy the1190 * The caller provided custom DMA operations. Copy the
@@ -1203,7 +1205,9 @@ static void setup_dma_device(struct ib_device *device) · 14 unmodified lines
1203 else1205 else
1204 WARN_ON_ONCE(true);1206 WARN_ON_ONCE(true);
1205 }1207 }
1206 } else {1208 } else
1209#endif /* CONFIG_DMA_OPS */
1210 {
1207 /*1211 /*
1208 * The caller did not provide custom DMA operations. Use the1212 * The caller did not provide custom DMA operations. Use the
1209 * DMA mapping operations of the parent device.1213 * DMA mapping operations of the parent device.
drivers/iommu/Kconfig+2−0
@@ -97,6 +97,7 @@ config OF_IOMMU · 96 unmodified lines
97# IOMMU-agnostic DMA-mapping layer97# IOMMU-agnostic DMA-mapping layer
98config IOMMU_DMA98config IOMMU_DMA
99 bool99 bool
100 select DMA_OPS
100 select IOMMU_API101 select IOMMU_API
101 select IOMMU_IOVA102 select IOMMU_IOVA
102 select IRQ_MSI_IOMMU103 select IRQ_MSI_IOMMU
@@ -183,6 +184,7 @@ config DMAR_TABLE · 80 unmodified lines
183config INTEL_IOMMU184config INTEL_IOMMU
184 bool "Support for Intel IOMMU using DMA Remapping Devices"185 bool "Support for Intel IOMMU using DMA Remapping Devices"
185 depends on PCI_MSI && ACPI && (X86 || IA64)186 depends on PCI_MSI && ACPI && (X86 || IA64)
187 select DMA_OPS
186 select IOMMU_API188 select IOMMU_API
187 select IOMMU_IOVA189 select IOMMU_IOVA
188 select NEED_DMA_MAP_STATE190 select NEED_DMA_MAP_STATE
drivers/macintosh/macio_asic.c+2−2
@@ -382,7 +382,7 @@ static struct macio_dev * macio_add_one_device(struct macio_chip *chip, · 381 unmodified lines
382 dma_set_max_seg_size(&dev->ofdev.dev, 65536);382 dma_set_max_seg_size(&dev->ofdev.dev, 65536);
383 dma_set_seg_boundary(&dev->ofdev.dev, 0xffffffff);383 dma_set_seg_boundary(&dev->ofdev.dev, 0xffffffff);
384 384
385#ifdef CONFIG_PCI385#if defined(CONFIG_PCI) && defined(CONFIG_DMA_OPS)
386 /* Set the DMA ops to the ones from the PCI device, this could be386 /* Set the DMA ops to the ones from the PCI device, this could be
387 * fishy if we didn't know that on PowerMac it's always direct ops387 * fishy if we didn't know that on PowerMac it's always direct ops
388 * or iommu ops that will work fine388 * or iommu ops that will work fine
@@ -391,7 +391,7 @@ static struct macio_dev * macio_add_one_device(struct macio_chip *chip, · 2 unmodified lines
391 */391 */
392 dev->ofdev.dev.archdata = chip->lbus.pdev->dev.archdata;392 dev->ofdev.dev.archdata = chip->lbus.pdev->dev.archdata;
393 dev->ofdev.dev.dma_ops = chip->lbus.pdev->dev.dma_ops;393 dev->ofdev.dev.dma_ops = chip->lbus.pdev->dev.dma_ops;
394#endif /* CONFIG_PCI */394#endif /* CONFIG_PCI && CONFIG_DMA_OPS */
395 395
396#ifdef DEBUG396#ifdef DEBUG
397 printk("preparing mdev @%p, ofdev @%p, dev @%p, kobj @%p\n",397 printk("preparing mdev @%p, ofdev @%p, dev @%p, kobj @%p\n",
drivers/misc/mic/Kconfig+4−0
@@ -4,6 +4,7 @@ menu "Intel MIC & related support" · 3 unmodified lines
4config INTEL_MIC_BUS4config INTEL_MIC_BUS
5 tristate "Intel MIC Bus Driver"5 tristate "Intel MIC Bus Driver"
6 depends on 64BIT && PCI && X866 depends on 64BIT && PCI && X86
7 select DMA_OPS
7 help8 help
8 This option is selected by any driver which registers a9 This option is selected by any driver which registers a
9 device or driver on the MIC Bus, such as CONFIG_INTEL_MIC_HOST,10 device or driver on the MIC Bus, such as CONFIG_INTEL_MIC_HOST,
@@ -19,6 +20,7 @@ config INTEL_MIC_BUS · 9 unmodified lines
19config SCIF_BUS20config SCIF_BUS
20 tristate "SCIF Bus Driver"21 tristate "SCIF Bus Driver"
21 depends on 64BIT && PCI && X8622 depends on 64BIT && PCI && X86
23 select DMA_OPS
22 help24 help
23 This option is selected by any driver which registers a25 This option is selected by any driver which registers a
24 device or driver on the SCIF Bus, such as CONFIG_INTEL_MIC_HOST26 device or driver on the SCIF Bus, such as CONFIG_INTEL_MIC_HOST
@@ -33,6 +35,7 @@ config SCIF_BUS · 8 unmodified lines
33 35
34config VOP_BUS36config VOP_BUS
35 tristate "VOP Bus Driver"37 tristate "VOP Bus Driver"
38 select DMA_OPS
36 help39 help
37 This option is selected by any driver which registers a40 This option is selected by any driver which registers a
38 device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST41 device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST
@@ -49,6 +52,7 @@ config INTEL_MIC_HOST · 10 unmodified lines
49 tristate "Intel MIC Host Driver"52 tristate "Intel MIC Host Driver"
50 depends on 64BIT && PCI && X8653 depends on 64BIT && PCI && X86
51 depends on INTEL_MIC_BUS && SCIF_BUS && MIC_COSM && VOP_BUS54 depends on INTEL_MIC_BUS && SCIF_BUS && MIC_COSM && VOP_BUS
55 select DMA_OPS
52 help56 help
53 This enables Host Driver support for the Intel Many Integrated57 This enables Host Driver support for the Intel Many Integrated
54 Core (MIC) family of PCIe form factor coprocessor devices that58 Core (MIC) family of PCIe form factor coprocessor devices that
drivers/vdpa/Kconfig+1−0
@@ -11,6 +11,7 @@ if VDPA · 10 unmodified lines
11config VDPA_SIM11config VDPA_SIM
12 tristate "vDPA device simulator"12 tristate "vDPA device simulator"
13 depends on RUNTIME_TESTING_MENU && HAS_DMA13 depends on RUNTIME_TESTING_MENU && HAS_DMA
14 select DMA_OPS
14 select VHOST_RING15 select VHOST_RING
15 default n16 default n
16 help17 help
drivers/xen/Kconfig+1−0
@@ -179,6 +179,7 @@ config XEN_GRANT_DMA_ALLOC · 178 unmodified lines
179 179
180config SWIOTLB_XEN180config SWIOTLB_XEN
181 def_bool y181 def_bool y
182 select DMA_OPS
182 select SWIOTLB183 select SWIOTLB
183 184
184config XEN_PCIDEV_BACKEND185config XEN_PCIDEV_BACKEND
include/linux/device.h+10−1
@@ -525,6 +525,11 @@ struct dev_links_info { · 524 unmodified lines
525 * sync_state() callback.525 * sync_state() callback.
526 * @dma_coherent: this particular device is dma coherent, even if the526 * @dma_coherent: this particular device is dma coherent, even if the
527 * architecture supports non-coherent devices.527 * architecture supports non-coherent devices.
528 * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
529 * streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
530 * and optionall (if the coherent mask is large enough) also
531 * for dma allocations. This flag is managed by the dma ops
532 * instance from ->dma_supported.
528 *533 *
529 * At the lowest level, every device in a Linux system is represented by an534 * At the lowest level, every device in a Linux system is represented by an
530 * instance of struct device. The device structure contains the information535 * instance of struct device. The device structure contains the information
@@ -574,8 +579,9 @@ struct device { · 43 unmodified lines
574#ifdef CONFIG_GENERIC_MSI_IRQ579#ifdef CONFIG_GENERIC_MSI_IRQ
575 struct list_head msi_list;580 struct list_head msi_list;
576#endif581#endif
577 582#ifdef CONFIG_DMA_OPS
578 const struct dma_map_ops *dma_ops;583 const struct dma_map_ops *dma_ops;
584#endif
579 u64 *dma_mask; /* dma mask (if dma'able device) */585 u64 *dma_mask; /* dma mask (if dma'able device) */
580 u64 coherent_dma_mask;/* Like dma_mask, but for586 u64 coherent_dma_mask;/* Like dma_mask, but for
581 alloc_coherent mappings as587 alloc_coherent mappings as
@@ -628,6 +634,9 @@ struct device { · 46 unmodified lines
628 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)634 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
629 bool dma_coherent:1;635 bool dma_coherent:1;
630#endif636#endif
637#ifdef CONFIG_DMA_OPS_BYPASS
638 bool dma_ops_bypass : 1;
639#endif
631};640};
632 641
633static inline struct device *kobj_to_dev(struct kobject *kobj)642static inline struct device *kobj_to_dev(struct kobject *kobj)
include/linux/dma-direct.h+104−0
@@ -1,10 +1,16 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Internals of the DMA direct mapping implementation. Only for use by the
4 * DMA mapping code and IOMMU drivers.
5 */
2#ifndef _LINUX_DMA_DIRECT_H6#ifndef _LINUX_DMA_DIRECT_H
3#define _LINUX_DMA_DIRECT_H 17#define _LINUX_DMA_DIRECT_H 1
4 8
5#include <linux/dma-mapping.h>9#include <linux/dma-mapping.h>
10#include <linux/dma-noncoherent.h>
6#include <linux/memblock.h> /* for min_low_pfn */11#include <linux/memblock.h> /* for min_low_pfn */
7#include <linux/mem_encrypt.h>12#include <linux/mem_encrypt.h>
13#include <linux/swiotlb.h>
8 14
9extern unsigned int zone_dma_bits;15extern unsigned int zone_dma_bits;
10 16
@@ -87,4 +93,102 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, · 76 unmodified lines
87 unsigned long attrs);93 unsigned long attrs);
88int dma_direct_supported(struct device *dev, u64 mask);94int dma_direct_supported(struct device *dev, u64 mask);
89bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);95bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);
96int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
97 enum dma_data_direction dir, unsigned long attrs);
98dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
99 size_t size, enum dma_data_direction dir, unsigned long attrs);
100size_t dma_direct_max_mapping_size(struct device *dev);
101
102#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
103 defined(CONFIG_SWIOTLB)
104void dma_direct_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
105 int nents, enum dma_data_direction dir);
106#else
107static inline void dma_direct_sync_sg_for_device(struct device *dev,
108 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
109{
110}
111#endif
112
113#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
114 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
115 defined(CONFIG_SWIOTLB)
116void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
117 int nents, enum dma_data_direction dir, unsigned long attrs);
118void dma_direct_sync_sg_for_cpu(struct device *dev,
119 struct scatterlist *sgl, int nents, enum dma_data_direction dir);
120#else
121static inline void dma_direct_unmap_sg(struct device *dev,
122 struct scatterlist *sgl, int nents, enum dma_data_direction dir,
123 unsigned long attrs)
124{
125}
126static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
127 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
128{
129}
130#endif
131
132static inline void dma_direct_sync_single_for_device(struct device *dev,
133 dma_addr_t addr, size_t size, enum dma_data_direction dir)
134{
135 phys_addr_t paddr = dma_to_phys(dev, addr);
136
137 if (unlikely(is_swiotlb_buffer(paddr)))
138 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
139
140 if (!dev_is_dma_coherent(dev))
141 arch_sync_dma_for_device(paddr, size, dir);
142}
143
144static inline void dma_direct_sync_single_for_cpu(struct device *dev,
145 dma_addr_t addr, size_t size, enum dma_data_direction dir)
146{
147 phys_addr_t paddr = dma_to_phys(dev, addr);
148
149 if (!dev_is_dma_coherent(dev)) {
150 arch_sync_dma_for_cpu(paddr, size, dir);
151 arch_sync_dma_for_cpu_all();
152 }
153
154 if (unlikely(is_swiotlb_buffer(paddr)))
155 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
156}
157
158static inline dma_addr_t dma_direct_map_page(struct device *dev,
159 struct page *page, unsigned long offset, size_t size,
160 enum dma_data_direction dir, unsigned long attrs)
161{
162 phys_addr_t phys = page_to_phys(page) + offset;
163 dma_addr_t dma_addr = phys_to_dma(dev, phys);
164
165 if (unlikely(swiotlb_force == SWIOTLB_FORCE))
166 return swiotlb_map(dev, phys, size, dir, attrs);
167
168 if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
169 if (swiotlb_force != SWIOTLB_NO_FORCE)
170 return swiotlb_map(dev, phys, size, dir, attrs);
171
172 dev_WARN_ONCE(dev, 1,
173 "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
174 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
175 return DMA_MAPPING_ERROR;
176 }
177
178 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
179 arch_sync_dma_for_device(phys, size, dir);
180 return dma_addr;
181}
182
183static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
184 size_t size, enum dma_data_direction dir, unsigned long attrs)
185{
186 phys_addr_t phys = dma_to_phys(dev, addr);
187
188 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
189 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
190
191 if (unlikely(is_swiotlb_buffer(phys)))
192 swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
193}
90#endif /* _LINUX_DMA_DIRECT_H */194#endif /* _LINUX_DMA_DIRECT_H */
include/linux/dma-mapping.h+29−222
@@ -188,76 +188,10 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma, · 187 unmodified lines
188}188}
189#endif /* CONFIG_DMA_DECLARE_COHERENT */189#endif /* CONFIG_DMA_DECLARE_COHERENT */
190 190
191static inline bool dma_is_direct(const struct dma_map_ops *ops)
192{
193 return likely(!ops);
194}
195
196/*
197 * All the dma_direct_* declarations are here just for the indirect call bypass,
198 * and must not be used directly drivers!
199 */
200dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
201 unsigned long offset, size_t size, enum dma_data_direction dir,
202 unsigned long attrs);
203int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
204 enum dma_data_direction dir, unsigned long attrs);
205dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
206 size_t size, enum dma_data_direction dir, unsigned long attrs);
207
208#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
209 defined(CONFIG_SWIOTLB)
210void dma_direct_sync_single_for_device(struct device *dev,
211 dma_addr_t addr, size_t size, enum dma_data_direction dir);
212void dma_direct_sync_sg_for_device(struct device *dev,
213 struct scatterlist *sgl, int nents, enum dma_data_direction dir);
214#else
215static inline void dma_direct_sync_single_for_device(struct device *dev,
216 dma_addr_t addr, size_t size, enum dma_data_direction dir)
217{
218}
219static inline void dma_direct_sync_sg_for_device(struct device *dev,
220 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
221{
222}
223#endif
224
225#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
226 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
227 defined(CONFIG_SWIOTLB)
228void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
229 size_t size, enum dma_data_direction dir, unsigned long attrs);
230void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
231 int nents, enum dma_data_direction dir, unsigned long attrs);
232void dma_direct_sync_single_for_cpu(struct device *dev,
233 dma_addr_t addr, size_t size, enum dma_data_direction dir);
234void dma_direct_sync_sg_for_cpu(struct device *dev,
235 struct scatterlist *sgl, int nents, enum dma_data_direction dir);
236#else
237static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
238 size_t size, enum dma_data_direction dir, unsigned long attrs)
239{
240}
241static inline void dma_direct_unmap_sg(struct device *dev,
242 struct scatterlist *sgl, int nents, enum dma_data_direction dir,
243 unsigned long attrs)
244{
245}
246static inline void dma_direct_sync_single_for_cpu(struct device *dev,
247 dma_addr_t addr, size_t size, enum dma_data_direction dir)
248{
249}
250static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
251 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
252{
253}
254#endif
255
256size_t dma_direct_max_mapping_size(struct device *dev);
257
258#ifdef CONFIG_HAS_DMA191#ifdef CONFIG_HAS_DMA
259#include <asm/dma-mapping.h>192#include <asm/dma-mapping.h>
260 193
194#ifdef CONFIG_DMA_OPS
261static inline const struct dma_map_ops *get_dma_ops(struct device *dev)195static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
262{196{
263 if (dev->dma_ops)197 if (dev->dma_ops)
@@ -270,165 +204,16 @@ static inline void set_dma_ops(struct device *dev, · 6 unmodified lines
270{204{
271 dev->dma_ops = dma_ops;205 dev->dma_ops = dma_ops;
272}206}
273 207#else /* CONFIG_DMA_OPS */
274static inline dma_addr_t dma_map_page_attrs(struct device *dev,208static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
275 struct page *page, size_t offset, size_t size,
276 enum dma_data_direction dir, unsigned long attrs)
277{
278 const struct dma_map_ops *ops = get_dma_ops(dev);
279 dma_addr_t addr;
280
281 BUG_ON(!valid_dma_direction(dir));
282 if (dma_is_direct(ops))
283 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
284 else
285 addr = ops->map_page(dev, page, offset, size, dir, attrs);
286 debug_dma_map_page(dev, page, offset, size, dir, addr);
287
288 return addr;
289}
290
291static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
292 size_t size, enum dma_data_direction dir, unsigned long attrs)
293{
294 const struct dma_map_ops *ops = get_dma_ops(dev);
295
296 BUG_ON(!valid_dma_direction(dir));
297 if (dma_is_direct(ops))
298 dma_direct_unmap_page(dev, addr, size, dir, attrs);
299 else if (ops->unmap_page)
300 ops->unmap_page(dev, addr, size, dir, attrs);
301 debug_dma_unmap_page(dev, addr, size, dir);
302}
303
304/*
305 * dma_maps_sg_attrs returns 0 on error and > 0 on success.
306 * It should never return a value < 0.
307 */
308static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
309 int nents, enum dma_data_direction dir,
310 unsigned long attrs)
311{
312 const struct dma_map_ops *ops = get_dma_ops(dev);
313 int ents;
314
315 BUG_ON(!valid_dma_direction(dir));
316 if (dma_is_direct(ops))
317 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
318 else
319 ents = ops->map_sg(dev, sg, nents, dir, attrs);
320 BUG_ON(ents < 0);
321 debug_dma_map_sg(dev, sg, nents, ents, dir);
322
323 return ents;
324}
325
326static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
327 int nents, enum dma_data_direction dir,
328 unsigned long attrs)
329{
330 const struct dma_map_ops *ops = get_dma_ops(dev);
331
332 BUG_ON(!valid_dma_direction(dir));
333 debug_dma_unmap_sg(dev, sg, nents, dir);
334 if (dma_is_direct(ops))
335 dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
336 else if (ops->unmap_sg)
337 ops->unmap_sg(dev, sg, nents, dir, attrs);
338}
339
340static inline dma_addr_t dma_map_resource(struct device *dev,
341 phys_addr_t phys_addr,
342 size_t size,
343 enum dma_data_direction dir,
344 unsigned long attrs)
345{
346 const struct dma_map_ops *ops = get_dma_ops(dev);
347 dma_addr_t addr = DMA_MAPPING_ERROR;
348
349 BUG_ON(!valid_dma_direction(dir));
350
351 /* Don't allow RAM to be mapped */
352 if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
353 return DMA_MAPPING_ERROR;
354
355 if (dma_is_direct(ops))
356 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
357 else if (ops->map_resource)
358 addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
359
360 debug_dma_map_resource(dev, phys_addr, size, dir, addr);
361 return addr;
362}
363
364static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
365 size_t size, enum dma_data_direction dir,
366 unsigned long attrs)
367{
368 const struct dma_map_ops *ops = get_dma_ops(dev);
369
370 BUG_ON(!valid_dma_direction(dir));
371 if (!dma_is_direct(ops) && ops->unmap_resource)
372 ops->unmap_resource(dev, addr, size, dir, attrs);
373 debug_dma_unmap_resource(dev, addr, size, dir);
374}
375
376static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
377 size_t size,
378 enum dma_data_direction dir)
379{
380 const struct dma_map_ops *ops = get_dma_ops(dev);
381
382 BUG_ON(!valid_dma_direction(dir));
383 if (dma_is_direct(ops))
384 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
385 else if (ops->sync_single_for_cpu)
386 ops->sync_single_for_cpu(dev, addr, size, dir);
387 debug_dma_sync_single_for_cpu(dev, addr, size, dir);
388}
389
390static inline void dma_sync_single_for_device(struct device *dev,
391 dma_addr_t addr, size_t size,
392 enum dma_data_direction dir)
393{
394 const struct dma_map_ops *ops = get_dma_ops(dev);
395
396 BUG_ON(!valid_dma_direction(dir));
397 if (dma_is_direct(ops))
398 dma_direct_sync_single_for_device(dev, addr, size, dir);
399 else if (ops->sync_single_for_device)
400 ops->sync_single_for_device(dev, addr, size, dir);
401 debug_dma_sync_single_for_device(dev, addr, size, dir);
402}
403
404static inline void
405dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
406 int nelems, enum dma_data_direction dir)
407{209{
408 const struct dma_map_ops *ops = get_dma_ops(dev);210 return NULL;
409
410 BUG_ON(!valid_dma_direction(dir));
411 if (dma_is_direct(ops))
412 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
413 else if (ops->sync_sg_for_cpu)
414 ops->sync_sg_for_cpu(dev, sg, nelems, dir);
415 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
416}211}
417 212static inline void set_dma_ops(struct device *dev,
418static inline void213 const struct dma_map_ops *dma_ops)
419dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
420 int nelems, enum dma_data_direction dir)
421{214{
422 const struct dma_map_ops *ops = get_dma_ops(dev);
423
424 BUG_ON(!valid_dma_direction(dir));
425 if (dma_is_direct(ops))
426 dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
427 else if (ops->sync_sg_for_device)
428 ops->sync_sg_for_device(dev, sg, nelems, dir);
429 debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
430
431}215}
216#endif /* CONFIG_DMA_OPS */
432 217
433static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)218static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
434{219{
@@ -439,6 +224,28 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) · 4 unmodified lines
439 return 0;224 return 0;
440}225}
441 226
227dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
228 size_t offset, size_t size, enum dma_data_direction dir,
229 unsigned long attrs);
230void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
231 enum dma_data_direction dir, unsigned long attrs);
232int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
233 enum dma_data_direction dir, unsigned long attrs);
234void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
235 int nents, enum dma_data_direction dir,
236 unsigned long attrs);
237dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
238 size_t size, enum dma_data_direction dir, unsigned long attrs);
239void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
240 enum dma_data_direction dir, unsigned long attrs);
241void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
242 enum dma_data_direction dir);
243void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
244 size_t size, enum dma_data_direction dir);
245void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
246 int nelems, enum dma_data_direction dir);
247void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
248 int nelems, enum dma_data_direction dir);
442void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,249void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
443 gfp_t flag, unsigned long attrs);250 gfp_t flag, unsigned long attrs);
444void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,251void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
kernel/dma/Kconfig+12−0
@@ -5,6 +5,17 @@ config HAS_DMA · 4 unmodified lines
5 depends on !NO_DMA5 depends on !NO_DMA
6 default y6 default y
7 7
8config DMA_OPS
9 bool
10
11#
12# IOMMU drivers that can bypass the IOMMU code and optionally use the direct
13# mapping fast path should select this option and set the dma_ops_bypass
14# flag in struct device where applicable
15#
16config DMA_OPS_BYPASS
17 bool
18
8config NEED_SG_DMA_LENGTH19config NEED_SG_DMA_LENGTH
9 bool20 bool
10 21
@@ -60,6 +71,7 @@ config DMA_NONCOHERENT_CACHE_SYNC · 49 unmodified lines
60config DMA_VIRT_OPS71config DMA_VIRT_OPS
61 bool72 bool
62 depends on HAS_DMA73 depends on HAS_DMA
74 select DMA_OPS
63 75
64config SWIOTLB76config SWIOTLB
65 bool77 bool
kernel/dma/Makefile+2−1
@@ -1,6 +1,7 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2 2
3obj-$(CONFIG_HAS_DMA) += mapping.o direct.o dummy.o3obj-$(CONFIG_HAS_DMA) += mapping.o direct.o
4obj-$(CONFIG_DMA_OPS) += dummy.o
4obj-$(CONFIG_DMA_CMA) += contiguous.o5obj-$(CONFIG_DMA_CMA) += contiguous.o
5obj-$(CONFIG_DMA_DECLARE_COHERENT) += coherent.o6obj-$(CONFIG_DMA_DECLARE_COHERENT) += coherent.o
6obj-$(CONFIG_DMA_VIRT_OPS) += virt.o7obj-$(CONFIG_DMA_VIRT_OPS) += virt.o
kernel/dma/contiguous.c+14−17
@@ -215,6 +215,13 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, · 214 unmodified lines
215 return cma_release(dev_get_cma_area(dev), pages, count);215 return cma_release(dev_get_cma_area(dev), pages, count);
216}216}
217 217
218static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp)
219{
220 unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT);
221
222 return cma_alloc(cma, size >> PAGE_SHIFT, align, gfp & __GFP_NOWARN);
223}
224
218/**225/**
219 * dma_alloc_contiguous() - allocate contiguous pages226 * dma_alloc_contiguous() - allocate contiguous pages
220 * @dev: Pointer to device for which the allocation is performed.227 * @dev: Pointer to device for which the allocation is performed.
@@ -231,24 +238,14 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, · 10 unmodified lines
231 */238 */
232struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)239struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
233{240{
234 size_t count = size >> PAGE_SHIFT;
235 struct page *page = NULL;
236 struct cma *cma = NULL;
237
238 if (dev && dev->cma_area)
239 cma = dev->cma_area;
240 else if (count > 1)
241 cma = dma_contiguous_default_area;
242
243 /* CMA can be used only in the context which permits sleeping */241 /* CMA can be used only in the context which permits sleeping */
244 if (cma && gfpflags_allow_blocking(gfp)) {242 if (!gfpflags_allow_blocking(gfp))
245 size_t align = get_order(size);243 return NULL;
246 size_t cma_align = min_t(size_t, align, CONFIG_CMA_ALIGNMENT);244 if (dev->cma_area)
247 245 return cma_alloc_aligned(dev->cma_area, size, gfp);
248 page = cma_alloc(cma, count, cma_align, gfp & __GFP_NOWARN);246 if (size <= PAGE_SIZE || !dma_contiguous_default_area)
249 }247 return NULL;
250 248 return cma_alloc_aligned(dma_contiguous_default_area, size, gfp);
251 return page;
252}249}
253 250
254/**251/**
kernel/dma/debug.c+6−2
@@ -144,8 +144,12 @@ static const char *type2name[] = { · 143 unmodified lines
144 [dma_debug_resource] = "resource",144 [dma_debug_resource] = "resource",
145};145};
146 146
147static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",147static const char *dir2name[] = {
148 "DMA_FROM_DEVICE", "DMA_NONE" };148 [DMA_BIDIRECTIONAL] = "DMA_BIDIRECTIONAL",
149 [DMA_TO_DEVICE] = "DMA_TO_DEVICE",
150 [DMA_FROM_DEVICE] = "DMA_FROM_DEVICE",
151 [DMA_NONE] = "DMA_NONE",
152};
149 153
150/*154/*
151 * The access to some variables in this macro is racy. We can't use atomic_t155 * The access to some variables in this macro is racy. We can't use atomic_t
kernel/dma/direct.c+0−74
@@ -10,11 +10,9 @@ · 9 unmodified lines
10#include <linux/dma-direct.h>10#include <linux/dma-direct.h>
11#include <linux/scatterlist.h>11#include <linux/scatterlist.h>
12#include <linux/dma-contiguous.h>12#include <linux/dma-contiguous.h>
13#include <linux/dma-noncoherent.h>
14#include <linux/pfn.h>13#include <linux/pfn.h>
15#include <linux/vmalloc.h>14#include <linux/vmalloc.h>
16#include <linux/set_memory.h>15#include <linux/set_memory.h>
17#include <linux/swiotlb.h>
18 16
19/*17/*
20 * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it18 * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it
@@ -304,19 +302,6 @@ void dma_direct_free(struct device *dev, size_t size, · 283 unmodified lines
304 302
305#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \303#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
306 defined(CONFIG_SWIOTLB)304 defined(CONFIG_SWIOTLB)
307void dma_direct_sync_single_for_device(struct device *dev,
308 dma_addr_t addr, size_t size, enum dma_data_direction dir)
309{
310 phys_addr_t paddr = dma_to_phys(dev, addr);
311
312 if (unlikely(is_swiotlb_buffer(paddr)))
313 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
314
315 if (!dev_is_dma_coherent(dev))
316 arch_sync_dma_for_device(paddr, size, dir);
317}
318EXPORT_SYMBOL(dma_direct_sync_single_for_device);
319
320void dma_direct_sync_sg_for_device(struct device *dev,305void dma_direct_sync_sg_for_device(struct device *dev,
321 struct scatterlist *sgl, int nents, enum dma_data_direction dir)306 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
322{307{
@@ -335,27 +320,11 @@ void dma_direct_sync_sg_for_device(struct device *dev, · 12 unmodified lines
335 dir);320 dir);
336 }321 }
337}322}
338EXPORT_SYMBOL(dma_direct_sync_sg_for_device);
339#endif323#endif
340 324
341#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \325#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
342 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \326 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
343 defined(CONFIG_SWIOTLB)327 defined(CONFIG_SWIOTLB)
344void dma_direct_sync_single_for_cpu(struct device *dev,
345 dma_addr_t addr, size_t size, enum dma_data_direction dir)
346{
347 phys_addr_t paddr = dma_to_phys(dev, addr);
348
349 if (!dev_is_dma_coherent(dev)) {
350 arch_sync_dma_for_cpu(paddr, size, dir);
351 arch_sync_dma_for_cpu_all();
352 }
353
354 if (unlikely(is_swiotlb_buffer(paddr)))
355 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
356}
357EXPORT_SYMBOL(dma_direct_sync_single_for_cpu);
358
359void dma_direct_sync_sg_for_cpu(struct device *dev,328void dma_direct_sync_sg_for_cpu(struct device *dev,
360 struct scatterlist *sgl, int nents, enum dma_data_direction dir)329 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
361{330{
@@ -376,20 +345,6 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, · 14 unmodified lines
376 if (!dev_is_dma_coherent(dev))345 if (!dev_is_dma_coherent(dev))
377 arch_sync_dma_for_cpu_all();346 arch_sync_dma_for_cpu_all();
378}347}
379EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu);
380
381void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
382 size_t size, enum dma_data_direction dir, unsigned long attrs)
383{
384 phys_addr_t phys = dma_to_phys(dev, addr);
385
386 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
387 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
388
389 if (unlikely(is_swiotlb_buffer(phys)))
390 swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
391}
392EXPORT_SYMBOL(dma_direct_unmap_page);
393 348
394void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,349void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
395 int nents, enum dma_data_direction dir, unsigned long attrs)350 int nents, enum dma_data_direction dir, unsigned long attrs)
@@ -401,35 +356,8 @@ void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, · 5 unmodified lines
401 dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,356 dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,
402 attrs);357 attrs);
403}358}
404EXPORT_SYMBOL(dma_direct_unmap_sg);
405#endif359#endif
406 360
407dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
408 unsigned long offset, size_t size, enum dma_data_direction dir,
409 unsigned long attrs)
410{
411 phys_addr_t phys = page_to_phys(page) + offset;
412 dma_addr_t dma_addr = phys_to_dma(dev, phys);
413
414 if (unlikely(swiotlb_force == SWIOTLB_FORCE))
415 return swiotlb_map(dev, phys, size, dir, attrs);
416
417 if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
418 if (swiotlb_force != SWIOTLB_NO_FORCE)
419 return swiotlb_map(dev, phys, size, dir, attrs);
420
421 dev_WARN_ONCE(dev, 1,
422 "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
423 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
424 return DMA_MAPPING_ERROR;
425 }
426
427 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
428 arch_sync_dma_for_device(phys, size, dir);
429 return dma_addr;
430}
431EXPORT_SYMBOL(dma_direct_map_page);
432
433int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,361int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
434 enum dma_data_direction dir, unsigned long attrs)362 enum dma_data_direction dir, unsigned long attrs)
435{363{
@@ -450,7 +378,6 @@ out_unmap: · 14 unmodified lines
450 dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);378 dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
451 return 0;379 return 0;
452}380}
453EXPORT_SYMBOL(dma_direct_map_sg);
454 381
455dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,382dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
456 size_t size, enum dma_data_direction dir, unsigned long attrs)383 size_t size, enum dma_data_direction dir, unsigned long attrs)
@@ -467,7 +394,6 @@ dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, · 10 unmodified lines
467 394
468 return dma_addr;395 return dma_addr;
469}396}
470EXPORT_SYMBOL(dma_direct_map_resource);
471 397
472int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,398int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,
473 void *cpu_addr, dma_addr_t dma_addr, size_t size,399 void *cpu_addr, dma_addr_t dma_addr, size_t size,
kernel/dma/mapping.c+204−10
@@ -105,6 +105,196 @@ void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · 104 unmodified lines
105}105}
106EXPORT_SYMBOL(dmam_alloc_attrs);106EXPORT_SYMBOL(dmam_alloc_attrs);
107 107
108static bool dma_go_direct(struct device *dev, dma_addr_t mask,
109 const struct dma_map_ops *ops)
110{
111 if (likely(!ops))
112 return true;
113#ifdef CONFIG_DMA_OPS_BYPASS
114 if (dev->dma_ops_bypass)
115 return min_not_zero(mask, dev->bus_dma_limit) >=
116 dma_direct_get_required_mask(dev);
117#endif
118 return false;
119}
120
121
122/*
123 * Check if the devices uses a direct mapping for streaming DMA operations.
124 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
125 * enough.
126 */
127static inline bool dma_alloc_direct(struct device *dev,
128 const struct dma_map_ops *ops)
129{
130 return dma_go_direct(dev, dev->coherent_dma_mask, ops);
131}
132
133static inline bool dma_map_direct(struct device *dev,
134 const struct dma_map_ops *ops)
135{
136 return dma_go_direct(dev, *dev->dma_mask, ops);
137}
138
139dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
140 size_t offset, size_t size, enum dma_data_direction dir,
141 unsigned long attrs)
142{
143 const struct dma_map_ops *ops = get_dma_ops(dev);
144 dma_addr_t addr;
145
146 BUG_ON(!valid_dma_direction(dir));
147 if (dma_map_direct(dev, ops))
148 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
149 else
150 addr = ops->map_page(dev, page, offset, size, dir, attrs);
151 debug_dma_map_page(dev, page, offset, size, dir, addr);
152
153 return addr;
154}
155EXPORT_SYMBOL(dma_map_page_attrs);
156
157void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
158 enum dma_data_direction dir, unsigned long attrs)
159{
160 const struct dma_map_ops *ops = get_dma_ops(dev);
161
162 BUG_ON(!valid_dma_direction(dir));
163 if (dma_map_direct(dev, ops))
164 dma_direct_unmap_page(dev, addr, size, dir, attrs);
165 else if (ops->unmap_page)
166 ops->unmap_page(dev, addr, size, dir, attrs);
167 debug_dma_unmap_page(dev, addr, size, dir);
168}
169EXPORT_SYMBOL(dma_unmap_page_attrs);
170
171/*
172 * dma_maps_sg_attrs returns 0 on error and > 0 on success.
173 * It should never return a value < 0.
174 */
175int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
176 enum dma_data_direction dir, unsigned long attrs)
177{
178 const struct dma_map_ops *ops = get_dma_ops(dev);
179 int ents;
180
181 BUG_ON(!valid_dma_direction(dir));
182 if (dma_map_direct(dev, ops))
183 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
184 else
185 ents = ops->map_sg(dev, sg, nents, dir, attrs);
186 BUG_ON(ents < 0);
187 debug_dma_map_sg(dev, sg, nents, ents, dir);
188
189 return ents;
190}
191EXPORT_SYMBOL(dma_map_sg_attrs);
192
193void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
194 int nents, enum dma_data_direction dir,
195 unsigned long attrs)
196{
197 const struct dma_map_ops *ops = get_dma_ops(dev);
198
199 BUG_ON(!valid_dma_direction(dir));
200 debug_dma_unmap_sg(dev, sg, nents, dir);
201 if (dma_map_direct(dev, ops))
202 dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
203 else if (ops->unmap_sg)
204 ops->unmap_sg(dev, sg, nents, dir, attrs);
205}
206EXPORT_SYMBOL(dma_unmap_sg_attrs);
207
208dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
209 size_t size, enum dma_data_direction dir, unsigned long attrs)
210{
211 const struct dma_map_ops *ops = get_dma_ops(dev);
212 dma_addr_t addr = DMA_MAPPING_ERROR;
213
214 BUG_ON(!valid_dma_direction(dir));
215
216 /* Don't allow RAM to be mapped */
217 if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
218 return DMA_MAPPING_ERROR;
219
220 if (dma_map_direct(dev, ops))
221 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
222 else if (ops->map_resource)
223 addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
224
225 debug_dma_map_resource(dev, phys_addr, size, dir, addr);
226 return addr;
227}
228EXPORT_SYMBOL(dma_map_resource);
229
230void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
231 enum dma_data_direction dir, unsigned long attrs)
232{
233 const struct dma_map_ops *ops = get_dma_ops(dev);
234
235 BUG_ON(!valid_dma_direction(dir));
236 if (!dma_map_direct(dev, ops) && ops->unmap_resource)
237 ops->unmap_resource(dev, addr, size, dir, attrs);
238 debug_dma_unmap_resource(dev, addr, size, dir);
239}
240EXPORT_SYMBOL(dma_unmap_resource);
241
242void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
243 enum dma_data_direction dir)
244{
245 const struct dma_map_ops *ops = get_dma_ops(dev);
246
247 BUG_ON(!valid_dma_direction(dir));
248 if (dma_map_direct(dev, ops))
249 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
250 else if (ops->sync_single_for_cpu)
251 ops->sync_single_for_cpu(dev, addr, size, dir);
252 debug_dma_sync_single_for_cpu(dev, addr, size, dir);
253}
254EXPORT_SYMBOL(dma_sync_single_for_cpu);
255
256void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
257 size_t size, enum dma_data_direction dir)
258{
259 const struct dma_map_ops *ops = get_dma_ops(dev);
260
261 BUG_ON(!valid_dma_direction(dir));
262 if (dma_map_direct(dev, ops))
263 dma_direct_sync_single_for_device(dev, addr, size, dir);
264 else if (ops->sync_single_for_device)
265 ops->sync_single_for_device(dev, addr, size, dir);
266 debug_dma_sync_single_for_device(dev, addr, size, dir);
267}
268EXPORT_SYMBOL(dma_sync_single_for_device);
269
270void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
271 int nelems, enum dma_data_direction dir)
272{
273 const struct dma_map_ops *ops = get_dma_ops(dev);
274
275 BUG_ON(!valid_dma_direction(dir));
276 if (dma_map_direct(dev, ops))
277 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
278 else if (ops->sync_sg_for_cpu)
279 ops->sync_sg_for_cpu(dev, sg, nelems, dir);
280 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
281}
282EXPORT_SYMBOL(dma_sync_sg_for_cpu);
283
284void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
285 int nelems, enum dma_data_direction dir)
286{
287 const struct dma_map_ops *ops = get_dma_ops(dev);
288
289 BUG_ON(!valid_dma_direction(dir));
290 if (dma_map_direct(dev, ops))
291 dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
292 else if (ops->sync_sg_for_device)
293 ops->sync_sg_for_device(dev, sg, nelems, dir);
294 debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
295}
296EXPORT_SYMBOL(dma_sync_sg_for_device);
297
108/*298/*
109 * Create scatter-list for the already allocated DMA buffer.299 * Create scatter-list for the already allocated DMA buffer.
110 */300 */
@@ -138,7 +328,7 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, · 27 unmodified lines
138{328{
139 const struct dma_map_ops *ops = get_dma_ops(dev);329 const struct dma_map_ops *ops = get_dma_ops(dev);
140 330
141 if (dma_is_direct(ops))331 if (dma_alloc_direct(dev, ops))
142 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,332 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
143 size, attrs);333 size, attrs);
144 if (!ops->get_sgtable)334 if (!ops->get_sgtable)
@@ -208,7 +398,7 @@ bool dma_can_mmap(struct device *dev) · 63 unmodified lines
208{398{
209 const struct dma_map_ops *ops = get_dma_ops(dev);399 const struct dma_map_ops *ops = get_dma_ops(dev);
210 400
211 if (dma_is_direct(ops))401 if (dma_alloc_direct(dev, ops))
212 return dma_direct_can_mmap(dev);402 return dma_direct_can_mmap(dev);
213 return ops->mmap != NULL;403 return ops->mmap != NULL;
214}404}
@@ -233,7 +423,7 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, · 18 unmodified lines
233{423{
234 const struct dma_map_ops *ops = get_dma_ops(dev);424 const struct dma_map_ops *ops = get_dma_ops(dev);
235 425
236 if (dma_is_direct(ops))426 if (dma_alloc_direct(dev, ops))
237 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,427 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
238 attrs);428 attrs);
239 if (!ops->mmap)429 if (!ops->mmap)
@@ -246,7 +436,7 @@ u64 dma_get_required_mask(struct device *dev) · 6 unmodified lines
246{436{
247 const struct dma_map_ops *ops = get_dma_ops(dev);437 const struct dma_map_ops *ops = get_dma_ops(dev);
248 438
249 if (dma_is_direct(ops))439 if (dma_alloc_direct(dev, ops))
250 return dma_direct_get_required_mask(dev);440 return dma_direct_get_required_mask(dev);
251 if (ops->get_required_mask)441 if (ops->get_required_mask)
252 return ops->get_required_mask(dev);442 return ops->get_required_mask(dev);
@@ -277,7 +467,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · 24 unmodified lines
277 /* let the implementation decide on the zone to allocate from: */467 /* let the implementation decide on the zone to allocate from: */
278 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);468 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
279 469
280 if (dma_is_direct(ops))470 if (dma_alloc_direct(dev, ops))
281 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);471 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
282 else if (ops->alloc)472 else if (ops->alloc)
283 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);473 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
@@ -309,7 +499,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, · 25 unmodified lines
309 return;499 return;
310 500
311 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);501 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
312 if (dma_is_direct(ops))502 if (dma_alloc_direct(dev, ops))
313 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);503 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
314 else if (ops->free)504 else if (ops->free)
315 ops->free(dev, size, cpu_addr, dma_handle, attrs);505 ops->free(dev, size, cpu_addr, dma_handle, attrs);
@@ -320,7 +510,11 @@ int dma_supported(struct device *dev, u64 mask) · 4 unmodified lines
320{510{
321 const struct dma_map_ops *ops = get_dma_ops(dev);511 const struct dma_map_ops *ops = get_dma_ops(dev);
322 512
323 if (dma_is_direct(ops))513 /*
514 * ->dma_supported sets the bypass flag, so we must always call
515 * into the method here unless the device is truly direct mapped.
516 */
517 if (!ops)
324 return dma_direct_supported(dev, mask);518 return dma_direct_supported(dev, mask);
325 if (!ops->dma_supported)519 if (!ops->dma_supported)
326 return 1;520 return 1;
@@ -376,7 +570,7 @@ void dma_cache_sync(struct device *dev, void *vaddr, size_t size, · 49 unmodified lines
376 570
377 BUG_ON(!valid_dma_direction(dir));571 BUG_ON(!valid_dma_direction(dir));
378 572
379 if (dma_is_direct(ops))573 if (dma_alloc_direct(dev, ops))
380 arch_dma_cache_sync(dev, vaddr, size, dir);574 arch_dma_cache_sync(dev, vaddr, size, dir);
381 else if (ops->cache_sync)575 else if (ops->cache_sync)
382 ops->cache_sync(dev, vaddr, size, dir);576 ops->cache_sync(dev, vaddr, size, dir);
@@ -388,7 +582,7 @@ size_t dma_max_mapping_size(struct device *dev) · 5 unmodified lines
388 const struct dma_map_ops *ops = get_dma_ops(dev);582 const struct dma_map_ops *ops = get_dma_ops(dev);
389 size_t size = SIZE_MAX;583 size_t size = SIZE_MAX;
390 584
391 if (dma_is_direct(ops))585 if (dma_map_direct(dev, ops))
392 size = dma_direct_max_mapping_size(dev);586 size = dma_direct_max_mapping_size(dev);
393 else if (ops && ops->max_mapping_size)587 else if (ops && ops->max_mapping_size)
394 size = ops->max_mapping_size(dev);588 size = ops->max_mapping_size(dev);
@@ -401,7 +595,7 @@ bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) · 6 unmodified lines
401{595{
402 const struct dma_map_ops *ops = get_dma_ops(dev);596 const struct dma_map_ops *ops = get_dma_ops(dev);
403 597
404 if (dma_is_direct(ops))598 if (dma_map_direct(dev, ops))
405 return dma_direct_need_sync(dev, dma_addr);599 return dma_direct_need_sync(dev, dma_addr);
406 return ops->sync_single_for_cpu || ops->sync_single_for_device;600 return ops->sync_single_for_cpu || ops->sync_single_for_device;
407}601}