History

Merge tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping updates from Christoph Hellwig: - make support for dma_ops optional - move more code out of line - add generic support for a dma_ops bypass mode - misc cleanups * tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping: dma-contiguous: cleanup dma_alloc_contiguous dma-debug: use named initializers for dir2name powerpc: use the generic dma_ops_bypass mode dma-mapping: add a dma_ops_bypass flag to struct device dma-mapping: make support for dma ops optional dma-mapping: inline the fast path dma-direct calls dma-mapping: move the remaining DMA API calls out of line

Linus Torvalds committed 6 years ago

Commit 2ed90db2 parents 9fa867d 274b3f726 files · +415 −416Browse files

Showing the changes to include/linux (3 of 26 files). Show all

Unified · Split

3 files
  1. include/linux/device.h +10 −1
  2. include/linux/dma-direct.h +104 −0
  3. include/linux/dma-mapping.h +29 −222
include/linux/device.h+10−1
@@ -525,6 +525,11 @@ struct dev_links_info { · 524 unmodified lines
525 * sync_state() callback.525 * sync_state() callback.
526 * @dma_coherent: this particular device is dma coherent, even if the526 * @dma_coherent: this particular device is dma coherent, even if the
527 * architecture supports non-coherent devices.527 * architecture supports non-coherent devices.
528 * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
529 * streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
530 * and optionall (if the coherent mask is large enough) also
531 * for dma allocations. This flag is managed by the dma ops
532 * instance from ->dma_supported.
528 *533 *
529 * At the lowest level, every device in a Linux system is represented by an534 * At the lowest level, every device in a Linux system is represented by an
530 * instance of struct device. The device structure contains the information535 * instance of struct device. The device structure contains the information
@@ -574,8 +579,9 @@ struct device { · 43 unmodified lines
574#ifdef CONFIG_GENERIC_MSI_IRQ579#ifdef CONFIG_GENERIC_MSI_IRQ
575 struct list_head msi_list;580 struct list_head msi_list;
576#endif581#endif
577 582#ifdef CONFIG_DMA_OPS
578 const struct dma_map_ops *dma_ops;583 const struct dma_map_ops *dma_ops;
584#endif
579 u64 *dma_mask; /* dma mask (if dma'able device) */585 u64 *dma_mask; /* dma mask (if dma'able device) */
580 u64 coherent_dma_mask;/* Like dma_mask, but for586 u64 coherent_dma_mask;/* Like dma_mask, but for
581 alloc_coherent mappings as587 alloc_coherent mappings as
@@ -628,6 +634,9 @@ struct device { · 46 unmodified lines
628 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)634 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
629 bool dma_coherent:1;635 bool dma_coherent:1;
630#endif636#endif
637#ifdef CONFIG_DMA_OPS_BYPASS
638 bool dma_ops_bypass : 1;
639#endif
631};640};
632 641
633static inline struct device *kobj_to_dev(struct kobject *kobj)642static inline struct device *kobj_to_dev(struct kobject *kobj)
include/linux/dma-direct.h+104−0
@@ -1,10 +1,16 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Internals of the DMA direct mapping implementation. Only for use by the
4 * DMA mapping code and IOMMU drivers.
5 */
2#ifndef _LINUX_DMA_DIRECT_H6#ifndef _LINUX_DMA_DIRECT_H
3#define _LINUX_DMA_DIRECT_H 17#define _LINUX_DMA_DIRECT_H 1
4 8
5#include <linux/dma-mapping.h>9#include <linux/dma-mapping.h>
10#include <linux/dma-noncoherent.h>
6#include <linux/memblock.h> /* for min_low_pfn */11#include <linux/memblock.h> /* for min_low_pfn */
7#include <linux/mem_encrypt.h>12#include <linux/mem_encrypt.h>
13#include <linux/swiotlb.h>
8 14
9extern unsigned int zone_dma_bits;15extern unsigned int zone_dma_bits;
10 16
@@ -87,4 +93,102 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, · 76 unmodified lines
87 unsigned long attrs);93 unsigned long attrs);
88int dma_direct_supported(struct device *dev, u64 mask);94int dma_direct_supported(struct device *dev, u64 mask);
89bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);95bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);
96int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
97 enum dma_data_direction dir, unsigned long attrs);
98dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
99 size_t size, enum dma_data_direction dir, unsigned long attrs);
100size_t dma_direct_max_mapping_size(struct device *dev);
101
102#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
103 defined(CONFIG_SWIOTLB)
104void dma_direct_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
105 int nents, enum dma_data_direction dir);
106#else
107static inline void dma_direct_sync_sg_for_device(struct device *dev,
108 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
109{
110}
111#endif
112
113#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
114 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
115 defined(CONFIG_SWIOTLB)
116void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
117 int nents, enum dma_data_direction dir, unsigned long attrs);
118void dma_direct_sync_sg_for_cpu(struct device *dev,
119 struct scatterlist *sgl, int nents, enum dma_data_direction dir);
120#else
121static inline void dma_direct_unmap_sg(struct device *dev,
122 struct scatterlist *sgl, int nents, enum dma_data_direction dir,
123 unsigned long attrs)
124{
125}
126static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
127 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
128{
129}
130#endif
131
132static inline void dma_direct_sync_single_for_device(struct device *dev,
133 dma_addr_t addr, size_t size, enum dma_data_direction dir)
134{
135 phys_addr_t paddr = dma_to_phys(dev, addr);
136
137 if (unlikely(is_swiotlb_buffer(paddr)))
138 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
139
140 if (!dev_is_dma_coherent(dev))
141 arch_sync_dma_for_device(paddr, size, dir);
142}
143
144static inline void dma_direct_sync_single_for_cpu(struct device *dev,
145 dma_addr_t addr, size_t size, enum dma_data_direction dir)
146{
147 phys_addr_t paddr = dma_to_phys(dev, addr);
148
149 if (!dev_is_dma_coherent(dev)) {
150 arch_sync_dma_for_cpu(paddr, size, dir);
151 arch_sync_dma_for_cpu_all();
152 }
153
154 if (unlikely(is_swiotlb_buffer(paddr)))
155 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
156}
157
158static inline dma_addr_t dma_direct_map_page(struct device *dev,
159 struct page *page, unsigned long offset, size_t size,
160 enum dma_data_direction dir, unsigned long attrs)
161{
162 phys_addr_t phys = page_to_phys(page) + offset;
163 dma_addr_t dma_addr = phys_to_dma(dev, phys);
164
165 if (unlikely(swiotlb_force == SWIOTLB_FORCE))
166 return swiotlb_map(dev, phys, size, dir, attrs);
167
168 if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
169 if (swiotlb_force != SWIOTLB_NO_FORCE)
170 return swiotlb_map(dev, phys, size, dir, attrs);
171
172 dev_WARN_ONCE(dev, 1,
173 "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
174 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
175 return DMA_MAPPING_ERROR;
176 }
177
178 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
179 arch_sync_dma_for_device(phys, size, dir);
180 return dma_addr;
181}
182
183static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
184 size_t size, enum dma_data_direction dir, unsigned long attrs)
185{
186 phys_addr_t phys = dma_to_phys(dev, addr);
187
188 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
189 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
190
191 if (unlikely(is_swiotlb_buffer(phys)))
192 swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
193}
90#endif /* _LINUX_DMA_DIRECT_H */194#endif /* _LINUX_DMA_DIRECT_H */
include/linux/dma-mapping.h+29−222
@@ -188,76 +188,10 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma, · 187 unmodified lines
188}188}
189#endif /* CONFIG_DMA_DECLARE_COHERENT */189#endif /* CONFIG_DMA_DECLARE_COHERENT */
190 190
191static inline bool dma_is_direct(const struct dma_map_ops *ops)
192{
193 return likely(!ops);
194}
195
196/*
197 * All the dma_direct_* declarations are here just for the indirect call bypass,
198 * and must not be used directly drivers!
199 */
200dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
201 unsigned long offset, size_t size, enum dma_data_direction dir,
202 unsigned long attrs);
203int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
204 enum dma_data_direction dir, unsigned long attrs);
205dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
206 size_t size, enum dma_data_direction dir, unsigned long attrs);
207
208#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
209 defined(CONFIG_SWIOTLB)
210void dma_direct_sync_single_for_device(struct device *dev,
211 dma_addr_t addr, size_t size, enum dma_data_direction dir);
212void dma_direct_sync_sg_for_device(struct device *dev,
213 struct scatterlist *sgl, int nents, enum dma_data_direction dir);
214#else
215static inline void dma_direct_sync_single_for_device(struct device *dev,
216 dma_addr_t addr, size_t size, enum dma_data_direction dir)
217{
218}
219static inline void dma_direct_sync_sg_for_device(struct device *dev,
220 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
221{
222}
223#endif
224
225#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
226 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
227 defined(CONFIG_SWIOTLB)
228void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
229 size_t size, enum dma_data_direction dir, unsigned long attrs);
230void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
231 int nents, enum dma_data_direction dir, unsigned long attrs);
232void dma_direct_sync_single_for_cpu(struct device *dev,
233 dma_addr_t addr, size_t size, enum dma_data_direction dir);
234void dma_direct_sync_sg_for_cpu(struct device *dev,
235 struct scatterlist *sgl, int nents, enum dma_data_direction dir);
236#else
237static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
238 size_t size, enum dma_data_direction dir, unsigned long attrs)
239{
240}
241static inline void dma_direct_unmap_sg(struct device *dev,
242 struct scatterlist *sgl, int nents, enum dma_data_direction dir,
243 unsigned long attrs)
244{
245}
246static inline void dma_direct_sync_single_for_cpu(struct device *dev,
247 dma_addr_t addr, size_t size, enum dma_data_direction dir)
248{
249}
250static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
251 struct scatterlist *sgl, int nents, enum dma_data_direction dir)
252{
253}
254#endif
255
256size_t dma_direct_max_mapping_size(struct device *dev);
257
258#ifdef CONFIG_HAS_DMA191#ifdef CONFIG_HAS_DMA
259#include <asm/dma-mapping.h>192#include <asm/dma-mapping.h>
260 193
194#ifdef CONFIG_DMA_OPS
261static inline const struct dma_map_ops *get_dma_ops(struct device *dev)195static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
262{196{
263 if (dev->dma_ops)197 if (dev->dma_ops)
@@ -270,165 +204,16 @@ static inline void set_dma_ops(struct device *dev, · 6 unmodified lines
270{204{
271 dev->dma_ops = dma_ops;205 dev->dma_ops = dma_ops;
272}206}
273 207#else /* CONFIG_DMA_OPS */
274static inline dma_addr_t dma_map_page_attrs(struct device *dev,208static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
275 struct page *page, size_t offset, size_t size,
276 enum dma_data_direction dir, unsigned long attrs)
277{
278 const struct dma_map_ops *ops = get_dma_ops(dev);
279 dma_addr_t addr;
280
281 BUG_ON(!valid_dma_direction(dir));
282 if (dma_is_direct(ops))
283 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
284 else
285 addr = ops->map_page(dev, page, offset, size, dir, attrs);
286 debug_dma_map_page(dev, page, offset, size, dir, addr);
287
288 return addr;
289}
290
291static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
292 size_t size, enum dma_data_direction dir, unsigned long attrs)
293{
294 const struct dma_map_ops *ops = get_dma_ops(dev);
295
296 BUG_ON(!valid_dma_direction(dir));
297 if (dma_is_direct(ops))
298 dma_direct_unmap_page(dev, addr, size, dir, attrs);
299 else if (ops->unmap_page)
300 ops->unmap_page(dev, addr, size, dir, attrs);
301 debug_dma_unmap_page(dev, addr, size, dir);
302}
303
304/*
305 * dma_maps_sg_attrs returns 0 on error and > 0 on success.
306 * It should never return a value < 0.
307 */
308static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
309 int nents, enum dma_data_direction dir,
310 unsigned long attrs)
311{
312 const struct dma_map_ops *ops = get_dma_ops(dev);
313 int ents;
314
315 BUG_ON(!valid_dma_direction(dir));
316 if (dma_is_direct(ops))
317 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
318 else
319 ents = ops->map_sg(dev, sg, nents, dir, attrs);
320 BUG_ON(ents < 0);
321 debug_dma_map_sg(dev, sg, nents, ents, dir);
322
323 return ents;
324}
325
326static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
327 int nents, enum dma_data_direction dir,
328 unsigned long attrs)
329{
330 const struct dma_map_ops *ops = get_dma_ops(dev);
331
332 BUG_ON(!valid_dma_direction(dir));
333 debug_dma_unmap_sg(dev, sg, nents, dir);
334 if (dma_is_direct(ops))
335 dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
336 else if (ops->unmap_sg)
337 ops->unmap_sg(dev, sg, nents, dir, attrs);
338}
339
340static inline dma_addr_t dma_map_resource(struct device *dev,
341 phys_addr_t phys_addr,
342 size_t size,
343 enum dma_data_direction dir,
344 unsigned long attrs)
345{
346 const struct dma_map_ops *ops = get_dma_ops(dev);
347 dma_addr_t addr = DMA_MAPPING_ERROR;
348
349 BUG_ON(!valid_dma_direction(dir));
350
351 /* Don't allow RAM to be mapped */
352 if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
353 return DMA_MAPPING_ERROR;
354
355 if (dma_is_direct(ops))
356 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
357 else if (ops->map_resource)
358 addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
359
360 debug_dma_map_resource(dev, phys_addr, size, dir, addr);
361 return addr;
362}
363
364static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
365 size_t size, enum dma_data_direction dir,
366 unsigned long attrs)
367{
368 const struct dma_map_ops *ops = get_dma_ops(dev);
369
370 BUG_ON(!valid_dma_direction(dir));
371 if (!dma_is_direct(ops) && ops->unmap_resource)
372 ops->unmap_resource(dev, addr, size, dir, attrs);
373 debug_dma_unmap_resource(dev, addr, size, dir);
374}
375
376static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
377 size_t size,
378 enum dma_data_direction dir)
379{
380 const struct dma_map_ops *ops = get_dma_ops(dev);
381
382 BUG_ON(!valid_dma_direction(dir));
383 if (dma_is_direct(ops))
384 dma_direct_sync_single_for_cpu(dev, addr, size, dir);
385 else if (ops->sync_single_for_cpu)
386 ops->sync_single_for_cpu(dev, addr, size, dir);
387 debug_dma_sync_single_for_cpu(dev, addr, size, dir);
388}
389
390static inline void dma_sync_single_for_device(struct device *dev,
391 dma_addr_t addr, size_t size,
392 enum dma_data_direction dir)
393{
394 const struct dma_map_ops *ops = get_dma_ops(dev);
395
396 BUG_ON(!valid_dma_direction(dir));
397 if (dma_is_direct(ops))
398 dma_direct_sync_single_for_device(dev, addr, size, dir);
399 else if (ops->sync_single_for_device)
400 ops->sync_single_for_device(dev, addr, size, dir);
401 debug_dma_sync_single_for_device(dev, addr, size, dir);
402}
403
404static inline void
405dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
406 int nelems, enum dma_data_direction dir)
407{209{
408 const struct dma_map_ops *ops = get_dma_ops(dev);210 return NULL;
409
410 BUG_ON(!valid_dma_direction(dir));
411 if (dma_is_direct(ops))
412 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
413 else if (ops->sync_sg_for_cpu)
414 ops->sync_sg_for_cpu(dev, sg, nelems, dir);
415 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
416}211}
417 212static inline void set_dma_ops(struct device *dev,
418static inline void213 const struct dma_map_ops *dma_ops)
419dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
420 int nelems, enum dma_data_direction dir)
421{214{
422 const struct dma_map_ops *ops = get_dma_ops(dev);
423
424 BUG_ON(!valid_dma_direction(dir));
425 if (dma_is_direct(ops))
426 dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
427 else if (ops->sync_sg_for_device)
428 ops->sync_sg_for_device(dev, sg, nelems, dir);
429 debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
430
431}215}
216#endif /* CONFIG_DMA_OPS */
432 217
433static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)218static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
434{219{
@@ -439,6 +224,28 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) · 4 unmodified lines
439 return 0;224 return 0;
440}225}
441 226
227dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
228 size_t offset, size_t size, enum dma_data_direction dir,
229 unsigned long attrs);
230void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
231 enum dma_data_direction dir, unsigned long attrs);
232int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
233 enum dma_data_direction dir, unsigned long attrs);
234void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
235 int nents, enum dma_data_direction dir,
236 unsigned long attrs);
237dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
238 size_t size, enum dma_data_direction dir, unsigned long attrs);
239void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
240 enum dma_data_direction dir, unsigned long attrs);
241void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
242 enum dma_data_direction dir);
243void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
244 size_t size, enum dma_data_direction dir);
245void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
246 int nelems, enum dma_data_direction dir);
247void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
248 int nelems, enum dma_data_direction dir);
442void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,249void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
443 gfp_t flag, unsigned long attrs);250 gfp_t flag, unsigned long attrs);
444void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,251void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,