History

Merge tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping updates from Christoph Hellwig: - make support for dma_ops optional - move more code out of line - add generic support for a dma_ops bypass mode - misc cleanups * tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping: dma-contiguous: cleanup dma_alloc_contiguous dma-debug: use named initializers for dir2name powerpc: use the generic dma_ops_bypass mode dma-mapping: add a dma_ops_bypass flag to struct device dma-mapping: make support for dma ops optional dma-mapping: inline the fast path dma-direct calls dma-mapping: move the remaining DMA API calls out of line

Linus Torvalds committed 6 years ago

Commit 2ed90db2 parents 9fa867d 274b3f726 files · +415 −416Browse files

Showing the changes to include/linux (3 of 26 files). Show all

Unified · Split

3 files
  1. include/linux/device.h +10 −1
  2. include/linux/dma-direct.h +104 −0
  3. include/linux/dma-mapping.h +29 −222
include/linux/device.h+10−1
@@ -525,6 +525,11 @@ struct dev_links_info { · 524 unmodified lines
525525 * sync_state() callback.
526526 * @dma_coherent: this particular device is dma coherent, even if the
527527 * architecture supports non-coherent devices.
528+ * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
529+ * streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
530+ * and optionall (if the coherent mask is large enough) also
531+ * for dma allocations. This flag is managed by the dma ops
532+ * instance from ->dma_supported.
528533 *
529534 * At the lowest level, every device in a Linux system is represented by an
530535 * instance of struct device. The device structure contains the information
@@ -574,8 +579,9 @@ struct device { · 43 unmodified lines
574579 #ifdef CONFIG_GENERIC_MSI_IRQ
575580 struct list_head msi_list;
576581 #endif
577−
582+#ifdef CONFIG_DMA_OPS
578583 const struct dma_map_ops *dma_ops;
584+#endif
579585 u64 *dma_mask; /* dma mask (if dma'able device) */
580586 u64 coherent_dma_mask;/* Like dma_mask, but for
581587 alloc_coherent mappings as
@@ -628,6 +634,9 @@ struct device { · 46 unmodified lines
628634 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
629635 bool dma_coherent:1;
630636 #endif
637+#ifdef CONFIG_DMA_OPS_BYPASS
638+ bool dma_ops_bypass : 1;
639+#endif
631640 };
632641
633642 static inline struct device *kobj_to_dev(struct kobject *kobj)
include/linux/dma-direct.h+104−0
@@ -1,10 +1,16 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
2+/*
3+ * Internals of the DMA direct mapping implementation. Only for use by the
4+ * DMA mapping code and IOMMU drivers.
5+ */
26 #ifndef _LINUX_DMA_DIRECT_H
37 #define _LINUX_DMA_DIRECT_H 1
48
59 #include <linux/dma-mapping.h>
10+#include <linux/dma-noncoherent.h>
611 #include <linux/memblock.h> /* for min_low_pfn */
712 #include <linux/mem_encrypt.h>
13+#include <linux/swiotlb.h>
814
915 extern unsigned int zone_dma_bits;
1016
@@ -87,4 +93,102 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, · 76 unmodified lines
8793 unsigned long attrs);
8894 int dma_direct_supported(struct device *dev, u64 mask);
8995 bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);
96+int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
97+ enum dma_data_direction dir, unsigned long attrs);
98+dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
99+ size_t size, enum dma_data_direction dir, unsigned long attrs);
100+size_t dma_direct_max_mapping_size(struct device *dev);
101+
102+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
103+ defined(CONFIG_SWIOTLB)
104+void dma_direct_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
105+ int nents, enum dma_data_direction dir);
106+#else
107+static inline void dma_direct_sync_sg_for_device(struct device *dev,
108+ struct scatterlist *sgl, int nents, enum dma_data_direction dir)
109+{
110+}
111+#endif
112+
113+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
114+ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
115+ defined(CONFIG_SWIOTLB)
116+void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
117+ int nents, enum dma_data_direction dir, unsigned long attrs);
118+void dma_direct_sync_sg_for_cpu(struct device *dev,
119+ struct scatterlist *sgl, int nents, enum dma_data_direction dir);
120+#else
121+static inline void dma_direct_unmap_sg(struct device *dev,
122+ struct scatterlist *sgl, int nents, enum dma_data_direction dir,
123+ unsigned long attrs)
124+{
125+}
126+static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
127+ struct scatterlist *sgl, int nents, enum dma_data_direction dir)
128+{
129+}
130+#endif
131+
132+static inline void dma_direct_sync_single_for_device(struct device *dev,
133+ dma_addr_t addr, size_t size, enum dma_data_direction dir)
134+{
135+ phys_addr_t paddr = dma_to_phys(dev, addr);
136+
137+ if (unlikely(is_swiotlb_buffer(paddr)))
138+ swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
139+
140+ if (!dev_is_dma_coherent(dev))
141+ arch_sync_dma_for_device(paddr, size, dir);
142+}
143+
144+static inline void dma_direct_sync_single_for_cpu(struct device *dev,
145+ dma_addr_t addr, size_t size, enum dma_data_direction dir)
146+{
147+ phys_addr_t paddr = dma_to_phys(dev, addr);
148+
149+ if (!dev_is_dma_coherent(dev)) {
150+ arch_sync_dma_for_cpu(paddr, size, dir);
151+ arch_sync_dma_for_cpu_all();
152+ }
153+
154+ if (unlikely(is_swiotlb_buffer(paddr)))
155+ swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
156+}
157+
158+static inline dma_addr_t dma_direct_map_page(struct device *dev,
159+ struct page *page, unsigned long offset, size_t size,
160+ enum dma_data_direction dir, unsigned long attrs)
161+{
162+ phys_addr_t phys = page_to_phys(page) + offset;
163+ dma_addr_t dma_addr = phys_to_dma(dev, phys);
164+
165+ if (unlikely(swiotlb_force == SWIOTLB_FORCE))
166+ return swiotlb_map(dev, phys, size, dir, attrs);
167+
168+ if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
169+ if (swiotlb_force != SWIOTLB_NO_FORCE)
170+ return swiotlb_map(dev, phys, size, dir, attrs);
171+
172+ dev_WARN_ONCE(dev, 1,
173+ "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
174+ &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
175+ return DMA_MAPPING_ERROR;
176+ }
177+
178+ if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
179+ arch_sync_dma_for_device(phys, size, dir);
180+ return dma_addr;
181+}
182+
183+static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
184+ size_t size, enum dma_data_direction dir, unsigned long attrs)
185+{
186+ phys_addr_t phys = dma_to_phys(dev, addr);
187+
188+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
189+ dma_direct_sync_single_for_cpu(dev, addr, size, dir);
190+
191+ if (unlikely(is_swiotlb_buffer(phys)))
192+ swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
193+}
90194 #endif /* _LINUX_DMA_DIRECT_H */
include/linux/dma-mapping.h+29−222
@@ -188,76 +188,10 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma, · 187 unmodified lines
188188 }
189189 #endif /* CONFIG_DMA_DECLARE_COHERENT */
190190
191−static inline bool dma_is_direct(const struct dma_map_ops *ops)
192−{
193− return likely(!ops);
194−}
195−
196−/*
197− * All the dma_direct_* declarations are here just for the indirect call bypass,
198− * and must not be used directly drivers!
199− */
200−dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
201− unsigned long offset, size_t size, enum dma_data_direction dir,
202− unsigned long attrs);
203−int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
204− enum dma_data_direction dir, unsigned long attrs);
205−dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
206− size_t size, enum dma_data_direction dir, unsigned long attrs);
207−
208−#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
209− defined(CONFIG_SWIOTLB)
210−void dma_direct_sync_single_for_device(struct device *dev,
211− dma_addr_t addr, size_t size, enum dma_data_direction dir);
212−void dma_direct_sync_sg_for_device(struct device *dev,
213− struct scatterlist *sgl, int nents, enum dma_data_direction dir);
214−#else
215−static inline void dma_direct_sync_single_for_device(struct device *dev,
216− dma_addr_t addr, size_t size, enum dma_data_direction dir)
217−{
218−}
219−static inline void dma_direct_sync_sg_for_device(struct device *dev,
220− struct scatterlist *sgl, int nents, enum dma_data_direction dir)
221−{
222−}
223−#endif
224−
225−#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
226− defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
227− defined(CONFIG_SWIOTLB)
228−void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
229− size_t size, enum dma_data_direction dir, unsigned long attrs);
230−void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
231− int nents, enum dma_data_direction dir, unsigned long attrs);
232−void dma_direct_sync_single_for_cpu(struct device *dev,
233− dma_addr_t addr, size_t size, enum dma_data_direction dir);
234−void dma_direct_sync_sg_for_cpu(struct device *dev,
235− struct scatterlist *sgl, int nents, enum dma_data_direction dir);
236−#else
237−static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
238− size_t size, enum dma_data_direction dir, unsigned long attrs)
239−{
240−}
241−static inline void dma_direct_unmap_sg(struct device *dev,
242− struct scatterlist *sgl, int nents, enum dma_data_direction dir,
243− unsigned long attrs)
244−{
245−}
246−static inline void dma_direct_sync_single_for_cpu(struct device *dev,
247− dma_addr_t addr, size_t size, enum dma_data_direction dir)
248−{
249−}
250−static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
251− struct scatterlist *sgl, int nents, enum dma_data_direction dir)
252−{
253−}
254−#endif
255−
256−size_t dma_direct_max_mapping_size(struct device *dev);
257−
258191 #ifdef CONFIG_HAS_DMA
259192 #include <asm/dma-mapping.h>
260193
194+#ifdef CONFIG_DMA_OPS
261195 static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
262196 {
263197 if (dev->dma_ops)
@@ -270,165 +204,16 @@ static inline void set_dma_ops(struct device *dev, · 6 unmodified lines
270204 {
271205 dev->dma_ops = dma_ops;
272206 }
273−
274−static inline dma_addr_t dma_map_page_attrs(struct device *dev,
275− struct page *page, size_t offset, size_t size,
276− enum dma_data_direction dir, unsigned long attrs)
277−{
278− const struct dma_map_ops *ops = get_dma_ops(dev);
279− dma_addr_t addr;
280−
281− BUG_ON(!valid_dma_direction(dir));
282− if (dma_is_direct(ops))
283− addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
284− else
285− addr = ops->map_page(dev, page, offset, size, dir, attrs);
286− debug_dma_map_page(dev, page, offset, size, dir, addr);
287−
288− return addr;
289−}
290−
291−static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
292− size_t size, enum dma_data_direction dir, unsigned long attrs)
293−{
294− const struct dma_map_ops *ops = get_dma_ops(dev);
295−
296− BUG_ON(!valid_dma_direction(dir));
297− if (dma_is_direct(ops))
298− dma_direct_unmap_page(dev, addr, size, dir, attrs);
299− else if (ops->unmap_page)
300− ops->unmap_page(dev, addr, size, dir, attrs);
301− debug_dma_unmap_page(dev, addr, size, dir);
302−}
303−
304−/*
305− * dma_maps_sg_attrs returns 0 on error and > 0 on success.
306− * It should never return a value < 0.
307− */
308−static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
309− int nents, enum dma_data_direction dir,
310− unsigned long attrs)
311−{
312− const struct dma_map_ops *ops = get_dma_ops(dev);
313− int ents;
314−
315− BUG_ON(!valid_dma_direction(dir));
316− if (dma_is_direct(ops))
317− ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
318− else
319− ents = ops->map_sg(dev, sg, nents, dir, attrs);
320− BUG_ON(ents < 0);
321− debug_dma_map_sg(dev, sg, nents, ents, dir);
322−
323− return ents;
324−}
325−
326−static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
327− int nents, enum dma_data_direction dir,
328− unsigned long attrs)
329−{
330− const struct dma_map_ops *ops = get_dma_ops(dev);
331−
332− BUG_ON(!valid_dma_direction(dir));
333− debug_dma_unmap_sg(dev, sg, nents, dir);
334− if (dma_is_direct(ops))
335− dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
336− else if (ops->unmap_sg)
337− ops->unmap_sg(dev, sg, nents, dir, attrs);
338−}
339−
340−static inline dma_addr_t dma_map_resource(struct device *dev,
341− phys_addr_t phys_addr,
342− size_t size,
343− enum dma_data_direction dir,
344− unsigned long attrs)
345−{
346− const struct dma_map_ops *ops = get_dma_ops(dev);
347− dma_addr_t addr = DMA_MAPPING_ERROR;
348−
349− BUG_ON(!valid_dma_direction(dir));
350−
351− /* Don't allow RAM to be mapped */
352− if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
353− return DMA_MAPPING_ERROR;
354−
355− if (dma_is_direct(ops))
356− addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
357− else if (ops->map_resource)
358− addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
359−
360− debug_dma_map_resource(dev, phys_addr, size, dir, addr);
361− return addr;
362−}
363−
364−static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
365− size_t size, enum dma_data_direction dir,
366− unsigned long attrs)
367−{
368− const struct dma_map_ops *ops = get_dma_ops(dev);
369−
370− BUG_ON(!valid_dma_direction(dir));
371− if (!dma_is_direct(ops) && ops->unmap_resource)
372− ops->unmap_resource(dev, addr, size, dir, attrs);
373− debug_dma_unmap_resource(dev, addr, size, dir);
374−}
375−
376−static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
377− size_t size,
378− enum dma_data_direction dir)
379−{
380− const struct dma_map_ops *ops = get_dma_ops(dev);
381−
382− BUG_ON(!valid_dma_direction(dir));
383− if (dma_is_direct(ops))
384− dma_direct_sync_single_for_cpu(dev, addr, size, dir);
385− else if (ops->sync_single_for_cpu)
386− ops->sync_single_for_cpu(dev, addr, size, dir);
387− debug_dma_sync_single_for_cpu(dev, addr, size, dir);
388−}
389−
390−static inline void dma_sync_single_for_device(struct device *dev,
391− dma_addr_t addr, size_t size,
392− enum dma_data_direction dir)
393−{
394− const struct dma_map_ops *ops = get_dma_ops(dev);
395−
396− BUG_ON(!valid_dma_direction(dir));
397− if (dma_is_direct(ops))
398− dma_direct_sync_single_for_device(dev, addr, size, dir);
399− else if (ops->sync_single_for_device)
400− ops->sync_single_for_device(dev, addr, size, dir);
401− debug_dma_sync_single_for_device(dev, addr, size, dir);
402−}
403−
404−static inline void
405−dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
406− int nelems, enum dma_data_direction dir)
207+#else /* CONFIG_DMA_OPS */
208+static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
407209 {
408− const struct dma_map_ops *ops = get_dma_ops(dev);
409−
410− BUG_ON(!valid_dma_direction(dir));
411− if (dma_is_direct(ops))
412− dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
413− else if (ops->sync_sg_for_cpu)
414− ops->sync_sg_for_cpu(dev, sg, nelems, dir);
415− debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
210+ return NULL;
416211 }
417−
418−static inline void
419−dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
420− int nelems, enum dma_data_direction dir)
212+static inline void set_dma_ops(struct device *dev,
213+ const struct dma_map_ops *dma_ops)
421214 {
422− const struct dma_map_ops *ops = get_dma_ops(dev);
423−
424− BUG_ON(!valid_dma_direction(dir));
425− if (dma_is_direct(ops))
426− dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
427− else if (ops->sync_sg_for_device)
428− ops->sync_sg_for_device(dev, sg, nelems, dir);
429− debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
430−
431215 }
216+#endif /* CONFIG_DMA_OPS */
432217
433218 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
434219 {
@@ -439,6 +224,28 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) · 4 unmodified lines
439224 return 0;
440225 }
441226
227+dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
228+ size_t offset, size_t size, enum dma_data_direction dir,
229+ unsigned long attrs);
230+void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
231+ enum dma_data_direction dir, unsigned long attrs);
232+int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
233+ enum dma_data_direction dir, unsigned long attrs);
234+void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
235+ int nents, enum dma_data_direction dir,
236+ unsigned long attrs);
237+dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
238+ size_t size, enum dma_data_direction dir, unsigned long attrs);
239+void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
240+ enum dma_data_direction dir, unsigned long attrs);
241+void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
242+ enum dma_data_direction dir);
243+void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
244+ size_t size, enum dma_data_direction dir);
245+void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
246+ int nelems, enum dma_data_direction dir);
247+void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
248+ int nelems, enum dma_data_direction dir);
442249 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
443250 gfp_t flag, unsigned long attrs);
444251 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,