History

dma-mapping: add a new dma_alloc_pages API

This API is the equivalent of alloc_pages, except that the returned memory is guaranteed to be DMA addressable by the passed in device. The implementation will also be used to provide a more sensible replacement for DMA_ATTR_NON_CONSISTENT flag. Additionally dma_alloc_noncoherent is switched over to use dma_alloc_pages as its backend.

Christoph Hellwig committed 6 years ago

Commit efa70f21 parent 5a8429223 files · +206 −37Browse files

Showing the changes to kernel (4 of 23 files). Show all

Unified · Split

4 files
  1. kernel/dma/direct.c +51 −1
  2. kernel/dma/mapping.c +60 −3
  3. kernel/dma/ops_helpers.c +35 −0
  4. kernel/dma/virt.c +2 −0
kernel/dma/direct.c+51−1
@@ -1,6 +1,6 @@
11 // SPDX-License-Identifier: GPL-2.0
22 /*
3− * Copyright (C) 2018 Christoph Hellwig.
3+ * Copyright (C) 2018-2020 Christoph Hellwig.
44 *
55 * DMA operations that map physical memory directly without using an IOMMU.
66 */
@@ -292,6 +292,56 @@ void dma_direct_free(struct device *dev, size_t size, · 285 unmodified lines
292292 dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);
293293 }
294294
295+struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
296+ dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
297+{
298+ struct page *page;
299+ void *ret;
300+
301+ if (dma_should_alloc_from_pool(dev, gfp, 0)) {
302+ page = dma_alloc_from_pool(dev, size, &ret, gfp,
303+ dma_coherent_ok);
304+ if (!page)
305+ return NULL;
306+ goto done;
307+ }
308+
309+ page = __dma_direct_alloc_pages(dev, size, gfp);
310+ if (!page)
311+ return NULL;
312+ ret = page_address(page);
313+ if (force_dma_unencrypted(dev)) {
314+ if (set_memory_decrypted((unsigned long)ret,
315+ 1 << get_order(size)))
316+ goto out_free_pages;
317+ }
318+ memset(ret, 0, size);
319+done:
320+ *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
321+ return page;
322+out_free_pages:
323+ dma_free_contiguous(dev, page, size);
324+ return NULL;
325+}
326+
327+void dma_direct_free_pages(struct device *dev, size_t size,
328+ struct page *page, dma_addr_t dma_addr,
329+ enum dma_data_direction dir)
330+{
331+ unsigned int page_order = get_order(size);
332+ void *vaddr = page_address(page);
333+
334+ /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
335+ if (dma_should_free_from_pool(dev, 0) &&
336+ dma_free_from_pool(dev, vaddr, size))
337+ return;
338+
339+ if (force_dma_unencrypted(dev))
340+ set_memory_encrypted((unsigned long)vaddr, 1 << page_order);
341+
342+ dma_free_contiguous(dev, page, size);
343+}
344+
295345 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
296346 defined(CONFIG_SWIOTLB)
297347 void dma_direct_sync_sg_for_device(struct device *dev,
kernel/dma/mapping.c+60−3
@@ -341,9 +341,7 @@ pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) · 340 unmodified lines
341341 {
342342 if (force_dma_unencrypted(dev))
343343 prot = pgprot_decrypted(prot);
344− if (dev_is_dma_coherent(dev) ||
345− (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) &&
346− (attrs & DMA_ATTR_NON_CONSISTENT)))
344+ if (dev_is_dma_coherent(dev))
347345 return prot;
348346 #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
349347 if (attrs & DMA_ATTR_WRITE_COMBINE)
@@ -472,6 +470,65 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, · 122 unmodified lines
472470 }
473471 EXPORT_SYMBOL(dma_free_attrs);
474472
473+struct page *dma_alloc_pages(struct device *dev, size_t size,
474+ dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
475+{
476+ const struct dma_map_ops *ops = get_dma_ops(dev);
477+ struct page *page;
478+
479+ if (WARN_ON_ONCE(!dev->coherent_dma_mask))
480+ return NULL;
481+ if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)))
482+ return NULL;
483+
484+ size = PAGE_ALIGN(size);
485+ if (dma_alloc_direct(dev, ops))
486+ page = dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);
487+ else if (ops->alloc_pages)
488+ page = ops->alloc_pages(dev, size, dma_handle, dir, gfp);
489+ else
490+ return NULL;
491+
492+ debug_dma_map_page(dev, page, 0, size, dir, *dma_handle);
493+
494+ return page;
495+}
496+EXPORT_SYMBOL_GPL(dma_alloc_pages);
497+
498+void dma_free_pages(struct device *dev, size_t size, struct page *page,
499+ dma_addr_t dma_handle, enum dma_data_direction dir)
500+{
501+ const struct dma_map_ops *ops = get_dma_ops(dev);
502+
503+ size = PAGE_ALIGN(size);
504+ debug_dma_unmap_page(dev, dma_handle, size, dir);
505+
506+ if (dma_alloc_direct(dev, ops))
507+ dma_direct_free_pages(dev, size, page, dma_handle, dir);
508+ else if (ops->free_pages)
509+ ops->free_pages(dev, size, page, dma_handle, dir);
510+}
511+EXPORT_SYMBOL_GPL(dma_free_pages);
512+
513+void *dma_alloc_noncoherent(struct device *dev, size_t size,
514+ dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
515+{
516+ struct page *page;
517+
518+ page = dma_alloc_pages(dev, size, dma_handle, dir, gfp);
519+ if (!page)
520+ return NULL;
521+ return page_address(page);
522+}
523+EXPORT_SYMBOL_GPL(dma_alloc_noncoherent);
524+
525+void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
526+ dma_addr_t dma_handle, enum dma_data_direction dir)
527+{
528+ dma_free_pages(dev, size, virt_to_page(vaddr), dma_handle, dir);
529+}
530+EXPORT_SYMBOL_GPL(dma_free_noncoherent);
531+
475532 int dma_supported(struct device *dev, u64 mask)
476533 {
477534 const struct dma_map_ops *ops = get_dma_ops(dev);
kernel/dma/ops_helpers.c+35−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 * Helpers for DMA ops implementations. These generally rely on the fact that
44 * the allocated memory contains normal pages in the direct kernel mapping.
55 */
6+#include <linux/dma-contiguous.h>
67 #include <linux/dma-noncoherent.h>
78
89 /*
@@ -49,3 +50,37 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, · 40 unmodified lines
4950 return -ENXIO;
5051 #endif /* CONFIG_MMU */
5152 }
53+
54+struct page *dma_common_alloc_pages(struct device *dev, size_t size,
55+ dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
56+{
57+ const struct dma_map_ops *ops = get_dma_ops(dev);
58+ struct page *page;
59+
60+ page = dma_alloc_contiguous(dev, size, gfp);
61+ if (!page)
62+ page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size));
63+ if (!page)
64+ return NULL;
65+
66+ *dma_handle = ops->map_page(dev, page, 0, size, dir,
67+ DMA_ATTR_SKIP_CPU_SYNC);
68+ if (*dma_handle == DMA_MAPPING_ERROR) {
69+ dma_free_contiguous(dev, page, size);
70+ return NULL;
71+ }
72+
73+ memset(page_address(page), 0, size);
74+ return page;
75+}
76+
77+void dma_common_free_pages(struct device *dev, size_t size, struct page *page,
78+ dma_addr_t dma_handle, enum dma_data_direction dir)
79+{
80+ const struct dma_map_ops *ops = get_dma_ops(dev);
81+
82+ if (ops->unmap_page)
83+ ops->unmap_page(dev, dma_handle, size, dir,
84+ DMA_ATTR_SKIP_CPU_SYNC);
85+ dma_free_contiguous(dev, page, size);
86+}
kernel/dma/virt.c+2−0
@@ -55,5 +55,7 @@ const struct dma_map_ops dma_virt_ops = { · 54 unmodified lines
5555 .free = dma_virt_free,
5656 .map_page = dma_virt_map_page,
5757 .map_sg = dma_virt_map_sg,
58+ .alloc_pages = dma_common_alloc_pages,
59+ .free_pages = dma_common_free_pages,
5860 };
5961 EXPORT_SYMBOL(dma_virt_ops);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t