History

dma-mapping: add a new dma_alloc_pages API

This API is the equivalent of alloc_pages, except that the returned memory is guaranteed to be DMA addressable by the passed in device. The implementation will also be used to provide a more sensible replacement for DMA_ATTR_NON_CONSISTENT flag. Additionally dma_alloc_noncoherent is switched over to use dma_alloc_pages as its backend.

Christoph Hellwig committed 6 years ago

Commit efa70f21 parent 5a8429223 files · +206 −37Browse files

Unified · Split

23 files
  1. Documentation/core-api/dma-attributes.rst +0 −8
  2. arch/alpha/kernel/pci_iommu.c +2 −0
  3. arch/arm/mm/dma-mapping-nommu.c +2 −0
  4. arch/arm/mm/dma-mapping.c +4 −0
  5. arch/ia64/hp/common/sba_iommu.c +2 −0
  6. arch/mips/jazz/jazzdma.c +2 −5
  7. arch/powerpc/kernel/dma-iommu.c +2 −0
  8. arch/powerpc/platforms/ps3/system-bus.c +4 −0
  9. arch/powerpc/platforms/pseries/vio.c +2 −0
  10. arch/s390/pci/pci_dma.c +2 −0
  11. arch/x86/kernel/amd_gart_64.c +2 −0
  12. drivers/iommu/dma-iommu.c +2 −0
  13. drivers/iommu/intel/iommu.c +4 −0
  14. drivers/parisc/ccio-dma.c +2 −0
  15. drivers/parisc/sba_iommu.c +2 −0
  16. drivers/xen/swiotlb-xen.c +2 −0
  17. include/linux/dma-direct.h +5 −0
  18. include/linux/dma-mapping.h +17 −17
  19. include/linux/dma-noncoherent.h +0 −3
  20. kernel/dma/direct.c +51 −1
  21. kernel/dma/mapping.c +60 −3
  22. kernel/dma/ops_helpers.c +35 −0
  23. kernel/dma/virt.c +2 −0
Documentation/core-api/dma-attributes.rst+0−8
@@ -25,14 +25,6 @@ Since it is optional for platforms to implement DMA_ATTR_WRITE_COMBINE, · 24 unmodified lines
25those that do not will simply ignore the attribute and exhibit default25those that do not will simply ignore the attribute and exhibit default
26behavior.26behavior.
27 27
28DMA_ATTR_NON_CONSISTENT
29-----------------------
30
31DMA_ATTR_NON_CONSISTENT lets the platform to choose to return either
32consistent or non-consistent memory as it sees fit. By using this API,
33you are guaranteeing to the platform that you have all the correct and
34necessary sync points for this memory in the driver.
35
36DMA_ATTR_NO_KERNEL_MAPPING28DMA_ATTR_NO_KERNEL_MAPPING
37--------------------------29--------------------------
38 30
arch/alpha/kernel/pci_iommu.c+2−0
@@ -952,5 +952,7 @@ const struct dma_map_ops alpha_pci_ops = { · 951 unmodified lines
952 .dma_supported = alpha_pci_supported,952 .dma_supported = alpha_pci_supported,
953 .mmap = dma_common_mmap,953 .mmap = dma_common_mmap,
954 .get_sgtable = dma_common_get_sgtable,954 .get_sgtable = dma_common_get_sgtable,
955 .alloc_pages = dma_common_alloc_pages,
956 .free_pages = dma_common_free_pages,
955};957};
956EXPORT_SYMBOL(alpha_pci_ops);958EXPORT_SYMBOL(alpha_pci_ops);
arch/arm/mm/dma-mapping-nommu.c+2−0
@@ -176,6 +176,8 @@ static void arm_nommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist · 175 unmodified lines
176const struct dma_map_ops arm_nommu_dma_ops = {176const struct dma_map_ops arm_nommu_dma_ops = {
177 .alloc = arm_nommu_dma_alloc,177 .alloc = arm_nommu_dma_alloc,
178 .free = arm_nommu_dma_free,178 .free = arm_nommu_dma_free,
179 .alloc_pages = dma_direct_alloc_pages,
180 .free_pages = dma_direct_free_pages,
179 .mmap = arm_nommu_dma_mmap,181 .mmap = arm_nommu_dma_mmap,
180 .map_page = arm_nommu_dma_map_page,182 .map_page = arm_nommu_dma_map_page,
181 .unmap_page = arm_nommu_dma_unmap_page,183 .unmap_page = arm_nommu_dma_unmap_page,
arch/arm/mm/dma-mapping.c+4−0
@@ -199,6 +199,8 @@ static int arm_dma_supported(struct device *dev, u64 mask) · 198 unmodified lines
199const struct dma_map_ops arm_dma_ops = {199const struct dma_map_ops arm_dma_ops = {
200 .alloc = arm_dma_alloc,200 .alloc = arm_dma_alloc,
201 .free = arm_dma_free,201 .free = arm_dma_free,
202 .alloc_pages = dma_direct_alloc_pages,
203 .free_pages = dma_direct_free_pages,
202 .mmap = arm_dma_mmap,204 .mmap = arm_dma_mmap,
203 .get_sgtable = arm_dma_get_sgtable,205 .get_sgtable = arm_dma_get_sgtable,
204 .map_page = arm_dma_map_page,206 .map_page = arm_dma_map_page,
@@ -226,6 +228,8 @@ static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, · 21 unmodified lines
226const struct dma_map_ops arm_coherent_dma_ops = {228const struct dma_map_ops arm_coherent_dma_ops = {
227 .alloc = arm_coherent_dma_alloc,229 .alloc = arm_coherent_dma_alloc,
228 .free = arm_coherent_dma_free,230 .free = arm_coherent_dma_free,
231 .alloc_pages = dma_direct_alloc_pages,
232 .free_pages = dma_direct_free_pages,
229 .mmap = arm_coherent_dma_mmap,233 .mmap = arm_coherent_dma_mmap,
230 .get_sgtable = arm_dma_get_sgtable,234 .get_sgtable = arm_dma_get_sgtable,
231 .map_page = arm_coherent_dma_map_page,235 .map_page = arm_coherent_dma_map_page,
arch/ia64/hp/common/sba_iommu.c+2−0
@@ -2070,6 +2070,8 @@ static const struct dma_map_ops sba_dma_ops = { · 2069 unmodified lines
2070 .dma_supported = sba_dma_supported,2070 .dma_supported = sba_dma_supported,
2071 .mmap = dma_common_mmap,2071 .mmap = dma_common_mmap,
2072 .get_sgtable = dma_common_get_sgtable,2072 .get_sgtable = dma_common_get_sgtable,
2073 .alloc_pages = dma_common_alloc_pages,
2074 .free_pages = dma_common_free_pages,
2073};2075};
2074 2076
2075static int __init2077static int __init
arch/mips/jazz/jazzdma.c+2−5
@@ -506,9 +506,6 @@ static void *jazz_dma_alloc(struct device *dev, size_t size, · 505 unmodified lines
506 *dma_handle = vdma_alloc(virt_to_phys(ret), size);506 *dma_handle = vdma_alloc(virt_to_phys(ret), size);
507 if (*dma_handle == DMA_MAPPING_ERROR)507 if (*dma_handle == DMA_MAPPING_ERROR)
508 goto out_free_pages;508 goto out_free_pages;
509
510 if (attrs & DMA_ATTR_NON_CONSISTENT)
511 return ret;
512 arch_dma_prep_coherent(page, size);509 arch_dma_prep_coherent(page, size);
513 return (void *)(UNCAC_BASE + __pa(ret));510 return (void *)(UNCAC_BASE + __pa(ret));
514 511
@@ -521,8 +518,6 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, · 6 unmodified lines
521 dma_addr_t dma_handle, unsigned long attrs)518 dma_addr_t dma_handle, unsigned long attrs)
522{519{
523 vdma_free(dma_handle);520 vdma_free(dma_handle);
524 if (!(attrs & DMA_ATTR_NON_CONSISTENT))
525 vaddr = __va(vaddr - UNCAC_BASE);
526 __free_pages(virt_to_page(vaddr), get_order(size));521 __free_pages(virt_to_page(vaddr), get_order(size));
527}522}
528 523
@@ -622,5 +617,7 @@ const struct dma_map_ops jazz_dma_ops = { · 93 unmodified lines
622 .sync_sg_for_device = jazz_dma_sync_sg_for_device,617 .sync_sg_for_device = jazz_dma_sync_sg_for_device,
623 .mmap = dma_common_mmap,618 .mmap = dma_common_mmap,
624 .get_sgtable = dma_common_get_sgtable,619 .get_sgtable = dma_common_get_sgtable,
620 .alloc_pages = dma_common_alloc_pages,
621 .free_pages = dma_common_free_pages,
625};622};
626EXPORT_SYMBOL(jazz_dma_ops);623EXPORT_SYMBOL(jazz_dma_ops);
arch/powerpc/kernel/dma-iommu.c+2−0
@@ -138,4 +138,6 @@ const struct dma_map_ops dma_iommu_ops = { · 137 unmodified lines
138 .get_required_mask = dma_iommu_get_required_mask,138 .get_required_mask = dma_iommu_get_required_mask,
139 .mmap = dma_common_mmap,139 .mmap = dma_common_mmap,
140 .get_sgtable = dma_common_get_sgtable,140 .get_sgtable = dma_common_get_sgtable,
141 .alloc_pages = dma_common_alloc_pages,
142 .free_pages = dma_common_free_pages,
141};143};
arch/powerpc/platforms/ps3/system-bus.c+4−0
@@ -696,6 +696,8 @@ static const struct dma_map_ops ps3_sb_dma_ops = { · 695 unmodified lines
696 .unmap_page = ps3_unmap_page,696 .unmap_page = ps3_unmap_page,
697 .mmap = dma_common_mmap,697 .mmap = dma_common_mmap,
698 .get_sgtable = dma_common_get_sgtable,698 .get_sgtable = dma_common_get_sgtable,
699 .alloc_pages = dma_common_alloc_pages,
700 .free_pages = dma_common_free_pages,
699};701};
700 702
701static const struct dma_map_ops ps3_ioc0_dma_ops = {703static const struct dma_map_ops ps3_ioc0_dma_ops = {
@@ -708,6 +710,8 @@ static const struct dma_map_ops ps3_ioc0_dma_ops = { · 6 unmodified lines
708 .unmap_page = ps3_unmap_page,710 .unmap_page = ps3_unmap_page,
709 .mmap = dma_common_mmap,711 .mmap = dma_common_mmap,
710 .get_sgtable = dma_common_get_sgtable,712 .get_sgtable = dma_common_get_sgtable,
713 .alloc_pages = dma_common_alloc_pages,
714 .free_pages = dma_common_free_pages,
711};715};
712 716
713/**717/**
arch/powerpc/platforms/pseries/vio.c+2−0
@@ -608,6 +608,8 @@ static const struct dma_map_ops vio_dma_mapping_ops = { · 607 unmodified lines
608 .get_required_mask = dma_iommu_get_required_mask,608 .get_required_mask = dma_iommu_get_required_mask,
609 .mmap = dma_common_mmap,609 .mmap = dma_common_mmap,
610 .get_sgtable = dma_common_get_sgtable,610 .get_sgtable = dma_common_get_sgtable,
611 .alloc_pages = dma_common_alloc_pages,
612 .free_pages = dma_common_free_pages,
611};613};
612 614
613/**615/**
arch/s390/pci/pci_dma.c+2−0
@@ -668,6 +668,8 @@ const struct dma_map_ops s390_pci_dma_ops = { · 667 unmodified lines
668 .unmap_page = s390_dma_unmap_pages,668 .unmap_page = s390_dma_unmap_pages,
669 .mmap = dma_common_mmap,669 .mmap = dma_common_mmap,
670 .get_sgtable = dma_common_get_sgtable,670 .get_sgtable = dma_common_get_sgtable,
671 .alloc_pages = dma_common_alloc_pages,
672 .free_pages = dma_common_free_pages,
671 /* dma_supported is unconditionally true without a callback */673 /* dma_supported is unconditionally true without a callback */
672};674};
673EXPORT_SYMBOL_GPL(s390_pci_dma_ops);675EXPORT_SYMBOL_GPL(s390_pci_dma_ops);
arch/x86/kernel/amd_gart_64.c+2−0
@@ -677,6 +677,8 @@ static const struct dma_map_ops gart_dma_ops = { · 676 unmodified lines
677 .get_sgtable = dma_common_get_sgtable,677 .get_sgtable = dma_common_get_sgtable,
678 .dma_supported = dma_direct_supported,678 .dma_supported = dma_direct_supported,
679 .get_required_mask = dma_direct_get_required_mask,679 .get_required_mask = dma_direct_get_required_mask,
680 .alloc_pages = dma_direct_alloc_pages,
681 .free_pages = dma_direct_free_pages,
680};682};
681 683
682static void gart_iommu_shutdown(void)684static void gart_iommu_shutdown(void)
drivers/iommu/dma-iommu.c+2−0
@@ -1120,6 +1120,8 @@ static unsigned long iommu_dma_get_merge_boundary(struct device *dev) · 1119 unmodified lines
1120static const struct dma_map_ops iommu_dma_ops = {1120static const struct dma_map_ops iommu_dma_ops = {
1121 .alloc = iommu_dma_alloc,1121 .alloc = iommu_dma_alloc,
1122 .free = iommu_dma_free,1122 .free = iommu_dma_free,
1123 .alloc_pages = dma_common_alloc_pages,
1124 .free_pages = dma_common_free_pages,
1123 .mmap = iommu_dma_mmap,1125 .mmap = iommu_dma_mmap,
1124 .get_sgtable = iommu_dma_get_sgtable,1126 .get_sgtable = iommu_dma_get_sgtable,
1125 .map_page = iommu_dma_map_page,1127 .map_page = iommu_dma_map_page,
drivers/iommu/intel/iommu.c+4−0
@@ -3712,6 +3712,8 @@ static const struct dma_map_ops intel_dma_ops = { · 3711 unmodified lines
3712 .dma_supported = dma_direct_supported,3712 .dma_supported = dma_direct_supported,
3713 .mmap = dma_common_mmap,3713 .mmap = dma_common_mmap,
3714 .get_sgtable = dma_common_get_sgtable,3714 .get_sgtable = dma_common_get_sgtable,
3715 .alloc_pages = dma_common_alloc_pages,
3716 .free_pages = dma_common_free_pages,
3715 .get_required_mask = intel_get_required_mask,3717 .get_required_mask = intel_get_required_mask,
3716};3718};
3717 3719
@@ -3965,6 +3967,8 @@ static const struct dma_map_ops bounce_dma_ops = { · 247 unmodified lines
3965 .sync_sg_for_device = bounce_sync_sg_for_device,3967 .sync_sg_for_device = bounce_sync_sg_for_device,
3966 .map_resource = bounce_map_resource,3968 .map_resource = bounce_map_resource,
3967 .unmap_resource = bounce_unmap_resource,3969 .unmap_resource = bounce_unmap_resource,
3970 .alloc_pages = dma_common_alloc_pages,
3971 .free_pages = dma_common_free_pages,
3968 .dma_supported = dma_direct_supported,3972 .dma_supported = dma_direct_supported,
3969};3973};
3970 3974
drivers/parisc/ccio-dma.c+2−0
@@ -1024,6 +1024,8 @@ static const struct dma_map_ops ccio_ops = { · 1023 unmodified lines
1024 .map_sg = ccio_map_sg,1024 .map_sg = ccio_map_sg,
1025 .unmap_sg = ccio_unmap_sg,1025 .unmap_sg = ccio_unmap_sg,
1026 .get_sgtable = dma_common_get_sgtable,1026 .get_sgtable = dma_common_get_sgtable,
1027 .alloc_pages = dma_common_alloc_pages,
1028 .free_pages = dma_common_free_pages,
1027};1029};
1028 1030
1029#ifdef CONFIG_PROC_FS1031#ifdef CONFIG_PROC_FS
drivers/parisc/sba_iommu.c+2−0
@@ -1076,6 +1076,8 @@ static const struct dma_map_ops sba_ops = { · 1075 unmodified lines
1076 .map_sg = sba_map_sg,1076 .map_sg = sba_map_sg,
1077 .unmap_sg = sba_unmap_sg,1077 .unmap_sg = sba_unmap_sg,
1078 .get_sgtable = dma_common_get_sgtable,1078 .get_sgtable = dma_common_get_sgtable,
1079 .alloc_pages = dma_common_alloc_pages,
1080 .free_pages = dma_common_free_pages,
1079};1081};
1080 1082
1081 1083
drivers/xen/swiotlb-xen.c+2−0
@@ -578,4 +578,6 @@ const struct dma_map_ops xen_swiotlb_dma_ops = { · 577 unmodified lines
578 .dma_supported = xen_swiotlb_dma_supported,578 .dma_supported = xen_swiotlb_dma_supported,
579 .mmap = dma_common_mmap,579 .mmap = dma_common_mmap,
580 .get_sgtable = dma_common_get_sgtable,580 .get_sgtable = dma_common_get_sgtable,
581 .alloc_pages = dma_common_alloc_pages,
582 .free_pages = dma_common_free_pages,
581};583};
include/linux/dma-direct.h+5−0
@@ -115,6 +115,11 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, · 114 unmodified lines
115 gfp_t gfp, unsigned long attrs);115 gfp_t gfp, unsigned long attrs);
116void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,116void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
117 dma_addr_t dma_addr, unsigned long attrs);117 dma_addr_t dma_addr, unsigned long attrs);
118struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
119 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
120void dma_direct_free_pages(struct device *dev, size_t size,
121 struct page *page, dma_addr_t dma_addr,
122 enum dma_data_direction dir);
118int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,123int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt,
119 void *cpu_addr, dma_addr_t dma_addr, size_t size,124 void *cpu_addr, dma_addr_t dma_addr, size_t size,
120 unsigned long attrs);125 unsigned long attrs);
include/linux/dma-mapping.h+17−17
@@ -27,11 +27,6 @@ · 26 unmodified lines
27 * buffered to improve performance.27 * buffered to improve performance.
28 */28 */
29#define DMA_ATTR_WRITE_COMBINE (1UL << 2)29#define DMA_ATTR_WRITE_COMBINE (1UL << 2)
30/*
31 * DMA_ATTR_NON_CONSISTENT: Lets the platform to choose to return either
32 * consistent or non-consistent memory as it sees fit.
33 */
34#define DMA_ATTR_NON_CONSISTENT (1UL << 3)
35/*30/*
36 * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel31 * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel
37 * virtual mapping for the allocated buffer.32 * virtual mapping for the allocated buffer.
@@ -74,6 +69,11 @@ struct dma_map_ops { · 36 unmodified lines
74 void (*free)(struct device *dev, size_t size,69 void (*free)(struct device *dev, size_t size,
75 void *vaddr, dma_addr_t dma_handle,70 void *vaddr, dma_addr_t dma_handle,
76 unsigned long attrs);71 unsigned long attrs);
72 struct page *(*alloc_pages)(struct device *dev, size_t size,
73 dma_addr_t *dma_handle, enum dma_data_direction dir,
74 gfp_t gfp);
75 void (*free_pages)(struct device *dev, size_t size, struct page *vaddr,
76 dma_addr_t dma_handle, enum dma_data_direction dir);
77 int (*mmap)(struct device *, struct vm_area_struct *,77 int (*mmap)(struct device *, struct vm_area_struct *,
78 void *, dma_addr_t, size_t,78 void *, dma_addr_t, size_t,
79 unsigned long attrs);79 unsigned long attrs);
@@ -376,17 +376,14 @@ static inline unsigned long dma_get_merge_boundary(struct device *dev) · 296 unmodified lines
376}376}
377#endif /* CONFIG_HAS_DMA */377#endif /* CONFIG_HAS_DMA */
378 378
379static inline void *dma_alloc_noncoherent(struct device *dev, size_t size,379struct page *dma_alloc_pages(struct device *dev, size_t size,
380 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)380 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
381{381void dma_free_pages(struct device *dev, size_t size, struct page *page,
382 return dma_alloc_attrs(dev, size, dma_handle, gfp,382 dma_addr_t dma_handle, enum dma_data_direction dir);
383 DMA_ATTR_NON_CONSISTENT);383void *dma_alloc_noncoherent(struct device *dev, size_t size,
384}384 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
385static inline void dma_free_noncoherent(struct device *dev, size_t size,385void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
386 void *vaddr, dma_addr_t dma_handle, enum dma_data_direction dir)386 dma_addr_t dma_handle, enum dma_data_direction dir);
387{
388 dma_free_attrs(dev, size, vaddr, dma_handle, DMA_ATTR_NON_CONSISTENT);
389}
390 387
391static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,388static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
392 size_t size, enum dma_data_direction dir, unsigned long attrs)389 size_t size, enum dma_data_direction dir, unsigned long attrs)
@@ -512,7 +509,10 @@ static inline void dma_sync_sgtable_for_device(struct device *dev, · 119 unmodified lines
512extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,509extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
513 void *cpu_addr, dma_addr_t dma_addr, size_t size,510 void *cpu_addr, dma_addr_t dma_addr, size_t size,
514 unsigned long attrs);511 unsigned long attrs);
515 512struct page *dma_common_alloc_pages(struct device *dev, size_t size,
513 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
514void dma_common_free_pages(struct device *dev, size_t size, struct page *vaddr,
515 dma_addr_t dma_handle, enum dma_data_direction dir);
516struct page **dma_common_find_pages(void *cpu_addr);516struct page **dma_common_find_pages(void *cpu_addr);
517void *dma_common_contiguous_remap(struct page *page, size_t size,517void *dma_common_contiguous_remap(struct page *page, size_t size,
518 pgprot_t prot, const void *caller);518 pgprot_t prot, const void *caller);
include/linux/dma-noncoherent.h+0−3
@@ -31,9 +31,6 @@ static __always_inline bool dma_alloc_need_uncached(struct device *dev, · 30 unmodified lines
31 return false;31 return false;
32 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)32 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
33 return false;33 return false;
34 if (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) &&
35 (attrs & DMA_ATTR_NON_CONSISTENT))
36 return false;
37 return true;34 return true;
38}35}
39 36
kernel/dma/direct.c+51−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/*2/*
3 * Copyright (C) 2018 Christoph Hellwig.3 * Copyright (C) 2018-2020 Christoph Hellwig.
4 *4 *
5 * DMA operations that map physical memory directly without using an IOMMU.5 * DMA operations that map physical memory directly without using an IOMMU.
6 */6 */
@@ -292,6 +292,56 @@ void dma_direct_free(struct device *dev, size_t size, · 285 unmodified lines
292 dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);292 dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);
293}293}
294 294
295struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
296 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
297{
298 struct page *page;
299 void *ret;
300
301 if (dma_should_alloc_from_pool(dev, gfp, 0)) {
302 page = dma_alloc_from_pool(dev, size, &ret, gfp,
303 dma_coherent_ok);
304 if (!page)
305 return NULL;
306 goto done;
307 }
308
309 page = __dma_direct_alloc_pages(dev, size, gfp);
310 if (!page)
311 return NULL;
312 ret = page_address(page);
313 if (force_dma_unencrypted(dev)) {
314 if (set_memory_decrypted((unsigned long)ret,
315 1 << get_order(size)))
316 goto out_free_pages;
317 }
318 memset(ret, 0, size);
319done:
320 *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
321 return page;
322out_free_pages:
323 dma_free_contiguous(dev, page, size);
324 return NULL;
325}
326
327void dma_direct_free_pages(struct device *dev, size_t size,
328 struct page *page, dma_addr_t dma_addr,
329 enum dma_data_direction dir)
330{
331 unsigned int page_order = get_order(size);
332 void *vaddr = page_address(page);
333
334 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
335 if (dma_should_free_from_pool(dev, 0) &&
336 dma_free_from_pool(dev, vaddr, size))
337 return;
338
339 if (force_dma_unencrypted(dev))
340 set_memory_encrypted((unsigned long)vaddr, 1 << page_order);
341
342 dma_free_contiguous(dev, page, size);
343}
344
295#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \345#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
296 defined(CONFIG_SWIOTLB)346 defined(CONFIG_SWIOTLB)
297void dma_direct_sync_sg_for_device(struct device *dev,347void dma_direct_sync_sg_for_device(struct device *dev,
kernel/dma/mapping.c+60−3
@@ -341,9 +341,7 @@ pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) · 340 unmodified lines
341{341{
342 if (force_dma_unencrypted(dev))342 if (force_dma_unencrypted(dev))
343 prot = pgprot_decrypted(prot);343 prot = pgprot_decrypted(prot);
344 if (dev_is_dma_coherent(dev) ||344 if (dev_is_dma_coherent(dev))
345 (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) &&
346 (attrs & DMA_ATTR_NON_CONSISTENT)))
347 return prot;345 return prot;
348#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE346#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
349 if (attrs & DMA_ATTR_WRITE_COMBINE)347 if (attrs & DMA_ATTR_WRITE_COMBINE)
@@ -472,6 +470,65 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, · 122 unmodified lines
472}470}
473EXPORT_SYMBOL(dma_free_attrs);471EXPORT_SYMBOL(dma_free_attrs);
474 472
473struct page *dma_alloc_pages(struct device *dev, size_t size,
474 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
475{
476 const struct dma_map_ops *ops = get_dma_ops(dev);
477 struct page *page;
478
479 if (WARN_ON_ONCE(!dev->coherent_dma_mask))
480 return NULL;
481 if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)))
482 return NULL;
483
484 size = PAGE_ALIGN(size);
485 if (dma_alloc_direct(dev, ops))
486 page = dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);
487 else if (ops->alloc_pages)
488 page = ops->alloc_pages(dev, size, dma_handle, dir, gfp);
489 else
490 return NULL;
491
492 debug_dma_map_page(dev, page, 0, size, dir, *dma_handle);
493
494 return page;
495}
496EXPORT_SYMBOL_GPL(dma_alloc_pages);
497
498void dma_free_pages(struct device *dev, size_t size, struct page *page,
499 dma_addr_t dma_handle, enum dma_data_direction dir)
500{
501 const struct dma_map_ops *ops = get_dma_ops(dev);
502
503 size = PAGE_ALIGN(size);
504 debug_dma_unmap_page(dev, dma_handle, size, dir);
505
506 if (dma_alloc_direct(dev, ops))
507 dma_direct_free_pages(dev, size, page, dma_handle, dir);
508 else if (ops->free_pages)
509 ops->free_pages(dev, size, page, dma_handle, dir);
510}
511EXPORT_SYMBOL_GPL(dma_free_pages);
512
513void *dma_alloc_noncoherent(struct device *dev, size_t size,
514 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
515{
516 struct page *page;
517
518 page = dma_alloc_pages(dev, size, dma_handle, dir, gfp);
519 if (!page)
520 return NULL;
521 return page_address(page);
522}
523EXPORT_SYMBOL_GPL(dma_alloc_noncoherent);
524
525void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
526 dma_addr_t dma_handle, enum dma_data_direction dir)
527{
528 dma_free_pages(dev, size, virt_to_page(vaddr), dma_handle, dir);
529}
530EXPORT_SYMBOL_GPL(dma_free_noncoherent);
531
475int dma_supported(struct device *dev, u64 mask)532int dma_supported(struct device *dev, u64 mask)
476{533{
477 const struct dma_map_ops *ops = get_dma_ops(dev);534 const struct dma_map_ops *ops = get_dma_ops(dev);
kernel/dma/ops_helpers.c+35−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
3 * Helpers for DMA ops implementations. These generally rely on the fact that3 * Helpers for DMA ops implementations. These generally rely on the fact that
4 * the allocated memory contains normal pages in the direct kernel mapping.4 * the allocated memory contains normal pages in the direct kernel mapping.
5 */5 */
6#include <linux/dma-contiguous.h>
6#include <linux/dma-noncoherent.h>7#include <linux/dma-noncoherent.h>
7 8
8/*9/*
@@ -49,3 +50,37 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, · 40 unmodified lines
49 return -ENXIO;50 return -ENXIO;
50#endif /* CONFIG_MMU */51#endif /* CONFIG_MMU */
51}52}
53
54struct page *dma_common_alloc_pages(struct device *dev, size_t size,
55 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
56{
57 const struct dma_map_ops *ops = get_dma_ops(dev);
58 struct page *page;
59
60 page = dma_alloc_contiguous(dev, size, gfp);
61 if (!page)
62 page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size));
63 if (!page)
64 return NULL;
65
66 *dma_handle = ops->map_page(dev, page, 0, size, dir,
67 DMA_ATTR_SKIP_CPU_SYNC);
68 if (*dma_handle == DMA_MAPPING_ERROR) {
69 dma_free_contiguous(dev, page, size);
70 return NULL;
71 }
72
73 memset(page_address(page), 0, size);
74 return page;
75}
76
77void dma_common_free_pages(struct device *dev, size_t size, struct page *page,
78 dma_addr_t dma_handle, enum dma_data_direction dir)
79{
80 const struct dma_map_ops *ops = get_dma_ops(dev);
81
82 if (ops->unmap_page)
83 ops->unmap_page(dev, dma_handle, size, dir,
84 DMA_ATTR_SKIP_CPU_SYNC);
85 dma_free_contiguous(dev, page, size);
86}
kernel/dma/virt.c+2−0
@@ -55,5 +55,7 @@ const struct dma_map_ops dma_virt_ops = { · 54 unmodified lines
55 .free = dma_virt_free,55 .free = dma_virt_free,
56 .map_page = dma_virt_map_page,56 .map_page = dma_virt_map_page,
57 .map_sg = dma_virt_map_sg,57 .map_sg = dma_virt_map_sg,
58 .alloc_pages = dma_common_alloc_pages,
59 .free_pages = dma_common_free_pages,
58};60};
59EXPORT_SYMBOL(dma_virt_ops);61EXPORT_SYMBOL(dma_virt_ops);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t