// SPDX-License-Identifier: GPL-2.0/* * arch-independent dma-mapping routines * * Copyright (c) 2006 SUSE Linux Products GmbH * Copyright (c) 2006 Tejun Heo <teheo@suse.de> */#include <linux/memblock.h> /* for max_pfn */#include <linux/acpi.h>#include <linux/dma-map-ops.h>#include <linux/export.h>#include <linux/gfp.h>#include <linux/iommu-dma.h>#include <linux/kmsan.h>#include <linux/of_device.h>#include <linux/slab.h>#include <linux/vmalloc.h>#include "debug.h"#include "direct.h" #define CREATE_TRACE_POINTS#include <trace/events/dma.h> #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)bool dma_default_coherent = IS_ENABLED(CONFIG_ARCH_DMA_DEFAULT_COHERENT);#endif /* * Managed DMA API */struct dma_devres { size_t size; void *vaddr; dma_addr_t dma_handle; unsigned long attrs;}; static void dmam_release(struct device *dev, void *res){ struct dma_devres *this = res; dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, this->attrs);} static int dmam_match(struct device *dev, void *res, void *match_data){ struct dma_devres *this = res, *match = match_data; if (this->vaddr == match->vaddr) { WARN_ON(this->size != match->size || this->dma_handle != match->dma_handle); return 1; } return 0;} /** * dmam_free_coherent - Managed dma_free_coherent() * @dev: Device to free coherent memory for * @size: Size of allocation * @vaddr: Virtual address of the memory to free * @dma_handle: DMA handle of the memory to free * * Managed dma_free_coherent(). */void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle){ struct dma_devres match_data = { size, vaddr, dma_handle }; WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); dma_free_coherent(dev, size, vaddr, dma_handle);}EXPORT_SYMBOL(dmam_free_coherent); /** * dmam_alloc_attrs - Managed dma_alloc_attrs() * @dev: Device to allocate non_coherent memory for * @size: Size of allocation * @dma_handle: Out argument for allocated DMA handle * @gfp: Allocation flags * @attrs: Flags in the DMA_ATTR_* namespace. * * Managed dma_alloc_attrs(). Memory allocated using this function will be * automatically released on driver detach. * * RETURNS: * Pointer to allocated memory on success, NULL on failure. */void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs){ struct dma_devres *dr; void *vaddr; dr = devres_alloc(dmam_release, sizeof(*dr), gfp); if (!dr) return NULL; vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); if (!vaddr) { devres_free(dr); return NULL; } dr->vaddr = vaddr; dr->dma_handle = *dma_handle; dr->size = size; dr->attrs = attrs; devres_add(dev, dr); return vaddr;}EXPORT_SYMBOL(dmam_alloc_attrs); static bool dma_go_direct(struct device *dev, dma_addr_t mask, const struct dma_map_ops *ops){ if (use_dma_iommu(dev)) return false; if (likely(!ops)) return true; #ifdef CONFIG_DMA_OPS_BYPASS if (dev->dma_ops_bypass) return min_not_zero(mask, dev->bus_dma_limit) >= dma_direct_get_required_mask(dev);#endif return false;} /* * Check if the devices uses a direct mapping for streaming DMA operations. * This allows IOMMU drivers to set a bypass mode if the DMA mask is large * enough. */static inline bool dma_alloc_direct(struct device *dev, const struct dma_map_ops *ops){ return dma_go_direct(dev, dev->coherent_dma_mask, ops);} static inline bool dma_map_direct(struct device *dev, const struct dma_map_ops *ops){ return dma_go_direct(dev, *dev->dma_mask, ops);} dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs){ const struct dma_map_ops *ops = get_dma_ops(dev); bool is_mmio = attrs & DMA_ATTR_MMIO; dma_addr_t addr; BUG_ON(!valid_dma_direction(dir)); if (WARN_ON_ONCE(!dev->dma_mask)) return DMA_MAPPING_ERROR; if (dma_map_direct(dev, ops) || (!is_mmio && arch_dma_map_phys_direct(dev, phys + size))) addr = dma_direct_map_phys(dev, phys, size, dir, attrs); else if (use_dma_iommu(dev)) addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); else if (is_mmio) { if (!ops->map_resource) return DMA_MAPPING_ERROR; addr = ops->map_resource(dev, phys, size, dir, attrs); } else { struct page *page = phys_to_page(phys); size_t offset = offset_in_page(phys); /* * The dma_ops API contract for ops->map_page() requires * kmappable memory, while ops->map_resource() does not. */ addr = ops->map_page(dev, page, offset, size, dir, attrs); } if (!is_mmio) kmsan_handle_dma(phys, size, dir); trace_dma_map_phys(dev, phys, addr, size, dir, attrs); debug_dma_map_phys(dev, phys, size, dir, addr, attrs); return addr;}EXPORT_SYMBOL_GPL(dma_map_phys); dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, size_t size, enum dma_data_direction dir, unsigned long attrs){ phys_addr_t phys = page_to_phys(page) + offset; if (unlikely(attrs & DMA_ATTR_MMIO)) return DMA_MAPPING_ERROR; if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && WARN_ON_ONCE(is_zone_device_page(page))) return DMA_MAPPING_ERROR; return dma_map_phys(dev, phys, size, dir, attrs);}EXPORT_SYMBOL(dma_map_page_attrs); void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs){ const struct dma_map_ops *ops = get_dma_ops(dev); bool is_mmio = attrs & DMA_ATTR_MMIO; BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops) || (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size))) dma_direct_unmap_phys(dev, addr, size, dir, attrs); else if (use_dma_iommu(dev)) iommu_dma_unmap_phys(dev, addr, size, dir, attrs); else if (is_mmio) { if (ops->unmap_resource) ops->unmap_resource(dev, addr, size, dir, attrs); } else ops->unmap_page(dev, addr, size, dir, attrs); trace_dma_unmap_phys(dev, addr, size, dir, attrs); debug_dma_unmap_phys(dev, addr, size, dir);}EXPORT_SYMBOL_GPL(dma_unmap_phys); void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs){ if (unlikely(attrs & DMA_ATTR_MMIO)) return; dma_unmap_phys(dev, addr, size, dir, attrs);}EXPORT_SYMBOL(dma_unmap_page_attrs); static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs){ const struct dma_map_ops *ops = get_dma_ops(dev); int ents; BUG_ON(!valid_dma_direction(dir)); if (WARN_ON_ONCE(!dev->dma_mask)) return 0; if (dma_map_direct(dev, ops) || arch_dma_map_sg_direct(dev, sg, nents)) ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); else if (use_dma_iommu(dev)) ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); else ents = ops->map_sg(dev, sg, nents, dir, attrs); if (ents > 0) { kmsan_handle_dma_sg(sg, nents, dir); trace_dma_map_sg(dev, sg, nents, ents, dir, attrs); debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && ents != -EIO && ents != -EREMOTEIO)) { trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs); return -EIO; } return ents;} /** * dma_map_sg_attrs - Map the given buffer for DMA * @dev: The device for which to perform the DMA operation * @sg: The sg_table object describing the buffer * @nents: Number of entries to map * @dir: DMA direction * @attrs: Optional DMA attributes for the map operation * * Maps a buffer described by a scatterlist passed in the sg argument with * nents segments for the @dir DMA operation by the @dev device. * * Returns the number of mapped entries (which can be less than nents) * on success. Zero is returned for any error. * * dma_unmap_sg_attrs() should be used to unmap the buffer with the * original sg and original nents (not the value returned by this funciton). */unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs){ int ret; ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); if (ret < 0) return 0; return ret;}EXPORT_SYMBOL(dma_map_sg_attrs); /** * dma_map_sgtable - Map the given buffer for DMA * @dev: The device for which to perform the DMA operation * @sgt: The sg_table object describing the buffer * @dir: DMA direction * @attrs: Optional DMA attributes for the map operation * * Maps a buffer described by a scatterlist stored in the given sg_table * object for the @dir DMA operation by the @dev device. After success, the * ownership for the buffer is transferred to the DMA domain. One has to * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the * ownership of the buffer back to the CPU domain before touching the * buffer by the CPU. * * Returns 0 on success or a negative error code on error. The following * error codes are supported with the given meaning: * * -EINVAL An invalid argument, unaligned access or other error * in usage. Will not succeed if retried. * -ENOMEM Insufficient resources (like memory or IOVA space) to * complete the mapping. Should succeed if retried later. * -EIO Legacy error code with an unknown meaning. eg. this is * returned if a lower level call returned * DMA_MAPPING_ERROR. * -EREMOTEIO The DMA device cannot access P2PDMA memory specified * in the sg_table. This will not succeed if retried. */int dma_map_sgtable(struct device *dev, struct sg_table *sgt, enum dma_data_direction dir, unsigned long attrs){ int nents; nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); if (nents < 0) return nents; sgt->nents = nents; return 0;}EXPORT_SYMBOL_GPL(dma_map_sgtable); void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs){ const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); trace_dma_unmap_sg(dev, sg, nents, dir, attrs); debug_dma_unmap_sg(dev, sg, nents, dir); if (dma_map_direct(dev, ops) || arch_dma_unmap_sg_direct(dev, sg, nents)) dma_direct_unmap_sg(dev, sg, nents, dir, attrs); else if (use_dma_iommu(dev)) iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); else if (ops->unmap_sg) ops->unmap_sg(dev, sg, nents, dir, attrs);}EXPORT_SYMBOL(dma_unmap_sg_attrs); dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, unsigned long attrs){ if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) return DMA_MAPPING_ERROR; return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO);}EXPORT_SYMBOL(dma_map_resource); void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs){ dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO);}EXPORT_SYMBOL(dma_unmap_resource); #ifdef CONFIG_DMA_NEED_SYNCvoid __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir){ const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_single_for_cpu(dev, addr, size, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_single_for_cpu(dev, addr, size, dir); else if (ops->sync_single_for_cpu) ops->sync_single_for_cpu(dev, addr, size, dir); trace_dma_sync_single_for_cpu(dev, addr, size, dir); debug_dma_sync_single_for_cpu(dev, addr, size, dir);}EXPORT_SYMBOL(__dma_sync_single_for_cpu); void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir){ const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_single_for_device(dev, addr, size, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_single_for_device(dev, addr, size, dir); else if (ops->sync_single_for_device) ops->sync_single_for_device(dev, addr, size, dir); trace_dma_sync_single_for_device(dev, addr, size, dir); debug_dma_sync_single_for_device(dev, addr, size, dir);}EXPORT_SYMBOL(__dma_sync_single_for_device); void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir){ const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); else if (ops->sync_sg_for_cpu) ops->sync_sg_for_cpu(dev, sg, nelems, dir); trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir); debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);}EXPORT_SYMBOL(__dma_sync_sg_for_cpu); void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir){ const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_sg_for_device(dev, sg, nelems, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); else if (ops->sync_sg_for_device) ops->sync_sg_for_device(dev, sg, nelems, dir); trace_dma_sync_sg_for_device(dev, sg, nelems, dir); debug_dma_sync_sg_for_device(dev, sg, nelems, dir);}EXPORT_SYMBOL(__dma_sync_sg_for_device); bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr){ const struct dma_map_ops *ops = get_dma_ops(dev); if (dma_map_direct(dev, ops)) /* * dma_skip_sync could've been reset on first SWIOTLB buffer * mapping, but @dma_addr is not necessary an SWIOTLB buffer. * In this case, fall back to more granular check. */ return dma_direct_need_sync(dev, dma_addr); return true;}EXPORT_SYMBOL_GPL(__dma_need_sync); /** * dma_need_unmap - does this device need dma_unmap_* operations * @dev: device to check * * If this function returns %false, drivers can skip calling dma_unmap_* after * finishing an I/O. This function must be called after all mappings that might * need to be unmapped have been performed. */bool dma_need_unmap(struct device *dev){ if (!dma_map_direct(dev, get_dma_ops(dev))) return true; if (!dev->dma_skip_sync) return true; return IS_ENABLED(CONFIG_DMA_API_DEBUG);}EXPORT_SYMBOL_GPL(dma_need_unmap); static void dma_setup_need_sync(struct device *dev){ const struct dma_map_ops *ops = get_dma_ops(dev); if (dma_map_direct(dev, ops) || use_dma_iommu(dev)) /* * dma_skip_sync will be reset to %false on first SWIOTLB buffer * mapping, if any. During the device initialization, it's * enough to check only for the DMA coherence. */ dev->dma_skip_sync = dev_is_dma_coherent(dev); else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) /* * Synchronization is not possible when none of DMA sync opsLines 1–500 of 1026. Load more
// SPDX-License-Identifier: GPL-2.0 /* * arch-independent dma-mapping routines * * Copyright (c) 2006 SUSE Linux Products GmbH * Copyright (c) 2006 Tejun Heo <teheo@suse.de> */ #include <linux/memblock.h> /* for max_pfn */ #include <linux/acpi.h> #include <linux/dma-map-ops.h> #include <linux/export.h> #include <linux/gfp.h> #include <linux/iommu-dma.h> #include <linux/kmsan.h> #include <linux/of_device.h> #include <linux/slab.h> #include <linux/vmalloc.h> #include "debug.h" #include "direct.h" #define CREATE_TRACE_POINTS #include <trace/events/dma.h> #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) bool dma_default_coherent = IS_ENABLED(CONFIG_ARCH_DMA_DEFAULT_COHERENT); #endif /* * Managed DMA API */ struct dma_devres { size_t size; void *vaddr; dma_addr_t dma_handle; unsigned long attrs; }; static void dmam_release(struct device *dev, void *res) { struct dma_devres *this = res; dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, this->attrs); } static int dmam_match(struct device *dev, void *res, void *match_data) { struct dma_devres *this = res, *match = match_data; if (this->vaddr == match->vaddr) { WARN_ON(this->size != match->size || this->dma_handle != match->dma_handle); return 1; } return 0; } /** * dmam_free_coherent - Managed dma_free_coherent() * @dev: Device to free coherent memory for * @size: Size of allocation * @vaddr: Virtual address of the memory to free * @dma_handle: DMA handle of the memory to free * * Managed dma_free_coherent(). */ void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle) { struct dma_devres match_data = { size, vaddr, dma_handle }; WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); dma_free_coherent(dev, size, vaddr, dma_handle); } EXPORT_SYMBOL(dmam_free_coherent); /** * dmam_alloc_attrs - Managed dma_alloc_attrs() * @dev: Device to allocate non_coherent memory for * @size: Size of allocation * @dma_handle: Out argument for allocated DMA handle * @gfp: Allocation flags * @attrs: Flags in the DMA_ATTR_* namespace. * * Managed dma_alloc_attrs(). Memory allocated using this function will be * automatically released on driver detach. * * RETURNS: * Pointer to allocated memory on success, NULL on failure. */ void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { struct dma_devres *dr; void *vaddr; dr = devres_alloc(dmam_release, sizeof(*dr), gfp); if (!dr) return NULL; vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); if (!vaddr) { devres_free(dr); return NULL; } dr->vaddr = vaddr; dr->dma_handle = *dma_handle; dr->size = size; dr->attrs = attrs; devres_add(dev, dr); return vaddr; } EXPORT_SYMBOL(dmam_alloc_attrs); static bool dma_go_direct(struct device *dev, dma_addr_t mask, const struct dma_map_ops *ops) { if (use_dma_iommu(dev)) return false; if (likely(!ops)) return true; #ifdef CONFIG_DMA_OPS_BYPASS if (dev->dma_ops_bypass) return min_not_zero(mask, dev->bus_dma_limit) >= dma_direct_get_required_mask(dev); #endif return false; } /* * Check if the devices uses a direct mapping for streaming DMA operations. * This allows IOMMU drivers to set a bypass mode if the DMA mask is large * enough. */ static inline bool dma_alloc_direct(struct device *dev, const struct dma_map_ops *ops) { return dma_go_direct(dev, dev->coherent_dma_mask, ops); } static inline bool dma_map_direct(struct device *dev, const struct dma_map_ops *ops) { return dma_go_direct(dev, *dev->dma_mask, ops); } dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); bool is_mmio = attrs & DMA_ATTR_MMIO; dma_addr_t addr; BUG_ON(!valid_dma_direction(dir)); if (WARN_ON_ONCE(!dev->dma_mask)) return DMA_MAPPING_ERROR; if (dma_map_direct(dev, ops) || (!is_mmio && arch_dma_map_phys_direct(dev, phys + size))) addr = dma_direct_map_phys(dev, phys, size, dir, attrs); else if (use_dma_iommu(dev)) addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); else if (is_mmio) { if (!ops->map_resource) return DMA_MAPPING_ERROR; addr = ops->map_resource(dev, phys, size, dir, attrs); } else { struct page *page = phys_to_page(phys); size_t offset = offset_in_page(phys); /* * The dma_ops API contract for ops->map_page() requires * kmappable memory, while ops->map_resource() does not. */ addr = ops->map_page(dev, page, offset, size, dir, attrs); } if (!is_mmio) kmsan_handle_dma(phys, size, dir); trace_dma_map_phys(dev, phys, addr, size, dir, attrs); debug_dma_map_phys(dev, phys, size, dir, addr, attrs); return addr; } EXPORT_SYMBOL_GPL(dma_map_phys); dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { phys_addr_t phys = page_to_phys(page) + offset; if (unlikely(attrs & DMA_ATTR_MMIO)) return DMA_MAPPING_ERROR; if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && WARN_ON_ONCE(is_zone_device_page(page))) return DMA_MAPPING_ERROR; return dma_map_phys(dev, phys, size, dir, attrs); } EXPORT_SYMBOL(dma_map_page_attrs); void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); bool is_mmio = attrs & DMA_ATTR_MMIO; BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops) || (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size))) dma_direct_unmap_phys(dev, addr, size, dir, attrs); else if (use_dma_iommu(dev)) iommu_dma_unmap_phys(dev, addr, size, dir, attrs); else if (is_mmio) { if (ops->unmap_resource) ops->unmap_resource(dev, addr, size, dir, attrs); } else ops->unmap_page(dev, addr, size, dir, attrs); trace_dma_unmap_phys(dev, addr, size, dir, attrs); debug_dma_unmap_phys(dev, addr, size, dir); } EXPORT_SYMBOL_GPL(dma_unmap_phys); void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { if (unlikely(attrs & DMA_ATTR_MMIO)) return; dma_unmap_phys(dev, addr, size, dir, attrs); } EXPORT_SYMBOL(dma_unmap_page_attrs); static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); int ents; BUG_ON(!valid_dma_direction(dir)); if (WARN_ON_ONCE(!dev->dma_mask)) return 0; if (dma_map_direct(dev, ops) || arch_dma_map_sg_direct(dev, sg, nents)) ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); else if (use_dma_iommu(dev)) ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); else ents = ops->map_sg(dev, sg, nents, dir, attrs); if (ents > 0) { kmsan_handle_dma_sg(sg, nents, dir); trace_dma_map_sg(dev, sg, nents, ents, dir, attrs); debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && ents != -EIO && ents != -EREMOTEIO)) { trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs); return -EIO; } return ents; } /** * dma_map_sg_attrs - Map the given buffer for DMA * @dev: The device for which to perform the DMA operation * @sg: The sg_table object describing the buffer * @nents: Number of entries to map * @dir: DMA direction * @attrs: Optional DMA attributes for the map operation * * Maps a buffer described by a scatterlist passed in the sg argument with * nents segments for the @dir DMA operation by the @dev device. * * Returns the number of mapped entries (which can be less than nents) * on success. Zero is returned for any error. * * dma_unmap_sg_attrs() should be used to unmap the buffer with the * original sg and original nents (not the value returned by this funciton). */ unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs) { int ret; ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); if (ret < 0) return 0; return ret; } EXPORT_SYMBOL(dma_map_sg_attrs); /** * dma_map_sgtable - Map the given buffer for DMA * @dev: The device for which to perform the DMA operation * @sgt: The sg_table object describing the buffer * @dir: DMA direction * @attrs: Optional DMA attributes for the map operation * * Maps a buffer described by a scatterlist stored in the given sg_table * object for the @dir DMA operation by the @dev device. After success, the * ownership for the buffer is transferred to the DMA domain. One has to * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the * ownership of the buffer back to the CPU domain before touching the * buffer by the CPU. * * Returns 0 on success or a negative error code on error. The following * error codes are supported with the given meaning: * * -EINVAL An invalid argument, unaligned access or other error * in usage. Will not succeed if retried. * -ENOMEM Insufficient resources (like memory or IOVA space) to * complete the mapping. Should succeed if retried later. * -EIO Legacy error code with an unknown meaning. eg. this is * returned if a lower level call returned * DMA_MAPPING_ERROR. * -EREMOTEIO The DMA device cannot access P2PDMA memory specified * in the sg_table. This will not succeed if retried. */ int dma_map_sgtable(struct device *dev, struct sg_table *sgt, enum dma_data_direction dir, unsigned long attrs) { int nents; nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); if (nents < 0) return nents; sgt->nents = nents; return 0; } EXPORT_SYMBOL_GPL(dma_map_sgtable); void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); trace_dma_unmap_sg(dev, sg, nents, dir, attrs); debug_dma_unmap_sg(dev, sg, nents, dir); if (dma_map_direct(dev, ops) || arch_dma_unmap_sg_direct(dev, sg, nents)) dma_direct_unmap_sg(dev, sg, nents, dir, attrs); else if (use_dma_iommu(dev)) iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); else if (ops->unmap_sg) ops->unmap_sg(dev, sg, nents, dir, attrs); } EXPORT_SYMBOL(dma_unmap_sg_attrs); dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) return DMA_MAPPING_ERROR; return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO); } EXPORT_SYMBOL(dma_map_resource); void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO); } EXPORT_SYMBOL(dma_unmap_resource); #ifdef CONFIG_DMA_NEED_SYNC void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_single_for_cpu(dev, addr, size, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_single_for_cpu(dev, addr, size, dir); else if (ops->sync_single_for_cpu) ops->sync_single_for_cpu(dev, addr, size, dir); trace_dma_sync_single_for_cpu(dev, addr, size, dir); debug_dma_sync_single_for_cpu(dev, addr, size, dir); } EXPORT_SYMBOL(__dma_sync_single_for_cpu); void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_single_for_device(dev, addr, size, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_single_for_device(dev, addr, size, dir); else if (ops->sync_single_for_device) ops->sync_single_for_device(dev, addr, size, dir); trace_dma_sync_single_for_device(dev, addr, size, dir); debug_dma_sync_single_for_device(dev, addr, size, dir); } EXPORT_SYMBOL(__dma_sync_single_for_device); void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); else if (ops->sync_sg_for_cpu) ops->sync_sg_for_cpu(dev, sg, nelems, dir); trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir); debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); } EXPORT_SYMBOL(__dma_sync_sg_for_cpu); void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_sync_sg_for_device(dev, sg, nelems, dir); else if (use_dma_iommu(dev)) iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); else if (ops->sync_sg_for_device) ops->sync_sg_for_device(dev, sg, nelems, dir); trace_dma_sync_sg_for_device(dev, sg, nelems, dir); debug_dma_sync_sg_for_device(dev, sg, nelems, dir); } EXPORT_SYMBOL(__dma_sync_sg_for_device); bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr) { const struct dma_map_ops *ops = get_dma_ops(dev); if (dma_map_direct(dev, ops)) /* * dma_skip_sync could've been reset on first SWIOTLB buffer * mapping, but @dma_addr is not necessary an SWIOTLB buffer. * In this case, fall back to more granular check. */ return dma_direct_need_sync(dev, dma_addr); return true; } EXPORT_SYMBOL_GPL(__dma_need_sync); /** * dma_need_unmap - does this device need dma_unmap_* operations * @dev: device to check * * If this function returns %false, drivers can skip calling dma_unmap_* after * finishing an I/O. This function must be called after all mappings that might * need to be unmapped have been performed. */ bool dma_need_unmap(struct device *dev) { if (!dma_map_direct(dev, get_dma_ops(dev))) return true; if (!dev->dma_skip_sync) return true; return IS_ENABLED(CONFIG_DMA_API_DEBUG); } EXPORT_SYMBOL_GPL(dma_need_unmap); static void dma_setup_need_sync(struct device *dev) { const struct dma_map_ops *ops = get_dma_ops(dev); if (dma_map_direct(dev, ops) || use_dma_iommu(dev)) /* * dma_skip_sync will be reset to %false on first SWIOTLB buffer * mapping, if any. During the device initialization, it's * enough to check only for the DMA coherence. */ dev->dma_skip_sync = dev_is_dma_coherent(dev); else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) /* * Synchronization is not possible when none of DMA sync ops