Update is_swiotlb_buffer to add a struct device argument. This will be useful later to allow for different pools.
Unified · Split
| @@ -506,7 +506,7 @@ static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr, · 505 unmodified lines | |||
| 506 | 506 | ||
| 507 | __iommu_dma_unmap(dev, dma_addr, size); | 507 | __iommu_dma_unmap(dev, dma_addr, size); |
| 508 | 508 | ||
| 509 | if (unlikely(is_swiotlb_buffer(phys))) | 509 | if (unlikely(is_swiotlb_buffer(dev, phys))) |
| 510 | swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); | 510 | swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); |
| 511 | } | 511 | } |
| 512 | 512 | ||
| @@ -577,7 +577,7 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys, · 64 unmodified lines | |||
| 577 | } | 577 | } |
| 578 | 578 | ||
| 579 | iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); | 579 | iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); |
| 580 | if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(phys)) | 580 | if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys)) |
| 581 | swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs); | 581 | swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs); |
| 582 | return iova; | 582 | return iova; |
| 583 | } | 583 | } |
| @@ -783,7 +783,7 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev, · 199 unmodified lines | |||
| 783 | if (!dev_is_dma_coherent(dev)) | 783 | if (!dev_is_dma_coherent(dev)) |
| 784 | arch_sync_dma_for_cpu(phys, size, dir); | 784 | arch_sync_dma_for_cpu(phys, size, dir); |
| 785 | 785 | ||
| 786 | if (is_swiotlb_buffer(phys)) | 786 | if (is_swiotlb_buffer(dev, phys)) |
| 787 | swiotlb_sync_single_for_cpu(dev, phys, size, dir); | 787 | swiotlb_sync_single_for_cpu(dev, phys, size, dir); |
| 788 | } | 788 | } |
| 789 | 789 | ||
| @@ -796,7 +796,7 @@ static void iommu_dma_sync_single_for_device(struct device *dev, · 6 unmodified lines | |||
| 796 | return; | 796 | return; |
| 797 | 797 | ||
| 798 | phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); | 798 | phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); |
| 799 | if (is_swiotlb_buffer(phys)) | 799 | if (is_swiotlb_buffer(dev, phys)) |
| 800 | swiotlb_sync_single_for_device(dev, phys, size, dir); | 800 | swiotlb_sync_single_for_device(dev, phys, size, dir); |
| 801 | 801 | ||
| 802 | if (!dev_is_dma_coherent(dev)) | 802 | if (!dev_is_dma_coherent(dev)) |
| @@ -817,7 +817,7 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev, · 14 unmodified lines | |||
| 817 | if (!dev_is_dma_coherent(dev)) | 817 | if (!dev_is_dma_coherent(dev)) |
| 818 | arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); | 818 | arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); |
| 819 | 819 | ||
| 820 | if (is_swiotlb_buffer(sg_phys(sg))) | 820 | if (is_swiotlb_buffer(dev, sg_phys(sg))) |
| 821 | swiotlb_sync_single_for_cpu(dev, sg_phys(sg), | 821 | swiotlb_sync_single_for_cpu(dev, sg_phys(sg), |
| 822 | sg->length, dir); | 822 | sg->length, dir); |
| 823 | } | 823 | } |
| @@ -834,7 +834,7 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, · 10 unmodified lines | |||
| 834 | return; | 834 | return; |
| 835 | 835 | ||
| 836 | for_each_sg(sgl, sg, nelems, i) { | 836 | for_each_sg(sgl, sg, nelems, i) { |
| 837 | if (is_swiotlb_buffer(sg_phys(sg))) | 837 | if (is_swiotlb_buffer(dev, sg_phys(sg))) |
| 838 | swiotlb_sync_single_for_device(dev, sg_phys(sg), | 838 | swiotlb_sync_single_for_device(dev, sg_phys(sg), |
| 839 | sg->length, dir); | 839 | sg->length, dir); |
| 840 | 840 | ||
| @@ -100,7 +100,7 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr) · 99 unmodified lines | |||
| 100 | * in our domain. Therefore _only_ check address within our domain. | 100 | * in our domain. Therefore _only_ check address within our domain. |
| 101 | */ | 101 | */ |
| 102 | if (pfn_valid(PFN_DOWN(paddr))) | 102 | if (pfn_valid(PFN_DOWN(paddr))) |
| 103 | return is_swiotlb_buffer(paddr); | 103 | return is_swiotlb_buffer(dev, paddr); |
| 104 | return 0; | 104 | return 0; |
| 105 | } | 105 | } |
| 106 | 106 | ||
| @@ -2,6 +2,7 @@ · 1 unmodified line | |||
| 2 | #ifndef __LINUX_SWIOTLB_H | 2 | #ifndef __LINUX_SWIOTLB_H |
| 3 | #define __LINUX_SWIOTLB_H | 3 | #define __LINUX_SWIOTLB_H |
| 4 | 4 | ||
| 5 | #include <linux/device.h> | ||
| 5 | #include <linux/dma-direction.h> | 6 | #include <linux/dma-direction.h> |
| 6 | #include <linux/init.h> | 7 | #include <linux/init.h> |
| 7 | #include <linux/types.h> | 8 | #include <linux/types.h> |
| @@ -101,9 +102,9 @@ struct io_tlb_mem { · 93 unmodified lines | |||
| 101 | }; | 102 | }; |
| 102 | extern struct io_tlb_mem *io_tlb_default_mem; | 103 | extern struct io_tlb_mem *io_tlb_default_mem; |
| 103 | 104 | ||
| 104 | static inline bool is_swiotlb_buffer(phys_addr_t paddr) | 105 | static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) |
| 105 | { | 106 | { |
| 106 | struct io_tlb_mem *mem = io_tlb_default_mem; | 107 | struct io_tlb_mem *mem = dev->dma_io_tlb_mem; |
| 107 | 108 | ||
| 108 | return mem && paddr >= mem->start && paddr < mem->end; | 109 | return mem && paddr >= mem->start && paddr < mem->end; |
| 109 | } | 110 | } |
| @@ -115,7 +116,7 @@ bool is_swiotlb_active(void); · 5 unmodified lines | |||
| 115 | void __init swiotlb_adjust_size(unsigned long size); | 116 | void __init swiotlb_adjust_size(unsigned long size); |
| 116 | #else | 117 | #else |
| 117 | #define swiotlb_force SWIOTLB_NO_FORCE | 118 | #define swiotlb_force SWIOTLB_NO_FORCE |
| 118 | static inline bool is_swiotlb_buffer(phys_addr_t paddr) | 119 | static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr) |
| 119 | { | 120 | { |
| 120 | return false; | 121 | return false; |
| 121 | } | 122 | } |
| @@ -343,7 +343,7 @@ void dma_direct_sync_sg_for_device(struct device *dev, · 342 unmodified lines | |||
| 343 | for_each_sg(sgl, sg, nents, i) { | 343 | for_each_sg(sgl, sg, nents, i) { |
| 344 | phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); | 344 | phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); |
| 345 | 345 | ||
| 346 | if (unlikely(is_swiotlb_buffer(paddr))) | 346 | if (unlikely(is_swiotlb_buffer(dev, paddr))) |
| 347 | swiotlb_sync_single_for_device(dev, paddr, sg->length, | 347 | swiotlb_sync_single_for_device(dev, paddr, sg->length, |
| 348 | dir); | 348 | dir); |
| 349 | 349 | ||
| @@ -369,7 +369,7 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, · 19 unmodified lines | |||
| 369 | if (!dev_is_dma_coherent(dev)) | 369 | if (!dev_is_dma_coherent(dev)) |
| 370 | arch_sync_dma_for_cpu(paddr, sg->length, dir); | 370 | arch_sync_dma_for_cpu(paddr, sg->length, dir); |
| 371 | 371 | ||
| 372 | if (unlikely(is_swiotlb_buffer(paddr))) | 372 | if (unlikely(is_swiotlb_buffer(dev, paddr))) |
| 373 | swiotlb_sync_single_for_cpu(dev, paddr, sg->length, | 373 | swiotlb_sync_single_for_cpu(dev, paddr, sg->length, |
| 374 | dir); | 374 | dir); |
| 375 | 375 | ||
| @@ -504,7 +504,7 @@ size_t dma_direct_max_mapping_size(struct device *dev) · 128 unmodified lines | |||
| 504 | bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr) | 504 | bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr) |
| 505 | { | 505 | { |
| 506 | return !dev_is_dma_coherent(dev) || | 506 | return !dev_is_dma_coherent(dev) || |
| 507 | is_swiotlb_buffer(dma_to_phys(dev, dma_addr)); | 507 | is_swiotlb_buffer(dev, dma_to_phys(dev, dma_addr)); |
| 508 | } | 508 | } |
| 509 | 509 | ||
| 510 | /** | 510 | /** |
| @@ -56,7 +56,7 @@ static inline void dma_direct_sync_single_for_device(struct device *dev, · 55 unmodified lines | |||
| 56 | { | 56 | { |
| 57 | phys_addr_t paddr = dma_to_phys(dev, addr); | 57 | phys_addr_t paddr = dma_to_phys(dev, addr); |
| 58 | 58 | ||
| 59 | if (unlikely(is_swiotlb_buffer(paddr))) | 59 | if (unlikely(is_swiotlb_buffer(dev, paddr))) |
| 60 | swiotlb_sync_single_for_device(dev, paddr, size, dir); | 60 | swiotlb_sync_single_for_device(dev, paddr, size, dir); |
| 61 | 61 | ||
| 62 | if (!dev_is_dma_coherent(dev)) | 62 | if (!dev_is_dma_coherent(dev)) |
| @@ -73,7 +73,7 @@ static inline void dma_direct_sync_single_for_cpu(struct device *dev, · 10 unmodified lines | |||
| 73 | arch_sync_dma_for_cpu_all(); | 73 | arch_sync_dma_for_cpu_all(); |
| 74 | } | 74 | } |
| 75 | 75 | ||
| 76 | if (unlikely(is_swiotlb_buffer(paddr))) | 76 | if (unlikely(is_swiotlb_buffer(dev, paddr))) |
| 77 | swiotlb_sync_single_for_cpu(dev, paddr, size, dir); | 77 | swiotlb_sync_single_for_cpu(dev, paddr, size, dir); |
| 78 | 78 | ||
| 79 | if (dir == DMA_FROM_DEVICE) | 79 | if (dir == DMA_FROM_DEVICE) |
| @@ -113,7 +113,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, · 33 unmodified lines | |||
| 113 | if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) | 113 | if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) |
| 114 | dma_direct_sync_single_for_cpu(dev, addr, size, dir); | 114 | dma_direct_sync_single_for_cpu(dev, addr, size, dir); |
| 115 | 115 | ||
| 116 | if (unlikely(is_swiotlb_buffer(phys))) | 116 | if (unlikely(is_swiotlb_buffer(dev, phys))) |
| 117 | swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); | 117 | swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); |
| 118 | } | 118 | } |
| 119 | #endif /* _KERNEL_DMA_DIRECT_H */ | 119 | #endif /* _KERNEL_DMA_DIRECT_H */ |