Merge tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping updates from Christoph Hellwig: - make support for dma_ops optional - move more code out of line - add generic support for a dma_ops bypass mode - misc cleanups * tag 'dma-mapping-5.9' of git://git.infradead.org/users/hch/dma-mapping: dma-contiguous: cleanup dma_alloc_contiguous dma-debug: use named initializers for dir2name powerpc: use the generic dma_ops_bypass mode dma-mapping: add a dma_ops_bypass flag to struct device dma-mapping: make support for dma ops optional dma-mapping: inline the fast path dma-direct calls dma-mapping: move the remaining DMA API calls out of line

Linus Torvalds committed 6 years ago

Commit 2ed90db2 parents 9fa867d 274b3f726 files · +415 −416Browse files

Unified · Split

26 files

arch/alpha/Kconfig +1 −0
arch/arm/Kconfig +1 −0
arch/ia64/Kconfig +1 −0
arch/mips/Kconfig +1 −0
arch/parisc/Kconfig +1 −0
arch/powerpc/Kconfig +2 −0
arch/powerpc/include/asm/device.h +0 −5
arch/powerpc/kernel/dma-iommu.c +9 −81
arch/s390/Kconfig +1 −0
arch/sparc/Kconfig +1 −0
arch/x86/Kconfig +1 −0
drivers/infiniband/core/device.c +5 −1
drivers/iommu/Kconfig +2 −0
drivers/macintosh/macio_asic.c +2 −2
drivers/misc/mic/Kconfig +4 −0
drivers/vdpa/Kconfig +1 −0
drivers/xen/Kconfig +1 −0
include/linux/device.h +10 −1
include/linux/dma-direct.h +104 −0
include/linux/dma-mapping.h +29 −222
kernel/dma/Kconfig +12 −0
kernel/dma/Makefile +2 −1
kernel/dma/contiguous.c +14 −17
kernel/dma/debug.c +6 −2
kernel/dma/direct.c +0 −74
kernel/dma/mapping.c +204 −10

arch/alpha/Kconfig+1−0

@@ -7,6 +7,7 @@ config ALPHA · 6 unmodified lines
7	7	select ARCH_NO_PREEMPT
8	8	select ARCH_NO_SG_CHAIN
9	9	select ARCH_USE_CMPXCHG_LOCKREF
	10	+ select DMA_OPS if PCI
10	11	select FORCE_PCI if !ALPHA_JENSEN
11	12	select PCI_DOMAINS if PCI
12	13	select PCI_SYSCALL if PCI

arch/arm/Kconfig+1−0

@@ -41,6 +41,7 @@ config ARM · 40 unmodified lines
41	41	select CPU_PM if SUSPEND \|\| CPU_IDLE
42	42	select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS
43	43	select DMA_DECLARE_COHERENT
	44	+ select DMA_OPS
44	45	select DMA_REMAP if MMU
45	46	select EDAC_SUPPORT
46	47	select EDAC_ATOMIC_SCRUB

arch/ia64/Kconfig+1−0

@@ -192,6 +192,7 @@ config IA64_SGI_UV · 191 unmodified lines
192	192
193	193	config IA64_HP_SBA_IOMMU
194	194	bool "HP SBA IOMMU support"
	195	+ select DMA_OPS
195	196	default y
196	197	help
197	198	Say Y here to add support for the SBA IOMMU found on HP zx1 and

arch/mips/Kconfig+1−0

@@ -366,6 +366,7 @@ config MACH_JAZZ · 365 unmodified lines
366	366	select ARC_PROMLIB
367	367	select ARCH_MIGHT_HAVE_PC_PARPORT
368	368	select ARCH_MIGHT_HAVE_PC_SERIO
	369	+ select DMA_OPS
369	370	select FW_ARC
370	371	select FW_ARC32
371	372	select ARCH_MAY_HAVE_PC_FDC

arch/parisc/Kconfig+1−0

@@ -14,6 +14,7 @@ config PARISC · 13 unmodified lines
14	14	select ARCH_HAS_UBSAN_SANITIZE_ALL
15	15	select ARCH_NO_SG_CHAIN
16	16	select ARCH_SUPPORTS_MEMORY_FAILURE
	17	+ select DMA_OPS
17	18	select RTC_CLASS
18	19	select RTC_DRV_GENERIC
19	20	select INIT_ALL_POSSIBLE

arch/powerpc/Kconfig+2−0

@@ -151,6 +151,8 @@ config PPC · 150 unmodified lines
151	151	select BUILDTIME_TABLE_SORT
152	152	select CLONE_BACKWARDS
153	153	select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN
	154	+ select DMA_OPS if PPC64
	155	+ select DMA_OPS_BYPASS if PPC64
154	156	select DYNAMIC_FTRACE if FUNCTION_TRACER
155	157	select EDAC_ATOMIC_SCRUB
156	158	select EDAC_SUPPORT

arch/powerpc/include/asm/device.h+0−5

@@ -18,11 +18,6 @@ struct iommu_table; · 17 unmodified lines
18	18	* drivers/macintosh/macio_asic.c
19	19	*/
20	20	struct dev_archdata {
21		− /*
22		− * Set to %true if the dma_iommu_ops are requested to use a direct
23		− * window instead of dynamically mapping memory.
24		− */
25		− bool iommu_bypass : 1;
26	21	/*
27	22	* These two used to be a union. However, with the hybrid ops we need
28	23	* both so here we store both a DMA offset for direct mappings and

arch/powerpc/kernel/dma-iommu.c+9−81

@@ -14,23 +14,6 @@ · 13 unmodified lines
14	14	* Generic iommu implementation
15	15	*/
16	16
17		−/*
18		− * The coherent mask may be smaller than the real mask, check if we can
19		− * really use a direct window.
20		− */
21		−static inline bool dma_iommu_alloc_bypass(struct device *dev)
22		−{
23		− return dev->archdata.iommu_bypass && !iommu_fixed_is_weak &&
24		− dma_direct_supported(dev, dev->coherent_dma_mask);
25		−}
26		−
27		−static inline bool dma_iommu_map_bypass(struct device *dev,
28		− unsigned long attrs)
29		−{
30		− return dev->archdata.iommu_bypass &&
31		− (!iommu_fixed_is_weak \|\| (attrs & DMA_ATTR_WEAK_ORDERING));
32		−}
33		−
34	17	/* Allocates a contiguous real buffer and creates mappings over it.
35	18	* Returns the virtual address of the buffer and sets dma_handle
36	19	* to the dma address (mapping) of the first page.
@@ -39,8 +22,6 @@ static void dma_iommu_alloc_coherent(struct device dev, size_t size, · 2 unmodified lines
39	22	dma_addr_t *dma_handle, gfp_t flag,
40	23	unsigned long attrs)
41	24	{
42		− if (dma_iommu_alloc_bypass(dev))
43		− return dma_direct_alloc(dev, size, dma_handle, flag, attrs);
44	25	return iommu_alloc_coherent(dev, get_iommu_table_base(dev), size,
45	26	dma_handle, dev->coherent_dma_mask, flag,
46	27	dev_to_node(dev));
@@ -50,11 +31,7 @@ static void dma_iommu_free_coherent(struct device *dev, size_t size, · 3 unmodified lines
50	31	void *vaddr, dma_addr_t dma_handle,
51	32	unsigned long attrs)
52	33	{
53		− if (dma_iommu_alloc_bypass(dev))
54		− dma_direct_free(dev, size, vaddr, dma_handle, attrs);
55		− else
56		− iommu_free_coherent(get_iommu_table_base(dev), size, vaddr,
57		− dma_handle);
	34	+ iommu_free_coherent(get_iommu_table_base(dev), size, vaddr, dma_handle);
58	35	}
59	36
60	37	/* Creates TCEs for a user provided buffer. The user buffer must be
@@ -67,9 +44,6 @@ static dma_addr_t dma_iommu_map_page(struct device dev, struct page page, · 6 unmodified lines
67	44	enum dma_data_direction direction,
68	45	unsigned long attrs)
69	46	{
70		− if (dma_iommu_map_bypass(dev, attrs))
71		− return dma_direct_map_page(dev, page, offset, size, direction,
72		− attrs);
73	47	return iommu_map_page(dev, get_iommu_table_base(dev), page, offset,
74	48	size, dma_get_mask(dev), direction, attrs);
75	49	}
@@ -79,11 +53,8 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, · 3 unmodified lines
79	53	size_t size, enum dma_data_direction direction,
80	54	unsigned long attrs)
81	55	{
82		− if (!dma_iommu_map_bypass(dev, attrs))
83		− iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size,
84		− direction, attrs);
85		− else
86		− dma_direct_unmap_page(dev, dma_handle, size, direction, attrs);
	56	+ iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction,
	57	+ attrs);
87	58	}
88	59
89	60
@@ -91,8 +62,6 @@ static int dma_iommu_map_sg(struct device dev, struct scatterlist sglist, · 1 unmodified line
91	62	int nelems, enum dma_data_direction direction,
92	63	unsigned long attrs)
93	64	{
94		− if (dma_iommu_map_bypass(dev, attrs))
95		− return dma_direct_map_sg(dev, sglist, nelems, direction, attrs);
96	65	return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems,
97	66	dma_get_mask(dev), direction, attrs);
98	67	}
@@ -101,11 +70,8 @@ static void dma_iommu_unmap_sg(struct device dev, struct scatterlist sglist, · 2 unmodified lines
101	70	int nelems, enum dma_data_direction direction,
102	71	unsigned long attrs)
103	72	{
104		− if (!dma_iommu_map_bypass(dev, attrs))
105		− ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
	73	+ ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
106	74	direction, attrs);
107		− else
108		− dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs);
109	75	}
110	76
111	77	static bool dma_iommu_bypass_supported(struct device *dev, u64 mask)
@@ -113,8 +79,9 @@ static bool dma_iommu_bypass_supported(struct device *dev, u64 mask) · 1 unmodified line
113	79	struct pci_dev *pdev = to_pci_dev(dev);
114	80	struct pci_controller *phb = pci_bus_to_host(pdev->bus);
115	81
116		− return phb->controller_ops.iommu_bypass_supported &&
117		− phb->controller_ops.iommu_bypass_supported(pdev, mask);
	82	+ if (iommu_fixed_is_weak \|\| !phb->controller_ops.iommu_bypass_supported)
	83	+ return false;
	84	+ return phb->controller_ops.iommu_bypass_supported(pdev, mask);
118	85	}
119	86
120	87	/* We support DMA to/from any memory page via the iommu */
@@ -123,7 +90,7 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask) · 2 unmodified lines
123	90	struct iommu_table *tbl = get_iommu_table_base(dev);
124	91
125	92	if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {
126		− dev->archdata.iommu_bypass = true;
	93	+ dev->dma_ops_bypass = true;
127	94	dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
128	95	return 1;
129	96	}
@@ -141,7 +108,7 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask) · 11 unmodified lines
141	108	}
142	109
143	110	dev_dbg(dev, "iommu: not 64-bit, using default ops\n");
144		− dev->archdata.iommu_bypass = false;
	111	+ dev->dma_ops_bypass = false;
145	112	return 1;
146	113	}
147	114
@@ -153,47 +120,12 @@ u64 dma_iommu_get_required_mask(struct device *dev) · 5 unmodified lines
153	120	if (!tbl)
154	121	return 0;
155	122
156		− if (dev_is_pci(dev)) {
157		− u64 bypass_mask = dma_direct_get_required_mask(dev);
158		−
159		− if (dma_iommu_bypass_supported(dev, bypass_mask))
160		− return bypass_mask;
161		− }
162		−
163	123	mask = 1ULL < (fls_long(tbl->it_offset + tbl->it_size) - 1);
164	124	mask += mask - 1;
165	125
166	126	return mask;
167	127	}
168	128
169		−static void dma_iommu_sync_for_cpu(struct device *dev, dma_addr_t addr,
170		− size_t size, enum dma_data_direction dir)
171		−{
172		− if (dma_iommu_alloc_bypass(dev))
173		− dma_direct_sync_single_for_cpu(dev, addr, size, dir);
174		−}
175		−
176		−static void dma_iommu_sync_for_device(struct device *dev, dma_addr_t addr,
177		− size_t sz, enum dma_data_direction dir)
178		−{
179		− if (dma_iommu_alloc_bypass(dev))
180		− dma_direct_sync_single_for_device(dev, addr, sz, dir);
181		−}
182		−
183		−extern void dma_iommu_sync_sg_for_cpu(struct device *dev,
184		− struct scatterlist *sgl, int nents, enum dma_data_direction dir)
185		−{
186		− if (dma_iommu_alloc_bypass(dev))
187		− dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir);
188		−}
189		−
190		−extern void dma_iommu_sync_sg_for_device(struct device *dev,
191		− struct scatterlist *sgl, int nents, enum dma_data_direction dir)
192		−{
193		− if (dma_iommu_alloc_bypass(dev))
194		− dma_direct_sync_sg_for_device(dev, sgl, nents, dir);
195		−}
196		−
197	129	const struct dma_map_ops dma_iommu_ops = {
198	130	.alloc = dma_iommu_alloc_coherent,
199	131	.free = dma_iommu_free_coherent,
@@ -203,10 +135,6 @@ const struct dma_map_ops dma_iommu_ops = { · 3 unmodified lines
203	135	.map_page = dma_iommu_map_page,
204	136	.unmap_page = dma_iommu_unmap_page,
205	137	.get_required_mask = dma_iommu_get_required_mask,
206		− .sync_single_for_cpu = dma_iommu_sync_for_cpu,
207		− .sync_single_for_device = dma_iommu_sync_for_device,
208		− .sync_sg_for_cpu = dma_iommu_sync_sg_for_cpu,
209		− .sync_sg_for_device = dma_iommu_sync_sg_for_device,
210	138	.mmap = dma_common_mmap,
211	139	.get_sgtable = dma_common_get_sgtable,
212	140	};

arch/s390/Kconfig+1−0

@@ -112,6 +112,7 @@ config S390 · 111 unmodified lines
112	112	select ARCH_WANT_IPC_PARSE_VERSION
113	113	select BUILDTIME_TABLE_SORT
114	114	select CLONE_BACKWARDS2
	115	+ select DMA_OPS if PCI
115	116	select DYNAMIC_FTRACE if FUNCTION_TRACER
116	117	select GENERIC_CLOCKEVENTS
117	118	select GENERIC_CPU_AUTOPROBE

arch/sparc/Kconfig+1−0

@@ -15,6 +15,7 @@ config SPARC · 14 unmodified lines
15	15	default y
16	16	select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI
17	17	select ARCH_MIGHT_HAVE_PC_SERIO
	18	+ select DMA_OPS
18	19	select OF
19	20	select OF_PROMTREE
20	21	select HAVE_ASM_MODVERSIONS

arch/x86/Kconfig+1−0

@@ -909,6 +909,7 @@ config DMI · 908 unmodified lines
909	909
910	910	config GART_IOMMU
911	911	bool "Old AMD GART IOMMU support"
	912	+ select DMA_OPS
912	913	select IOMMU_HELPER
913	914	select SWIOTLB
914	915	depends on X86_64 && PCI && AMD_NB

drivers/infiniband/core/device.c+5−1

@@ -1183,6 +1183,8 @@ static void setup_dma_device(struct ib_device *device) · 1182 unmodified lines
1183	1183	struct device *parent = device->dev.parent;
1184	1184
1185	1185	WARN_ON_ONCE(device->dma_device);
	1186	+
	1187	+#ifdef CONFIG_DMA_OPS
1186	1188	if (device->dev.dma_ops) {
1187	1189	/*
1188	1190	* The caller provided custom DMA operations. Copy the
@@ -1203,7 +1205,9 @@ static void setup_dma_device(struct ib_device *device) · 14 unmodified lines
1203	1205	else
1204	1206	WARN_ON_ONCE(true);
1205	1207	}
1206		− } else {
	1208	+ } else
	1209	+#endif /* CONFIG_DMA_OPS */
	1210	+ {
1207	1211	/*
1208	1212	* The caller did not provide custom DMA operations. Use the
1209	1213	* DMA mapping operations of the parent device.

drivers/iommu/Kconfig+2−0

@@ -97,6 +97,7 @@ config OF_IOMMU · 96 unmodified lines
97	97	# IOMMU-agnostic DMA-mapping layer
98	98	config IOMMU_DMA
99	99	bool
	100	+ select DMA_OPS
100	101	select IOMMU_API
101	102	select IOMMU_IOVA
102	103	select IRQ_MSI_IOMMU
@@ -183,6 +184,7 @@ config DMAR_TABLE · 80 unmodified lines
183	184	config INTEL_IOMMU
184	185	bool "Support for Intel IOMMU using DMA Remapping Devices"
185	186	depends on PCI_MSI && ACPI && (X86 \|\| IA64)
	187	+ select DMA_OPS
186	188	select IOMMU_API
187	189	select IOMMU_IOVA
188	190	select NEED_DMA_MAP_STATE

drivers/macintosh/macio_asic.c+2−2

@@ -382,7 +382,7 @@ static struct macio_dev * macio_add_one_device(struct macio_chip *chip, · 381 unmodified lines
382	382	dma_set_max_seg_size(&dev->ofdev.dev, 65536);
383	383	dma_set_seg_boundary(&dev->ofdev.dev, 0xffffffff);
384	384
385		−#ifdef CONFIG_PCI
	385	+#if defined(CONFIG_PCI) && defined(CONFIG_DMA_OPS)
386	386	/* Set the DMA ops to the ones from the PCI device, this could be
387	387	* fishy if we didn't know that on PowerMac it's always direct ops
388	388	* or iommu ops that will work fine
@@ -391,7 +391,7 @@ static struct macio_dev * macio_add_one_device(struct macio_chip *chip, · 2 unmodified lines
391	391	*/
392	392	dev->ofdev.dev.archdata = chip->lbus.pdev->dev.archdata;
393	393	dev->ofdev.dev.dma_ops = chip->lbus.pdev->dev.dma_ops;
394		−#endif /* CONFIG_PCI */
	394	+#endif /* CONFIG_PCI && CONFIG_DMA_OPS */
395	395
396	396	#ifdef DEBUG
397	397	printk("preparing mdev @%p, ofdev @%p, dev @%p, kobj @%p\n",

drivers/misc/mic/Kconfig+4−0

@@ -4,6 +4,7 @@ menu "Intel MIC & related support" · 3 unmodified lines
4	4	config INTEL_MIC_BUS
5	5	tristate "Intel MIC Bus Driver"
6	6	depends on 64BIT && PCI && X86
	7	+ select DMA_OPS
7	8	help
8	9	This option is selected by any driver which registers a
9	10	device or driver on the MIC Bus, such as CONFIG_INTEL_MIC_HOST,
@@ -19,6 +20,7 @@ config INTEL_MIC_BUS · 9 unmodified lines
19	20	config SCIF_BUS
20	21	tristate "SCIF Bus Driver"
21	22	depends on 64BIT && PCI && X86
	23	+ select DMA_OPS
22	24	help
23	25	This option is selected by any driver which registers a
24	26	device or driver on the SCIF Bus, such as CONFIG_INTEL_MIC_HOST
@@ -33,6 +35,7 @@ config SCIF_BUS · 8 unmodified lines
33	35
34	36	config VOP_BUS
35	37	tristate "VOP Bus Driver"
	38	+ select DMA_OPS
36	39	help
37	40	This option is selected by any driver which registers a
38	41	device or driver on the VOP Bus, such as CONFIG_INTEL_MIC_HOST
@@ -49,6 +52,7 @@ config INTEL_MIC_HOST · 10 unmodified lines
49	52	tristate "Intel MIC Host Driver"
50	53	depends on 64BIT && PCI && X86
51	54	depends on INTEL_MIC_BUS && SCIF_BUS && MIC_COSM && VOP_BUS
	55	+ select DMA_OPS
52	56	help
53	57	This enables Host Driver support for the Intel Many Integrated
54	58	Core (MIC) family of PCIe form factor coprocessor devices that

drivers/vdpa/Kconfig+1−0

@@ -11,6 +11,7 @@ if VDPA · 10 unmodified lines
11	11	config VDPA_SIM
12	12	tristate "vDPA device simulator"
13	13	depends on RUNTIME_TESTING_MENU && HAS_DMA
	14	+ select DMA_OPS
14	15	select VHOST_RING
15	16	default n
16	17	help

drivers/xen/Kconfig+1−0

@@ -179,6 +179,7 @@ config XEN_GRANT_DMA_ALLOC · 178 unmodified lines
179	179
180	180	config SWIOTLB_XEN
181	181	def_bool y
	182	+ select DMA_OPS
182	183	select SWIOTLB
183	184
184	185	config XEN_PCIDEV_BACKEND

include/linux/device.h+10−1

@@ -525,6 +525,11 @@ struct dev_links_info { · 524 unmodified lines
525	525	* sync_state() callback.
526	526	* @dma_coherent: this particular device is dma coherent, even if the
527	527	* architecture supports non-coherent devices.
	528	+ * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
	529	+ * streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
	530	+ * and optionall (if the coherent mask is large enough) also
	531	+ * for dma allocations. This flag is managed by the dma ops
	532	+ * instance from ->dma_supported.
528	533	*
529	534	* At the lowest level, every device in a Linux system is represented by an
530	535	* instance of struct device. The device structure contains the information
@@ -574,8 +579,9 @@ struct device { · 43 unmodified lines
574	579	#ifdef CONFIG_GENERIC_MSI_IRQ
575	580	struct list_head msi_list;
576	581	#endif
577		−
	582	+#ifdef CONFIG_DMA_OPS
578	583	const struct dma_map_ops *dma_ops;
	584	+#endif
579	585	u64 dma_mask; / dma mask (if dma'able device) */
580	586	u64 coherent_dma_mask;/* Like dma_mask, but for
581	587	alloc_coherent mappings as
@@ -628,6 +634,9 @@ struct device { · 46 unmodified lines
628	634	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
629	635	bool dma_coherent:1;
630	636	#endif
	637	+#ifdef CONFIG_DMA_OPS_BYPASS
	638	+ bool dma_ops_bypass : 1;
	639	+#endif
631	640	};
632	641
633	642	static inline struct device kobj_to_dev(struct kobject kobj)

include/linux/dma-direct.h+104−0

@@ -1,10 +1,16 @@
1	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * Internals of the DMA direct mapping implementation. Only for use by the
	4	+ * DMA mapping code and IOMMU drivers.
	5	+ */
2	6	#ifndef _LINUX_DMA_DIRECT_H
3	7	#define _LINUX_DMA_DIRECT_H 1
4	8
5	9	#include <linux/dma-mapping.h>
	10	+#include <linux/dma-noncoherent.h>
6	11	#include <linux/memblock.h> /* for min_low_pfn */
7	12	#include <linux/mem_encrypt.h>
	13	+#include <linux/swiotlb.h>
8	14
9	15	extern unsigned int zone_dma_bits;
10	16
@@ -87,4 +93,102 @@ int dma_direct_mmap(struct device dev, struct vm_area_struct vma, · 76 unmodified lines
87	93	unsigned long attrs);
88	94	int dma_direct_supported(struct device *dev, u64 mask);
89	95	bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);
	96	+int dma_direct_map_sg(struct device dev, struct scatterlist sgl, int nents,
	97	+ enum dma_data_direction dir, unsigned long attrs);
	98	+dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
	99	+ size_t size, enum dma_data_direction dir, unsigned long attrs);
	100	+size_t dma_direct_max_mapping_size(struct device *dev);
	101	+
	102	+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) \|\| \
	103	+ defined(CONFIG_SWIOTLB)
	104	+void dma_direct_sync_sg_for_device(struct device dev, struct scatterlist sgl,
	105	+ int nents, enum dma_data_direction dir);
	106	+#else
	107	+static inline void dma_direct_sync_sg_for_device(struct device *dev,
	108	+ struct scatterlist *sgl, int nents, enum dma_data_direction dir)
	109	+{
	110	+}
	111	+#endif
	112	+
	113	+#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) \|\| \
	114	+ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) \|\| \
	115	+ defined(CONFIG_SWIOTLB)
	116	+void dma_direct_unmap_sg(struct device dev, struct scatterlist sgl,
	117	+ int nents, enum dma_data_direction dir, unsigned long attrs);
	118	+void dma_direct_sync_sg_for_cpu(struct device *dev,
	119	+ struct scatterlist *sgl, int nents, enum dma_data_direction dir);
	120	+#else
	121	+static inline void dma_direct_unmap_sg(struct device *dev,
	122	+ struct scatterlist *sgl, int nents, enum dma_data_direction dir,
	123	+ unsigned long attrs)
	124	+{
	125	+}
	126	+static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
	127	+ struct scatterlist *sgl, int nents, enum dma_data_direction dir)
	128	+{
	129	+}
	130	+#endif
	131	+
	132	+static inline void dma_direct_sync_single_for_device(struct device *dev,
	133	+ dma_addr_t addr, size_t size, enum dma_data_direction dir)
	134	+{
	135	+ phys_addr_t paddr = dma_to_phys(dev, addr);
	136	+
	137	+ if (unlikely(is_swiotlb_buffer(paddr)))
	138	+ swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
	139	+
	140	+ if (!dev_is_dma_coherent(dev))
	141	+ arch_sync_dma_for_device(paddr, size, dir);
	142	+}
	143	+
	144	+static inline void dma_direct_sync_single_for_cpu(struct device *dev,
	145	+ dma_addr_t addr, size_t size, enum dma_data_direction dir)
	146	+{
	147	+ phys_addr_t paddr = dma_to_phys(dev, addr);
	148	+
	149	+ if (!dev_is_dma_coherent(dev)) {
	150	+ arch_sync_dma_for_cpu(paddr, size, dir);
	151	+ arch_sync_dma_for_cpu_all();
	152	+ }
	153	+
	154	+ if (unlikely(is_swiotlb_buffer(paddr)))
	155	+ swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
	156	+}
	157	+
	158	+static inline dma_addr_t dma_direct_map_page(struct device *dev,
	159	+ struct page *page, unsigned long offset, size_t size,
	160	+ enum dma_data_direction dir, unsigned long attrs)
	161	+{
	162	+ phys_addr_t phys = page_to_phys(page) + offset;
	163	+ dma_addr_t dma_addr = phys_to_dma(dev, phys);
	164	+
	165	+ if (unlikely(swiotlb_force == SWIOTLB_FORCE))
	166	+ return swiotlb_map(dev, phys, size, dir, attrs);
	167	+
	168	+ if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
	169	+ if (swiotlb_force != SWIOTLB_NO_FORCE)
	170	+ return swiotlb_map(dev, phys, size, dir, attrs);
	171	+
	172	+ dev_WARN_ONCE(dev, 1,
	173	+ "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
	174	+ &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
	175	+ return DMA_MAPPING_ERROR;
	176	+ }
	177	+
	178	+ if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
	179	+ arch_sync_dma_for_device(phys, size, dir);
	180	+ return dma_addr;
	181	+}
	182	+
	183	+static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
	184	+ size_t size, enum dma_data_direction dir, unsigned long attrs)
	185	+{
	186	+ phys_addr_t phys = dma_to_phys(dev, addr);
	187	+
	188	+ if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
	189	+ dma_direct_sync_single_for_cpu(dev, addr, size, dir);
	190	+
	191	+ if (unlikely(is_swiotlb_buffer(phys)))
	192	+ swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
	193	+}
90	194	#endif /* _LINUX_DMA_DIRECT_H */

include/linux/dma-mapping.h+29−222

@@ -188,76 +188,10 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma, · 187 unmodified lines
188	188	}
189	189	#endif /* CONFIG_DMA_DECLARE_COHERENT */
190	190
191		−static inline bool dma_is_direct(const struct dma_map_ops *ops)
192		−{
193		− return likely(!ops);
194		−}
195		−
196		−/*
197		− * All the dma_direct_* declarations are here just for the indirect call bypass,
198		− * and must not be used directly drivers!
199		− */
200		−dma_addr_t dma_direct_map_page(struct device dev, struct page page,
201		− unsigned long offset, size_t size, enum dma_data_direction dir,
202		− unsigned long attrs);
203		−int dma_direct_map_sg(struct device dev, struct scatterlist sgl, int nents,
204		− enum dma_data_direction dir, unsigned long attrs);
205		−dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
206		− size_t size, enum dma_data_direction dir, unsigned long attrs);
207		−
208		−#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) \|\| \
209		− defined(CONFIG_SWIOTLB)
210		−void dma_direct_sync_single_for_device(struct device *dev,
211		− dma_addr_t addr, size_t size, enum dma_data_direction dir);
212		−void dma_direct_sync_sg_for_device(struct device *dev,
213		− struct scatterlist *sgl, int nents, enum dma_data_direction dir);
214		−#else
215		−static inline void dma_direct_sync_single_for_device(struct device *dev,
216		− dma_addr_t addr, size_t size, enum dma_data_direction dir)
217		−{
218		−}
219		−static inline void dma_direct_sync_sg_for_device(struct device *dev,
220		− struct scatterlist *sgl, int nents, enum dma_data_direction dir)
221		−{
222		−}
223		−#endif
224		−
225		−#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) \|\| \
226		− defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) \|\| \
227		− defined(CONFIG_SWIOTLB)
228		−void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
229		− size_t size, enum dma_data_direction dir, unsigned long attrs);
230		−void dma_direct_unmap_sg(struct device dev, struct scatterlist sgl,
231		− int nents, enum dma_data_direction dir, unsigned long attrs);
232		−void dma_direct_sync_single_for_cpu(struct device *dev,
233		− dma_addr_t addr, size_t size, enum dma_data_direction dir);
234		−void dma_direct_sync_sg_for_cpu(struct device *dev,
235		− struct scatterlist *sgl, int nents, enum dma_data_direction dir);
236		−#else
237		−static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
238		− size_t size, enum dma_data_direction dir, unsigned long attrs)
239		−{
240		−}
241		−static inline void dma_direct_unmap_sg(struct device *dev,
242		− struct scatterlist *sgl, int nents, enum dma_data_direction dir,
243		− unsigned long attrs)
244		−{
245		−}
246		−static inline void dma_direct_sync_single_for_cpu(struct device *dev,
247		− dma_addr_t addr, size_t size, enum dma_data_direction dir)
248		−{
249		−}
250		−static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
251		− struct scatterlist *sgl, int nents, enum dma_data_direction dir)
252		−{
253		−}
254		−#endif
255		−
256		−size_t dma_direct_max_mapping_size(struct device *dev);
257		−
258	191	#ifdef CONFIG_HAS_DMA
259	192	#include <asm/dma-mapping.h>
260	193
	194	+#ifdef CONFIG_DMA_OPS
261	195	static inline const struct dma_map_ops get_dma_ops(struct device dev)
262	196	{
263	197	if (dev->dma_ops)
@@ -270,165 +204,16 @@ static inline void set_dma_ops(struct device *dev, · 6 unmodified lines
270	204	{
271	205	dev->dma_ops = dma_ops;
272	206	}
273		−
274		−static inline dma_addr_t dma_map_page_attrs(struct device *dev,
275		− struct page *page, size_t offset, size_t size,
276		− enum dma_data_direction dir, unsigned long attrs)
277		−{
278		− const struct dma_map_ops *ops = get_dma_ops(dev);
279		− dma_addr_t addr;
280		−
281		− BUG_ON(!valid_dma_direction(dir));
282		− if (dma_is_direct(ops))
283		− addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
284		− else
285		− addr = ops->map_page(dev, page, offset, size, dir, attrs);
286		− debug_dma_map_page(dev, page, offset, size, dir, addr);
287		−
288		− return addr;
289		−}
290		−
291		−static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
292		− size_t size, enum dma_data_direction dir, unsigned long attrs)
293		−{
294		− const struct dma_map_ops *ops = get_dma_ops(dev);
295		−
296		− BUG_ON(!valid_dma_direction(dir));
297		− if (dma_is_direct(ops))
298		− dma_direct_unmap_page(dev, addr, size, dir, attrs);
299		− else if (ops->unmap_page)
300		− ops->unmap_page(dev, addr, size, dir, attrs);
301		− debug_dma_unmap_page(dev, addr, size, dir);
302		−}
303		−
304		−/*
305		− * dma_maps_sg_attrs returns 0 on error and > 0 on success.
306		− * It should never return a value < 0.
307		− */
308		−static inline int dma_map_sg_attrs(struct device dev, struct scatterlist sg,
309		− int nents, enum dma_data_direction dir,
310		− unsigned long attrs)
311		−{
312		− const struct dma_map_ops *ops = get_dma_ops(dev);
313		− int ents;
314		−
315		− BUG_ON(!valid_dma_direction(dir));
316		− if (dma_is_direct(ops))
317		− ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
318		− else
319		− ents = ops->map_sg(dev, sg, nents, dir, attrs);
320		− BUG_ON(ents < 0);
321		− debug_dma_map_sg(dev, sg, nents, ents, dir);
322		−
323		− return ents;
324		−}
325		−
326		−static inline void dma_unmap_sg_attrs(struct device dev, struct scatterlist sg,
327		− int nents, enum dma_data_direction dir,
328		− unsigned long attrs)
329		−{
330		− const struct dma_map_ops *ops = get_dma_ops(dev);
331		−
332		− BUG_ON(!valid_dma_direction(dir));
333		− debug_dma_unmap_sg(dev, sg, nents, dir);
334		− if (dma_is_direct(ops))
335		− dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
336		− else if (ops->unmap_sg)
337		− ops->unmap_sg(dev, sg, nents, dir, attrs);
338		−}
339		−
340		−static inline dma_addr_t dma_map_resource(struct device *dev,
341		− phys_addr_t phys_addr,
342		− size_t size,
343		− enum dma_data_direction dir,
344		− unsigned long attrs)
345		−{
346		− const struct dma_map_ops *ops = get_dma_ops(dev);
347		− dma_addr_t addr = DMA_MAPPING_ERROR;
348		−
349		− BUG_ON(!valid_dma_direction(dir));
350		−
351		− /* Don't allow RAM to be mapped */
352		− if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
353		− return DMA_MAPPING_ERROR;
354		−
355		− if (dma_is_direct(ops))
356		− addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
357		− else if (ops->map_resource)
358		− addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
359		−
360		− debug_dma_map_resource(dev, phys_addr, size, dir, addr);
361		− return addr;
362		−}
363		−
364		−static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
365		− size_t size, enum dma_data_direction dir,
366		− unsigned long attrs)
367		−{
368		− const struct dma_map_ops *ops = get_dma_ops(dev);
369		−
370		− BUG_ON(!valid_dma_direction(dir));
371		− if (!dma_is_direct(ops) && ops->unmap_resource)
372		− ops->unmap_resource(dev, addr, size, dir, attrs);
373		− debug_dma_unmap_resource(dev, addr, size, dir);
374		−}
375		−
376		−static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
377		− size_t size,
378		− enum dma_data_direction dir)
379		−{
380		− const struct dma_map_ops *ops = get_dma_ops(dev);
381		−
382		− BUG_ON(!valid_dma_direction(dir));
383		− if (dma_is_direct(ops))
384		− dma_direct_sync_single_for_cpu(dev, addr, size, dir);
385		− else if (ops->sync_single_for_cpu)
386		− ops->sync_single_for_cpu(dev, addr, size, dir);
387		− debug_dma_sync_single_for_cpu(dev, addr, size, dir);
388		−}
389		−
390		−static inline void dma_sync_single_for_device(struct device *dev,
391		− dma_addr_t addr, size_t size,
392		− enum dma_data_direction dir)
393		−{
394		− const struct dma_map_ops *ops = get_dma_ops(dev);
395		−
396		− BUG_ON(!valid_dma_direction(dir));
397		− if (dma_is_direct(ops))
398		− dma_direct_sync_single_for_device(dev, addr, size, dir);
399		− else if (ops->sync_single_for_device)
400		− ops->sync_single_for_device(dev, addr, size, dir);
401		− debug_dma_sync_single_for_device(dev, addr, size, dir);
402		−}
403		−
404		−static inline void
405		−dma_sync_sg_for_cpu(struct device dev, struct scatterlist sg,
406		− int nelems, enum dma_data_direction dir)
	207	+#else /* CONFIG_DMA_OPS */
	208	+static inline const struct dma_map_ops get_dma_ops(struct device dev)
407	209	{
408		− const struct dma_map_ops *ops = get_dma_ops(dev);
409		−
410		− BUG_ON(!valid_dma_direction(dir));
411		− if (dma_is_direct(ops))
412		− dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
413		− else if (ops->sync_sg_for_cpu)
414		− ops->sync_sg_for_cpu(dev, sg, nelems, dir);
415		− debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
	210	+ return NULL;
416	211	}
417		−
418		−static inline void
419		−dma_sync_sg_for_device(struct device dev, struct scatterlist sg,
420		− int nelems, enum dma_data_direction dir)
	212	+static inline void set_dma_ops(struct device *dev,
	213	+ const struct dma_map_ops *dma_ops)
421	214	{
422		− const struct dma_map_ops *ops = get_dma_ops(dev);
423		−
424		− BUG_ON(!valid_dma_direction(dir));
425		− if (dma_is_direct(ops))
426		− dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
427		− else if (ops->sync_sg_for_device)
428		− ops->sync_sg_for_device(dev, sg, nelems, dir);
429		− debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
430		−
431	215	}
	216	+#endif /* CONFIG_DMA_OPS */
432	217
433	218	static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
434	219	{
@@ -439,6 +224,28 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) · 4 unmodified lines
439	224	return 0;
440	225	}
441	226
	227	+dma_addr_t dma_map_page_attrs(struct device dev, struct page page,
	228	+ size_t offset, size_t size, enum dma_data_direction dir,
	229	+ unsigned long attrs);
	230	+void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
	231	+ enum dma_data_direction dir, unsigned long attrs);
	232	+int dma_map_sg_attrs(struct device dev, struct scatterlist sg, int nents,
	233	+ enum dma_data_direction dir, unsigned long attrs);
	234	+void dma_unmap_sg_attrs(struct device dev, struct scatterlist sg,
	235	+ int nents, enum dma_data_direction dir,
	236	+ unsigned long attrs);
	237	+dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
	238	+ size_t size, enum dma_data_direction dir, unsigned long attrs);
	239	+void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
	240	+ enum dma_data_direction dir, unsigned long attrs);
	241	+void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
	242	+ enum dma_data_direction dir);
	243	+void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
	244	+ size_t size, enum dma_data_direction dir);
	245	+void dma_sync_sg_for_cpu(struct device dev, struct scatterlist sg,
	246	+ int nelems, enum dma_data_direction dir);
	247	+void dma_sync_sg_for_device(struct device dev, struct scatterlist sg,
	248	+ int nelems, enum dma_data_direction dir);
442	249	void dma_alloc_attrs(struct device dev, size_t size, dma_addr_t *dma_handle,
443	250	gfp_t flag, unsigned long attrs);
444	251	void dma_free_attrs(struct device dev, size_t size, void cpu_addr,

kernel/dma/Kconfig+12−0

@@ -5,6 +5,17 @@ config HAS_DMA · 4 unmodified lines
5	5	depends on !NO_DMA
6	6	default y
7	7
	8	+config DMA_OPS
	9	+ bool
	10	+
	11	+#
	12	+# IOMMU drivers that can bypass the IOMMU code and optionally use the direct
	13	+# mapping fast path should select this option and set the dma_ops_bypass
	14	+# flag in struct device where applicable
	15	+#
	16	+config DMA_OPS_BYPASS
	17	+ bool
	18	+
8	19	config NEED_SG_DMA_LENGTH
9	20	bool
10	21
@@ -60,6 +71,7 @@ config DMA_NONCOHERENT_CACHE_SYNC · 49 unmodified lines
60	71	config DMA_VIRT_OPS
61	72	bool
62	73	depends on HAS_DMA
	74	+ select DMA_OPS
63	75
64	76	config SWIOTLB
65	77	bool

kernel/dma/Makefile+2−1

@@ -1,6 +1,7 @@
1	1	# SPDX-License-Identifier: GPL-2.0
2	2
3		−obj-$(CONFIG_HAS_DMA) += mapping.o direct.o dummy.o
	3	+obj-$(CONFIG_HAS_DMA) += mapping.o direct.o
	4	+obj-$(CONFIG_DMA_OPS) += dummy.o
4	5	obj-$(CONFIG_DMA_CMA) += contiguous.o
5	6	obj-$(CONFIG_DMA_DECLARE_COHERENT) += coherent.o
6	7	obj-$(CONFIG_DMA_VIRT_OPS) += virt.o

kernel/dma/contiguous.c+14−17

@@ -215,6 +215,13 @@ bool dma_release_from_contiguous(struct device dev, struct page pages, · 214 unmodified lines
215	215	return cma_release(dev_get_cma_area(dev), pages, count);
216	216	}
217	217
	218	+static struct page cma_alloc_aligned(struct cma cma, size_t size, gfp_t gfp)
	219	+{
	220	+ unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT);
	221	+
	222	+ return cma_alloc(cma, size >> PAGE_SHIFT, align, gfp & __GFP_NOWARN);
	223	+}
	224	+
218	225	/**
219	226	* dma_alloc_contiguous() - allocate contiguous pages
220	227	* @dev: Pointer to device for which the allocation is performed.
@@ -231,24 +238,14 @@ bool dma_release_from_contiguous(struct device dev, struct page pages, · 10 unmodified lines
231	238	*/
232	239	struct page dma_alloc_contiguous(struct device dev, size_t size, gfp_t gfp)
233	240	{
234		− size_t count = size >> PAGE_SHIFT;
235		− struct page *page = NULL;
236		− struct cma *cma = NULL;
237		−
238		− if (dev && dev->cma_area)
239		− cma = dev->cma_area;
240		− else if (count > 1)
241		− cma = dma_contiguous_default_area;
242		−
243	241	/* CMA can be used only in the context which permits sleeping */
244		− if (cma && gfpflags_allow_blocking(gfp)) {
245		− size_t align = get_order(size);
246		− size_t cma_align = min_t(size_t, align, CONFIG_CMA_ALIGNMENT);
247		−
248		− page = cma_alloc(cma, count, cma_align, gfp & __GFP_NOWARN);
249		− }
250		−
251		− return page;
	242	+ if (!gfpflags_allow_blocking(gfp))
	243	+ return NULL;
	244	+ if (dev->cma_area)
	245	+ return cma_alloc_aligned(dev->cma_area, size, gfp);
	246	+ if (size <= PAGE_SIZE \|\| !dma_contiguous_default_area)
	247	+ return NULL;
	248	+ return cma_alloc_aligned(dma_contiguous_default_area, size, gfp);
252	249	}
253	250
254	251	/**

kernel/dma/debug.c+6−2

@@ -144,8 +144,12 @@ static const char *type2name[] = { · 143 unmodified lines
144	144	[dma_debug_resource] = "resource",
145	145	};
146	146
147		−static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
148		− "DMA_FROM_DEVICE", "DMA_NONE" };
	147	+static const char *dir2name[] = {
	148	+ [DMA_BIDIRECTIONAL] = "DMA_BIDIRECTIONAL",
	149	+ [DMA_TO_DEVICE] = "DMA_TO_DEVICE",
	150	+ [DMA_FROM_DEVICE] = "DMA_FROM_DEVICE",
	151	+ [DMA_NONE] = "DMA_NONE",
	152	+};
149	153
150	154	/*
151	155	* The access to some variables in this macro is racy. We can't use atomic_t

kernel/dma/direct.c+0−74

@@ -10,11 +10,9 @@ · 9 unmodified lines
10	10	#include <linux/dma-direct.h>
11	11	#include <linux/scatterlist.h>
12	12	#include <linux/dma-contiguous.h>
13		−#include <linux/dma-noncoherent.h>
14	13	#include <linux/pfn.h>
15	14	#include <linux/vmalloc.h>
16	15	#include <linux/set_memory.h>
17		−#include <linux/swiotlb.h>
18	16
19	17	/*
20	18	* Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it
@@ -304,19 +302,6 @@ void dma_direct_free(struct device *dev, size_t size, · 283 unmodified lines
304	302
305	303	#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) \|\| \
306	304	defined(CONFIG_SWIOTLB)
307		−void dma_direct_sync_single_for_device(struct device *dev,
308		− dma_addr_t addr, size_t size, enum dma_data_direction dir)
309		−{
310		− phys_addr_t paddr = dma_to_phys(dev, addr);
311		−
312		− if (unlikely(is_swiotlb_buffer(paddr)))
313		− swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
314		−
315		− if (!dev_is_dma_coherent(dev))
316		− arch_sync_dma_for_device(paddr, size, dir);
317		−}
318		−EXPORT_SYMBOL(dma_direct_sync_single_for_device);
319		−
320	305	void dma_direct_sync_sg_for_device(struct device *dev,
321	306	struct scatterlist *sgl, int nents, enum dma_data_direction dir)
322	307	{
@@ -335,27 +320,11 @@ void dma_direct_sync_sg_for_device(struct device *dev, · 12 unmodified lines
335	320	dir);
336	321	}
337	322	}
338		−EXPORT_SYMBOL(dma_direct_sync_sg_for_device);
339	323	#endif
340	324
341	325	#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) \|\| \
342	326	defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) \|\| \
343	327	defined(CONFIG_SWIOTLB)
344		−void dma_direct_sync_single_for_cpu(struct device *dev,
345		− dma_addr_t addr, size_t size, enum dma_data_direction dir)
346		−{
347		− phys_addr_t paddr = dma_to_phys(dev, addr);
348		−
349		− if (!dev_is_dma_coherent(dev)) {
350		− arch_sync_dma_for_cpu(paddr, size, dir);
351		− arch_sync_dma_for_cpu_all();
352		− }
353		−
354		− if (unlikely(is_swiotlb_buffer(paddr)))
355		− swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
356		−}
357		−EXPORT_SYMBOL(dma_direct_sync_single_for_cpu);
358		−
359	328	void dma_direct_sync_sg_for_cpu(struct device *dev,
360	329	struct scatterlist *sgl, int nents, enum dma_data_direction dir)
361	330	{
@@ -376,20 +345,6 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, · 14 unmodified lines
376	345	if (!dev_is_dma_coherent(dev))
377	346	arch_sync_dma_for_cpu_all();
378	347	}
379		−EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu);
380		−
381		−void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
382		− size_t size, enum dma_data_direction dir, unsigned long attrs)
383		−{
384		− phys_addr_t phys = dma_to_phys(dev, addr);
385		−
386		− if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
387		− dma_direct_sync_single_for_cpu(dev, addr, size, dir);
388		−
389		− if (unlikely(is_swiotlb_buffer(phys)))
390		− swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
391		−}
392		−EXPORT_SYMBOL(dma_direct_unmap_page);
393	348
394	349	void dma_direct_unmap_sg(struct device dev, struct scatterlist sgl,
395	350	int nents, enum dma_data_direction dir, unsigned long attrs)
@@ -401,35 +356,8 @@ void dma_direct_unmap_sg(struct device dev, struct scatterlist sgl, · 5 unmodified lines
401	356	dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,
402	357	attrs);
403	358	}
404		−EXPORT_SYMBOL(dma_direct_unmap_sg);
405	359	#endif
406	360
407		−dma_addr_t dma_direct_map_page(struct device dev, struct page page,
408		− unsigned long offset, size_t size, enum dma_data_direction dir,
409		− unsigned long attrs)
410		−{
411		− phys_addr_t phys = page_to_phys(page) + offset;
412		− dma_addr_t dma_addr = phys_to_dma(dev, phys);
413		−
414		− if (unlikely(swiotlb_force == SWIOTLB_FORCE))
415		− return swiotlb_map(dev, phys, size, dir, attrs);
416		−
417		− if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
418		− if (swiotlb_force != SWIOTLB_NO_FORCE)
419		− return swiotlb_map(dev, phys, size, dir, attrs);
420		−
421		− dev_WARN_ONCE(dev, 1,
422		− "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
423		− &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
424		− return DMA_MAPPING_ERROR;
425		− }
426		−
427		− if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
428		− arch_sync_dma_for_device(phys, size, dir);
429		− return dma_addr;
430		−}
431		−EXPORT_SYMBOL(dma_direct_map_page);
432		−
433	361	int dma_direct_map_sg(struct device dev, struct scatterlist sgl, int nents,
434	362	enum dma_data_direction dir, unsigned long attrs)
435	363	{
@@ -450,7 +378,6 @@ out_unmap: · 14 unmodified lines
450	378	dma_direct_unmap_sg(dev, sgl, i, dir, attrs \| DMA_ATTR_SKIP_CPU_SYNC);
451	379	return 0;
452	380	}
453		−EXPORT_SYMBOL(dma_direct_map_sg);
454	381
455	382	dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
456	383	size_t size, enum dma_data_direction dir, unsigned long attrs)
@@ -467,7 +394,6 @@ dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, · 10 unmodified lines
467	394
468	395	return dma_addr;
469	396	}
470		−EXPORT_SYMBOL(dma_direct_map_resource);
471	397
472	398	int dma_direct_get_sgtable(struct device dev, struct sg_table sgt,
473	399	void *cpu_addr, dma_addr_t dma_addr, size_t size,

kernel/dma/mapping.c+204−10

@@ -105,6 +105,196 @@ void dmam_alloc_attrs(struct device dev, size_t size, dma_addr_t *dma_handle, · 104 unmodified lines
105	105	}
106	106	EXPORT_SYMBOL(dmam_alloc_attrs);
107	107
	108	+static bool dma_go_direct(struct device *dev, dma_addr_t mask,
	109	+ const struct dma_map_ops *ops)
	110	+{
	111	+ if (likely(!ops))
	112	+ return true;
	113	+#ifdef CONFIG_DMA_OPS_BYPASS
	114	+ if (dev->dma_ops_bypass)
	115	+ return min_not_zero(mask, dev->bus_dma_limit) >=
	116	+ dma_direct_get_required_mask(dev);
	117	+#endif
	118	+ return false;
	119	+}
	120	+
	121	+
	122	+/*
	123	+ * Check if the devices uses a direct mapping for streaming DMA operations.
	124	+ * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
	125	+ * enough.
	126	+ */
	127	+static inline bool dma_alloc_direct(struct device *dev,
	128	+ const struct dma_map_ops *ops)
	129	+{
	130	+ return dma_go_direct(dev, dev->coherent_dma_mask, ops);
	131	+}
	132	+
	133	+static inline bool dma_map_direct(struct device *dev,
	134	+ const struct dma_map_ops *ops)
	135	+{
	136	+ return dma_go_direct(dev, *dev->dma_mask, ops);
	137	+}
	138	+
	139	+dma_addr_t dma_map_page_attrs(struct device dev, struct page page,
	140	+ size_t offset, size_t size, enum dma_data_direction dir,
	141	+ unsigned long attrs)
	142	+{
	143	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	144	+ dma_addr_t addr;
	145	+
	146	+ BUG_ON(!valid_dma_direction(dir));
	147	+ if (dma_map_direct(dev, ops))
	148	+ addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
	149	+ else
	150	+ addr = ops->map_page(dev, page, offset, size, dir, attrs);
	151	+ debug_dma_map_page(dev, page, offset, size, dir, addr);
	152	+
	153	+ return addr;
	154	+}
	155	+EXPORT_SYMBOL(dma_map_page_attrs);
	156	+
	157	+void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
	158	+ enum dma_data_direction dir, unsigned long attrs)
	159	+{
	160	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	161	+
	162	+ BUG_ON(!valid_dma_direction(dir));
	163	+ if (dma_map_direct(dev, ops))
	164	+ dma_direct_unmap_page(dev, addr, size, dir, attrs);
	165	+ else if (ops->unmap_page)
	166	+ ops->unmap_page(dev, addr, size, dir, attrs);
	167	+ debug_dma_unmap_page(dev, addr, size, dir);
	168	+}
	169	+EXPORT_SYMBOL(dma_unmap_page_attrs);
	170	+
	171	+/*
	172	+ * dma_maps_sg_attrs returns 0 on error and > 0 on success.
	173	+ * It should never return a value < 0.
	174	+ */
	175	+int dma_map_sg_attrs(struct device dev, struct scatterlist sg, int nents,
	176	+ enum dma_data_direction dir, unsigned long attrs)
	177	+{
	178	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	179	+ int ents;
	180	+
	181	+ BUG_ON(!valid_dma_direction(dir));
	182	+ if (dma_map_direct(dev, ops))
	183	+ ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
	184	+ else
	185	+ ents = ops->map_sg(dev, sg, nents, dir, attrs);
	186	+ BUG_ON(ents < 0);
	187	+ debug_dma_map_sg(dev, sg, nents, ents, dir);
	188	+
	189	+ return ents;
	190	+}
	191	+EXPORT_SYMBOL(dma_map_sg_attrs);
	192	+
	193	+void dma_unmap_sg_attrs(struct device dev, struct scatterlist sg,
	194	+ int nents, enum dma_data_direction dir,
	195	+ unsigned long attrs)
	196	+{
	197	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	198	+
	199	+ BUG_ON(!valid_dma_direction(dir));
	200	+ debug_dma_unmap_sg(dev, sg, nents, dir);
	201	+ if (dma_map_direct(dev, ops))
	202	+ dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
	203	+ else if (ops->unmap_sg)
	204	+ ops->unmap_sg(dev, sg, nents, dir, attrs);
	205	+}
	206	+EXPORT_SYMBOL(dma_unmap_sg_attrs);
	207	+
	208	+dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
	209	+ size_t size, enum dma_data_direction dir, unsigned long attrs)
	210	+{
	211	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	212	+ dma_addr_t addr = DMA_MAPPING_ERROR;
	213	+
	214	+ BUG_ON(!valid_dma_direction(dir));
	215	+
	216	+ /* Don't allow RAM to be mapped */
	217	+ if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
	218	+ return DMA_MAPPING_ERROR;
	219	+
	220	+ if (dma_map_direct(dev, ops))
	221	+ addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
	222	+ else if (ops->map_resource)
	223	+ addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
	224	+
	225	+ debug_dma_map_resource(dev, phys_addr, size, dir, addr);
	226	+ return addr;
	227	+}
	228	+EXPORT_SYMBOL(dma_map_resource);
	229	+
	230	+void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
	231	+ enum dma_data_direction dir, unsigned long attrs)
	232	+{
	233	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	234	+
	235	+ BUG_ON(!valid_dma_direction(dir));
	236	+ if (!dma_map_direct(dev, ops) && ops->unmap_resource)
	237	+ ops->unmap_resource(dev, addr, size, dir, attrs);
	238	+ debug_dma_unmap_resource(dev, addr, size, dir);
	239	+}
	240	+EXPORT_SYMBOL(dma_unmap_resource);
	241	+
	242	+void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
	243	+ enum dma_data_direction dir)
	244	+{
	245	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	246	+
	247	+ BUG_ON(!valid_dma_direction(dir));
	248	+ if (dma_map_direct(dev, ops))
	249	+ dma_direct_sync_single_for_cpu(dev, addr, size, dir);
	250	+ else if (ops->sync_single_for_cpu)
	251	+ ops->sync_single_for_cpu(dev, addr, size, dir);
	252	+ debug_dma_sync_single_for_cpu(dev, addr, size, dir);
	253	+}
	254	+EXPORT_SYMBOL(dma_sync_single_for_cpu);
	255	+
	256	+void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
	257	+ size_t size, enum dma_data_direction dir)
	258	+{
	259	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	260	+
	261	+ BUG_ON(!valid_dma_direction(dir));
	262	+ if (dma_map_direct(dev, ops))
	263	+ dma_direct_sync_single_for_device(dev, addr, size, dir);
	264	+ else if (ops->sync_single_for_device)
	265	+ ops->sync_single_for_device(dev, addr, size, dir);
	266	+ debug_dma_sync_single_for_device(dev, addr, size, dir);
	267	+}
	268	+EXPORT_SYMBOL(dma_sync_single_for_device);
	269	+
	270	+void dma_sync_sg_for_cpu(struct device dev, struct scatterlist sg,
	271	+ int nelems, enum dma_data_direction dir)
	272	+{
	273	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	274	+
	275	+ BUG_ON(!valid_dma_direction(dir));
	276	+ if (dma_map_direct(dev, ops))
	277	+ dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
	278	+ else if (ops->sync_sg_for_cpu)
	279	+ ops->sync_sg_for_cpu(dev, sg, nelems, dir);
	280	+ debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
	281	+}
	282	+EXPORT_SYMBOL(dma_sync_sg_for_cpu);
	283	+
	284	+void dma_sync_sg_for_device(struct device dev, struct scatterlist sg,
	285	+ int nelems, enum dma_data_direction dir)
	286	+{
	287	+ const struct dma_map_ops *ops = get_dma_ops(dev);
	288	+
	289	+ BUG_ON(!valid_dma_direction(dir));
	290	+ if (dma_map_direct(dev, ops))
	291	+ dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
	292	+ else if (ops->sync_sg_for_device)
	293	+ ops->sync_sg_for_device(dev, sg, nelems, dir);
	294	+ debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
	295	+}
	296	+EXPORT_SYMBOL(dma_sync_sg_for_device);
	297	+
108	298	/*
109	299	* Create scatter-list for the already allocated DMA buffer.
110	300	*/
@@ -138,7 +328,7 @@ int dma_get_sgtable_attrs(struct device dev, struct sg_table sgt, · 27 unmodified lines
138	328	{
139	329	const struct dma_map_ops *ops = get_dma_ops(dev);
140	330
141		− if (dma_is_direct(ops))
	331	+ if (dma_alloc_direct(dev, ops))
142	332	return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
143	333	size, attrs);
144	334	if (!ops->get_sgtable)
@@ -208,7 +398,7 @@ bool dma_can_mmap(struct device *dev) · 63 unmodified lines
208	398	{
209	399	const struct dma_map_ops *ops = get_dma_ops(dev);
210	400
211		− if (dma_is_direct(ops))
	401	+ if (dma_alloc_direct(dev, ops))
212	402	return dma_direct_can_mmap(dev);
213	403	return ops->mmap != NULL;
214	404	}
@@ -233,7 +423,7 @@ int dma_mmap_attrs(struct device dev, struct vm_area_struct vma, · 18 unmodified lines
233	423	{
234	424	const struct dma_map_ops *ops = get_dma_ops(dev);
235	425
236		− if (dma_is_direct(ops))
	426	+ if (dma_alloc_direct(dev, ops))
237	427	return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
238	428	attrs);
239	429	if (!ops->mmap)
@@ -246,7 +436,7 @@ u64 dma_get_required_mask(struct device *dev) · 6 unmodified lines
246	436	{
247	437	const struct dma_map_ops *ops = get_dma_ops(dev);
248	438
249		− if (dma_is_direct(ops))
	439	+ if (dma_alloc_direct(dev, ops))
250	440	return dma_direct_get_required_mask(dev);
251	441	if (ops->get_required_mask)
252	442	return ops->get_required_mask(dev);
@@ -277,7 +467,7 @@ void dma_alloc_attrs(struct device dev, size_t size, dma_addr_t *dma_handle, · 24 unmodified lines
277	467	/* let the implementation decide on the zone to allocate from: */
278	468	flag &= ~(__GFP_DMA \| __GFP_DMA32 \| __GFP_HIGHMEM);
279	469
280		− if (dma_is_direct(ops))
	470	+ if (dma_alloc_direct(dev, ops))
281	471	cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
282	472	else if (ops->alloc)
283	473	cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
@@ -309,7 +499,7 @@ void dma_free_attrs(struct device dev, size_t size, void cpu_addr, · 25 unmodified lines
309	499	return;
310	500
311	501	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
312		− if (dma_is_direct(ops))
	502	+ if (dma_alloc_direct(dev, ops))
313	503	dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
314	504	else if (ops->free)
315	505	ops->free(dev, size, cpu_addr, dma_handle, attrs);
@@ -320,7 +510,11 @@ int dma_supported(struct device *dev, u64 mask) · 4 unmodified lines
320	510	{
321	511	const struct dma_map_ops *ops = get_dma_ops(dev);
322	512
323		− if (dma_is_direct(ops))
	513	+ /*
	514	+ * ->dma_supported sets the bypass flag, so we must always call
	515	+ * into the method here unless the device is truly direct mapped.
	516	+ */
	517	+ if (!ops)
324	518	return dma_direct_supported(dev, mask);
325	519	if (!ops->dma_supported)
326	520	return 1;
@@ -376,7 +570,7 @@ void dma_cache_sync(struct device dev, void vaddr, size_t size, · 49 unmodified lines
376	570
377	571	BUG_ON(!valid_dma_direction(dir));
378	572
379		− if (dma_is_direct(ops))
	573	+ if (dma_alloc_direct(dev, ops))
380	574	arch_dma_cache_sync(dev, vaddr, size, dir);
381	575	else if (ops->cache_sync)
382	576	ops->cache_sync(dev, vaddr, size, dir);
@@ -388,7 +582,7 @@ size_t dma_max_mapping_size(struct device *dev) · 5 unmodified lines
388	582	const struct dma_map_ops *ops = get_dma_ops(dev);
389	583	size_t size = SIZE_MAX;
390	584
391		− if (dma_is_direct(ops))
	585	+ if (dma_map_direct(dev, ops))
392	586	size = dma_direct_max_mapping_size(dev);
393	587	else if (ops && ops->max_mapping_size)
394	588	size = ops->max_mapping_size(dev);
@@ -401,7 +595,7 @@ bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) · 6 unmodified lines
401	595	{
402	596	const struct dma_map_ops *ops = get_dma_ops(dev);
403	597
404		− if (dma_is_direct(ops))
	598	+ if (dma_map_direct(dev, ops))
405	599	return dma_direct_need_sync(dev, dma_addr);
406	600	return ops->sync_single_for_cpu \|\| ops->sync_single_for_device;
407	601	}