dma-direct: provide mmap and get_sgtable method overrides

For dma-direct we know that the DMA address is an encoding of the physical address that we can trivially decode. Use that fact to provide implementations that do not need the arch_dma_coherent_to_pfn architecture hook. Note that we still can only support mmap of non-coherent memory only if the architecture provides a way to set an uncached bit in the page tables. This must be true for architectures that use the generic remap helpers, but other architectures can also manually select it.

Christoph Hellwig committed 7 years ago

Commit 34dc0ea1 parent 4e1003a16 files · +85 −75Browse files

Unified · Split

16 files

arch/arc/Kconfig +0 −1
arch/arm/Kconfig +0 −1
arch/arm/mm/dma-mapping.c +0 −6
arch/arm64/Kconfig +0 −1
arch/ia64/Kconfig +1 −1
arch/ia64/kernel/dma-mapping.c +0 −6
arch/microblaze/Kconfig +0 −1
arch/mips/Kconfig +2 −2
arch/mips/mm/dma-noncoherent.c +0 −6
arch/powerpc/platforms/Kconfig.cputype +0 −1
include/linux/dma-direct.h +7 −0
include/linux/dma-noncoherent.h +0 −2
kernel/dma/Kconfig +9 −3
kernel/dma/direct.c +59 −0
kernel/dma/mapping.c +7 −38
kernel/dma/remap.c +0 −6

arch/arc/Kconfig+0−1

@@ -6,7 +6,6 @@ · 5 unmodified lines
6	6	config ARC
7	7	def_bool y
8	8	select ARC_TIMERS
9		− select ARCH_HAS_DMA_COHERENT_TO_PFN
10	9	select ARCH_HAS_DMA_PREP_COHERENT
11	10	select ARCH_HAS_PTE_SPECIAL
12	11	select ARCH_HAS_SETUP_DMA_OPS

arch/arm/Kconfig+0−1

@@ -7,7 +7,6 @@ config ARM · 6 unmodified lines
7	7	select ARCH_HAS_BINFMT_FLAT
8	8	select ARCH_HAS_DEBUG_VIRTUAL if MMU
9	9	select ARCH_HAS_DEVMEM_IS_ALLOWED
10		− select ARCH_HAS_DMA_COHERENT_TO_PFN if SWIOTLB
11	10	select ARCH_HAS_DMA_WRITE_COMBINE if !ARM_DMA_MEM_BUFFERABLE
12	11	select ARCH_HAS_ELF_RANDOMIZE
13	12	select ARCH_HAS_FORTIFY_SOURCE

arch/arm/mm/dma-mapping.c+0−6

@@ -2346,12 +2346,6 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr, · 2345 unmodified lines
2346	2346	size, dir);
2347	2347	}
2348	2348
2349		−long arch_dma_coherent_to_pfn(struct device dev, void cpu_addr,
2350		− dma_addr_t dma_addr)
2351		−{
2352		− return dma_to_pfn(dev, dma_addr);
2353		−}
2354		−
2355	2349	void arch_dma_alloc(struct device dev, size_t size, dma_addr_t *dma_handle,
2356	2350	gfp_t gfp, unsigned long attrs)
2357	2351	{

arch/arm64/Kconfig+0−1

@@ -12,7 +12,6 @@ config ARM64 · 11 unmodified lines
12	12	select ARCH_CLOCKSOURCE_DATA
13	13	select ARCH_HAS_DEBUG_VIRTUAL
14	14	select ARCH_HAS_DEVMEM_IS_ALLOWED
15		− select ARCH_HAS_DMA_COHERENT_TO_PFN
16	15	select ARCH_HAS_DMA_PREP_COHERENT
17	16	select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI
18	17	select ARCH_HAS_FAST_MULTIPLIER

arch/ia64/Kconfig+1−1

@@ -33,7 +33,7 @@ config IA64 · 32 unmodified lines
33	33	select HAVE_ARCH_TRACEHOOK
34	34	select HAVE_MEMBLOCK_NODE_MAP
35	35	select HAVE_VIRT_CPU_ACCOUNTING
36		− select ARCH_HAS_DMA_COHERENT_TO_PFN
	36	+ select DMA_NONCOHERENT_MMAP
37	37	select ARCH_HAS_SYNC_DMA_FOR_CPU
38	38	select VIRT_TO_BUS
39	39	select GENERIC_IRQ_PROBE

arch/ia64/kernel/dma-mapping.c+0−6

@@ -19,9 +19,3 @@ void arch_dma_free(struct device dev, size_t size, void cpu_addr, · 18 unmodified lines
19	19	{
20	20	dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
21	21	}
22		−
23		−long arch_dma_coherent_to_pfn(struct device dev, void cpu_addr,
24		− dma_addr_t dma_addr)
25		−{
26		− return page_to_pfn(virt_to_page(cpu_addr));
27		−}

arch/microblaze/Kconfig+0−1

@@ -4,7 +4,6 @@ config MICROBLAZE · 3 unmodified lines
4	4	select ARCH_32BIT_OFF_T
5	5	select ARCH_NO_SWAP
6	6	select ARCH_HAS_BINFMT_FLAT if !MMU
7		− select ARCH_HAS_DMA_COHERENT_TO_PFN if MMU
8	7	select ARCH_HAS_DMA_PREP_COHERENT
9	8	select ARCH_HAS_GCOV_PROFILE_ALL
10	9	select ARCH_HAS_SYNC_DMA_FOR_CPU

arch/mips/Kconfig+2−2

@@ -1134,9 +1134,9 @@ config DMA_NONCOHERENT · 1133 unmodified lines
1134	1134	select ARCH_HAS_DMA_WRITE_COMBINE
1135	1135	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
1136	1136	select ARCH_HAS_UNCACHED_SEGMENT
1137		− select NEED_DMA_MAP_STATE
1138		− select ARCH_HAS_DMA_COHERENT_TO_PFN
	1137	+ select DMA_NONCOHERENT_MMAP
1139	1138	select DMA_NONCOHERENT_CACHE_SYNC
	1139	+ select NEED_DMA_MAP_STATE
1140	1140
1141	1141	config SYS_HAS_EARLY_PRINTK
1142	1142	bool

arch/mips/mm/dma-noncoherent.c+0−6

@@ -59,12 +59,6 @@ void cached_kernel_address(void addr) · 58 unmodified lines
59	59	return __va(addr) - UNCAC_BASE;
60	60	}
61	61
62		−long arch_dma_coherent_to_pfn(struct device dev, void cpu_addr,
63		− dma_addr_t dma_addr)
64		−{
65		− return page_to_pfn(virt_to_page(cached_kernel_address(cpu_addr)));
66		−}
67		−
68	62	static inline void dma_sync_virt(void *addr, size_t size,
69	63	enum dma_data_direction dir)
70	64	{

arch/powerpc/platforms/Kconfig.cputype+0−1

@@ -459,7 +459,6 @@ config NOT_COHERENT_CACHE · 458 unmodified lines
459	459	bool
460	460	depends on 4xx \|\| PPC_8xx \|\| E200 \|\| PPC_MPC512x \|\| \
461	461	GAMECUBE_COMMON \|\| AMIGAONE
462		− select ARCH_HAS_DMA_COHERENT_TO_PFN
463	462	select ARCH_HAS_DMA_PREP_COHERENT
464	463	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
465	464	select ARCH_HAS_SYNC_DMA_FOR_CPU

include/linux/dma-direct.h+7−0

@@ -68,5 +68,12 @@ void dma_direct_free_pages(struct device dev, size_t size, void cpu_addr, · 67 unmodified lines
68	68	dma_addr_t dma_addr, unsigned long attrs);
69	69	struct page __dma_direct_alloc_pages(struct device dev, size_t size,
70	70	gfp_t gfp, unsigned long attrs);
	71	+int dma_direct_get_sgtable(struct device dev, struct sg_table sgt,
	72	+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
	73	+ unsigned long attrs);
	74	+bool dma_direct_can_mmap(struct device *dev);
	75	+int dma_direct_mmap(struct device dev, struct vm_area_struct vma,
	76	+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
	77	+ unsigned long attrs);
71	78	int dma_direct_supported(struct device *dev, u64 mask);
72	79	#endif /* _LINUX_DMA_DIRECT_H */

include/linux/dma-noncoherent.h+0−2

@@ -41,8 +41,6 @@ void arch_dma_alloc(struct device dev, size_t size, dma_addr_t *dma_handle, · 40 unmodified lines
41	41	gfp_t gfp, unsigned long attrs);
42	42	void arch_dma_free(struct device dev, size_t size, void cpu_addr,
43	43	dma_addr_t dma_addr, unsigned long attrs);
44		−long arch_dma_coherent_to_pfn(struct device dev, void cpu_addr,
45		− dma_addr_t dma_addr);
46	44
47	45	#ifdef CONFIG_MMU
48	46	/*

kernel/dma/Kconfig+9−3

@@ -51,9 +51,6 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL · 50 unmodified lines
51	51	config ARCH_HAS_DMA_PREP_COHERENT
52	52	bool
53	53
54		−config ARCH_HAS_DMA_COHERENT_TO_PFN
55		− bool
56		−
57	54	config ARCH_HAS_FORCE_DMA_UNENCRYPTED
58	55	bool
59	56
@@ -68,9 +65,18 @@ config SWIOTLB · 8 unmodified lines
68	65	bool
69	66	select NEED_DMA_MAP_STATE
70	67
	68	+#
	69	+# Should be selected if we can mmap non-coherent mappings to userspace.
	70	+# The only thing that is really required is a way to set an uncached bit
	71	+# in the pagetables
	72	+#
	73	+config DMA_NONCOHERENT_MMAP
	74	+ bool
	75	+
71	76	config DMA_REMAP
72	77	depends on MMU
73	78	select GENERIC_ALLOCATOR
	79	+ select DMA_NONCOHERENT_MMAP
74	80	bool
75	81
76	82	config DMA_DIRECT_REMAP

kernel/dma/direct.c+59−0

@@ -43,6 +43,12 @@ static inline dma_addr_t phys_to_dma_direct(struct device *dev, · 42 unmodified lines
43	43	return phys_to_dma(dev, phys);
44	44	}
45	45
	46	+static inline struct page dma_direct_to_page(struct device dev,
	47	+ dma_addr_t dma_addr)
	48	+{
	49	+ return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr)));
	50	+}
	51	+
46	52	u64 dma_direct_get_required_mask(struct device *dev)
47	53	{
48	54	u64 max_dma = phys_to_dma_direct(dev, (max_pfn - 1) << PAGE_SHIFT);
@@ -379,6 +385,59 @@ dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, · 330 unmodified lines
379	385	}
380	386	EXPORT_SYMBOL(dma_direct_map_resource);
381	387
	388	+int dma_direct_get_sgtable(struct device dev, struct sg_table sgt,
	389	+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
	390	+ unsigned long attrs)
	391	+{
	392	+ struct page *page = dma_direct_to_page(dev, dma_addr);
	393	+ int ret;
	394	+
	395	+ ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
	396	+ if (!ret)
	397	+ sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
	398	+ return ret;
	399	+}
	400	+
	401	+#ifdef CONFIG_MMU
	402	+bool dma_direct_can_mmap(struct device *dev)
	403	+{
	404	+ return dev_is_dma_coherent(dev) \|\|
	405	+ IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP);
	406	+}
	407	+
	408	+int dma_direct_mmap(struct device dev, struct vm_area_struct vma,
	409	+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
	410	+ unsigned long attrs)
	411	+{
	412	+ unsigned long user_count = vma_pages(vma);
	413	+ unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	414	+ unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr));
	415	+ int ret = -ENXIO;
	416	+
	417	+ vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
	418	+
	419	+ if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
	420	+ return ret;
	421	+
	422	+ if (vma->vm_pgoff >= count \|\| user_count > count - vma->vm_pgoff)
	423	+ return -ENXIO;
	424	+ return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
	425	+ user_count << PAGE_SHIFT, vma->vm_page_prot);
	426	+}
	427	+#else /* CONFIG_MMU */
	428	+bool dma_direct_can_mmap(struct device *dev)
	429	+{
	430	+ return false;
	431	+}
	432	+
	433	+int dma_direct_mmap(struct device dev, struct vm_area_struct vma,
	434	+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
	435	+ unsigned long attrs)
	436	+{
	437	+ return -ENXIO;
	438	+}
	439	+#endif /* CONFIG_MMU */
	440	+
382	441	/*
383	442	* Because 32-bit DMA masks are so common we expect every architecture to be
384	443	* able to satisfy them - either by not supporting more physical memory, or by

kernel/dma/mapping.c+7−38

@@ -112,24 +112,9 @@ int dma_common_get_sgtable(struct device dev, struct sg_table sgt, · 111 unmodified lines
112	112	void *cpu_addr, dma_addr_t dma_addr, size_t size,
113	113	unsigned long attrs)
114	114	{
115		− struct page *page;
	115	+ struct page *page = virt_to_page(cpu_addr);
116	116	int ret;
117	117
118		− if (!dev_is_dma_coherent(dev)) {
119		− unsigned long pfn;
120		−
121		− if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
122		− return -ENXIO;
123		−
124		− /* If the PFN is not valid, we do not have a struct page */
125		− pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
126		− if (!pfn_valid(pfn))
127		− return -ENXIO;
128		− page = pfn_to_page(pfn);
129		− } else {
130		− page = virt_to_page(cpu_addr);
131		− }
132		−
133	118	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
134	119	if (!ret)
135	120	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
@@ -154,7 +139,7 @@ int dma_get_sgtable_attrs(struct device dev, struct sg_table sgt, · 18 unmodified lines
154	139	const struct dma_map_ops *ops = get_dma_ops(dev);
155	140
156	141	if (dma_is_direct(ops))
157		− return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr,
	142	+ return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
158	143	size, attrs);
159	144	if (!ops->get_sgtable)
160	145	return -ENXIO;
@@ -192,7 +177,6 @@ int dma_common_mmap(struct device dev, struct vm_area_struct vma, · 31 unmodified lines
192	177	unsigned long user_count = vma_pages(vma);
193	178	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
194	179	unsigned long off = vma->vm_pgoff;
195		− unsigned long pfn;
196	180	int ret = -ENXIO;
197	181
198	182	vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
@@ -203,19 +187,8 @@ int dma_common_mmap(struct device dev, struct vm_area_struct vma, · 4 unmodified lines
203	187	if (off >= count \|\| user_count > count - off)
204	188	return -ENXIO;
205	189
206		− if (!dev_is_dma_coherent(dev)) {
207		− if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN))
208		− return -ENXIO;
209		−
210		− /* If the PFN is not valid, we do not have a struct page */
211		− pfn = arch_dma_coherent_to_pfn(dev, cpu_addr, dma_addr);
212		− if (!pfn_valid(pfn))
213		− return -ENXIO;
214		− } else {
215		− pfn = page_to_pfn(virt_to_page(cpu_addr));
216		− }
217		−
218		− return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
	190	+ return remap_pfn_range(vma, vma->vm_start,
	191	+ page_to_pfn(virt_to_page(cpu_addr)) + vma->vm_pgoff,
219	192	user_count << PAGE_SHIFT, vma->vm_page_prot);
220	193	#else
221	194	return -ENXIO;
@@ -233,12 +206,8 @@ bool dma_can_mmap(struct device *dev) · 11 unmodified lines
233	206	{
234	207	const struct dma_map_ops *ops = get_dma_ops(dev);
235	208
236		− if (dma_is_direct(ops)) {
237		− return IS_ENABLED(CONFIG_MMU) &&
238		− (dev_is_dma_coherent(dev) \|\|
239		− IS_ENABLED(CONFIG_ARCH_HAS_DMA_COHERENT_TO_PFN));
240		− }
241		−
	209	+ if (dma_is_direct(ops))
	210	+ return dma_direct_can_mmap(dev);
242	211	return ops->mmap != NULL;
243	212	}
244	213	EXPORT_SYMBOL_GPL(dma_can_mmap);
@@ -263,7 +232,7 @@ int dma_mmap_attrs(struct device dev, struct vm_area_struct vma, · 18 unmodified lines
263	232	const struct dma_map_ops *ops = get_dma_ops(dev);
264	233
265	234	if (dma_is_direct(ops))
266		− return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size,
	235	+ return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
267	236	attrs);
268	237	if (!ops->mmap)
269	238	return -ENXIO;

kernel/dma/remap.c+0−6

@@ -259,10 +259,4 @@ void arch_dma_free(struct device dev, size_t size, void vaddr, · 258 unmodified lines
259	259	dma_free_contiguous(dev, page, size);
260	260	}
261	261	}
262		−
263		−long arch_dma_coherent_to_pfn(struct device dev, void cpu_addr,
264		− dma_addr_t dma_addr)
265		−{
266		− return __phys_to_pfn(dma_to_phys(dev, dma_addr));
267		−}
268	262	#endif /* CONFIG_DMA_DIRECT_REMAP */

dma-direct: provide mmap and get_sgtable method overrides

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code