dma-pool: fix coherent pool allocations for IOMMU mappings

When allocating coherent pool memory for an IOMMU mapping we don't care about the DMA mask. Move the guess for the initial GFP mask into the dma_direct_alloc_pages and pass dma_coherent_ok as a function pointer argument so that it doesn't get applied to the IOMMU case.

Christoph Hellwig committed 6 years ago

Commit 94201391 parent a1d21085 files · +62 −77Browse files

Unified · Split

5 files

drivers/iommu/dma-iommu.c +2 −2
include/linux/dma-direct.h +0 −3
include/linux/dma-mapping.h +3 −2
kernel/dma/direct.c +9 −4
kernel/dma/pool.c +48 −66

drivers/iommu/dma-iommu.c+2−2

@@ -1035,8 +1035,8 @@ static void iommu_dma_alloc(struct device dev, size_t size, · 1034 unmodified lines
1035	1035
1036	1036	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1037	1037	!gfpflags_allow_blocking(gfp) && !coherent)
1038		− cpu_addr = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page,
1039		− gfp);
	1038	+ page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
	1039	+ gfp, NULL);
1040	1040	else
1041	1041	cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
1042	1042	if (!cpu_addr)

include/linux/dma-direct.h+0−3

@@ -73,9 +73,6 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size, · 72 unmodified lines
73	73	}
74	74
75	75	u64 dma_direct_get_required_mask(struct device *dev);
76		−gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
77		− u64 *phys_mask);
78		−bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size);
79	76	void dma_direct_alloc(struct device dev, size_t size, dma_addr_t *dma_handle,
80	77	gfp_t gfp, unsigned long attrs);
81	78	void dma_direct_free(struct device dev, size_t size, void cpu_addr,

include/linux/dma-mapping.h+3−2

@@ -522,8 +522,9 @@ void dma_common_pages_remap(struct page *pages, size_t size, · 521 unmodified lines
522	522	pgprot_t prot, const void *caller);
523	523	void dma_common_free_remap(void *cpu_addr, size_t size);
524	524
525		−void dma_alloc_from_pool(struct device dev, size_t size,
526		− struct page **ret_page, gfp_t flags);
	525	+struct page dma_alloc_from_pool(struct device dev, size_t size,
	526	+ void **cpu_addr, gfp_t flags,
	527	+ bool (phys_addr_ok)(struct device , phys_addr_t, size_t));
527	528	bool dma_free_from_pool(struct device dev, void start, size_t size);
528	529
529	530	int

kernel/dma/direct.c+9−4

@@ -43,7 +43,7 @@ u64 dma_direct_get_required_mask(struct device *dev) · 42 unmodified lines
43	43	return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
44	44	}
45	45
46		−gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
	46	+static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
47	47	u64 *phys_limit)
48	48	{
49	49	u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);
@@ -68,7 +68,7 @@ gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, · 18 unmodified lines
68	68	return 0;
69	69	}
70	70
71		−bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
	71	+static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
72	72	{
73	73	return phys_to_dma_direct(dev, phys) + size - 1 <=
74	74	min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);
@@ -161,8 +161,13 @@ void dma_direct_alloc_pages(struct device dev, size_t size, · 86 unmodified lines
161	161	size = PAGE_ALIGN(size);
162	162
163	163	if (dma_should_alloc_from_pool(dev, gfp, attrs)) {
164		− ret = dma_alloc_from_pool(dev, size, &page, gfp);
165		− if (!ret)
	164	+ u64 phys_mask;
	165	+
	166	+ gfp \|= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
	167	+ &phys_mask);
	168	+ page = dma_alloc_from_pool(dev, size, &ret, gfp,
	169	+ dma_coherent_ok);
	170	+ if (!page)
166	171	return NULL;
167	172	goto done;
168	173	}

kernel/dma/pool.c+48−66

@@ -196,93 +196,75 @@ static int __init dma_atomic_pool_init(void) · 195 unmodified lines
196	196	}
197	197	postcore_initcall(dma_atomic_pool_init);
198	198
199		−static inline struct gen_pool dma_guess_pool_from_device(struct device dev)
	199	+static inline struct gen_pool dma_guess_pool(struct gen_pool prev, gfp_t gfp)
200	200	{
201		− u64 phys_mask;
202		− gfp_t gfp;
203		−
204		− gfp = dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
205		− &phys_mask);
206		− if (IS_ENABLED(CONFIG_ZONE_DMA) && gfp == GFP_DMA)
	201	+ if (prev == NULL) {
	202	+ if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32))
	203	+ return atomic_pool_dma32;
	204	+ if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp & GFP_DMA))
	205	+ return atomic_pool_dma;
	206	+ return atomic_pool_kernel;
	207	+ }
	208	+ if (prev == atomic_pool_kernel)
	209	+ return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
	210	+ if (prev == atomic_pool_dma32)
207	211	return atomic_pool_dma;
208		− if (IS_ENABLED(CONFIG_ZONE_DMA32) && gfp == GFP_DMA32)
209		− return atomic_pool_dma32;
210		− return atomic_pool_kernel;
	212	+ return NULL;
211	213	}
212	214
213		−static inline struct gen_pool dma_get_safer_pool(struct gen_pool bad_pool)
	215	+static struct page __dma_alloc_from_pool(struct device dev, size_t size,
	216	+ struct gen_pool pool, void *cpu_addr,
	217	+ bool (phys_addr_ok)(struct device , phys_addr_t, size_t))
214	218	{
215		− if (bad_pool == atomic_pool_kernel)
216		− return atomic_pool_dma32 ? : atomic_pool_dma;
	219	+ unsigned long addr;
	220	+ phys_addr_t phys;
217	221
218		− if (bad_pool == atomic_pool_dma32)
219		− return atomic_pool_dma;
	222	+ addr = gen_pool_alloc(pool, size);
	223	+ if (!addr)
	224	+ return NULL;
220	225
221		− return NULL;
222		−}
	226	+ phys = gen_pool_virt_to_phys(pool, addr);
	227	+ if (phys_addr_ok && !phys_addr_ok(dev, phys, size)) {
	228	+ gen_pool_free(pool, addr, size);
	229	+ return NULL;
	230	+ }
223	231
224		−static inline struct gen_pool dma_guess_pool(struct device dev,
225		− struct gen_pool *bad_pool)
226		−{
227		− if (bad_pool)
228		− return dma_get_safer_pool(bad_pool);
	232	+ if (gen_pool_avail(pool) < atomic_pool_size)
	233	+ schedule_work(&atomic_pool_work);
229	234
230		− return dma_guess_pool_from_device(dev);
	235	+ cpu_addr = (void )addr;
	236	+ memset(*cpu_addr, 0, size);
	237	+ return pfn_to_page(__phys_to_pfn(phys));
231	238	}
232	239
233		−void dma_alloc_from_pool(struct device dev, size_t size,
234		− struct page **ret_page, gfp_t flags)
	240	+struct page dma_alloc_from_pool(struct device dev, size_t size,
	241	+ void **cpu_addr, gfp_t gfp,
	242	+ bool (phys_addr_ok)(struct device , phys_addr_t, size_t))
235	243	{
236	244	struct gen_pool *pool = NULL;
237		− unsigned long val = 0;
238		− void *ptr = NULL;
239		− phys_addr_t phys;
240		−
241		− while (1) {
242		− pool = dma_guess_pool(dev, pool);
243		− if (!pool) {
244		− WARN(1, "Failed to get suitable pool for %s\n",
245		− dev_name(dev));
246		− break;
247		− }
248		−
249		− val = gen_pool_alloc(pool, size);
250		− if (!val)
251		− continue;
252		−
253		− phys = gen_pool_virt_to_phys(pool, val);
254		− if (dma_coherent_ok(dev, phys, size))
255		− break;
256		−
257		− gen_pool_free(pool, val, size);
258		− val = 0;
259		− }
260		−
261		−
262		− if (val) {
263		− *ret_page = pfn_to_page(__phys_to_pfn(phys));
264		− ptr = (void *)val;
265		− memset(ptr, 0, size);
	245	+ struct page *page;
266	246
267		− if (gen_pool_avail(pool) < atomic_pool_size)
268		− schedule_work(&atomic_pool_work);
	247	+ while ((pool = dma_guess_pool(pool, gfp))) {
	248	+ page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
	249	+ phys_addr_ok);
	250	+ if (page)
	251	+ return page;
269	252	}
270	253
271		− return ptr;
	254	+ WARN(1, "Failed to get suitable pool for %s\n", dev_name(dev));
	255	+ return NULL;
272	256	}
273	257
274	258	bool dma_free_from_pool(struct device dev, void start, size_t size)
275	259	{
276	260	struct gen_pool *pool = NULL;
277	261
278		− while (1) {
279		− pool = dma_guess_pool(dev, pool);
280		− if (!pool)
281		− return false;
282		−
283		− if (gen_pool_has_addr(pool, (unsigned long)start, size)) {
284		− gen_pool_free(pool, (unsigned long)start, size);
285		− return true;
286		− }
	262	+ while ((pool = dma_guess_pool(pool, 0))) {
	263	+ if (!gen_pool_has_addr(pool, (unsigned long)start, size))
	264	+ continue;
	265	+ gen_pool_free(pool, (unsigned long)start, size);
	266	+ return true;
287	267	}
	268	+
	269	+ return false;
288	270	}

dma-pool: fix coherent pool allocations for IOMMU mappings

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code