History

dma-pool: fix coherent pool allocations for IOMMU mappings

When allocating coherent pool memory for an IOMMU mapping we don't care about the DMA mask. Move the guess for the initial GFP mask into the dma_direct_alloc_pages and pass dma_coherent_ok as a function pointer argument so that it doesn't get applied to the IOMMU case.

Christoph Hellwig committed 6 years ago

Commit 94201391 parent a1d21085 files · +62 −77Browse files

Unified · Split

5 files
  1. drivers/iommu/dma-iommu.c +2 −2
  2. include/linux/dma-direct.h +0 −3
  3. include/linux/dma-mapping.h +3 −2
  4. kernel/dma/direct.c +9 −4
  5. kernel/dma/pool.c +48 −66
drivers/iommu/dma-iommu.c+2−2
@@ -1035,8 +1035,8 @@ static void *iommu_dma_alloc(struct device *dev, size_t size, · 1034 unmodified lines
1035 1035
1036 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&1036 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1037 !gfpflags_allow_blocking(gfp) && !coherent)1037 !gfpflags_allow_blocking(gfp) && !coherent)
1038 cpu_addr = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page,1038 page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
1039 gfp);1039 gfp, NULL);
1040 else1040 else
1041 cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);1041 cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
1042 if (!cpu_addr)1042 if (!cpu_addr)
include/linux/dma-direct.h+0−3
@@ -73,9 +73,6 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size, · 72 unmodified lines
73}73}
74 74
75u64 dma_direct_get_required_mask(struct device *dev);75u64 dma_direct_get_required_mask(struct device *dev);
76gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
77 u64 *phys_mask);
78bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size);
79void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,76void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
80 gfp_t gfp, unsigned long attrs);77 gfp_t gfp, unsigned long attrs);
81void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,78void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
include/linux/dma-mapping.h+3−2
@@ -522,8 +522,9 @@ void *dma_common_pages_remap(struct page **pages, size_t size, · 521 unmodified lines
522 pgprot_t prot, const void *caller);522 pgprot_t prot, const void *caller);
523void dma_common_free_remap(void *cpu_addr, size_t size);523void dma_common_free_remap(void *cpu_addr, size_t size);
524 524
525void *dma_alloc_from_pool(struct device *dev, size_t size,525struct page *dma_alloc_from_pool(struct device *dev, size_t size,
526 struct page **ret_page, gfp_t flags);526 void **cpu_addr, gfp_t flags,
527 bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
527bool dma_free_from_pool(struct device *dev, void *start, size_t size);528bool dma_free_from_pool(struct device *dev, void *start, size_t size);
528 529
529int530int
kernel/dma/direct.c+9−4
@@ -43,7 +43,7 @@ u64 dma_direct_get_required_mask(struct device *dev) · 42 unmodified lines
43 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;43 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
44}44}
45 45
46gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,46static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask,
47 u64 *phys_limit)47 u64 *phys_limit)
48{48{
49 u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);49 u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);
@@ -68,7 +68,7 @@ gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, · 18 unmodified lines
68 return 0;68 return 0;
69}69}
70 70
71bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)71static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
72{72{
73 return phys_to_dma_direct(dev, phys) + size - 1 <=73 return phys_to_dma_direct(dev, phys) + size - 1 <=
74 min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);74 min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);
@@ -161,8 +161,13 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, · 86 unmodified lines
161 size = PAGE_ALIGN(size);161 size = PAGE_ALIGN(size);
162 162
163 if (dma_should_alloc_from_pool(dev, gfp, attrs)) {163 if (dma_should_alloc_from_pool(dev, gfp, attrs)) {
164 ret = dma_alloc_from_pool(dev, size, &page, gfp);164 u64 phys_mask;
165 if (!ret)165
166 gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
167 &phys_mask);
168 page = dma_alloc_from_pool(dev, size, &ret, gfp,
169 dma_coherent_ok);
170 if (!page)
166 return NULL;171 return NULL;
167 goto done;172 goto done;
168 }173 }
kernel/dma/pool.c+48−66
@@ -196,93 +196,75 @@ static int __init dma_atomic_pool_init(void) · 195 unmodified lines
196}196}
197postcore_initcall(dma_atomic_pool_init);197postcore_initcall(dma_atomic_pool_init);
198 198
199static inline struct gen_pool *dma_guess_pool_from_device(struct device *dev)199static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
200{200{
201 u64 phys_mask;201 if (prev == NULL) {
202 gfp_t gfp;202 if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32))
203 203 return atomic_pool_dma32;
204 gfp = dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,204 if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp & GFP_DMA))
205 &phys_mask);205 return atomic_pool_dma;
206 if (IS_ENABLED(CONFIG_ZONE_DMA) && gfp == GFP_DMA)206 return atomic_pool_kernel;
207 }
208 if (prev == atomic_pool_kernel)
209 return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
210 if (prev == atomic_pool_dma32)
207 return atomic_pool_dma;211 return atomic_pool_dma;
208 if (IS_ENABLED(CONFIG_ZONE_DMA32) && gfp == GFP_DMA32)212 return NULL;
209 return atomic_pool_dma32;
210 return atomic_pool_kernel;
211}213}
212 214
213static inline struct gen_pool *dma_get_safer_pool(struct gen_pool *bad_pool)215static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,
216 struct gen_pool *pool, void **cpu_addr,
217 bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
214{218{
215 if (bad_pool == atomic_pool_kernel)219 unsigned long addr;
216 return atomic_pool_dma32 ? : atomic_pool_dma;220 phys_addr_t phys;
217 221
218 if (bad_pool == atomic_pool_dma32)222 addr = gen_pool_alloc(pool, size);
219 return atomic_pool_dma;223 if (!addr)
224 return NULL;
220 225
221 return NULL;226 phys = gen_pool_virt_to_phys(pool, addr);
222}227 if (phys_addr_ok && !phys_addr_ok(dev, phys, size)) {
228 gen_pool_free(pool, addr, size);
229 return NULL;
230 }
223 231
224static inline struct gen_pool *dma_guess_pool(struct device *dev,232 if (gen_pool_avail(pool) < atomic_pool_size)
225 struct gen_pool *bad_pool)233 schedule_work(&atomic_pool_work);
226{
227 if (bad_pool)
228 return dma_get_safer_pool(bad_pool);
229 234
230 return dma_guess_pool_from_device(dev);235 *cpu_addr = (void *)addr;
236 memset(*cpu_addr, 0, size);
237 return pfn_to_page(__phys_to_pfn(phys));
231}238}
232 239
233void *dma_alloc_from_pool(struct device *dev, size_t size,240struct page *dma_alloc_from_pool(struct device *dev, size_t size,
234 struct page **ret_page, gfp_t flags)241 void **cpu_addr, gfp_t gfp,
242 bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
235{243{
236 struct gen_pool *pool = NULL;244 struct gen_pool *pool = NULL;
237 unsigned long val = 0;245 struct page *page;
238 void *ptr = NULL;
239 phys_addr_t phys;
240
241 while (1) {
242 pool = dma_guess_pool(dev, pool);
243 if (!pool) {
244 WARN(1, "Failed to get suitable pool for %s\n",
245 dev_name(dev));
246 break;
247 }
248
249 val = gen_pool_alloc(pool, size);
250 if (!val)
251 continue;
252
253 phys = gen_pool_virt_to_phys(pool, val);
254 if (dma_coherent_ok(dev, phys, size))
255 break;
256
257 gen_pool_free(pool, val, size);
258 val = 0;
259 }
260
261
262 if (val) {
263 *ret_page = pfn_to_page(__phys_to_pfn(phys));
264 ptr = (void *)val;
265 memset(ptr, 0, size);
266 246
267 if (gen_pool_avail(pool) < atomic_pool_size)247 while ((pool = dma_guess_pool(pool, gfp))) {
268 schedule_work(&atomic_pool_work);248 page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
249 phys_addr_ok);
250 if (page)
251 return page;
269 }252 }
270 253
271 return ptr;254 WARN(1, "Failed to get suitable pool for %s\n", dev_name(dev));
255 return NULL;
272}256}
273 257
274bool dma_free_from_pool(struct device *dev, void *start, size_t size)258bool dma_free_from_pool(struct device *dev, void *start, size_t size)
275{259{
276 struct gen_pool *pool = NULL;260 struct gen_pool *pool = NULL;
277 261
278 while (1) {262 while ((pool = dma_guess_pool(pool, 0))) {
279 pool = dma_guess_pool(dev, pool);263 if (!gen_pool_has_addr(pool, (unsigned long)start, size))
280 if (!pool)264 continue;
281 return false;265 gen_pool_free(pool, (unsigned long)start, size);
282 266 return true;
283 if (gen_pool_has_addr(pool, (unsigned long)start, size)) {
284 gen_pool_free(pool, (unsigned long)start, size);
285 return true;
286 }
287 }267 }
268
269 return false;
288}270}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t