When allocating coherent pool memory for an IOMMU mapping we don't care about the DMA mask. Move the guess for the initial GFP mask into the dma_direct_alloc_pages and pass dma_coherent_ok as a function pointer argument so that it doesn't get applied to the IOMMU case.
Showing the changes to kernel (2 of 5 files). Show all
Unified · Split
| @@ -43,7 +43,7 @@ u64 dma_direct_get_required_mask(struct device *dev) · 42 unmodified lines | ||
| 43 | 43 | return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; |
| 44 | 44 | } |
| 45 | 45 | |
| 46 | −gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, | |
| 46 | +static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, | |
| 47 | 47 | u64 *phys_limit) |
| 48 | 48 | { |
| 49 | 49 | u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit); |
| @@ -68,7 +68,7 @@ gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, · 18 unmodified lines | ||
| 68 | 68 | return 0; |
| 69 | 69 | } |
| 70 | 70 | |
| 71 | −bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) | |
| 71 | +static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) | |
| 72 | 72 | { |
| 73 | 73 | return phys_to_dma_direct(dev, phys) + size - 1 <= |
| 74 | 74 | min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); |
| @@ -161,8 +161,13 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, · 86 unmodified lines | ||
| 161 | 161 | size = PAGE_ALIGN(size); |
| 162 | 162 | |
| 163 | 163 | if (dma_should_alloc_from_pool(dev, gfp, attrs)) { |
| 164 | − ret = dma_alloc_from_pool(dev, size, &page, gfp); | |
| 165 | − if (!ret) | |
| 164 | + u64 phys_mask; | |
| 165 | + | |
| 166 | + gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, | |
| 167 | + &phys_mask); | |
| 168 | + page = dma_alloc_from_pool(dev, size, &ret, gfp, | |
| 169 | + dma_coherent_ok); | |
| 170 | + if (!page) | |
| 166 | 171 | return NULL; |
| 167 | 172 | goto done; |
| 168 | 173 | } |
| @@ -196,93 +196,75 @@ static int __init dma_atomic_pool_init(void) · 195 unmodified lines | ||
| 196 | 196 | } |
| 197 | 197 | postcore_initcall(dma_atomic_pool_init); |
| 198 | 198 | |
| 199 | −static inline struct gen_pool *dma_guess_pool_from_device(struct device *dev) | |
| 199 | +static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp) | |
| 200 | 200 | { |
| 201 | − u64 phys_mask; | |
| 202 | − gfp_t gfp; | |
| 203 | − | |
| 204 | − gfp = dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, | |
| 205 | − &phys_mask); | |
| 206 | − if (IS_ENABLED(CONFIG_ZONE_DMA) && gfp == GFP_DMA) | |
| 201 | + if (prev == NULL) { | |
| 202 | + if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32)) | |
| 203 | + return atomic_pool_dma32; | |
| 204 | + if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp & GFP_DMA)) | |
| 205 | + return atomic_pool_dma; | |
| 206 | + return atomic_pool_kernel; | |
| 207 | + } | |
| 208 | + if (prev == atomic_pool_kernel) | |
| 209 | + return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma; | |
| 210 | + if (prev == atomic_pool_dma32) | |
| 207 | 211 | return atomic_pool_dma; |
| 208 | − if (IS_ENABLED(CONFIG_ZONE_DMA32) && gfp == GFP_DMA32) | |
| 209 | − return atomic_pool_dma32; | |
| 210 | − return atomic_pool_kernel; | |
| 212 | + return NULL; | |
| 211 | 213 | } |
| 212 | 214 | |
| 213 | −static inline struct gen_pool *dma_get_safer_pool(struct gen_pool *bad_pool) | |
| 215 | +static struct page *__dma_alloc_from_pool(struct device *dev, size_t size, | |
| 216 | + struct gen_pool *pool, void **cpu_addr, | |
| 217 | + bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)) | |
| 214 | 218 | { |
| 215 | − if (bad_pool == atomic_pool_kernel) | |
| 216 | − return atomic_pool_dma32 ? : atomic_pool_dma; | |
| 219 | + unsigned long addr; | |
| 220 | + phys_addr_t phys; | |
| 217 | 221 | |
| 218 | − if (bad_pool == atomic_pool_dma32) | |
| 219 | − return atomic_pool_dma; | |
| 222 | + addr = gen_pool_alloc(pool, size); | |
| 223 | + if (!addr) | |
| 224 | + return NULL; | |
| 220 | 225 | |
| 221 | − return NULL; | |
| 222 | −} | |
| 226 | + phys = gen_pool_virt_to_phys(pool, addr); | |
| 227 | + if (phys_addr_ok && !phys_addr_ok(dev, phys, size)) { | |
| 228 | + gen_pool_free(pool, addr, size); | |
| 229 | + return NULL; | |
| 230 | + } | |
| 223 | 231 | |
| 224 | −static inline struct gen_pool *dma_guess_pool(struct device *dev, | |
| 225 | − struct gen_pool *bad_pool) | |
| 226 | −{ | |
| 227 | − if (bad_pool) | |
| 228 | − return dma_get_safer_pool(bad_pool); | |
| 232 | + if (gen_pool_avail(pool) < atomic_pool_size) | |
| 233 | + schedule_work(&atomic_pool_work); | |
| 229 | 234 | |
| 230 | − return dma_guess_pool_from_device(dev); | |
| 235 | + *cpu_addr = (void *)addr; | |
| 236 | + memset(*cpu_addr, 0, size); | |
| 237 | + return pfn_to_page(__phys_to_pfn(phys)); | |
| 231 | 238 | } |
| 232 | 239 | |
| 233 | −void *dma_alloc_from_pool(struct device *dev, size_t size, | |
| 234 | − struct page **ret_page, gfp_t flags) | |
| 240 | +struct page *dma_alloc_from_pool(struct device *dev, size_t size, | |
| 241 | + void **cpu_addr, gfp_t gfp, | |
| 242 | + bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)) | |
| 235 | 243 | { |
| 236 | 244 | struct gen_pool *pool = NULL; |
| 237 | − unsigned long val = 0; | |
| 238 | − void *ptr = NULL; | |
| 239 | − phys_addr_t phys; | |
| 240 | − | |
| 241 | − while (1) { | |
| 242 | − pool = dma_guess_pool(dev, pool); | |
| 243 | − if (!pool) { | |
| 244 | − WARN(1, "Failed to get suitable pool for %s\n", | |
| 245 | − dev_name(dev)); | |
| 246 | − break; | |
| 247 | − } | |
| 248 | − | |
| 249 | − val = gen_pool_alloc(pool, size); | |
| 250 | − if (!val) | |
| 251 | − continue; | |
| 252 | − | |
| 253 | − phys = gen_pool_virt_to_phys(pool, val); | |
| 254 | − if (dma_coherent_ok(dev, phys, size)) | |
| 255 | − break; | |
| 256 | − | |
| 257 | − gen_pool_free(pool, val, size); | |
| 258 | − val = 0; | |
| 259 | − } | |
| 260 | − | |
| 261 | − | |
| 262 | − if (val) { | |
| 263 | − *ret_page = pfn_to_page(__phys_to_pfn(phys)); | |
| 264 | − ptr = (void *)val; | |
| 265 | − memset(ptr, 0, size); | |
| 245 | + struct page *page; | |
| 266 | 246 | |
| 267 | − if (gen_pool_avail(pool) < atomic_pool_size) | |
| 268 | − schedule_work(&atomic_pool_work); | |
| 247 | + while ((pool = dma_guess_pool(pool, gfp))) { | |
| 248 | + page = __dma_alloc_from_pool(dev, size, pool, cpu_addr, | |
| 249 | + phys_addr_ok); | |
| 250 | + if (page) | |
| 251 | + return page; | |
| 269 | 252 | } |
| 270 | 253 | |
| 271 | − return ptr; | |
| 254 | + WARN(1, "Failed to get suitable pool for %s\n", dev_name(dev)); | |
| 255 | + return NULL; | |
| 272 | 256 | } |
| 273 | 257 | |
| 274 | 258 | bool dma_free_from_pool(struct device *dev, void *start, size_t size) |
| 275 | 259 | { |
| 276 | 260 | struct gen_pool *pool = NULL; |
| 277 | 261 | |
| 278 | − while (1) { | |
| 279 | − pool = dma_guess_pool(dev, pool); | |
| 280 | − if (!pool) | |
| 281 | − return false; | |
| 282 | − | |
| 283 | − if (gen_pool_has_addr(pool, (unsigned long)start, size)) { | |
| 284 | − gen_pool_free(pool, (unsigned long)start, size); | |
| 285 | − return true; | |
| 286 | − } | |
| 262 | + while ((pool = dma_guess_pool(pool, 0))) { | |
| 263 | + if (!gen_pool_has_addr(pool, (unsigned long)start, size)) | |
| 264 | + continue; | |
| 265 | + gen_pool_free(pool, (unsigned long)start, size); | |
| 266 | + return true; | |
| 287 | 267 | } |
| 268 | + | |
| 269 | + return false; | |
| 288 | 270 | } |