This API is the equivalent of alloc_pages, except that the returned memory is guaranteed to be DMA addressable by the passed in device. The implementation will also be used to provide a more sensible replacement for DMA_ATTR_NON_CONSISTENT flag. Additionally dma_alloc_noncoherent is switched over to use dma_alloc_pages as its backend.
Showing the changes to kernel (4 of 23 files). Show all
Unified · Split
| @@ -1,6 +1,6 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | /* | 2 | /* |
| 3 | * Copyright (C) 2018 Christoph Hellwig. | 3 | * Copyright (C) 2018-2020 Christoph Hellwig. |
| 4 | * | 4 | * |
| 5 | * DMA operations that map physical memory directly without using an IOMMU. | 5 | * DMA operations that map physical memory directly without using an IOMMU. |
| 6 | */ | 6 | */ |
| @@ -292,6 +292,56 @@ void dma_direct_free(struct device *dev, size_t size, · 285 unmodified lines | |||
| 292 | dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size); | 292 | dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size); |
| 293 | } | 293 | } |
| 294 | 294 | ||
| 295 | struct page *dma_direct_alloc_pages(struct device *dev, size_t size, | ||
| 296 | dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) | ||
| 297 | { | ||
| 298 | struct page *page; | ||
| 299 | void *ret; | ||
| 300 | |||
| 301 | if (dma_should_alloc_from_pool(dev, gfp, 0)) { | ||
| 302 | page = dma_alloc_from_pool(dev, size, &ret, gfp, | ||
| 303 | dma_coherent_ok); | ||
| 304 | if (!page) | ||
| 305 | return NULL; | ||
| 306 | goto done; | ||
| 307 | } | ||
| 308 | |||
| 309 | page = __dma_direct_alloc_pages(dev, size, gfp); | ||
| 310 | if (!page) | ||
| 311 | return NULL; | ||
| 312 | ret = page_address(page); | ||
| 313 | if (force_dma_unencrypted(dev)) { | ||
| 314 | if (set_memory_decrypted((unsigned long)ret, | ||
| 315 | 1 << get_order(size))) | ||
| 316 | goto out_free_pages; | ||
| 317 | } | ||
| 318 | memset(ret, 0, size); | ||
| 319 | done: | ||
| 320 | *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); | ||
| 321 | return page; | ||
| 322 | out_free_pages: | ||
| 323 | dma_free_contiguous(dev, page, size); | ||
| 324 | return NULL; | ||
| 325 | } | ||
| 326 | |||
| 327 | void dma_direct_free_pages(struct device *dev, size_t size, | ||
| 328 | struct page *page, dma_addr_t dma_addr, | ||
| 329 | enum dma_data_direction dir) | ||
| 330 | { | ||
| 331 | unsigned int page_order = get_order(size); | ||
| 332 | void *vaddr = page_address(page); | ||
| 333 | |||
| 334 | /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ | ||
| 335 | if (dma_should_free_from_pool(dev, 0) && | ||
| 336 | dma_free_from_pool(dev, vaddr, size)) | ||
| 337 | return; | ||
| 338 | |||
| 339 | if (force_dma_unencrypted(dev)) | ||
| 340 | set_memory_encrypted((unsigned long)vaddr, 1 << page_order); | ||
| 341 | |||
| 342 | dma_free_contiguous(dev, page, size); | ||
| 343 | } | ||
| 344 | |||
| 295 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ | 345 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ |
| 296 | defined(CONFIG_SWIOTLB) | 346 | defined(CONFIG_SWIOTLB) |
| 297 | void dma_direct_sync_sg_for_device(struct device *dev, | 347 | void dma_direct_sync_sg_for_device(struct device *dev, |
| @@ -341,9 +341,7 @@ pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) · 340 unmodified lines | |||
| 341 | { | 341 | { |
| 342 | if (force_dma_unencrypted(dev)) | 342 | if (force_dma_unencrypted(dev)) |
| 343 | prot = pgprot_decrypted(prot); | 343 | prot = pgprot_decrypted(prot); |
| 344 | if (dev_is_dma_coherent(dev) || | 344 | if (dev_is_dma_coherent(dev)) |
| 345 | (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && | ||
| 346 | (attrs & DMA_ATTR_NON_CONSISTENT))) | ||
| 347 | return prot; | 345 | return prot; |
| 348 | #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE | 346 | #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE |
| 349 | if (attrs & DMA_ATTR_WRITE_COMBINE) | 347 | if (attrs & DMA_ATTR_WRITE_COMBINE) |
| @@ -472,6 +470,65 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, · 122 unmodified lines | |||
| 472 | } | 470 | } |
| 473 | EXPORT_SYMBOL(dma_free_attrs); | 471 | EXPORT_SYMBOL(dma_free_attrs); |
| 474 | 472 | ||
| 473 | struct page *dma_alloc_pages(struct device *dev, size_t size, | ||
| 474 | dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) | ||
| 475 | { | ||
| 476 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 477 | struct page *page; | ||
| 478 | |||
| 479 | if (WARN_ON_ONCE(!dev->coherent_dma_mask)) | ||
| 480 | return NULL; | ||
| 481 | if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM))) | ||
| 482 | return NULL; | ||
| 483 | |||
| 484 | size = PAGE_ALIGN(size); | ||
| 485 | if (dma_alloc_direct(dev, ops)) | ||
| 486 | page = dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); | ||
| 487 | else if (ops->alloc_pages) | ||
| 488 | page = ops->alloc_pages(dev, size, dma_handle, dir, gfp); | ||
| 489 | else | ||
| 490 | return NULL; | ||
| 491 | |||
| 492 | debug_dma_map_page(dev, page, 0, size, dir, *dma_handle); | ||
| 493 | |||
| 494 | return page; | ||
| 495 | } | ||
| 496 | EXPORT_SYMBOL_GPL(dma_alloc_pages); | ||
| 497 | |||
| 498 | void dma_free_pages(struct device *dev, size_t size, struct page *page, | ||
| 499 | dma_addr_t dma_handle, enum dma_data_direction dir) | ||
| 500 | { | ||
| 501 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 502 | |||
| 503 | size = PAGE_ALIGN(size); | ||
| 504 | debug_dma_unmap_page(dev, dma_handle, size, dir); | ||
| 505 | |||
| 506 | if (dma_alloc_direct(dev, ops)) | ||
| 507 | dma_direct_free_pages(dev, size, page, dma_handle, dir); | ||
| 508 | else if (ops->free_pages) | ||
| 509 | ops->free_pages(dev, size, page, dma_handle, dir); | ||
| 510 | } | ||
| 511 | EXPORT_SYMBOL_GPL(dma_free_pages); | ||
| 512 | |||
| 513 | void *dma_alloc_noncoherent(struct device *dev, size_t size, | ||
| 514 | dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) | ||
| 515 | { | ||
| 516 | struct page *page; | ||
| 517 | |||
| 518 | page = dma_alloc_pages(dev, size, dma_handle, dir, gfp); | ||
| 519 | if (!page) | ||
| 520 | return NULL; | ||
| 521 | return page_address(page); | ||
| 522 | } | ||
| 523 | EXPORT_SYMBOL_GPL(dma_alloc_noncoherent); | ||
| 524 | |||
| 525 | void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr, | ||
| 526 | dma_addr_t dma_handle, enum dma_data_direction dir) | ||
| 527 | { | ||
| 528 | dma_free_pages(dev, size, virt_to_page(vaddr), dma_handle, dir); | ||
| 529 | } | ||
| 530 | EXPORT_SYMBOL_GPL(dma_free_noncoherent); | ||
| 531 | |||
| 475 | int dma_supported(struct device *dev, u64 mask) | 532 | int dma_supported(struct device *dev, u64 mask) |
| 476 | { | 533 | { |
| 477 | const struct dma_map_ops *ops = get_dma_ops(dev); | 534 | const struct dma_map_ops *ops = get_dma_ops(dev); |
| @@ -3,6 +3,7 @@ · 2 unmodified lines | |||
| 3 | * Helpers for DMA ops implementations. These generally rely on the fact that | 3 | * Helpers for DMA ops implementations. These generally rely on the fact that |
| 4 | * the allocated memory contains normal pages in the direct kernel mapping. | 4 | * the allocated memory contains normal pages in the direct kernel mapping. |
| 5 | */ | 5 | */ |
| 6 | #include <linux/dma-contiguous.h> | ||
| 6 | #include <linux/dma-noncoherent.h> | 7 | #include <linux/dma-noncoherent.h> |
| 7 | 8 | ||
| 8 | /* | 9 | /* |
| @@ -49,3 +50,37 @@ int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, · 40 unmodified lines | |||
| 49 | return -ENXIO; | 50 | return -ENXIO; |
| 50 | #endif /* CONFIG_MMU */ | 51 | #endif /* CONFIG_MMU */ |
| 51 | } | 52 | } |
| 53 | |||
| 54 | struct page *dma_common_alloc_pages(struct device *dev, size_t size, | ||
| 55 | dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) | ||
| 56 | { | ||
| 57 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 58 | struct page *page; | ||
| 59 | |||
| 60 | page = dma_alloc_contiguous(dev, size, gfp); | ||
| 61 | if (!page) | ||
| 62 | page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); | ||
| 63 | if (!page) | ||
| 64 | return NULL; | ||
| 65 | |||
| 66 | *dma_handle = ops->map_page(dev, page, 0, size, dir, | ||
| 67 | DMA_ATTR_SKIP_CPU_SYNC); | ||
| 68 | if (*dma_handle == DMA_MAPPING_ERROR) { | ||
| 69 | dma_free_contiguous(dev, page, size); | ||
| 70 | return NULL; | ||
| 71 | } | ||
| 72 | |||
| 73 | memset(page_address(page), 0, size); | ||
| 74 | return page; | ||
| 75 | } | ||
| 76 | |||
| 77 | void dma_common_free_pages(struct device *dev, size_t size, struct page *page, | ||
| 78 | dma_addr_t dma_handle, enum dma_data_direction dir) | ||
| 79 | { | ||
| 80 | const struct dma_map_ops *ops = get_dma_ops(dev); | ||
| 81 | |||
| 82 | if (ops->unmap_page) | ||
| 83 | ops->unmap_page(dev, dma_handle, size, dir, | ||
| 84 | DMA_ATTR_SKIP_CPU_SYNC); | ||
| 85 | dma_free_contiguous(dev, page, size); | ||
| 86 | } | ||
| @@ -55,5 +55,7 @@ const struct dma_map_ops dma_virt_ops = { · 54 unmodified lines | |||
| 55 | .free = dma_virt_free, | 55 | .free = dma_virt_free, |
| 56 | .map_page = dma_virt_map_page, | 56 | .map_page = dma_virt_map_page, |
| 57 | .map_sg = dma_virt_map_sg, | 57 | .map_sg = dma_virt_map_sg, |
| 58 | .alloc_pages = dma_common_alloc_pages, | ||
| 59 | .free_pages = dma_common_free_pages, | ||
| 58 | }; | 60 | }; |
| 59 | EXPORT_SYMBOL(dma_virt_ops); | 61 | EXPORT_SYMBOL(dma_virt_ops); |