Propagate force_dma_unencrypted() into __DMA_ATTR_ALLOC_CC_SHARED in the dma-direct allocation path and use the attribute to drive the related decisions. This updates dma_direct_alloc(), dma_direct_free(), and dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.
Showing the changes to kernel (2 of 2 files). Show all
Unified · Split
| @@ -192,16 +192,22 @@ void *dma_direct_alloc(struct device *dev, size_t size, · 191 unmodified lines | ||
| 192 | 192 | dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) |
| 193 | 193 | { |
| 194 | 194 | bool remap = false, set_uncached = false; |
| 195 | − bool mark_mem_decrypt = true; | |
| 195 | + bool mark_mem_decrypt = false; | |
| 196 | 196 | struct page *page; |
| 197 | 197 | void *ret; |
| 198 | 198 | |
| 199 | + if (force_dma_unencrypted(dev)) | |
| 200 | + attrs |= __DMA_ATTR_ALLOC_CC_SHARED; | |
| 201 | + | |
| 202 | + if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) | |
| 203 | + mark_mem_decrypt = true; | |
| 204 | + | |
| 199 | 205 | size = PAGE_ALIGN(size); |
| 200 | 206 | if (attrs & DMA_ATTR_NO_WARN) |
| 201 | 207 | gfp |= __GFP_NOWARN; |
| 202 | 208 | |
| 203 | − if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && | |
| 204 | − !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) | |
| 209 | + if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) == | |
| 210 | + DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) | |
| 205 | 211 | return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); |
| 206 | 212 | |
| 207 | 213 | if (!dev_is_dma_coherent(dev)) { |
| @@ -235,7 +241,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, · 27 unmodified lines | ||
| 235 | 241 | * Remapping or decrypting memory may block, allocate the memory from |
| 236 | 242 | * the atomic pools instead if we aren't allowed block. |
| 237 | 243 | */ |
| 238 | − if ((remap || force_dma_unencrypted(dev)) && | |
| 244 | + if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) && | |
| 239 | 245 | dma_direct_use_pool(dev, gfp)) { |
| 240 | 246 | page = dma_direct_alloc_from_pool(dev, size, dma_handle, |
| 241 | 247 | &ret, gfp); |
| @@ -314,12 +320,22 @@ void dma_direct_free(struct device *dev, size_t size, · 72 unmodified lines | ||
| 314 | 320 | void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) |
| 315 | 321 | { |
| 316 | 322 | phys_addr_t phys; |
| 317 | − bool mark_mem_encrypted = true; | |
| 323 | + bool mark_mem_encrypted = false; | |
| 318 | 324 | struct io_tlb_pool *swiotlb_pool; |
| 319 | 325 | unsigned int page_order = get_order(size); |
| 320 | 326 | |
| 321 | − if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && | |
| 322 | − !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) { | |
| 327 | + /* | |
| 328 | + * If the allocation used decrypted/shared backing pages, restore | |
| 329 | + * the encryption state on free. | |
| 330 | + */ | |
| 331 | + if (force_dma_unencrypted(dev)) | |
| 332 | + attrs |= __DMA_ATTR_ALLOC_CC_SHARED; | |
| 333 | + | |
| 334 | + if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) | |
| 335 | + mark_mem_encrypted = true; | |
| 336 | + | |
| 337 | + if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) == | |
| 338 | + DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) { | |
| 323 | 339 | /* cpu_addr is a struct page cookie, not a kernel address */ |
| 324 | 340 | dma_free_contiguous(dev, cpu_addr, size); |
| 325 | 341 | return; |
| @@ -368,10 +384,14 @@ void dma_direct_free(struct device *dev, size_t size, · 42 unmodified lines | ||
| 368 | 384 | struct page *dma_direct_alloc_pages(struct device *dev, size_t size, |
| 369 | 385 | dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) |
| 370 | 386 | { |
| 387 | + unsigned long attrs = 0; | |
| 371 | 388 | struct page *page; |
| 372 | 389 | void *ret; |
| 373 | 390 | |
| 374 | − if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) | |
| 391 | + if (force_dma_unencrypted(dev)) | |
| 392 | + attrs |= __DMA_ATTR_ALLOC_CC_SHARED; | |
| 393 | + | |
| 394 | + if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp)) | |
| 375 | 395 | return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp); |
| 376 | 396 | |
| 377 | 397 | if (is_swiotlb_for_alloc(dev)) { |
| @@ -405,7 +425,11 @@ void dma_direct_free_pages(struct device *dev, size_t size, · 27 unmodified lines | ||
| 405 | 425 | phys_addr_t phys; |
| 406 | 426 | void *vaddr = page_address(page); |
| 407 | 427 | struct io_tlb_pool *swiotlb_pool; |
| 408 | − bool mark_mem_encrypted = true; | |
| 428 | + /* | |
| 429 | + * if the device had requested for an unencrypted buffer, | |
| 430 | + * convert it to encrypted on free | |
| 431 | + */ | |
| 432 | + bool mark_mem_encrypted = force_dma_unencrypted(dev); | |
| 409 | 433 | |
| 410 | 434 | /* If page is not from an atomic pool, dma_free_from_pool_page() fails */ |
| 411 | 435 | if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && |
| @@ -638,6 +638,15 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · 637 unmodified lines | ||
| 638 | 638 | if (WARN_ON_ONCE(flag & __GFP_COMP)) |
| 639 | 639 | return NULL; |
| 640 | 640 | |
| 641 | + if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) { | |
| 642 | + trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag, | |
| 643 | + attrs); | |
| 644 | + return NULL; | |
| 645 | + } | |
| 646 | + | |
| 647 | + if (force_dma_unencrypted(dev)) | |
| 648 | + attrs |= __DMA_ATTR_ALLOC_CC_SHARED; | |
| 649 | + | |
| 641 | 650 | if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) { |
| 642 | 651 | trace_dma_alloc(dev, cpu_addr, *dma_handle, size, |
| 643 | 652 | DMA_BIDIRECTIONAL, flag, attrs); |