History

dma-direct: use __DMA_ATTR_ALLOC_CC_SHARED in alloc/free paths

Propagate force_dma_unencrypted() into __DMA_ATTR_ALLOC_CC_SHARED in the dma-direct allocation path and use the attribute to drive the related decisions. This updates dma_direct_alloc(), dma_direct_free(), and dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.

Aneesh Kumar K.V (Arm) committed 4 months ago

Commit b5ef0dd1 parent a6e40902 files · +42 −9Browse files

Unified · Split

2 files
  1. kernel/dma/direct.c +33 −9
  2. kernel/dma/mapping.c +9 −0
kernel/dma/direct.c+33−9
@@ -192,16 +192,22 @@ void *dma_direct_alloc(struct device *dev, size_t size, · 191 unmodified lines
192 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)192 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
193{193{
194 bool remap = false, set_uncached = false;194 bool remap = false, set_uncached = false;
195 bool mark_mem_decrypt = true;195 bool mark_mem_decrypt = false;
196 struct page *page;196 struct page *page;
197 void *ret;197 void *ret;
198 198
199 if (force_dma_unencrypted(dev))
200 attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
201
202 if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
203 mark_mem_decrypt = true;
204
199 size = PAGE_ALIGN(size);205 size = PAGE_ALIGN(size);
200 if (attrs & DMA_ATTR_NO_WARN)206 if (attrs & DMA_ATTR_NO_WARN)
201 gfp |= __GFP_NOWARN;207 gfp |= __GFP_NOWARN;
202 208
203 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&209 if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
204 !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))210 DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
205 return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);211 return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
206 212
207 if (!dev_is_dma_coherent(dev)) {213 if (!dev_is_dma_coherent(dev)) {
@@ -235,7 +241,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, · 27 unmodified lines
235 * Remapping or decrypting memory may block, allocate the memory from241 * Remapping or decrypting memory may block, allocate the memory from
236 * the atomic pools instead if we aren't allowed block.242 * the atomic pools instead if we aren't allowed block.
237 */243 */
238 if ((remap || force_dma_unencrypted(dev)) &&244 if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&
239 dma_direct_use_pool(dev, gfp)) {245 dma_direct_use_pool(dev, gfp)) {
240 page = dma_direct_alloc_from_pool(dev, size, dma_handle,246 page = dma_direct_alloc_from_pool(dev, size, dma_handle,
241 &ret, gfp);247 &ret, gfp);
@@ -314,12 +320,22 @@ void dma_direct_free(struct device *dev, size_t size, · 72 unmodified lines
314 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)320 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
315{321{
316 phys_addr_t phys;322 phys_addr_t phys;
317 bool mark_mem_encrypted = true;323 bool mark_mem_encrypted = false;
318 struct io_tlb_pool *swiotlb_pool;324 struct io_tlb_pool *swiotlb_pool;
319 unsigned int page_order = get_order(size);325 unsigned int page_order = get_order(size);
320 326
321 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&327 /*
322 !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {328 * If the allocation used decrypted/shared backing pages, restore
329 * the encryption state on free.
330 */
331 if (force_dma_unencrypted(dev))
332 attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
333
334 if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
335 mark_mem_encrypted = true;
336
337 if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
338 DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
323 /* cpu_addr is a struct page cookie, not a kernel address */339 /* cpu_addr is a struct page cookie, not a kernel address */
324 dma_free_contiguous(dev, cpu_addr, size);340 dma_free_contiguous(dev, cpu_addr, size);
325 return;341 return;
@@ -368,10 +384,14 @@ void dma_direct_free(struct device *dev, size_t size, · 42 unmodified lines
368struct page *dma_direct_alloc_pages(struct device *dev, size_t size,384struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
369 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)385 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
370{386{
387 unsigned long attrs = 0;
371 struct page *page;388 struct page *page;
372 void *ret;389 void *ret;
373 390
374 if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))391 if (force_dma_unencrypted(dev))
392 attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
393
394 if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
375 return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);395 return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
376 396
377 if (is_swiotlb_for_alloc(dev)) {397 if (is_swiotlb_for_alloc(dev)) {
@@ -405,7 +425,11 @@ void dma_direct_free_pages(struct device *dev, size_t size, · 27 unmodified lines
405 phys_addr_t phys;425 phys_addr_t phys;
406 void *vaddr = page_address(page);426 void *vaddr = page_address(page);
407 struct io_tlb_pool *swiotlb_pool;427 struct io_tlb_pool *swiotlb_pool;
408 bool mark_mem_encrypted = true;428 /*
429 * if the device had requested for an unencrypted buffer,
430 * convert it to encrypted on free
431 */
432 bool mark_mem_encrypted = force_dma_unencrypted(dev);
409 433
410 /* If page is not from an atomic pool, dma_free_from_pool_page() fails */434 /* If page is not from an atomic pool, dma_free_from_pool_page() fails */
411 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&435 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
kernel/dma/mapping.c+9−0
@@ -638,6 +638,15 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · 637 unmodified lines
638 if (WARN_ON_ONCE(flag & __GFP_COMP))638 if (WARN_ON_ONCE(flag & __GFP_COMP))
639 return NULL;639 return NULL;
640 640
641 if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) {
642 trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,
643 attrs);
644 return NULL;
645 }
646
647 if (force_dma_unencrypted(dev))
648 attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
649
641 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {650 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {
642 trace_dma_alloc(dev, cpu_addr, *dma_handle, size,651 trace_dma_alloc(dev, cpu_addr, *dma_handle, size,
643 DMA_BIDIRECTIONAL, flag, attrs);652 DMA_BIDIRECTIONAL, flag, attrs);