History

dma-direct: use __DMA_ATTR_ALLOC_CC_SHARED in alloc/free paths

Propagate force_dma_unencrypted() into __DMA_ATTR_ALLOC_CC_SHARED in the dma-direct allocation path and use the attribute to drive the related decisions. This updates dma_direct_alloc(), dma_direct_free(), and dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.

Aneesh Kumar K.V (Arm) committed 4 months ago

Commit b5ef0dd1 parent a6e40902 files · +42 −9Browse files

Unified · Split

2 files
  1. kernel/dma/direct.c +33 −9
  2. kernel/dma/mapping.c +9 −0
kernel/dma/direct.c+33−9
@@ -192,16 +192,22 @@ void *dma_direct_alloc(struct device *dev, size_t size, · 191 unmodified lines
192192 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
193193 {
194194 bool remap = false, set_uncached = false;
195− bool mark_mem_decrypt = true;
195+ bool mark_mem_decrypt = false;
196196 struct page *page;
197197 void *ret;
198198
199+ if (force_dma_unencrypted(dev))
200+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
201+
202+ if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
203+ mark_mem_decrypt = true;
204+
199205 size = PAGE_ALIGN(size);
200206 if (attrs & DMA_ATTR_NO_WARN)
201207 gfp |= __GFP_NOWARN;
202208
203− if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
204− !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))
209+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
210+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
205211 return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
206212
207213 if (!dev_is_dma_coherent(dev)) {
@@ -235,7 +241,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, · 27 unmodified lines
235241 * Remapping or decrypting memory may block, allocate the memory from
236242 * the atomic pools instead if we aren't allowed block.
237243 */
238− if ((remap || force_dma_unencrypted(dev)) &&
244+ if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&
239245 dma_direct_use_pool(dev, gfp)) {
240246 page = dma_direct_alloc_from_pool(dev, size, dma_handle,
241247 &ret, gfp);
@@ -314,12 +320,22 @@ void dma_direct_free(struct device *dev, size_t size, · 72 unmodified lines
314320 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
315321 {
316322 phys_addr_t phys;
317− bool mark_mem_encrypted = true;
323+ bool mark_mem_encrypted = false;
318324 struct io_tlb_pool *swiotlb_pool;
319325 unsigned int page_order = get_order(size);
320326
321− if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
322− !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {
327+ /*
328+ * If the allocation used decrypted/shared backing pages, restore
329+ * the encryption state on free.
330+ */
331+ if (force_dma_unencrypted(dev))
332+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
333+
334+ if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
335+ mark_mem_encrypted = true;
336+
337+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
338+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
323339 /* cpu_addr is a struct page cookie, not a kernel address */
324340 dma_free_contiguous(dev, cpu_addr, size);
325341 return;
@@ -368,10 +384,14 @@ void dma_direct_free(struct device *dev, size_t size, · 42 unmodified lines
368384 struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
369385 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
370386 {
387+ unsigned long attrs = 0;
371388 struct page *page;
372389 void *ret;
373390
374− if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
391+ if (force_dma_unencrypted(dev))
392+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
393+
394+ if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
375395 return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
376396
377397 if (is_swiotlb_for_alloc(dev)) {
@@ -405,7 +425,11 @@ void dma_direct_free_pages(struct device *dev, size_t size, · 27 unmodified lines
405425 phys_addr_t phys;
406426 void *vaddr = page_address(page);
407427 struct io_tlb_pool *swiotlb_pool;
408− bool mark_mem_encrypted = true;
428+ /*
429+ * if the device had requested for an unencrypted buffer,
430+ * convert it to encrypted on free
431+ */
432+ bool mark_mem_encrypted = force_dma_unencrypted(dev);
409433
410434 /* If page is not from an atomic pool, dma_free_from_pool_page() fails */
411435 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
kernel/dma/mapping.c+9−0
@@ -638,6 +638,15 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · 637 unmodified lines
638638 if (WARN_ON_ONCE(flag & __GFP_COMP))
639639 return NULL;
640640
641+ if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) {
642+ trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,
643+ attrs);
644+ return NULL;
645+ }
646+
647+ if (force_dma_unencrypted(dev))
648+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
649+
641650 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {
642651 trace_dma_alloc(dev, cpu_addr, *dma_handle, size,
643652 DMA_BIDIRECTIONAL, flag, attrs);