History

Merge tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping fixes from Christoph Hellwig: "Fix various regressions: - force unencrypted dma-coherent buffers if encryption bit can't fit into the dma coherent mask (Tom Lendacky) - avoid limiting request size if swiotlb is not used (me) - fix swiotlb handling in dma_direct_sync_sg_for_cpu/device (Fugang Duan)" * tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping: dma-direct: correct the physical addr in dma_direct_sync_sg_for_cpu/device dma-direct: only limit the mapping size if swiotlb could be used dma-mapping: add a dma_addressing_limited helper dma-direct: Force unencrypted DMA under SME for certain DMA masks

Linus Torvalds committed 7 years ago

Commit ac606022 parents c6dd78f 449fa548 files · +83 −26Browse files

Unified · Split

8 files
  1. arch/s390/Kconfig +1 −0
  2. arch/s390/mm/init.c +6 −1
  3. arch/x86/Kconfig +1 −0
  4. arch/x86/mm/mem_encrypt.c +30 −0
  5. include/linux/dma-direct.h +9 −0
  6. include/linux/dma-mapping.h +14 −0
  7. kernel/dma/Kconfig +3 −0
  8. kernel/dma/direct.c +19 −25
arch/s390/Kconfig+1−0
@@ -189,6 +189,7 @@ config S390 · 188 unmodified lines
189 select VIRT_CPU_ACCOUNTING189 select VIRT_CPU_ACCOUNTING
190 select ARCH_HAS_SCALED_CPUTIME190 select ARCH_HAS_SCALED_CPUTIME
191 select HAVE_NMI191 select HAVE_NMI
192 select ARCH_HAS_FORCE_DMA_UNENCRYPTED
192 select SWIOTLB193 select SWIOTLB
193 select GENERIC_ALLOCATOR194 select GENERIC_ALLOCATOR
194 195
arch/s390/mm/init.c+6−1
@@ -30,7 +30,7 @@ · 29 unmodified lines
30#include <linux/export.h>30#include <linux/export.h>
31#include <linux/cma.h>31#include <linux/cma.h>
32#include <linux/gfp.h>32#include <linux/gfp.h>
33#include <linux/dma-mapping.h>33#include <linux/dma-direct.h>
34#include <asm/processor.h>34#include <asm/processor.h>
35#include <linux/uaccess.h>35#include <linux/uaccess.h>
36#include <asm/pgtable.h>36#include <asm/pgtable.h>
@@ -161,6 +161,11 @@ bool sev_active(void) · 124 unmodified lines
161 return is_prot_virt_guest();161 return is_prot_virt_guest();
162}162}
163 163
164bool force_dma_unencrypted(struct device *dev)
165{
166 return sev_active();
167}
168
164/* protected virtualization */169/* protected virtualization */
165static void pv_init(void)170static void pv_init(void)
166{171{
arch/x86/Kconfig+1−0
@@ -1526,6 +1526,7 @@ config AMD_MEM_ENCRYPT · 1525 unmodified lines
1526 depends on X86_64 && CPU_SUP_AMD1526 depends on X86_64 && CPU_SUP_AMD
1527 select DYNAMIC_PHYSICAL_MASK1527 select DYNAMIC_PHYSICAL_MASK
1528 select ARCH_USE_MEMREMAP_PROT1528 select ARCH_USE_MEMREMAP_PROT
1529 select ARCH_HAS_FORCE_DMA_UNENCRYPTED
1529 ---help---1530 ---help---
1530 Say yes to enable support for the encryption of system memory.1531 Say yes to enable support for the encryption of system memory.
1531 This requires an AMD processor that supports Secure Memory1532 This requires an AMD processor that supports Secure Memory
arch/x86/mm/mem_encrypt.c+30−0
@@ -15,6 +15,10 @@ · 14 unmodified lines
15#include <linux/dma-direct.h>15#include <linux/dma-direct.h>
16#include <linux/swiotlb.h>16#include <linux/swiotlb.h>
17#include <linux/mem_encrypt.h>17#include <linux/mem_encrypt.h>
18#include <linux/device.h>
19#include <linux/kernel.h>
20#include <linux/bitops.h>
21#include <linux/dma-mapping.h>
18 22
19#include <asm/tlbflush.h>23#include <asm/tlbflush.h>
20#include <asm/fixmap.h>24#include <asm/fixmap.h>
@@ -348,6 +352,32 @@ bool sev_active(void) · 327 unmodified lines
348}352}
349EXPORT_SYMBOL(sev_active);353EXPORT_SYMBOL(sev_active);
350 354
355/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
356bool force_dma_unencrypted(struct device *dev)
357{
358 /*
359 * For SEV, all DMA must be to unencrypted addresses.
360 */
361 if (sev_active())
362 return true;
363
364 /*
365 * For SME, all DMA must be to unencrypted addresses if the
366 * device does not support DMA to addresses that include the
367 * encryption mask.
368 */
369 if (sme_active()) {
370 u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
371 u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
372 dev->bus_dma_mask);
373
374 if (dma_dev_mask <= dma_enc_mask)
375 return true;
376 }
377
378 return false;
379}
380
351/* Architecture __weak replacement functions */381/* Architecture __weak replacement functions */
352void __init mem_encrypt_free_decrypted_mem(void)382void __init mem_encrypt_free_decrypted_mem(void)
353{383{
include/linux/dma-direct.h+9−0
@@ -32,6 +32,15 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) · 31 unmodified lines
32}32}
33#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */33#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
34 34
35#ifdef CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED
36bool force_dma_unencrypted(struct device *dev);
37#else
38static inline bool force_dma_unencrypted(struct device *dev)
39{
40 return false;
41}
42#endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
43
35/*44/*
36 * If memory encryption is supported, phys_to_dma will set the memory encryption45 * If memory encryption is supported, phys_to_dma will set the memory encryption
37 * bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma46 * bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma
include/linux/dma-mapping.h+14−0
@@ -679,6 +679,20 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask) · 678 unmodified lines
679 return dma_set_mask_and_coherent(dev, mask);679 return dma_set_mask_and_coherent(dev, mask);
680}680}
681 681
682/**
683 * dma_addressing_limited - return if the device is addressing limited
684 * @dev: device to check
685 *
686 * Return %true if the devices DMA mask is too small to address all memory in
687 * the system, else %false. Lack of addressing bits is the prime reason for
688 * bounce buffering, but might not be the only one.
689 */
690static inline bool dma_addressing_limited(struct device *dev)
691{
692 return min_not_zero(*dev->dma_mask, dev->bus_dma_mask) <
693 dma_get_required_mask(dev);
694}
695
682#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS696#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
683void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,697void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
684 const struct iommu_ops *iommu, bool coherent);698 const struct iommu_ops *iommu, bool coherent);
kernel/dma/Kconfig+3−0
@@ -48,6 +48,9 @@ config ARCH_HAS_DMA_COHERENT_TO_PFN · 47 unmodified lines
48config ARCH_HAS_DMA_MMAP_PGPROT48config ARCH_HAS_DMA_MMAP_PGPROT
49 bool49 bool
50 50
51config ARCH_HAS_FORCE_DMA_UNENCRYPTED
52 bool
53
51config DMA_NONCOHERENT_CACHE_SYNC54config DMA_NONCOHERENT_CACHE_SYNC
52 bool55 bool
53 56
kernel/dma/direct.c+19−25
@@ -23,14 +23,6 @@ · 22 unmodified lines
23#define ARCH_ZONE_DMA_BITS 2423#define ARCH_ZONE_DMA_BITS 24
24#endif24#endif
25 25
26/*
27 * For AMD SEV all DMA must be to unencrypted addresses.
28 */
29static inline bool force_dma_unencrypted(void)
30{
31 return sev_active();
32}
33
34static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size)26static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size)
35{27{
36 if (!dev->dma_mask) {28 if (!dev->dma_mask) {
@@ -46,7 +38,7 @@ static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size) · 9 unmodified lines
46static inline dma_addr_t phys_to_dma_direct(struct device *dev,38static inline dma_addr_t phys_to_dma_direct(struct device *dev,
47 phys_addr_t phys)39 phys_addr_t phys)
48{40{
49 if (force_dma_unencrypted())41 if (force_dma_unencrypted(dev))
50 return __phys_to_dma(dev, phys);42 return __phys_to_dma(dev, phys);
51 return phys_to_dma(dev, phys);43 return phys_to_dma(dev, phys);
52}44}
@@ -67,7 +59,7 @@ static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, · 14 unmodified lines
67 if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)59 if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
68 dma_mask = dev->bus_dma_mask;60 dma_mask = dev->bus_dma_mask;
69 61
70 if (force_dma_unencrypted())62 if (force_dma_unencrypted(dev))
71 *phys_mask = __dma_to_phys(dev, dma_mask);63 *phys_mask = __dma_to_phys(dev, dma_mask);
72 else64 else
73 *phys_mask = dma_to_phys(dev, dma_mask);65 *phys_mask = dma_to_phys(dev, dma_mask);
@@ -159,7 +151,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, · 85 unmodified lines
159 }151 }
160 152
161 ret = page_address(page);153 ret = page_address(page);
162 if (force_dma_unencrypted()) {154 if (force_dma_unencrypted(dev)) {
163 set_memory_decrypted((unsigned long)ret, 1 << get_order(size));155 set_memory_decrypted((unsigned long)ret, 1 << get_order(size));
164 *dma_handle = __phys_to_dma(dev, page_to_phys(page));156 *dma_handle = __phys_to_dma(dev, page_to_phys(page));
165 } else {157 } else {
@@ -192,7 +184,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, · 26 unmodified lines
192 return;184 return;
193 }185 }
194 186
195 if (force_dma_unencrypted())187 if (force_dma_unencrypted(dev))
196 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);188 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
197 189
198 if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&190 if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
@@ -242,12 +234,14 @@ void dma_direct_sync_sg_for_device(struct device *dev, · 43 unmodified lines
242 int i;234 int i;
243 235
244 for_each_sg(sgl, sg, nents, i) {236 for_each_sg(sgl, sg, nents, i) {
245 if (unlikely(is_swiotlb_buffer(sg_phys(sg))))237 phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
246 swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length,238
239 if (unlikely(is_swiotlb_buffer(paddr)))
240 swiotlb_tbl_sync_single(dev, paddr, sg->length,
247 dir, SYNC_FOR_DEVICE);241 dir, SYNC_FOR_DEVICE);
248 242
249 if (!dev_is_dma_coherent(dev))243 if (!dev_is_dma_coherent(dev))
250 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length,244 arch_sync_dma_for_device(dev, paddr, sg->length,
251 dir);245 dir);
252 }246 }
253}247}
@@ -279,11 +273,13 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, · 25 unmodified lines
279 int i;273 int i;
280 274
281 for_each_sg(sgl, sg, nents, i) {275 for_each_sg(sgl, sg, nents, i) {
276 phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
277
282 if (!dev_is_dma_coherent(dev))278 if (!dev_is_dma_coherent(dev))
283 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);279 arch_sync_dma_for_cpu(dev, paddr, sg->length, dir);
284 280
285 if (unlikely(is_swiotlb_buffer(sg_phys(sg))))281 if (unlikely(is_swiotlb_buffer(paddr)))
286 swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length, dir,282 swiotlb_tbl_sync_single(dev, paddr, sg->length, dir,
287 SYNC_FOR_CPU);283 SYNC_FOR_CPU);
288 }284 }
289 285
@@ -407,11 +403,9 @@ int dma_direct_supported(struct device *dev, u64 mask) · 117 unmodified lines
407 403
408size_t dma_direct_max_mapping_size(struct device *dev)404size_t dma_direct_max_mapping_size(struct device *dev)
409{405{
410 size_t size = SIZE_MAX;
411
412 /* If SWIOTLB is active, use its maximum mapping size */406 /* If SWIOTLB is active, use its maximum mapping size */
413 if (is_swiotlb_active())407 if (is_swiotlb_active() &&
414 size = swiotlb_max_mapping_size(dev);408 (dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE))
415 409 return swiotlb_max_mapping_size(dev);
416 return size;410 return SIZE_MAX;
417}411}