History

Merge tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping fixes from Christoph Hellwig: "Fix various regressions: - force unencrypted dma-coherent buffers if encryption bit can't fit into the dma coherent mask (Tom Lendacky) - avoid limiting request size if swiotlb is not used (me) - fix swiotlb handling in dma_direct_sync_sg_for_cpu/device (Fugang Duan)" * tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping: dma-direct: correct the physical addr in dma_direct_sync_sg_for_cpu/device dma-direct: only limit the mapping size if swiotlb could be used dma-mapping: add a dma_addressing_limited helper dma-direct: Force unencrypted DMA under SME for certain DMA masks

Linus Torvalds committed 7 years ago

Commit ac606022 parents c6dd78f 449fa548 files · +83 −26Browse files

Unified · Split

8 files
  1. arch/s390/Kconfig +1 −0
  2. arch/s390/mm/init.c +6 −1
  3. arch/x86/Kconfig +1 −0
  4. arch/x86/mm/mem_encrypt.c +30 −0
  5. include/linux/dma-direct.h +9 −0
  6. include/linux/dma-mapping.h +14 −0
  7. kernel/dma/Kconfig +3 −0
  8. kernel/dma/direct.c +19 −25
arch/s390/Kconfig+1−0
@@ -189,6 +189,7 @@ config S390 · 188 unmodified lines
189189 select VIRT_CPU_ACCOUNTING
190190 select ARCH_HAS_SCALED_CPUTIME
191191 select HAVE_NMI
192+ select ARCH_HAS_FORCE_DMA_UNENCRYPTED
192193 select SWIOTLB
193194 select GENERIC_ALLOCATOR
194195
arch/s390/mm/init.c+6−1
@@ -30,7 +30,7 @@ · 29 unmodified lines
3030 #include <linux/export.h>
3131 #include <linux/cma.h>
3232 #include <linux/gfp.h>
33−#include <linux/dma-mapping.h>
33+#include <linux/dma-direct.h>
3434 #include <asm/processor.h>
3535 #include <linux/uaccess.h>
3636 #include <asm/pgtable.h>
@@ -161,6 +161,11 @@ bool sev_active(void) · 124 unmodified lines
161161 return is_prot_virt_guest();
162162 }
163163
164+bool force_dma_unencrypted(struct device *dev)
165+{
166+ return sev_active();
167+}
168+
164169 /* protected virtualization */
165170 static void pv_init(void)
166171 {
arch/x86/Kconfig+1−0
@@ -1526,6 +1526,7 @@ config AMD_MEM_ENCRYPT · 1525 unmodified lines
15261526 depends on X86_64 && CPU_SUP_AMD
15271527 select DYNAMIC_PHYSICAL_MASK
15281528 select ARCH_USE_MEMREMAP_PROT
1529+ select ARCH_HAS_FORCE_DMA_UNENCRYPTED
15291530 ---help---
15301531 Say yes to enable support for the encryption of system memory.
15311532 This requires an AMD processor that supports Secure Memory
arch/x86/mm/mem_encrypt.c+30−0
@@ -15,6 +15,10 @@ · 14 unmodified lines
1515 #include <linux/dma-direct.h>
1616 #include <linux/swiotlb.h>
1717 #include <linux/mem_encrypt.h>
18+#include <linux/device.h>
19+#include <linux/kernel.h>
20+#include <linux/bitops.h>
21+#include <linux/dma-mapping.h>
1822
1923 #include <asm/tlbflush.h>
2024 #include <asm/fixmap.h>
@@ -348,6 +352,32 @@ bool sev_active(void) · 327 unmodified lines
348352 }
349353 EXPORT_SYMBOL(sev_active);
350354
355+/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
356+bool force_dma_unencrypted(struct device *dev)
357+{
358+ /*
359+ * For SEV, all DMA must be to unencrypted addresses.
360+ */
361+ if (sev_active())
362+ return true;
363+
364+ /*
365+ * For SME, all DMA must be to unencrypted addresses if the
366+ * device does not support DMA to addresses that include the
367+ * encryption mask.
368+ */
369+ if (sme_active()) {
370+ u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
371+ u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
372+ dev->bus_dma_mask);
373+
374+ if (dma_dev_mask <= dma_enc_mask)
375+ return true;
376+ }
377+
378+ return false;
379+}
380+
351381 /* Architecture __weak replacement functions */
352382 void __init mem_encrypt_free_decrypted_mem(void)
353383 {
include/linux/dma-direct.h+9−0
@@ -32,6 +32,15 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) · 31 unmodified lines
3232 }
3333 #endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
3434
35+#ifdef CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED
36+bool force_dma_unencrypted(struct device *dev);
37+#else
38+static inline bool force_dma_unencrypted(struct device *dev)
39+{
40+ return false;
41+}
42+#endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
43+
3544 /*
3645 * If memory encryption is supported, phys_to_dma will set the memory encryption
3746 * bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma
include/linux/dma-mapping.h+14−0
@@ -679,6 +679,20 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask) · 678 unmodified lines
679679 return dma_set_mask_and_coherent(dev, mask);
680680 }
681681
682+/**
683+ * dma_addressing_limited - return if the device is addressing limited
684+ * @dev: device to check
685+ *
686+ * Return %true if the devices DMA mask is too small to address all memory in
687+ * the system, else %false. Lack of addressing bits is the prime reason for
688+ * bounce buffering, but might not be the only one.
689+ */
690+static inline bool dma_addressing_limited(struct device *dev)
691+{
692+ return min_not_zero(*dev->dma_mask, dev->bus_dma_mask) <
693+ dma_get_required_mask(dev);
694+}
695+
682696 #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
683697 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
684698 const struct iommu_ops *iommu, bool coherent);
kernel/dma/Kconfig+3−0
@@ -48,6 +48,9 @@ config ARCH_HAS_DMA_COHERENT_TO_PFN · 47 unmodified lines
4848 config ARCH_HAS_DMA_MMAP_PGPROT
4949 bool
5050
51+config ARCH_HAS_FORCE_DMA_UNENCRYPTED
52+ bool
53+
5154 config DMA_NONCOHERENT_CACHE_SYNC
5255 bool
5356
kernel/dma/direct.c+19−25
@@ -23,14 +23,6 @@ · 22 unmodified lines
2323 #define ARCH_ZONE_DMA_BITS 24
2424 #endif
2525
26−/*
27− * For AMD SEV all DMA must be to unencrypted addresses.
28− */
29−static inline bool force_dma_unencrypted(void)
30−{
31− return sev_active();
32−}
33−
3426 static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size)
3527 {
3628 if (!dev->dma_mask) {
@@ -46,7 +38,7 @@ static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size) · 9 unmodified lines
4638 static inline dma_addr_t phys_to_dma_direct(struct device *dev,
4739 phys_addr_t phys)
4840 {
49− if (force_dma_unencrypted())
41+ if (force_dma_unencrypted(dev))
5042 return __phys_to_dma(dev, phys);
5143 return phys_to_dma(dev, phys);
5244 }
@@ -67,7 +59,7 @@ static gfp_t __dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, · 14 unmodified lines
6759 if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
6860 dma_mask = dev->bus_dma_mask;
6961
70− if (force_dma_unencrypted())
62+ if (force_dma_unencrypted(dev))
7163 *phys_mask = __dma_to_phys(dev, dma_mask);
7264 else
7365 *phys_mask = dma_to_phys(dev, dma_mask);
@@ -159,7 +151,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, · 85 unmodified lines
159151 }
160152
161153 ret = page_address(page);
162− if (force_dma_unencrypted()) {
154+ if (force_dma_unencrypted(dev)) {
163155 set_memory_decrypted((unsigned long)ret, 1 << get_order(size));
164156 *dma_handle = __phys_to_dma(dev, page_to_phys(page));
165157 } else {
@@ -192,7 +184,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, · 26 unmodified lines
192184 return;
193185 }
194186
195− if (force_dma_unencrypted())
187+ if (force_dma_unencrypted(dev))
196188 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
197189
198190 if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
@@ -242,12 +234,14 @@ void dma_direct_sync_sg_for_device(struct device *dev, · 43 unmodified lines
242234 int i;
243235
244236 for_each_sg(sgl, sg, nents, i) {
245− if (unlikely(is_swiotlb_buffer(sg_phys(sg))))
246− swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length,
237+ phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
238+
239+ if (unlikely(is_swiotlb_buffer(paddr)))
240+ swiotlb_tbl_sync_single(dev, paddr, sg->length,
247241 dir, SYNC_FOR_DEVICE);
248242
249243 if (!dev_is_dma_coherent(dev))
250− arch_sync_dma_for_device(dev, sg_phys(sg), sg->length,
244+ arch_sync_dma_for_device(dev, paddr, sg->length,
251245 dir);
252246 }
253247 }
@@ -279,11 +273,13 @@ void dma_direct_sync_sg_for_cpu(struct device *dev, · 25 unmodified lines
279273 int i;
280274
281275 for_each_sg(sgl, sg, nents, i) {
276+ phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
277+
282278 if (!dev_is_dma_coherent(dev))
283− arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
284−
285− if (unlikely(is_swiotlb_buffer(sg_phys(sg))))
286− swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length, dir,
279+ arch_sync_dma_for_cpu(dev, paddr, sg->length, dir);
280+
281+ if (unlikely(is_swiotlb_buffer(paddr)))
282+ swiotlb_tbl_sync_single(dev, paddr, sg->length, dir,
287283 SYNC_FOR_CPU);
288284 }
289285
@@ -407,11 +403,9 @@ int dma_direct_supported(struct device *dev, u64 mask) · 117 unmodified lines
407403
408404 size_t dma_direct_max_mapping_size(struct device *dev)
409405 {
410− size_t size = SIZE_MAX;
411−
412406 /* If SWIOTLB is active, use its maximum mapping size */
413− if (is_swiotlb_active())
414− size = swiotlb_max_mapping_size(dev);
415−
416− return size;
407+ if (is_swiotlb_active() &&
408+ (dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE))
409+ return swiotlb_max_mapping_size(dev);
410+ return SIZE_MAX;
417411 }