Before we rework the "pmem api" to stop abusing __copy_user_nocache() for memcpy_to_pmem() we need to fix cases where we may strand dirty data in the cpu cache. The problem occurs when copy_from_iter_pmem() is used for arbitrary data transfers from userspace. There is no guarantee that these transfers, performed by dax_iomap_actor(), will have aligned destinations or aligned transfer lengths. Backstop the usage __copy_user_nocache() with explicit cache management in these unaligned cases. Yes, copy_from_iter_pmem() is now too big for an inline, but addressing that is saved for a later patch that moves the entirety of the "pmem api" into the pmem driver directly.
Unified · Split
| @@ -55,7 +55,8 @@ static inline int arch_memcpy_from_pmem(void *dst, const void *src, size_t n) · 54 unmodified lines | |||
| 55 | * @size: number of bytes to write back | 55 | * @size: number of bytes to write back |
| 56 | * | 56 | * |
| 57 | * Write back a cache range using the CLWB (cache line write back) | 57 | * Write back a cache range using the CLWB (cache line write back) |
| 58 | * instruction. | 58 | * instruction. Note that @size is internally rounded up to be cache |
| 59 | * line size aligned. | ||
| 59 | */ | 60 | */ |
| 60 | static inline void arch_wb_cache_pmem(void *addr, size_t size) | 61 | static inline void arch_wb_cache_pmem(void *addr, size_t size) |
| 61 | { | 62 | { |
| @@ -69,15 +70,6 @@ static inline void arch_wb_cache_pmem(void *addr, size_t size) · 7 unmodified lines | |||
| 69 | clwb(p); | 70 | clwb(p); |
| 70 | } | 71 | } |
| 71 | 72 | ||
| 72 | /* | ||
| 73 | * copy_from_iter_nocache() on x86 only uses non-temporal stores for iovec | ||
| 74 | * iterators, so for other types (bvec & kvec) we must do a cache write-back. | ||
| 75 | */ | ||
| 76 | static inline bool __iter_needs_pmem_wb(struct iov_iter *i) | ||
| 77 | { | ||
| 78 | return iter_is_iovec(i) == false; | ||
| 79 | } | ||
| 80 | |||
| 81 | /** | 73 | /** |
| 82 | * arch_copy_from_iter_pmem - copy data from an iterator to PMEM | 74 | * arch_copy_from_iter_pmem - copy data from an iterator to PMEM |
| 83 | * @addr: PMEM destination address | 75 | * @addr: PMEM destination address |
| @@ -94,7 +86,35 @@ static inline size_t arch_copy_from_iter_pmem(void *addr, size_t bytes, · 10 unmodified lines | |||
| 94 | /* TODO: skip the write-back by always using non-temporal stores */ | 86 | /* TODO: skip the write-back by always using non-temporal stores */ |
| 95 | len = copy_from_iter_nocache(addr, bytes, i); | 87 | len = copy_from_iter_nocache(addr, bytes, i); |
| 96 | 88 | ||
| 97 | if (__iter_needs_pmem_wb(i)) | 89 | /* |
| 90 | * In the iovec case on x86_64 copy_from_iter_nocache() uses | ||
| 91 | * non-temporal stores for the bulk of the transfer, but we need | ||
| 92 | * to manually flush if the transfer is unaligned. A cached | ||
| 93 | * memory copy is used when destination or size is not naturally | ||
| 94 | * aligned. That is: | ||
| 95 | * - Require 8-byte alignment when size is 8 bytes or larger. | ||
| 96 | * - Require 4-byte alignment when size is 4 bytes. | ||
| 97 | * | ||
| 98 | * In the non-iovec case the entire destination needs to be | ||
| 99 | * flushed. | ||
| 100 | */ | ||
| 101 | if (iter_is_iovec(i)) { | ||
| 102 | unsigned long flushed, dest = (unsigned long) addr; | ||
| 103 | |||
| 104 | if (bytes < 8) { | ||
| 105 | if (!IS_ALIGNED(dest, 4) || (bytes != 4)) | ||
| 106 | arch_wb_cache_pmem(addr, 1); | ||
| 107 | } else { | ||
| 108 | if (!IS_ALIGNED(dest, 8)) { | ||
| 109 | dest = ALIGN(dest, boot_cpu_data.x86_clflush_size); | ||
| 110 | arch_wb_cache_pmem(addr, 1); | ||
| 111 | } | ||
| 112 | |||
| 113 | flushed = dest - (unsigned long) addr; | ||
| 114 | if (bytes > flushed && !IS_ALIGNED(bytes - flushed, 8)) | ||
| 115 | arch_wb_cache_pmem(addr + bytes - 1, 1); | ||
| 116 | } | ||
| 117 | } else | ||
| 98 | arch_wb_cache_pmem(addr, bytes); | 118 | arch_wb_cache_pmem(addr, bytes); |
| 99 | 119 | ||
| 100 | return len; | 120 | return len; |