Before we rework the "pmem api" to stop abusing __copy_user_nocache() for memcpy_to_pmem() we need to fix cases where we may strand dirty data in the cpu cache. The problem occurs when copy_from_iter_pmem() is used for arbitrary data transfers from userspace. There is no guarantee that these transfers, performed by dax_iomap_actor(), will have aligned destinations or aligned transfer lengths. Backstop the usage __copy_user_nocache() with explicit cache management in these unaligned cases. Yes, copy_from_iter_pmem() is now too big for an inline, but addressing that is saved for a later patch that moves the entirety of the "pmem api" into the pmem driver directly.
Unified · Split
| @@ -55,7 +55,8 @@ static inline int arch_memcpy_from_pmem(void *dst, const void *src, size_t n) · 54 unmodified lines | ||
| 55 | 55 | * @size: number of bytes to write back |
| 56 | 56 | * |
| 57 | 57 | * Write back a cache range using the CLWB (cache line write back) |
| 58 | − * instruction. | |
| 58 | + * instruction. Note that @size is internally rounded up to be cache | |
| 59 | + * line size aligned. | |
| 59 | 60 | */ |
| 60 | 61 | static inline void arch_wb_cache_pmem(void *addr, size_t size) |
| 61 | 62 | { |
| @@ -69,15 +70,6 @@ static inline void arch_wb_cache_pmem(void *addr, size_t size) · 7 unmodified lines | ||
| 69 | 70 | clwb(p); |
| 70 | 71 | } |
| 71 | 72 | |
| 72 | −/* | |
| 73 | − * copy_from_iter_nocache() on x86 only uses non-temporal stores for iovec | |
| 74 | − * iterators, so for other types (bvec & kvec) we must do a cache write-back. | |
| 75 | − */ | |
| 76 | −static inline bool __iter_needs_pmem_wb(struct iov_iter *i) | |
| 77 | −{ | |
| 78 | − return iter_is_iovec(i) == false; | |
| 79 | −} | |
| 80 | − | |
| 81 | 73 | /** |
| 82 | 74 | * arch_copy_from_iter_pmem - copy data from an iterator to PMEM |
| 83 | 75 | * @addr: PMEM destination address |
| @@ -94,7 +86,35 @@ static inline size_t arch_copy_from_iter_pmem(void *addr, size_t bytes, · 10 unmodified lines | ||
| 94 | 86 | /* TODO: skip the write-back by always using non-temporal stores */ |
| 95 | 87 | len = copy_from_iter_nocache(addr, bytes, i); |
| 96 | 88 | |
| 97 | − if (__iter_needs_pmem_wb(i)) | |
| 89 | + /* | |
| 90 | + * In the iovec case on x86_64 copy_from_iter_nocache() uses | |
| 91 | + * non-temporal stores for the bulk of the transfer, but we need | |
| 92 | + * to manually flush if the transfer is unaligned. A cached | |
| 93 | + * memory copy is used when destination or size is not naturally | |
| 94 | + * aligned. That is: | |
| 95 | + * - Require 8-byte alignment when size is 8 bytes or larger. | |
| 96 | + * - Require 4-byte alignment when size is 4 bytes. | |
| 97 | + * | |
| 98 | + * In the non-iovec case the entire destination needs to be | |
| 99 | + * flushed. | |
| 100 | + */ | |
| 101 | + if (iter_is_iovec(i)) { | |
| 102 | + unsigned long flushed, dest = (unsigned long) addr; | |
| 103 | + | |
| 104 | + if (bytes < 8) { | |
| 105 | + if (!IS_ALIGNED(dest, 4) || (bytes != 4)) | |
| 106 | + arch_wb_cache_pmem(addr, 1); | |
| 107 | + } else { | |
| 108 | + if (!IS_ALIGNED(dest, 8)) { | |
| 109 | + dest = ALIGN(dest, boot_cpu_data.x86_clflush_size); | |
| 110 | + arch_wb_cache_pmem(addr, 1); | |
| 111 | + } | |
| 112 | + | |
| 113 | + flushed = dest - (unsigned long) addr; | |
| 114 | + if (bytes > flushed && !IS_ALIGNED(bytes - flushed, 8)) | |
| 115 | + arch_wb_cache_pmem(addr + bytes - 1, 1); | |
| 116 | + } | |
| 117 | + } else | |
| 98 | 118 | arch_wb_cache_pmem(addr, bytes); |
| 99 | 119 | |
| 100 | 120 | return len; |