Unshare the DAX and iomap buffered I/O page zeroing code. This code previously did a IS_DAX check deep inside the iomap code, which in fact was the only DAX check in the code. Instead move these checks into the callers. Most callers already have DAX special casing anyway and XFS will need it for reflink support as well.
Unified · Split
| @@ -1135,24 +1135,73 @@ static int dax_memzero(struct dax_device *dax_dev, pgoff_t pgoff, · 1134 unmodified lines | |||
| 1135 | return ret; | 1135 | return ret; |
| 1136 | } | 1136 | } |
| 1137 | 1137 | ||
| 1138 | s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap) | 1138 | static s64 dax_zero_iter(struct iomap_iter *iter, bool *did_zero) |
| 1139 | { | 1139 | { |
| 1140 | pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); | 1140 | const struct iomap *iomap = &iter->iomap; |
| 1141 | long rc, id; | 1141 | const struct iomap *srcmap = iomap_iter_srcmap(iter); |
| 1142 | unsigned offset = offset_in_page(pos); | 1142 | loff_t pos = iter->pos; |
| 1143 | unsigned size = min_t(u64, PAGE_SIZE - offset, length); | 1143 | u64 length = iomap_length(iter); |
| 1144 | s64 written = 0; | ||
| 1145 | |||
| 1146 | /* already zeroed? we're done. */ | ||
| 1147 | if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) | ||
| 1148 | return length; | ||
| 1149 | |||
| 1150 | do { | ||
| 1151 | unsigned offset = offset_in_page(pos); | ||
| 1152 | unsigned size = min_t(u64, PAGE_SIZE - offset, length); | ||
| 1153 | pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); | ||
| 1154 | long rc; | ||
| 1155 | int id; | ||
| 1144 | 1156 | ||
| 1145 | id = dax_read_lock(); | 1157 | id = dax_read_lock(); |
| 1146 | if (IS_ALIGNED(pos, PAGE_SIZE) && size == PAGE_SIZE) | 1158 | if (IS_ALIGNED(pos, PAGE_SIZE) && size == PAGE_SIZE) |
| 1147 | rc = dax_zero_page_range(iomap->dax_dev, pgoff, 1); | 1159 | rc = dax_zero_page_range(iomap->dax_dev, pgoff, 1); |
| 1148 | else | 1160 | else |
| 1149 | rc = dax_memzero(iomap->dax_dev, pgoff, offset, size); | 1161 | rc = dax_memzero(iomap->dax_dev, pgoff, offset, size); |
| 1150 | dax_read_unlock(id); | 1162 | dax_read_unlock(id); |
| 1151 | 1163 | ||
| 1152 | if (rc < 0) | 1164 | if (rc < 0) |
| 1153 | return rc; | 1165 | return rc; |
| 1154 | return size; | 1166 | pos += size; |
| 1167 | length -= size; | ||
| 1168 | written += size; | ||
| 1169 | if (did_zero) | ||
| 1170 | *did_zero = true; | ||
| 1171 | } while (length > 0); | ||
| 1172 | |||
| 1173 | return written; | ||
| 1174 | } | ||
| 1175 | |||
| 1176 | int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, | ||
| 1177 | const struct iomap_ops *ops) | ||
| 1178 | { | ||
| 1179 | struct iomap_iter iter = { | ||
| 1180 | .inode = inode, | ||
| 1181 | .pos = pos, | ||
| 1182 | .len = len, | ||
| 1183 | .flags = IOMAP_ZERO, | ||
| 1184 | }; | ||
| 1185 | int ret; | ||
| 1186 | |||
| 1187 | while ((ret = iomap_iter(&iter, ops)) > 0) | ||
| 1188 | iter.processed = dax_zero_iter(&iter, did_zero); | ||
| 1189 | return ret; | ||
| 1190 | } | ||
| 1191 | EXPORT_SYMBOL_GPL(dax_zero_range); | ||
| 1192 | |||
| 1193 | int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, | ||
| 1194 | const struct iomap_ops *ops) | ||
| 1195 | { | ||
| 1196 | unsigned int blocksize = i_blocksize(inode); | ||
| 1197 | unsigned int off = pos & (blocksize - 1); | ||
| 1198 | |||
| 1199 | /* Block boundary? Nothing to do */ | ||
| 1200 | if (!off) | ||
| 1201 | return 0; | ||
| 1202 | return dax_zero_range(inode, pos, blocksize - off, did_zero, ops); | ||
| 1155 | } | 1203 | } |
| 1204 | EXPORT_SYMBOL_GPL(dax_truncate_page); | ||
| 1156 | 1205 | ||
| 1157 | static loff_t dax_iomap_iter(const struct iomap_iter *iomi, | 1206 | static loff_t dax_iomap_iter(const struct iomap_iter *iomi, |
| 1158 | struct iov_iter *iter) | 1207 | struct iov_iter *iter) |
| @@ -36,6 +36,7 @@ · 35 unmodified lines | |||
| 36 | #include <linux/iomap.h> | 36 | #include <linux/iomap.h> |
| 37 | #include <linux/namei.h> | 37 | #include <linux/namei.h> |
| 38 | #include <linux/uio.h> | 38 | #include <linux/uio.h> |
| 39 | #include <linux/dax.h> | ||
| 39 | #include "ext2.h" | 40 | #include "ext2.h" |
| 40 | #include "acl.h" | 41 | #include "acl.h" |
| 41 | #include "xattr.h" | 42 | #include "xattr.h" |
| @@ -1297,9 +1298,9 @@ static int ext2_setsize(struct inode *inode, loff_t newsize) · 1255 unmodified lines | |||
| 1297 | inode_dio_wait(inode); | 1298 | inode_dio_wait(inode); |
| 1298 | 1299 | ||
| 1299 | if (IS_DAX(inode)) { | 1300 | if (IS_DAX(inode)) { |
| 1300 | error = iomap_zero_range(inode, newsize, | 1301 | error = dax_zero_range(inode, newsize, |
| 1301 | PAGE_ALIGN(newsize) - newsize, NULL, | 1302 | PAGE_ALIGN(newsize) - newsize, NULL, |
| 1302 | &ext2_iomap_ops); | 1303 | &ext2_iomap_ops); |
| 1303 | } else if (test_opt(inode->i_sb, NOBH)) | 1304 | } else if (test_opt(inode->i_sb, NOBH)) |
| 1304 | error = nobh_truncate_page(inode->i_mapping, | 1305 | error = nobh_truncate_page(inode->i_mapping, |
| 1305 | newsize, ext2_get_block); | 1306 | newsize, ext2_get_block); |
| @@ -41,6 +41,7 @@ · 40 unmodified lines | |||
| 41 | #include <linux/bitops.h> | 41 | #include <linux/bitops.h> |
| 42 | #include <linux/iomap.h> | 42 | #include <linux/iomap.h> |
| 43 | #include <linux/iversion.h> | 43 | #include <linux/iversion.h> |
| 44 | #include <linux/dax.h> | ||
| 44 | 45 | ||
| 45 | #include "ext4_jbd2.h" | 46 | #include "ext4_jbd2.h" |
| 46 | #include "xattr.h" | 47 | #include "xattr.h" |
| @@ -3780,8 +3781,8 @@ static int ext4_block_zero_page_range(handle_t *handle, · 3733 unmodified lines | |||
| 3780 | length = max; | 3781 | length = max; |
| 3781 | 3782 | ||
| 3782 | if (IS_DAX(inode)) { | 3783 | if (IS_DAX(inode)) { |
| 3783 | return iomap_zero_range(inode, from, length, NULL, | 3784 | return dax_zero_range(inode, from, length, NULL, |
| 3784 | &ext4_iomap_ops); | 3785 | &ext4_iomap_ops); |
| 3785 | } | 3786 | } |
| 3786 | return __ext4_block_zero_page_range(handle, mapping, from, length); | 3787 | return __ext4_block_zero_page_range(handle, mapping, from, length); |
| 3787 | } | 3788 | } |
| @@ -870,26 +870,8 @@ iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, · 869 unmodified lines | |||
| 870 | } | 870 | } |
| 871 | EXPORT_SYMBOL_GPL(iomap_file_unshare); | 871 | EXPORT_SYMBOL_GPL(iomap_file_unshare); |
| 872 | 872 | ||
| 873 | static s64 __iomap_zero_iter(struct iomap_iter *iter, loff_t pos, u64 length) | ||
| 874 | { | ||
| 875 | struct page *page; | ||
| 876 | int status; | ||
| 877 | unsigned offset = offset_in_page(pos); | ||
| 878 | unsigned bytes = min_t(u64, PAGE_SIZE - offset, length); | ||
| 879 | |||
| 880 | status = iomap_write_begin(iter, pos, bytes, &page); | ||
| 881 | if (status) | ||
| 882 | return status; | ||
| 883 | |||
| 884 | zero_user(page, offset, bytes); | ||
| 885 | mark_page_accessed(page); | ||
| 886 | |||
| 887 | return iomap_write_end(iter, pos, bytes, bytes, page); | ||
| 888 | } | ||
| 889 | |||
| 890 | static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero) | 873 | static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero) |
| 891 | { | 874 | { |
| 892 | struct iomap *iomap = &iter->iomap; | ||
| 893 | const struct iomap *srcmap = iomap_iter_srcmap(iter); | 875 | const struct iomap *srcmap = iomap_iter_srcmap(iter); |
| 894 | loff_t pos = iter->pos; | 876 | loff_t pos = iter->pos; |
| 895 | loff_t length = iomap_length(iter); | 877 | loff_t length = iomap_length(iter); |
| @@ -900,12 +882,19 @@ static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero) · 4 unmodified lines | |||
| 900 | return length; | 882 | return length; |
| 901 | 883 | ||
| 902 | do { | 884 | do { |
| 903 | s64 bytes; | 885 | unsigned offset = offset_in_page(pos); |
| 886 | size_t bytes = min_t(u64, PAGE_SIZE - offset, length); | ||
| 887 | struct page *page; | ||
| 888 | int status; | ||
| 904 | 889 | ||
| 905 | if (IS_DAX(iter->inode)) | 890 | status = iomap_write_begin(iter, pos, bytes, &page); |
| 906 | bytes = dax_iomap_zero(pos, length, iomap); | 891 | if (status) |
| 907 | else | 892 | return status; |
| 908 | bytes = __iomap_zero_iter(iter, pos, length); | 893 | |
| 894 | zero_user(page, offset, bytes); | ||
| 895 | mark_page_accessed(page); | ||
| 896 | |||
| 897 | bytes = iomap_write_end(iter, pos, bytes, bytes, page); | ||
| 909 | if (bytes < 0) | 898 | if (bytes < 0) |
| 910 | return bytes; | 899 | return bytes; |
| 911 | 900 | ||
| @@ -28,7 +28,6 @@ · 27 unmodified lines | |||
| 28 | #include "xfs_dquot.h" | 28 | #include "xfs_dquot.h" |
| 29 | #include "xfs_reflink.h" | 29 | #include "xfs_reflink.h" |
| 30 | 30 | ||
| 31 | |||
| 32 | #define XFS_ALLOC_ALIGN(mp, off) \ | 31 | #define XFS_ALLOC_ALIGN(mp, off) \ |
| 33 | (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log) | 32 | (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log) |
| 34 | 33 | ||
| @@ -1321,6 +1320,9 @@ xfs_zero_range( · 1286 unmodified lines | |||
| 1321 | { | 1320 | { |
| 1322 | struct inode *inode = VFS_I(ip); | 1321 | struct inode *inode = VFS_I(ip); |
| 1323 | 1322 | ||
| 1323 | if (IS_DAX(inode)) | ||
| 1324 | return dax_zero_range(inode, pos, len, did_zero, | ||
| 1325 | &xfs_buffered_write_iomap_ops); | ||
| 1324 | return iomap_zero_range(inode, pos, len, did_zero, | 1326 | return iomap_zero_range(inode, pos, len, did_zero, |
| 1325 | &xfs_buffered_write_iomap_ops); | 1327 | &xfs_buffered_write_iomap_ops); |
| 1326 | } | 1328 | } |
| @@ -1333,6 +1335,9 @@ xfs_truncate_page( · 6 unmodified lines | |||
| 1333 | { | 1335 | { |
| 1334 | struct inode *inode = VFS_I(ip); | 1336 | struct inode *inode = VFS_I(ip); |
| 1335 | 1337 | ||
| 1338 | if (IS_DAX(inode)) | ||
| 1339 | return dax_truncate_page(inode, pos, did_zero, | ||
| 1340 | &xfs_buffered_write_iomap_ops); | ||
| 1336 | return iomap_truncate_page(inode, pos, did_zero, | 1341 | return iomap_truncate_page(inode, pos, did_zero, |
| 1337 | &xfs_buffered_write_iomap_ops); | 1342 | &xfs_buffered_write_iomap_ops); |
| 1338 | } | 1343 | } |
| @@ -14,6 +14,7 @@ typedef unsigned long dax_entry_t; · 13 unmodified lines | |||
| 14 | struct dax_device; | 14 | struct dax_device; |
| 15 | struct gendisk; | 15 | struct gendisk; |
| 16 | struct iomap_ops; | 16 | struct iomap_ops; |
| 17 | struct iomap_iter; | ||
| 17 | struct iomap; | 18 | struct iomap; |
| 18 | 19 | ||
| 19 | struct dax_operations { | 20 | struct dax_operations { |
| @@ -170,6 +171,11 @@ static inline void dax_unlock_page(struct page *page, dax_entry_t cookie) · 150 unmodified lines | |||
| 170 | } | 171 | } |
| 171 | #endif | 172 | #endif |
| 172 | 173 | ||
| 174 | int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, | ||
| 175 | const struct iomap_ops *ops); | ||
| 176 | int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, | ||
| 177 | const struct iomap_ops *ops); | ||
| 178 | |||
| 173 | #if IS_ENABLED(CONFIG_DAX) | 179 | #if IS_ENABLED(CONFIG_DAX) |
| 174 | int dax_read_lock(void); | 180 | int dax_read_lock(void); |
| 175 | void dax_read_unlock(int id); | 181 | void dax_read_unlock(int id); |
| @@ -204,7 +210,6 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, · 28 unmodified lines | |||
| 204 | int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); | 210 | int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); |
| 205 | int dax_invalidate_mapping_entry_sync(struct address_space *mapping, | 211 | int dax_invalidate_mapping_entry_sync(struct address_space *mapping, |
| 206 | pgoff_t index); | 212 | pgoff_t index); |
| 207 | s64 dax_iomap_zero(loff_t pos, u64 length, struct iomap *iomap); | ||
| 208 | static inline bool dax_mapping(struct address_space *mapping) | 213 | static inline bool dax_mapping(struct address_space *mapping) |
| 209 | { | 214 | { |
| 210 | return mapping->host && IS_DAX(mapping->host); | 215 | return mapping->host && IS_DAX(mapping->host); |