iomap: lift the xfs writeback code to iomap

Take the xfs writeback code and move it to fs/iomap. A new structure with three methods is added as the abstraction from the generic writeback code to the file system. These methods are used to map blocks, submit an ioend, and cancel a page that encountered an error before it was added to an ioend.

Christoph Hellwig committed 7 years ago

Commit 598ecfb1 parent 9e91c577 files · +669 −648Browse files

Unified · Split

7 files

fs/iomap/buffered-io.c +550 −1
fs/iomap/trace.h +1 −0
fs/xfs/xfs_aops.c +58 −581
fs/xfs/xfs_aops.h +0 −17
fs/xfs/xfs_super.c +1 −10
fs/xfs/xfs_trace.h +0 −39
include/linux/iomap.h +59 −0

fs/iomap/buffered-io.c+550−1

@@ -1,7 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/*
3	3	* Copyright (C) 2010 Red Hat, Inc.
4		− * Copyright (c) 2016-2018 Christoph Hellwig.
	4	+ * Copyright (C) 2016-2019 Christoph Hellwig.
5	5	*/
6	6	#include <linux/module.h>
7	7	#include <linux/compiler.h>
@@ -12,6 +12,7 @@ · 4 unmodified lines
12	12	#include <linux/buffer_head.h>
13	13	#include <linux/dax.h>
14	14	#include <linux/writeback.h>
	15	+#include <linux/list_sort.h>
15	16	#include <linux/swap.h>
16	17	#include <linux/bio.h>
17	18	#include <linux/sched/signal.h>
@@ -20,6 +21,8 @@ · 2 unmodified lines
20	21
21	22	#include "../internal.h"
22	23
	24	+static struct bio_set iomap_ioend_bioset;
	25	+
23	26	static struct iomap_page *
24	27	iomap_page_create(struct inode inode, struct page page)
25	28	{
@@ -1088,3 +1091,549 @@ out_unlock: · 1062 unmodified lines
1088	1091	return block_page_mkwrite_return(ret);
1089	1092	}
1090	1093	EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
	1094	+
	1095	+static void
	1096	+iomap_finish_page_writeback(struct inode inode, struct bio_vec bvec,
	1097	+ int error)
	1098	+{
	1099	+ struct iomap_page *iop = to_iomap_page(bvec->bv_page);
	1100	+
	1101	+ if (error) {
	1102	+ SetPageError(bvec->bv_page);
	1103	+ mapping_set_error(inode->i_mapping, -EIO);
	1104	+ }
	1105	+
	1106	+ WARN_ON_ONCE(i_blocksize(inode) < PAGE_SIZE && !iop);
	1107	+ WARN_ON_ONCE(iop && atomic_read(&iop->write_count) <= 0);
	1108	+
	1109	+ if (!iop \|\| atomic_dec_and_test(&iop->write_count))
	1110	+ end_page_writeback(bvec->bv_page);
	1111	+}
	1112	+
	1113	+/*
	1114	+ * We're now finished for good with this ioend structure. Update the page
	1115	+ * state, release holds on bios, and finally free up memory. Do not use the
	1116	+ * ioend after this.
	1117	+ */
	1118	+static void
	1119	+iomap_finish_ioend(struct iomap_ioend *ioend, int error)
	1120	+{
	1121	+ struct inode *inode = ioend->io_inode;
	1122	+ struct bio *bio = &ioend->io_inline_bio;
	1123	+ struct bio last = ioend->io_bio, next;
	1124	+ u64 start = bio->bi_iter.bi_sector;
	1125	+ bool quiet = bio_flagged(bio, BIO_QUIET);
	1126	+
	1127	+ for (bio = &ioend->io_inline_bio; bio; bio = next) {
	1128	+ struct bio_vec *bv;
	1129	+ struct bvec_iter_all iter_all;
	1130	+
	1131	+ /*
	1132	+ * For the last bio, bi_private points to the ioend, so we
	1133	+ * need to explicitly end the iteration here.
	1134	+ */
	1135	+ if (bio == last)
	1136	+ next = NULL;
	1137	+ else
	1138	+ next = bio->bi_private;
	1139	+
	1140	+ /* walk each page on bio, ending page IO on them */
	1141	+ bio_for_each_segment_all(bv, bio, iter_all)
	1142	+ iomap_finish_page_writeback(inode, bv, error);
	1143	+ bio_put(bio);
	1144	+ }
	1145	+
	1146	+ if (unlikely(error && !quiet)) {
	1147	+ printk_ratelimited(KERN_ERR
	1148	+ "%s: writeback error on sector %llu",
	1149	+ inode->i_sb->s_id, start);
	1150	+ }
	1151	+}
	1152	+
	1153	+void
	1154	+iomap_finish_ioends(struct iomap_ioend *ioend, int error)
	1155	+{
	1156	+ struct list_head tmp;
	1157	+
	1158	+ list_replace_init(&ioend->io_list, &tmp);
	1159	+ iomap_finish_ioend(ioend, error);
	1160	+
	1161	+ while (!list_empty(&tmp)) {
	1162	+ ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
	1163	+ list_del_init(&ioend->io_list);
	1164	+ iomap_finish_ioend(ioend, error);
	1165	+ }
	1166	+}
	1167	+EXPORT_SYMBOL_GPL(iomap_finish_ioends);
	1168	+
	1169	+/*
	1170	+ * We can merge two adjacent ioends if they have the same set of work to do.
	1171	+ */
	1172	+static bool
	1173	+iomap_ioend_can_merge(struct iomap_ioend ioend, struct iomap_ioend next)
	1174	+{
	1175	+ if (ioend->io_bio->bi_status != next->io_bio->bi_status)
	1176	+ return false;
	1177	+ if ((ioend->io_flags & IOMAP_F_SHARED) ^
	1178	+ (next->io_flags & IOMAP_F_SHARED))
	1179	+ return false;
	1180	+ if ((ioend->io_type == IOMAP_UNWRITTEN) ^
	1181	+ (next->io_type == IOMAP_UNWRITTEN))
	1182	+ return false;
	1183	+ if (ioend->io_offset + ioend->io_size != next->io_offset)
	1184	+ return false;
	1185	+ return true;
	1186	+}
	1187	+
	1188	+void
	1189	+iomap_ioend_try_merge(struct iomap_ioend ioend, struct list_head more_ioends,
	1190	+ void (merge_private)(struct iomap_ioend ioend,
	1191	+ struct iomap_ioend *next))
	1192	+{
	1193	+ struct iomap_ioend *next;
	1194	+
	1195	+ INIT_LIST_HEAD(&ioend->io_list);
	1196	+
	1197	+ while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
	1198	+ io_list))) {
	1199	+ if (!iomap_ioend_can_merge(ioend, next))
	1200	+ break;
	1201	+ list_move_tail(&next->io_list, &ioend->io_list);
	1202	+ ioend->io_size += next->io_size;
	1203	+ if (next->io_private && merge_private)
	1204	+ merge_private(ioend, next);
	1205	+ }
	1206	+}
	1207	+EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
	1208	+
	1209	+static int
	1210	+iomap_ioend_compare(void priv, struct list_head a, struct list_head *b)
	1211	+{
	1212	+ struct iomap_ioend ia, ib;
	1213	+
	1214	+ ia = container_of(a, struct iomap_ioend, io_list);
	1215	+ ib = container_of(b, struct iomap_ioend, io_list);
	1216	+ if (ia->io_offset < ib->io_offset)
	1217	+ return -1;
	1218	+ else if (ia->io_offset > ib->io_offset)
	1219	+ return 1;
	1220	+ return 0;
	1221	+}
	1222	+
	1223	+void
	1224	+iomap_sort_ioends(struct list_head *ioend_list)
	1225	+{
	1226	+ list_sort(NULL, ioend_list, iomap_ioend_compare);
	1227	+}
	1228	+EXPORT_SYMBOL_GPL(iomap_sort_ioends);
	1229	+
	1230	+static void iomap_writepage_end_bio(struct bio *bio)
	1231	+{
	1232	+ struct iomap_ioend *ioend = bio->bi_private;
	1233	+
	1234	+ iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status));
	1235	+}
	1236	+
	1237	+/*
	1238	+ * Submit the final bio for an ioend.
	1239	+ *
	1240	+ * If @error is non-zero, it means that we have a situation where some part of
	1241	+ * the submission process has failed after we have marked paged for writeback
	1242	+ * and unlocked them. In this situation, we need to fail the bio instead of
	1243	+ * submitting it. This typically only happens on a filesystem shutdown.
	1244	+ */
	1245	+static int
	1246	+iomap_submit_ioend(struct iomap_writepage_ctx wpc, struct iomap_ioend ioend,
	1247	+ int error)
	1248	+{
	1249	+ ioend->io_bio->bi_private = ioend;
	1250	+ ioend->io_bio->bi_end_io = iomap_writepage_end_bio;
	1251	+
	1252	+ if (wpc->ops->prepare_ioend)
	1253	+ error = wpc->ops->prepare_ioend(ioend, error);
	1254	+ if (error) {
	1255	+ /*
	1256	+ * If we are failing the IO now, just mark the ioend with an
	1257	+ * error and finish it. This will run IO completion immediately
	1258	+ * as there is only one reference to the ioend at this point in
	1259	+ * time.
	1260	+ */
	1261	+ ioend->io_bio->bi_status = errno_to_blk_status(error);
	1262	+ bio_endio(ioend->io_bio);
	1263	+ return error;
	1264	+ }
	1265	+
	1266	+ submit_bio(ioend->io_bio);
	1267	+ return 0;
	1268	+}
	1269	+
	1270	+static struct iomap_ioend *
	1271	+iomap_alloc_ioend(struct inode inode, struct iomap_writepage_ctx wpc,
	1272	+ loff_t offset, sector_t sector, struct writeback_control *wbc)
	1273	+{
	1274	+ struct iomap_ioend *ioend;
	1275	+ struct bio *bio;
	1276	+
	1277	+ bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &iomap_ioend_bioset);
	1278	+ bio_set_dev(bio, wpc->iomap.bdev);
	1279	+ bio->bi_iter.bi_sector = sector;
	1280	+ bio->bi_opf = REQ_OP_WRITE \| wbc_to_write_flags(wbc);
	1281	+ bio->bi_write_hint = inode->i_write_hint;
	1282	+ wbc_init_bio(wbc, bio);
	1283	+
	1284	+ ioend = container_of(bio, struct iomap_ioend, io_inline_bio);
	1285	+ INIT_LIST_HEAD(&ioend->io_list);
	1286	+ ioend->io_type = wpc->iomap.type;
	1287	+ ioend->io_flags = wpc->iomap.flags;
	1288	+ ioend->io_inode = inode;
	1289	+ ioend->io_size = 0;
	1290	+ ioend->io_offset = offset;
	1291	+ ioend->io_private = NULL;
	1292	+ ioend->io_bio = bio;
	1293	+ return ioend;
	1294	+}
	1295	+
	1296	+/*
	1297	+ * Allocate a new bio, and chain the old bio to the new one.
	1298	+ *
	1299	+ * Note that we have to do perform the chaining in this unintuitive order
	1300	+ * so that the bi_private linkage is set up in the right direction for the
	1301	+ * traversal in iomap_finish_ioend().
	1302	+ */
	1303	+static struct bio *
	1304	+iomap_chain_bio(struct bio *prev)
	1305	+{
	1306	+ struct bio *new;
	1307	+
	1308	+ new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
	1309	+ bio_copy_dev(new, prev);/* also copies over blkcg information */
	1310	+ new->bi_iter.bi_sector = bio_end_sector(prev);
	1311	+ new->bi_opf = prev->bi_opf;
	1312	+ new->bi_write_hint = prev->bi_write_hint;
	1313	+
	1314	+ bio_chain(prev, new);
	1315	+ bio_get(prev); /* for iomap_finish_ioend */
	1316	+ submit_bio(prev);
	1317	+ return new;
	1318	+}
	1319	+
	1320	+static bool
	1321	+iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset,
	1322	+ sector_t sector)
	1323	+{
	1324	+ if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
	1325	+ (wpc->ioend->io_flags & IOMAP_F_SHARED))
	1326	+ return false;
	1327	+ if (wpc->iomap.type != wpc->ioend->io_type)
	1328	+ return false;
	1329	+ if (offset != wpc->ioend->io_offset + wpc->ioend->io_size)
	1330	+ return false;
	1331	+ if (sector != bio_end_sector(wpc->ioend->io_bio))
	1332	+ return false;
	1333	+ return true;
	1334	+}
	1335	+
	1336	+/*
	1337	+ * Test to see if we have an existing ioend structure that we could append to
	1338	+ * first, otherwise finish off the current ioend and start another.
	1339	+ */
	1340	+static void
	1341	+iomap_add_to_ioend(struct inode inode, loff_t offset, struct page page,
	1342	+ struct iomap_page iop, struct iomap_writepage_ctx wpc,
	1343	+ struct writeback_control wbc, struct list_head iolist)
	1344	+{
	1345	+ sector_t sector = iomap_sector(&wpc->iomap, offset);
	1346	+ unsigned len = i_blocksize(inode);
	1347	+ unsigned poff = offset & (PAGE_SIZE - 1);
	1348	+ bool merged, same_page = false;
	1349	+
	1350	+ if (!wpc->ioend \|\| !iomap_can_add_to_ioend(wpc, offset, sector)) {
	1351	+ if (wpc->ioend)
	1352	+ list_add(&wpc->ioend->io_list, iolist);
	1353	+ wpc->ioend = iomap_alloc_ioend(inode, wpc, offset, sector, wbc);
	1354	+ }
	1355	+
	1356	+ merged = __bio_try_merge_page(wpc->ioend->io_bio, page, len, poff,
	1357	+ &same_page);
	1358	+ if (iop && !same_page)
	1359	+ atomic_inc(&iop->write_count);
	1360	+
	1361	+ if (!merged) {
	1362	+ if (bio_full(wpc->ioend->io_bio, len)) {
	1363	+ wpc->ioend->io_bio =
	1364	+ iomap_chain_bio(wpc->ioend->io_bio);
	1365	+ }
	1366	+ bio_add_page(wpc->ioend->io_bio, page, len, poff);
	1367	+ }
	1368	+
	1369	+ wpc->ioend->io_size += len;
	1370	+ wbc_account_cgroup_owner(wbc, page, len);
	1371	+}
	1372	+
	1373	+/*
	1374	+ * We implement an immediate ioend submission policy here to avoid needing to
	1375	+ * chain multiple ioends and hence nest mempool allocations which can violate
	1376	+ * forward progress guarantees we need to provide. The current ioend we are
	1377	+ * adding blocks to is cached on the writepage context, and if the new block
	1378	+ * does not append to the cached ioend it will create a new ioend and cache that
	1379	+ * instead.
	1380	+ *
	1381	+ * If a new ioend is created and cached, the old ioend is returned and queued
	1382	+ * locally for submission once the entire page is processed or an error has been
	1383	+ * detected. While ioends are submitted immediately after they are completed,
	1384	+ * batching optimisations are provided by higher level block plugging.
	1385	+ *
	1386	+ * At the end of a writeback pass, there will be a cached ioend remaining on the
	1387	+ * writepage context that the caller will need to submit.
	1388	+ */
	1389	+static int
	1390	+iomap_writepage_map(struct iomap_writepage_ctx *wpc,
	1391	+ struct writeback_control wbc, struct inode inode,
	1392	+ struct page *page, u64 end_offset)
	1393	+{
	1394	+ struct iomap_page *iop = to_iomap_page(page);
	1395	+ struct iomap_ioend ioend, next;
	1396	+ unsigned len = i_blocksize(inode);
	1397	+ u64 file_offset; /* file offset of page */
	1398	+ int error = 0, count = 0, i;
	1399	+ LIST_HEAD(submit_list);
	1400	+
	1401	+ WARN_ON_ONCE(i_blocksize(inode) < PAGE_SIZE && !iop);
	1402	+ WARN_ON_ONCE(iop && atomic_read(&iop->write_count) != 0);
	1403	+
	1404	+ /*
	1405	+ * Walk through the page to find areas to write back. If we run off the
	1406	+ * end of the current map or find the current map invalid, grab a new
	1407	+ * one.
	1408	+ */
	1409	+ for (i = 0, file_offset = page_offset(page);
	1410	+ i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset;
	1411	+ i++, file_offset += len) {
	1412	+ if (iop && !test_bit(i, iop->uptodate))
	1413	+ continue;
	1414	+
	1415	+ error = wpc->ops->map_blocks(wpc, inode, file_offset);
	1416	+ if (error)
	1417	+ break;
	1418	+ if (wpc->iomap.type == IOMAP_HOLE)
	1419	+ continue;
	1420	+ iomap_add_to_ioend(inode, file_offset, page, iop, wpc, wbc,
	1421	+ &submit_list);
	1422	+ count++;
	1423	+ }
	1424	+
	1425	+ WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list));
	1426	+ WARN_ON_ONCE(!PageLocked(page));
	1427	+ WARN_ON_ONCE(PageWriteback(page));
	1428	+
	1429	+ /*
	1430	+ * We cannot cancel the ioend directly here on error. We may have
	1431	+ * already set other pages under writeback and hence we have to run I/O
	1432	+ * completion to mark the error state of the pages under writeback
	1433	+ * appropriately.
	1434	+ */
	1435	+ if (unlikely(error)) {
	1436	+ if (!count) {
	1437	+ /*
	1438	+ * If the current page hasn't been added to ioend, it
	1439	+ * won't be affected by I/O completions and we must
	1440	+ * discard and unlock it right here.
	1441	+ */
	1442	+ if (wpc->ops->discard_page)
	1443	+ wpc->ops->discard_page(page);
	1444	+ ClearPageUptodate(page);
	1445	+ unlock_page(page);
	1446	+ goto done;
	1447	+ }
	1448	+
	1449	+ /*
	1450	+ * If the page was not fully cleaned, we need to ensure that the
	1451	+ * higher layers come back to it correctly. That means we need
	1452	+ * to keep the page dirty, and for WB_SYNC_ALL writeback we need
	1453	+ * to ensure the PAGECACHE_TAG_TOWRITE index mark is not removed
	1454	+ * so another attempt to write this page in this writeback sweep
	1455	+ * will be made.
	1456	+ */
	1457	+ set_page_writeback_keepwrite(page);
	1458	+ } else {
	1459	+ clear_page_dirty_for_io(page);
	1460	+ set_page_writeback(page);
	1461	+ }
	1462	+
	1463	+ unlock_page(page);
	1464	+
	1465	+ /*
	1466	+ * Preserve the original error if there was one, otherwise catch
	1467	+ * submission errors here and propagate into subsequent ioend
	1468	+ * submissions.
	1469	+ */
	1470	+ list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
	1471	+ int error2;
	1472	+
	1473	+ list_del_init(&ioend->io_list);
	1474	+ error2 = iomap_submit_ioend(wpc, ioend, error);
	1475	+ if (error2 && !error)
	1476	+ error = error2;
	1477	+ }
	1478	+
	1479	+ /*
	1480	+ * We can end up here with no error and nothing to write only if we race
	1481	+ * with a partial page truncate on a sub-page block sized filesystem.
	1482	+ */
	1483	+ if (!count)
	1484	+ end_page_writeback(page);
	1485	+done:
	1486	+ mapping_set_error(page->mapping, error);
	1487	+ return error;
	1488	+}
	1489	+
	1490	+/*
	1491	+ * Write out a dirty page.
	1492	+ *
	1493	+ * For delalloc space on the page we need to allocate space and flush it.
	1494	+ * For unwritten space on the page we need to start the conversion to
	1495	+ * regular allocated space.
	1496	+ */
	1497	+static int
	1498	+iomap_do_writepage(struct page page, struct writeback_control wbc, void *data)
	1499	+{
	1500	+ struct iomap_writepage_ctx *wpc = data;
	1501	+ struct inode *inode = page->mapping->host;
	1502	+ pgoff_t end_index;
	1503	+ u64 end_offset;
	1504	+ loff_t offset;
	1505	+
	1506	+ trace_iomap_writepage(inode, page, 0, 0);
	1507	+
	1508	+ /*
	1509	+ * Refuse to write the page out if we are called from reclaim context.
	1510	+ *
	1511	+ * This avoids stack overflows when called from deeply used stacks in
	1512	+ * random callers for direct reclaim or memcg reclaim. We explicitly
	1513	+ * allow reclaim from kswapd as the stack usage there is relatively low.
	1514	+ *
	1515	+ * This should never happen except in the case of a VM regression so
	1516	+ * warn about it.
	1517	+ */
	1518	+ if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC\|PF_KSWAPD)) ==
	1519	+ PF_MEMALLOC))
	1520	+ goto redirty;
	1521	+
	1522	+ /*
	1523	+ * Given that we do not allow direct reclaim to call us, we should
	1524	+ * never be called in a recursive filesystem reclaim context.
	1525	+ */
	1526	+ if (WARN_ON_ONCE(current->flags & PF_MEMALLOC_NOFS))
	1527	+ goto redirty;
	1528	+
	1529	+ /*
	1530	+ * Is this page beyond the end of the file?
	1531	+ *
	1532	+ * The page index is less than the end_index, adjust the end_offset
	1533	+ * to the highest offset that this page should represent.
	1534	+ * -----------------------------------------------------
	1535	+ * \| file mapping \| <EOF> \|
	1536	+ * -----------------------------------------------------
	1537	+ * \| Page ... \| Page N-2 \| Page N-1 \| Page N \| \|
	1538	+ * ^--------------------------------^----------\|--------
	1539	+ * \| desired writeback range \| see else \|
	1540	+ * ---------------------------------^------------------\|
	1541	+ */
	1542	+ offset = i_size_read(inode);
	1543	+ end_index = offset >> PAGE_SHIFT;
	1544	+ if (page->index < end_index)
	1545	+ end_offset = (loff_t)(page->index + 1) << PAGE_SHIFT;
	1546	+ else {
	1547	+ /*
	1548	+ * Check whether the page to write out is beyond or straddles
	1549	+ * i_size or not.
	1550	+ * -------------------------------------------------------
	1551	+ * \| file mapping \| <EOF> \|
	1552	+ * -------------------------------------------------------
	1553	+ * \| Page ... \| Page N-2 \| Page N-1 \| Page N \| Beyond \|
	1554	+ * ^--------------------------------^-----------\|---------
	1555	+ * \| \| Straddles \|
	1556	+ * ---------------------------------^-----------\|--------\|
	1557	+ */
	1558	+ unsigned offset_into_page = offset & (PAGE_SIZE - 1);
	1559	+
	1560	+ /*
	1561	+ * Skip the page if it is fully outside i_size, e.g. due to a
	1562	+ * truncate operation that is in progress. We must redirty the
	1563	+ * page so that reclaim stops reclaiming it. Otherwise
	1564	+ * iomap_vm_releasepage() is called on it and gets confused.
	1565	+ *
	1566	+ * Note that the end_index is unsigned long, it would overflow
	1567	+ * if the given offset is greater than 16TB on 32-bit system
	1568	+ * and if we do check the page is fully outside i_size or not
	1569	+ * via "if (page->index >= end_index + 1)" as "end_index + 1"
	1570	+ * will be evaluated to 0. Hence this page will be redirtied
	1571	+ * and be written out repeatedly which would result in an
	1572	+ * infinite loop, the user program that perform this operation
	1573	+ * will hang. Instead, we can verify this situation by checking
	1574	+ * if the page to write is totally beyond the i_size or if it's
	1575	+ * offset is just equal to the EOF.
	1576	+ */
	1577	+ if (page->index > end_index \|\|
	1578	+ (page->index == end_index && offset_into_page == 0))
	1579	+ goto redirty;
	1580	+
	1581	+ /*
	1582	+ * The page straddles i_size. It must be zeroed out on each
	1583	+ * and every writepage invocation because it may be mmapped.
	1584	+ * "A file is mapped in multiples of the page size. For a file
	1585	+ * that is not a multiple of the page size, the remaining
	1586	+ * memory is zeroed when mapped, and writes to that region are
	1587	+ * not written out to the file."
	1588	+ */
	1589	+ zero_user_segment(page, offset_into_page, PAGE_SIZE);
	1590	+
	1591	+ /* Adjust the end_offset to the end of file */
	1592	+ end_offset = offset;
	1593	+ }
	1594	+
	1595	+ return iomap_writepage_map(wpc, wbc, inode, page, end_offset);
	1596	+
	1597	+redirty:
	1598	+ redirty_page_for_writepage(wbc, page);
	1599	+ unlock_page(page);
	1600	+ return 0;
	1601	+}
	1602	+
	1603	+int
	1604	+iomap_writepage(struct page page, struct writeback_control wbc,
	1605	+ struct iomap_writepage_ctx *wpc,
	1606	+ const struct iomap_writeback_ops *ops)
	1607	+{
	1608	+ int ret;
	1609	+
	1610	+ wpc->ops = ops;
	1611	+ ret = iomap_do_writepage(page, wbc, wpc);
	1612	+ if (!wpc->ioend)
	1613	+ return ret;
	1614	+ return iomap_submit_ioend(wpc, wpc->ioend, ret);
	1615	+}
	1616	+EXPORT_SYMBOL_GPL(iomap_writepage);
	1617	+
	1618	+int
	1619	+iomap_writepages(struct address_space mapping, struct writeback_control wbc,
	1620	+ struct iomap_writepage_ctx *wpc,
	1621	+ const struct iomap_writeback_ops *ops)
	1622	+{
	1623	+ int ret;
	1624	+
	1625	+ wpc->ops = ops;
	1626	+ ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc);
	1627	+ if (!wpc->ioend)
	1628	+ return ret;
	1629	+ return iomap_submit_ioend(wpc, wpc->ioend, ret);
	1630	+}
	1631	+EXPORT_SYMBOL_GPL(iomap_writepages);
	1632	+
	1633	+static int __init iomap_init(void)
	1634	+{
	1635	+ return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
	1636	+ offsetof(struct iomap_ioend, io_inline_bio),
	1637	+ BIOSET_NEED_BVECS);
	1638	+}
	1639	+fs_initcall(iomap_init);

fs/iomap/trace.h+1−0

@@ -76,6 +76,7 @@ DEFINE_EVENT(iomap_page_class, name, \ · 75 unmodified lines
76	76	TP_PROTO(struct inode inode, struct page page, unsigned long off, \
77	77	unsigned int len), \
78	78	TP_ARGS(inode, page, off, len))
	79	+DEFINE_PAGE_EVENT(iomap_writepage);
79	80	DEFINE_PAGE_EVENT(iomap_releasepage);
80	81	DEFINE_PAGE_EVENT(iomap_invalidatepage);
81	82

fs/xfs/xfs_aops.c+58−581

@@ -18,16 +18,18 @@ · 17 unmodified lines
18	18	#include "xfs_bmap_util.h"
19	19	#include "xfs_reflink.h"
20	20
21		−/*
22		− * structure owned by writepages passed to individual writepage calls
23		− */
24	21	struct xfs_writepage_ctx {
25		− struct iomap iomap;
	22	+ struct iomap_writepage_ctx ctx;
26	23	unsigned int data_seq;
27	24	unsigned int cow_seq;
28		− struct xfs_ioend *ioend;
29	25	};
30	26
	27	+static inline struct xfs_writepage_ctx *
	28	+XFS_WPC(struct iomap_writepage_ctx *ctx)
	29	+{
	30	+ return container_of(ctx, struct xfs_writepage_ctx, ctx);
	31	+}
	32	+
31	33	struct block_device *
32	34	xfs_find_bdev_for_inode(
33	35	struct inode *inode)
@@ -54,87 +56,10 @@ xfs_find_daxdev_for_inode( · 20 unmodified lines
54	56	return mp->m_ddev_targp->bt_daxdev;
55	57	}
56	58
57		−static void
58		−xfs_finish_page_writeback(
59		− struct inode *inode,
60		− struct bio_vec *bvec,
61		− int error)
62		−{
63		− struct iomap_page *iop = to_iomap_page(bvec->bv_page);
64		−
65		− if (error) {
66		− SetPageError(bvec->bv_page);
67		− mapping_set_error(inode->i_mapping, -EIO);
68		− }
69		−
70		− ASSERT(iop \|\| i_blocksize(inode) == PAGE_SIZE);
71		− ASSERT(!iop \|\| atomic_read(&iop->write_count) > 0);
72		−
73		− if (!iop \|\| atomic_dec_and_test(&iop->write_count))
74		− end_page_writeback(bvec->bv_page);
75		−}
76		−
77		−/*
78		− * We're now finished for good with this ioend structure. Update the page
79		− * state, release holds on bios, and finally free up memory. Do not use the
80		− * ioend after this.
81		− */
82		−STATIC void
83		−xfs_destroy_ioend(
84		− struct xfs_ioend *ioend,
85		− int error)
86		−{
87		− struct inode *inode = ioend->io_inode;
88		− struct bio *bio = &ioend->io_inline_bio;
89		− struct bio last = ioend->io_bio, next;
90		− u64 start = bio->bi_iter.bi_sector;
91		− bool quiet = bio_flagged(bio, BIO_QUIET);
92		−
93		− for (bio = &ioend->io_inline_bio; bio; bio = next) {
94		− struct bio_vec *bvec;
95		− struct bvec_iter_all iter_all;
96		−
97		− /*
98		− * For the last bio, bi_private points to the ioend, so we
99		− * need to explicitly end the iteration here.
100		− */
101		− if (bio == last)
102		− next = NULL;
103		− else
104		− next = bio->bi_private;
105		−
106		− /* walk each page on bio, ending page IO on them */
107		− bio_for_each_segment_all(bvec, bio, iter_all)
108		− xfs_finish_page_writeback(inode, bvec, error);
109		− bio_put(bio);
110		− }
111		−
112		− if (unlikely(error && !quiet)) {
113		− xfs_err_ratelimited(XFS_I(inode)->i_mount,
114		− "writeback error on sector %llu", start);
115		− }
116		−}
117		−
118		−static void
119		−xfs_destroy_ioends(
120		− struct xfs_ioend *ioend,
121		− int error)
122		−{
123		− struct list_head tmp;
124		−
125		− list_replace_init(&ioend->io_list, &tmp);
126		− xfs_destroy_ioend(ioend, error);
127		− while ((ioend = list_first_entry_or_null(&tmp, struct xfs_ioend,
128		− io_list))) {
129		− list_del_init(&ioend->io_list);
130		− xfs_destroy_ioend(ioend, error);
131		− }
132		−}
133		−
134	59	/*
135	60	* Fast and loose check if this write could update the on-disk inode size.
136	61	*/
137		−static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend)
	62	+static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
138	63	{
139	64	return ioend->io_offset + ioend->io_size >
140	65	XFS_I(ioend->io_inode)->i_d.di_size;
@@ -142,7 +67,7 @@ static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend) · 1 unmodified line
142	67
143	68	STATIC int
144	69	xfs_setfilesize_trans_alloc(
145		− struct xfs_ioend *ioend)
	70	+ struct iomap_ioend *ioend)
146	71	{
147	72	struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount;
148	73	struct xfs_trans *tp;
@@ -215,7 +140,7 @@ xfs_setfilesize( · 66 unmodified lines
215	140
216	141	STATIC int
217	142	xfs_setfilesize_ioend(
218		− struct xfs_ioend *ioend,
	143	+ struct iomap_ioend *ioend,
219	144	int error)
220	145	{
221	146	struct xfs_inode *ip = XFS_I(ioend->io_inode);
@@ -243,7 +168,7 @@ xfs_setfilesize_ioend( · 21 unmodified lines
243	168	*/
244	169	STATIC void
245	170	xfs_end_ioend(
246		− struct xfs_ioend *ioend)
	171	+ struct iomap_ioend *ioend)
247	172	{
248	173	struct xfs_inode *ip = XFS_I(ioend->io_inode);
249	174	xfs_off_t offset = ioend->io_offset;
@@ -289,31 +214,10 @@ xfs_end_ioend( · 39 unmodified lines
289	214	done:
290	215	if (ioend->io_private)
291	216	error = xfs_setfilesize_ioend(ioend, error);
292		− xfs_destroy_ioends(ioend, error);
	217	+ iomap_finish_ioends(ioend, error);
293	218	memalloc_nofs_restore(nofs_flag);
294	219	}
295	220
296		−/*
297		− * We can merge two adjacent ioends if they have the same set of work to do.
298		− */
299		−static bool
300		−xfs_ioend_can_merge(
301		− struct xfs_ioend *ioend,
302		− struct xfs_ioend *next)
303		−{
304		− if (ioend->io_bio->bi_status != next->io_bio->bi_status)
305		− return false;
306		− if ((ioend->io_flags & IOMAP_F_SHARED) ^
307		− (next->io_flags & IOMAP_F_SHARED))
308		− return false;
309		− if ((ioend->io_type == IOMAP_UNWRITTEN) ^
310		− (next->io_type == IOMAP_UNWRITTEN))
311		− return false;
312		− if (ioend->io_offset + ioend->io_size != next->io_offset)
313		− return false;
314		− return true;
315		−}
316		−
317	221	/*
318	222	* If the to be merged ioend has a preallocated transaction for file
319	223	* size updates we need to ensure the ioend it is merged into also
@@ -322,8 +226,8 @@ xfs_ioend_can_merge( · 2 unmodified lines
322	226	*/
323	227	static void
324	228	xfs_ioend_merge_private(
325		− struct xfs_ioend *ioend,
326		− struct xfs_ioend *next)
	229	+ struct iomap_ioend *ioend,
	230	+ struct iomap_ioend *next)
327	231	{
328	232	if (!ioend->io_private) {
329	233	ioend->io_private = next->io_private;
@@ -333,53 +237,6 @@ xfs_ioend_merge_private( · 3 unmodified lines
333	237	}
334	238	}
335	239
336		−/* Try to merge adjacent completions. */
337		−STATIC void
338		−xfs_ioend_try_merge(
339		− struct xfs_ioend *ioend,
340		− struct list_head *more_ioends)
341		−{
342		− struct xfs_ioend *next;
343		−
344		− INIT_LIST_HEAD(&ioend->io_list);
345		−
346		− while ((next = list_first_entry_or_null(more_ioends, struct xfs_ioend,
347		− io_list))) {
348		− if (!xfs_ioend_can_merge(ioend, next))
349		− break;
350		− list_move_tail(&next->io_list, &ioend->io_list);
351		− ioend->io_size += next->io_size;
352		− if (next->io_private)
353		− xfs_ioend_merge_private(ioend, next);
354		− }
355		−}
356		−
357		−/* list_sort compare function for ioends */
358		−static int
359		−xfs_ioend_compare(
360		− void *priv,
361		− struct list_head *a,
362		− struct list_head *b)
363		−{
364		− struct xfs_ioend *ia;
365		− struct xfs_ioend *ib;
366		−
367		− ia = container_of(a, struct xfs_ioend, io_list);
368		− ib = container_of(b, struct xfs_ioend, io_list);
369		− if (ia->io_offset < ib->io_offset)
370		− return -1;
371		− else if (ia->io_offset > ib->io_offset)
372		− return 1;
373		− return 0;
374		−}
375		−
376		−static void
377		−xfs_sort_ioends(
378		− struct list_head *ioend_list)
379		−{
380		− list_sort(NULL, ioend_list, xfs_ioend_compare);
381		−}
382		−
383	240	/* Finish all pending io completions. */
384	241	void
385	242	xfs_end_io(
@@ -387,7 +244,7 @@ xfs_end_io( · 1 unmodified line
387	244	{
388	245	struct xfs_inode *ip =
389	246	container_of(work, struct xfs_inode, i_ioend_work);
390		− struct xfs_ioend *ioend;
	247	+ struct iomap_ioend *ioend;
391	248	struct list_head tmp;
392	249	unsigned long flags;
393	250
@@ -395,16 +252,16 @@ xfs_end_io( · 1 unmodified line
395	252	list_replace_init(&ip->i_ioend_list, &tmp);
396	253	spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
397	254
398		− xfs_sort_ioends(&tmp);
399		− while ((ioend = list_first_entry_or_null(&tmp, struct xfs_ioend,
	255	+ iomap_sort_ioends(&tmp);
	256	+ while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
400	257	io_list))) {
401	258	list_del_init(&ioend->io_list);
402		− xfs_ioend_try_merge(ioend, &tmp);
	259	+ iomap_ioend_try_merge(ioend, &tmp, xfs_ioend_merge_private);
403	260	xfs_end_ioend(ioend);
404	261	}
405	262	}
406	263
407		−static inline bool xfs_ioend_needs_workqueue(struct xfs_ioend *ioend)
	264	+static inline bool xfs_ioend_needs_workqueue(struct iomap_ioend *ioend)
408	265	{
409	266	return ioend->io_private \|\|
410	267	ioend->io_type == IOMAP_UNWRITTEN \|\|
@@ -415,20 +272,18 @@ STATIC void · 4 unmodified lines
415	272	xfs_end_bio(
416	273	struct bio *bio)
417	274	{
418		− struct xfs_ioend *ioend = bio->bi_private;
	275	+ struct iomap_ioend *ioend = bio->bi_private;
419	276	struct xfs_inode *ip = XFS_I(ioend->io_inode);
420		− struct xfs_mount *mp = ip->i_mount;
421	277	unsigned long flags;
422	278
423		− if (xfs_ioend_needs_workqueue(ioend)) {
424		− spin_lock_irqsave(&ip->i_ioend_lock, flags);
425		− if (list_empty(&ip->i_ioend_list))
426		− WARN_ON_ONCE(!queue_work(mp->m_unwritten_workqueue,
427		− &ip->i_ioend_work));
428		− list_add_tail(&ioend->io_list, &ip->i_ioend_list);
429		− spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
430		− } else
431		− xfs_destroy_ioend(ioend, blk_status_to_errno(bio->bi_status));
	279	+ ASSERT(xfs_ioend_needs_workqueue(ioend));
	280	+
	281	+ spin_lock_irqsave(&ip->i_ioend_lock, flags);
	282	+ if (list_empty(&ip->i_ioend_list))
	283	+ WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
	284	+ &ip->i_ioend_work));
	285	+ list_add_tail(&ioend->io_list, &ip->i_ioend_list);
	286	+ spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
432	287	}
433	288
434	289	/*
@@ -437,7 +292,7 @@ xfs_end_bio( · 2 unmodified lines
437	292	*/
438	293	static bool
439	294	xfs_imap_valid(
440		− struct xfs_writepage_ctx *wpc,
	295	+ struct iomap_writepage_ctx *wpc,
441	296	struct xfs_inode *ip,
442	297	loff_t offset)
443	298	{
@@ -459,10 +314,10 @@ xfs_imap_valid( · 15 unmodified lines
459	314	* checked (and found nothing at this offset) could have added
460	315	* overlapping blocks.
461	316	*/
462		− if (wpc->data_seq != READ_ONCE(ip->i_df.if_seq))
	317	+ if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq))
463	318	return false;
464	319	if (xfs_inode_has_cow_data(ip) &&
465		− wpc->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
	320	+ XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
466	321	return false;
467	322	return true;
468	323	}
@@ -477,12 +332,18 @@ xfs_imap_valid( · 8 unmodified lines
477	332	*/
478	333	static int
479	334	xfs_convert_blocks(
480		− struct xfs_writepage_ctx *wpc,
	335	+ struct iomap_writepage_ctx *wpc,
481	336	struct xfs_inode *ip,
482	337	int whichfork,
483	338	loff_t offset)
484	339	{
485	340	int error;
	341	+ unsigned *seq;
	342	+
	343	+ if (whichfork == XFS_COW_FORK)
	344	+ seq = &XFS_WPC(wpc)->cow_seq;
	345	+ else
	346	+ seq = &XFS_WPC(wpc)->data_seq;
486	347
487	348	/*
488	349	* Attempt to allocate whatever delalloc extent currently backs offset
@@ -492,8 +353,7 @@ xfs_convert_blocks( · 3 unmodified lines
492	353	*/
493	354	do {
494	355	error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
495		− &wpc->iomap, whichfork == XFS_COW_FORK ?
496		− &wpc->cow_seq : &wpc->data_seq);
	356	+ &wpc->iomap, seq);
497	357	if (error)
498	358	return error;
499	359	} while (wpc->iomap.offset + wpc->iomap.length <= offset);
@@ -501,9 +361,9 @@ xfs_convert_blocks( · 1 unmodified line
501	361	return 0;
502	362	}
503	363
504		−STATIC int
	364	+static int
505	365	xfs_map_blocks(
506		− struct xfs_writepage_ctx *wpc,
	366	+ struct iomap_writepage_ctx *wpc,
507	367	struct inode *inode,
508	368	loff_t offset)
509	369	{
@@ -559,7 +419,7 @@ retry: · 49 unmodified lines
559	419	xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
560	420	cow_fsb = imap.br_startoff;
561	421	if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
562		− wpc->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
	422	+ XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
563	423	xfs_iunlock(ip, XFS_ILOCK_SHARED);
564	424
565	425	whichfork = XFS_COW_FORK;
@@ -582,7 +442,7 @@ retry: · 16 unmodified lines
582	442	*/
583	443	if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
584	444	imap.br_startoff = end_fsb; /* fake a hole past EOF */
585		− wpc->data_seq = READ_ONCE(ip->i_df.if_seq);
	445	+ XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
586	446	xfs_iunlock(ip, XFS_ILOCK_SHARED);
587	447
588	448	/* landed in a hole or beyond EOF? */
@@ -645,24 +505,9 @@ allocate_blocks: · 56 unmodified lines
645	505	return 0;
646	506	}
647	507
648		−/*
649		− * Submit the bio for an ioend. We are passed an ioend with a bio attached to
650		− * it, and we submit that bio. The ioend may be used for multiple bio
651		− * submissions, so we only want to allocate an append transaction for the ioend
652		− * once. In the case of multiple bio submission, each bio will take an IO
653		− * reference to the ioend to ensure that the ioend completion is only done once
654		− * all bios have been submitted and the ioend is really done.
655		− *
656		− * If @status is non-zero, it means that we have a situation where some part of
657		− * the submission process has failed after we have marked paged for writeback
658		− * and unlocked them. In this situation, we need to fail the bio and ioend
659		− * rather than submit it to IO. This typically only happens on a filesystem
660		− * shutdown.
661		− */
662		−STATIC int
663		−xfs_submit_ioend(
664		− struct writeback_control *wbc,
665		− struct xfs_ioend *ioend,
	508	+static int
	509	+xfs_prepare_ioend(
	510	+ struct iomap_ioend *ioend,
666	511	int status)
667	512	{
668	513	unsigned int nofs_flag;
@@ -690,137 +535,9 @@ xfs_submit_ioend( · 21 unmodified lines
690	535
691	536	memalloc_nofs_restore(nofs_flag);
692	537
693		− ioend->io_bio->bi_private = ioend;
694		− ioend->io_bio->bi_end_io = xfs_end_bio;
695		−
696		− /*
697		− * If we are failing the IO now, just mark the ioend with an
698		− * error and finish it. This will run IO completion immediately
699		− * as there is only one reference to the ioend at this point in
700		− * time.
701		− */
702		− if (status) {
703		− ioend->io_bio->bi_status = errno_to_blk_status(status);
704		− bio_endio(ioend->io_bio);
705		− return status;
706		− }
707		−
708		− submit_bio(ioend->io_bio);
709		− return 0;
710		−}
711		−
712		−static struct xfs_ioend *
713		−xfs_alloc_ioend(
714		− struct inode *inode,
715		− struct xfs_writepage_ctx *wpc,
716		− xfs_off_t offset,
717		− sector_t sector,
718		− struct writeback_control *wbc)
719		−{
720		− struct xfs_ioend *ioend;
721		− struct bio *bio;
722		−
723		− bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &xfs_ioend_bioset);
724		− bio_set_dev(bio, wpc->iomap.bdev);
725		− bio->bi_iter.bi_sector = sector;
726		− bio->bi_opf = REQ_OP_WRITE \| wbc_to_write_flags(wbc);
727		− bio->bi_write_hint = inode->i_write_hint;
728		− wbc_init_bio(wbc, bio);
729		−
730		− ioend = container_of(bio, struct xfs_ioend, io_inline_bio);
731		− INIT_LIST_HEAD(&ioend->io_list);
732		− ioend->io_type = wpc->iomap.type;
733		− ioend->io_flags = wpc->iomap.flags;
734		− ioend->io_inode = inode;
735		− ioend->io_size = 0;
736		− ioend->io_offset = offset;
737		− ioend->io_private = NULL;
738		− ioend->io_bio = bio;
739		− return ioend;
740		−}
741		−
742		−/*
743		− * Allocate a new bio, and chain the old bio to the new one.
744		− *
745		− * Note that we have to do perform the chaining in this unintuitive order
746		− * so that the bi_private linkage is set up in the right direction for the
747		− * traversal in xfs_destroy_ioend().
748		− */
749		−static struct bio *
750		−xfs_chain_bio(
751		− struct bio *prev)
752		−{
753		− struct bio *new;
754		−
755		− new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
756		− bio_copy_dev(new, prev);/* also copies over blkcg information */
757		− new->bi_iter.bi_sector = bio_end_sector(prev);
758		− new->bi_opf = prev->bi_opf;
759		− new->bi_write_hint = prev->bi_write_hint;
760		−
761		− bio_chain(prev, new);
762		− bio_get(prev); /* for xfs_destroy_ioend */
763		− submit_bio(prev);
764		− return new;
765		−}
766		−
767		−static bool
768		−xfs_can_add_to_ioend(
769		− struct xfs_writepage_ctx *wpc,
770		− xfs_off_t offset,
771		− sector_t sector)
772		−{
773		− if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
774		− (wpc->ioend->io_flags & IOMAP_F_SHARED))
775		− return false;
776		− if (wpc->iomap.type != wpc->ioend->io_type)
777		− return false;
778		− if (offset != wpc->ioend->io_offset + wpc->ioend->io_size)
779		− return false;
780		− if (sector != bio_end_sector(wpc->ioend->io_bio))
781		− return false;
782		− return true;
783		−}
784		−
785		−/*
786		− * Test to see if we have an existing ioend structure that we could append to
787		− * first, otherwise finish off the current ioend and start another.
788		− */
789		−STATIC void
790		−xfs_add_to_ioend(
791		− struct inode *inode,
792		− xfs_off_t offset,
793		− struct page *page,
794		− struct iomap_page *iop,
795		− struct xfs_writepage_ctx *wpc,
796		− struct writeback_control *wbc,
797		− struct list_head *iolist)
798		−{
799		− sector_t sector = iomap_sector(&wpc->iomap, offset);
800		− unsigned len = i_blocksize(inode);
801		− unsigned poff = offset & (PAGE_SIZE - 1);
802		− bool merged, same_page = false;
803		−
804		− if (!wpc->ioend \|\| !xfs_can_add_to_ioend(wpc, offset, sector)) {
805		− if (wpc->ioend)
806		− list_add(&wpc->ioend->io_list, iolist);
807		− wpc->ioend = xfs_alloc_ioend(inode, wpc, offset, sector, wbc);
808		− }
809		−
810		− merged = __bio_try_merge_page(wpc->ioend->io_bio, page, len, poff,
811		− &same_page);
812		−
813		− if (iop && !same_page)
814		− atomic_inc(&iop->write_count);
815		−
816		− if (!merged) {
817		− if (bio_full(wpc->ioend->io_bio, len))
818		− wpc->ioend->io_bio = xfs_chain_bio(wpc->ioend->io_bio);
819		− bio_add_page(wpc->ioend->io_bio, page, len, poff);
820		− }
821		−
822		− wpc->ioend->io_size += len;
823		− wbc_account_cgroup_owner(wbc, page, len);
	538	+ if (xfs_ioend_needs_workqueue(ioend))
	539	+ ioend->io_bio->bi_end_io = xfs_end_bio;
	540	+ return status;
824	541	}
825	542
826	543	/*
@@ -834,8 +551,8 @@ xfs_add_to_ioend( · 7 unmodified lines
834	551	* transaction as there is no space left for block reservation (typically why we
835	552	* see a ENOSPC in writeback).
836	553	*/
837		−STATIC void
838		−xfs_aops_discard_page(
	554	+static void
	555	+xfs_discard_page(
839	556	struct page *page)
840	557	{
841	558	struct inode *inode = page->mapping->host;
@@ -860,243 +577,11 @@ out_invalidate: · 18 unmodified lines
860	577	iomap_invalidatepage(page, 0, PAGE_SIZE);
861	578	}
862	579
863		−/*
864		− * We implement an immediate ioend submission policy here to avoid needing to
865		− * chain multiple ioends and hence nest mempool allocations which can violate
866		− * forward progress guarantees we need to provide. The current ioend we are
867		− * adding blocks to is cached on the writepage context, and if the new block
868		− * does not append to the cached ioend it will create a new ioend and cache that
869		− * instead.
870		− *
871		− * If a new ioend is created and cached, the old ioend is returned and queued
872		− * locally for submission once the entire page is processed or an error has been
873		− * detected. While ioends are submitted immediately after they are completed,
874		− * batching optimisations are provided by higher level block plugging.
875		− *
876		− * At the end of a writeback pass, there will be a cached ioend remaining on the
877		− * writepage context that the caller will need to submit.
878		− */
879		−static int
880		−xfs_writepage_map(
881		− struct xfs_writepage_ctx *wpc,
882		− struct writeback_control *wbc,
883		− struct inode *inode,
884		− struct page *page,
885		− uint64_t end_offset)
886		−{
887		− LIST_HEAD(submit_list);
888		− struct iomap_page *iop = to_iomap_page(page);
889		− unsigned len = i_blocksize(inode);
890		− struct xfs_ioend ioend, next;
891		− uint64_t file_offset; /* file offset of page */
892		− int error = 0, count = 0, i;
893		−
894		− ASSERT(iop \|\| i_blocksize(inode) == PAGE_SIZE);
895		− ASSERT(!iop \|\| atomic_read(&iop->write_count) == 0);
896		−
897		− /*
898		− * Walk through the page to find areas to write back. If we run off the
899		− * end of the current map or find the current map invalid, grab a new
900		− * one.
901		− */
902		− for (i = 0, file_offset = page_offset(page);
903		− i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset;
904		− i++, file_offset += len) {
905		− if (iop && !test_bit(i, iop->uptodate))
906		− continue;
907		−
908		− error = xfs_map_blocks(wpc, inode, file_offset);
909		− if (error)
910		− break;
911		− if (wpc->iomap.type == IOMAP_HOLE)
912		− continue;
913		− xfs_add_to_ioend(inode, file_offset, page, iop, wpc, wbc,
914		− &submit_list);
915		− count++;
916		− }
917		−
918		− ASSERT(wpc->ioend \|\| list_empty(&submit_list));
919		− ASSERT(PageLocked(page));
920		− ASSERT(!PageWriteback(page));
921		−
922		− /*
923		− * On error, we have to fail the ioend here because we may have set
924		− * pages under writeback, we have to make sure we run IO completion to
925		− * mark the error state of the IO appropriately, so we can't cancel the
926		− * ioend directly here. That means we have to mark this page as under
927		− * writeback if we included any blocks from it in the ioend chain so
928		− * that completion treats it correctly.
929		− *
930		− * If we didn't include the page in the ioend, the on error we can
931		− * simply discard and unlock it as there are no other users of the page
932		− * now. The caller will still need to trigger submission of outstanding
933		− * ioends on the writepage context so they are treated correctly on
934		− * error.
935		− */
936		− if (unlikely(error)) {
937		− if (!count) {
938		− xfs_aops_discard_page(page);
939		− ClearPageUptodate(page);
940		− unlock_page(page);
941		− goto done;
942		− }
943		−
944		− /*
945		− * If the page was not fully cleaned, we need to ensure that the
946		− * higher layers come back to it correctly. That means we need
947		− * to keep the page dirty, and for WB_SYNC_ALL writeback we need
948		− * to ensure the PAGECACHE_TAG_TOWRITE index mark is not removed
949		− * so another attempt to write this page in this writeback sweep
950		− * will be made.
951		− */
952		− set_page_writeback_keepwrite(page);
953		− } else {
954		− clear_page_dirty_for_io(page);
955		− set_page_writeback(page);
956		− }
957		−
958		− unlock_page(page);
959		−
960		− /*
961		− * Preserve the original error if there was one, otherwise catch
962		− * submission errors here and propagate into subsequent ioend
963		− * submissions.
964		− */
965		− list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
966		− int error2;
967		−
968		− list_del_init(&ioend->io_list);
969		− error2 = xfs_submit_ioend(wbc, ioend, error);
970		− if (error2 && !error)
971		− error = error2;
972		− }
973		−
974		− /*
975		− * We can end up here with no error and nothing to write only if we race
976		− * with a partial page truncate on a sub-page block sized filesystem.
977		− */
978		− if (!count)
979		− end_page_writeback(page);
980		−done:
981		− mapping_set_error(page->mapping, error);
982		− return error;
983		−}
984		−
985		−/*
986		− * Write out a dirty page.
987		− *
988		− * For delalloc space on the page we need to allocate space and flush it.
989		− * For unwritten space on the page we need to start the conversion to
990		− * regular allocated space.
991		− */
992		−STATIC int
993		−xfs_do_writepage(
994		− struct page *page,
995		− struct writeback_control *wbc,
996		− void *data)
997		−{
998		− struct xfs_writepage_ctx *wpc = data;
999		− struct inode *inode = page->mapping->host;
1000		− loff_t offset;
1001		− uint64_t end_offset;
1002		− pgoff_t end_index;
1003		−
1004		− trace_xfs_writepage(inode, page, 0, 0);
1005		−
1006		− /*
1007		− * Refuse to write the page out if we are called from reclaim context.
1008		− *
1009		− * This avoids stack overflows when called from deeply used stacks in
1010		− * random callers for direct reclaim or memcg reclaim. We explicitly
1011		− * allow reclaim from kswapd as the stack usage there is relatively low.
1012		− *
1013		− * This should never happen except in the case of a VM regression so
1014		− * warn about it.
1015		− */
1016		− if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC\|PF_KSWAPD)) ==
1017		− PF_MEMALLOC))
1018		− goto redirty;
1019		−
1020		− /*
1021		− * Given that we do not allow direct reclaim to call us, we should
1022		− * never be called while in a filesystem transaction.
1023		− */
1024		− if (WARN_ON_ONCE(current->flags & PF_MEMALLOC_NOFS))
1025		− goto redirty;
1026		−
1027		− /*
1028		− * Is this page beyond the end of the file?
1029		− *
1030		− * The page index is less than the end_index, adjust the end_offset
1031		− * to the highest offset that this page should represent.
1032		− * -----------------------------------------------------
1033		− * \| file mapping \| <EOF> \|
1034		− * -----------------------------------------------------
1035		− * \| Page ... \| Page N-2 \| Page N-1 \| Page N \| \|
1036		− * ^--------------------------------^----------\|--------
1037		− * \| desired writeback range \| see else \|
1038		− * ---------------------------------^------------------\|
1039		− */
1040		− offset = i_size_read(inode);
1041		− end_index = offset >> PAGE_SHIFT;
1042		− if (page->index < end_index)
1043		− end_offset = (xfs_off_t)(page->index + 1) << PAGE_SHIFT;
1044		− else {
1045		− /*
1046		− * Check whether the page to write out is beyond or straddles
1047		− * i_size or not.
1048		− * -------------------------------------------------------
1049		− * \| file mapping \| <EOF> \|
1050		− * -------------------------------------------------------
1051		− * \| Page ... \| Page N-2 \| Page N-1 \| Page N \| Beyond \|
1052		− * ^--------------------------------^-----------\|---------
1053		− * \| \| Straddles \|
1054		− * ---------------------------------^-----------\|--------\|
1055		− */
1056		− unsigned offset_into_page = offset & (PAGE_SIZE - 1);
1057		−
1058		− /*
1059		− * Skip the page if it is fully outside i_size, e.g. due to a
1060		− * truncate operation that is in progress. We must redirty the
1061		− * page so that reclaim stops reclaiming it. Otherwise
1062		− * xfs_vm_releasepage() is called on it and gets confused.
1063		− *
1064		− * Note that the end_index is unsigned long, it would overflow
1065		− * if the given offset is greater than 16TB on 32-bit system
1066		− * and if we do check the page is fully outside i_size or not
1067		− * via "if (page->index >= end_index + 1)" as "end_index + 1"
1068		− * will be evaluated to 0. Hence this page will be redirtied
1069		− * and be written out repeatedly which would result in an
1070		− * infinite loop, the user program that perform this operation
1071		− * will hang. Instead, we can verify this situation by checking
1072		− * if the page to write is totally beyond the i_size or if it's
1073		− * offset is just equal to the EOF.
1074		− */
1075		− if (page->index > end_index \|\|
1076		− (page->index == end_index && offset_into_page == 0))
1077		− goto redirty;
1078		−
1079		− /*
1080		− * The page straddles i_size. It must be zeroed out on each
1081		− * and every writepage invocation because it may be mmapped.
1082		− * "A file is mapped in multiples of the page size. For a file
1083		− * that is not a multiple of the page size, the remaining
1084		− * memory is zeroed when mapped, and writes to that region are
1085		− * not written out to the file."
1086		− */
1087		− zero_user_segment(page, offset_into_page, PAGE_SIZE);
1088		−
1089		− /* Adjust the end_offset to the end of file */
1090		− end_offset = offset;
1091		− }
1092		−
1093		− return xfs_writepage_map(wpc, wbc, inode, page, end_offset);
1094		−
1095		−redirty:
1096		− redirty_page_for_writepage(wbc, page);
1097		− unlock_page(page);
1098		− return 0;
1099		−}
	580	+static const struct iomap_writeback_ops xfs_writeback_ops = {
	581	+ .map_blocks = xfs_map_blocks,
	582	+ .prepare_ioend = xfs_prepare_ioend,
	583	+ .discard_page = xfs_discard_page,
	584	+};
1100	585
1101	586	STATIC int
1102	587	xfs_vm_writepage(
@@ -1104,12 +589,8 @@ xfs_vm_writepage( · 1 unmodified line
1104	589	struct writeback_control *wbc)
1105	590	{
1106	591	struct xfs_writepage_ctx wpc = { };
1107		− int ret;
1108	592
1109		− ret = xfs_do_writepage(page, wbc, &wpc);
1110		− if (wpc.ioend)
1111		− ret = xfs_submit_ioend(wbc, wpc.ioend, ret);
1112		− return ret;
	593	+ return iomap_writepage(page, wbc, &wpc.ctx, &xfs_writeback_ops);
1113	594	}
1114	595
1115	596	STATIC int
@@ -1118,13 +599,9 @@ xfs_vm_writepages( · 2 unmodified lines
1118	599	struct writeback_control *wbc)
1119	600	{
1120	601	struct xfs_writepage_ctx wpc = { };
1121		− int ret;
1122	602
1123	603	xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
1124		− ret = write_cache_pages(mapping, wbc, xfs_do_writepage, &wpc);
1125		− if (wpc.ioend)
1126		− ret = xfs_submit_ioend(wbc, wpc.ioend, ret);
1127		− return ret;
	604	+ return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
1128	605	}
1129	606
1130	607	STATIC int

fs/xfs/xfs_aops.h+0−17

@@ -6,23 +6,6 @@ · 5 unmodified lines
6	6	#ifndef __XFS_AOPS_H__
7	7	#define __XFS_AOPS_H__
8	8
9		−extern struct bio_set xfs_ioend_bioset;
10		−
11		−/*
12		− * Structure for buffered I/O completions.
13		− */
14		−struct xfs_ioend {
15		− struct list_head io_list; /* next ioend in chain */
16		− u16 io_type;
17		− u16 io_flags; /* IOMAP_F_* */
18		− struct inode io_inode; / file being written to */
19		− size_t io_size; /* size of the extent */
20		− xfs_off_t io_offset; /* offset in the file */
21		− void io_private; / file system private data */
22		− struct bio io_bio; / bio being built */
23		− struct bio io_inline_bio; /* MUST BE LAST! */
24		−};
25		−
26	9	extern const struct address_space_operations xfs_address_space_operations;
27	10	extern const struct address_space_operations xfs_dax_aops;
28	11

fs/xfs/xfs_super.c+1−10

@@ -40,7 +40,6 @@ · 39 unmodified lines
40	40	#include <linux/parser.h>
41	41
42	42	static const struct super_operations xfs_super_operations;
43		−struct bio_set xfs_ioend_bioset;
44	43
45	44	static struct kset xfs_kset; / top-level xfs sysfs dir */
46	45	#ifdef DEBUG
@@ -1853,15 +1852,10 @@ MODULE_ALIAS_FS("xfs"); · 1806 unmodified lines
1853	1852	STATIC int __init
1854	1853	xfs_init_zones(void)
1855	1854	{
1856		− if (bioset_init(&xfs_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
1857		− offsetof(struct xfs_ioend, io_inline_bio),
1858		− BIOSET_NEED_BVECS))
1859		− goto out;
1860		−
1861	1855	xfs_log_ticket_zone = kmem_zone_init(sizeof(xlog_ticket_t),
1862	1856	"xfs_log_ticket");
1863	1857	if (!xfs_log_ticket_zone)
1864		− goto out_free_ioend_bioset;
	1858	+ goto out;
1865	1859
1866	1860	xfs_bmap_free_item_zone = kmem_zone_init(
1867	1861	sizeof(struct xfs_extent_free_item),
@@ -1996,8 +1990,6 @@ xfs_init_zones(void) · 128 unmodified lines
1996	1990	kmem_zone_destroy(xfs_bmap_free_item_zone);
1997	1991	out_destroy_log_ticket_zone:
1998	1992	kmem_zone_destroy(xfs_log_ticket_zone);
1999		− out_free_ioend_bioset:
2000		− bioset_exit(&xfs_ioend_bioset);
2001	1993	out:
2002	1994	return -ENOMEM;
2003	1995	}
@@ -2028,7 +2020,6 @@ xfs_destroy_zones(void) · 24 unmodified lines
2028	2020	kmem_zone_destroy(xfs_btree_cur_zone);
2029	2021	kmem_zone_destroy(xfs_bmap_free_item_zone);
2030	2022	kmem_zone_destroy(xfs_log_ticket_zone);
2031		− bioset_exit(&xfs_ioend_bioset);
2032	2023	}
2033	2024
2034	2025	STATIC int __init

fs/xfs/xfs_trace.h+0−39

@@ -1158,45 +1158,6 @@ DEFINE_RW_EVENT(xfs_file_buffered_write); · 1157 unmodified lines
1158	1158	DEFINE_RW_EVENT(xfs_file_direct_write);
1159	1159	DEFINE_RW_EVENT(xfs_file_dax_write);
1160	1160
1161		−DECLARE_EVENT_CLASS(xfs_page_class,
1162		− TP_PROTO(struct inode inode, struct page page, unsigned long off,
1163		− unsigned int len),
1164		− TP_ARGS(inode, page, off, len),
1165		− TP_STRUCT__entry(
1166		− __field(dev_t, dev)
1167		− __field(xfs_ino_t, ino)
1168		− __field(pgoff_t, pgoff)
1169		− __field(loff_t, size)
1170		− __field(unsigned long, offset)
1171		− __field(unsigned int, length)
1172		− ),
1173		− TP_fast_assign(
1174		− __entry->dev = inode->i_sb->s_dev;
1175		− __entry->ino = XFS_I(inode)->i_ino;
1176		− __entry->pgoff = page_offset(page);
1177		− __entry->size = i_size_read(inode);
1178		− __entry->offset = off;
1179		− __entry->length = len;
1180		− ),
1181		− TP_printk("dev %d:%d ino 0x%llx pgoff 0x%lx size 0x%llx offset %lx "
1182		− "length %x",
1183		− MAJOR(__entry->dev), MINOR(__entry->dev),
1184		− __entry->ino,
1185		− __entry->pgoff,
1186		− __entry->size,
1187		− __entry->offset,
1188		− __entry->length)
1189		−)
1190		−
1191		−#define DEFINE_PAGE_EVENT(name) \
1192		−DEFINE_EVENT(xfs_page_class, name, \
1193		− TP_PROTO(struct inode inode, struct page page, unsigned long off, \
1194		− unsigned int len), \
1195		− TP_ARGS(inode, page, off, len))
1196		−DEFINE_PAGE_EVENT(xfs_writepage);
1197		−DEFINE_PAGE_EVENT(xfs_releasepage);
1198		−DEFINE_PAGE_EVENT(xfs_invalidatepage);
1199		−
1200	1161	DECLARE_EVENT_CLASS(xfs_imap_class,
1201	1162	TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count,
1202	1163	int whichfork, struct xfs_bmbt_irec *irec),

include/linux/iomap.h+59−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	4
5	5	#include <linux/atomic.h>
6	6	#include <linux/bitmap.h>
	7	+#include <linux/blk_types.h>
7	8	#include <linux/mm.h>
8	9	#include <linux/types.h>
9	10	#include <linux/mm_types.h>
@@ -12,6 +13,7 @@ · 2 unmodified lines
12	13	struct address_space;
13	14	struct fiemap_extent_info;
14	15	struct inode;
	16	+struct iomap_writepage_ctx;
15	17	struct iov_iter;
16	18	struct kiocb;
17	19	struct page;
@@ -185,6 +187,63 @@ loff_t iomap_seek_data(struct inode *inode, loff_t offset, · 167 unmodified lines
185	187	sector_t iomap_bmap(struct address_space *mapping, sector_t bno,
186	188	const struct iomap_ops *ops);
187	189
	190	+/*
	191	+ * Structure for writeback I/O completions.
	192	+ */
	193	+struct iomap_ioend {
	194	+ struct list_head io_list; /* next ioend in chain */
	195	+ u16 io_type;
	196	+ u16 io_flags; /* IOMAP_F_* */
	197	+ struct inode io_inode; / file being written to */
	198	+ size_t io_size; /* size of the extent */
	199	+ loff_t io_offset; /* offset in the file */
	200	+ void io_private; / file system private data */
	201	+ struct bio io_bio; / bio being built */
	202	+ struct bio io_inline_bio; /* MUST BE LAST! */
	203	+};
	204	+
	205	+struct iomap_writeback_ops {
	206	+ /*
	207	+ * Required, maps the blocks so that writeback can be performed on
	208	+ * the range starting at offset.
	209	+ */
	210	+ int (map_blocks)(struct iomap_writepage_ctx wpc, struct inode *inode,
	211	+ loff_t offset);
	212	+
	213	+ /*
	214	+ * Optional, allows the file systems to perform actions just before
	215	+ * submitting the bio and/or override the bio end_io handler for complex
	216	+ * operations like copy on write extent manipulation or unwritten extent
	217	+ * conversions.
	218	+ */
	219	+ int (prepare_ioend)(struct iomap_ioend ioend, int status);
	220	+
	221	+ /*
	222	+ * Optional, allows the file system to discard state on a page where
	223	+ * we failed to submit any I/O.
	224	+ */
	225	+ void (discard_page)(struct page page);
	226	+};
	227	+
	228	+struct iomap_writepage_ctx {
	229	+ struct iomap iomap;
	230	+ struct iomap_ioend *ioend;
	231	+ const struct iomap_writeback_ops *ops;
	232	+};
	233	+
	234	+void iomap_finish_ioends(struct iomap_ioend *ioend, int error);
	235	+void iomap_ioend_try_merge(struct iomap_ioend *ioend,
	236	+ struct list_head *more_ioends,
	237	+ void (merge_private)(struct iomap_ioend ioend,
	238	+ struct iomap_ioend *next));
	239	+void iomap_sort_ioends(struct list_head *ioend_list);
	240	+int iomap_writepage(struct page page, struct writeback_control wbc,
	241	+ struct iomap_writepage_ctx *wpc,
	242	+ const struct iomap_writeback_ops *ops);
	243	+int iomap_writepages(struct address_space *mapping,
	244	+ struct writeback_control wbc, struct iomap_writepage_ctx wpc,
	245	+ const struct iomap_writeback_ops *ops);
	246	+
188	247	/*
189	248	* Flags for direct I/O ->end_io:
190	249	*/

iomap: lift the xfs writeback code to iomap

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code