History

iomap: lift the xfs writeback code to iomap

Take the xfs writeback code and move it to fs/iomap. A new structure with three methods is added as the abstraction from the generic writeback code to the file system. These methods are used to map blocks, submit an ioend, and cancel a page that encountered an error before it was added to an ioend.

Christoph Hellwig committed 7 years ago

Commit 598ecfb1 parent 9e91c577 files · +669 −648Browse files

Unified · Split

7 files
  1. fs/iomap/buffered-io.c +550 −1
  2. fs/iomap/trace.h +1 −0
  3. fs/xfs/xfs_aops.c +58 −581
  4. fs/xfs/xfs_aops.h +0 −17
  5. fs/xfs/xfs_super.c +1 −10
  6. fs/xfs/xfs_trace.h +0 −39
  7. include/linux/iomap.h +59 −0
fs/iomap/buffered-io.c+550−1
@@ -1,7 +1,7 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/*2/*
3 * Copyright (C) 2010 Red Hat, Inc.3 * Copyright (C) 2010 Red Hat, Inc.
4 * Copyright (c) 2016-2018 Christoph Hellwig.4 * Copyright (C) 2016-2019 Christoph Hellwig.
5 */5 */
6#include <linux/module.h>6#include <linux/module.h>
7#include <linux/compiler.h>7#include <linux/compiler.h>
@@ -12,6 +12,7 @@ · 4 unmodified lines
12#include <linux/buffer_head.h>12#include <linux/buffer_head.h>
13#include <linux/dax.h>13#include <linux/dax.h>
14#include <linux/writeback.h>14#include <linux/writeback.h>
15#include <linux/list_sort.h>
15#include <linux/swap.h>16#include <linux/swap.h>
16#include <linux/bio.h>17#include <linux/bio.h>
17#include <linux/sched/signal.h>18#include <linux/sched/signal.h>
@@ -20,6 +21,8 @@ · 2 unmodified lines
20 21
21#include "../internal.h"22#include "../internal.h"
22 23
24static struct bio_set iomap_ioend_bioset;
25
23static struct iomap_page *26static struct iomap_page *
24iomap_page_create(struct inode *inode, struct page *page)27iomap_page_create(struct inode *inode, struct page *page)
25{28{
@@ -1088,3 +1091,549 @@ out_unlock: · 1062 unmodified lines
1088 return block_page_mkwrite_return(ret);1091 return block_page_mkwrite_return(ret);
1089}1092}
1090EXPORT_SYMBOL_GPL(iomap_page_mkwrite);1093EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1094
1095static void
1096iomap_finish_page_writeback(struct inode *inode, struct bio_vec *bvec,
1097 int error)
1098{
1099 struct iomap_page *iop = to_iomap_page(bvec->bv_page);
1100
1101 if (error) {
1102 SetPageError(bvec->bv_page);
1103 mapping_set_error(inode->i_mapping, -EIO);
1104 }
1105
1106 WARN_ON_ONCE(i_blocksize(inode) < PAGE_SIZE && !iop);
1107 WARN_ON_ONCE(iop && atomic_read(&iop->write_count) <= 0);
1108
1109 if (!iop || atomic_dec_and_test(&iop->write_count))
1110 end_page_writeback(bvec->bv_page);
1111}
1112
1113/*
1114 * We're now finished for good with this ioend structure. Update the page
1115 * state, release holds on bios, and finally free up memory. Do not use the
1116 * ioend after this.
1117 */
1118static void
1119iomap_finish_ioend(struct iomap_ioend *ioend, int error)
1120{
1121 struct inode *inode = ioend->io_inode;
1122 struct bio *bio = &ioend->io_inline_bio;
1123 struct bio *last = ioend->io_bio, *next;
1124 u64 start = bio->bi_iter.bi_sector;
1125 bool quiet = bio_flagged(bio, BIO_QUIET);
1126
1127 for (bio = &ioend->io_inline_bio; bio; bio = next) {
1128 struct bio_vec *bv;
1129 struct bvec_iter_all iter_all;
1130
1131 /*
1132 * For the last bio, bi_private points to the ioend, so we
1133 * need to explicitly end the iteration here.
1134 */
1135 if (bio == last)
1136 next = NULL;
1137 else
1138 next = bio->bi_private;
1139
1140 /* walk each page on bio, ending page IO on them */
1141 bio_for_each_segment_all(bv, bio, iter_all)
1142 iomap_finish_page_writeback(inode, bv, error);
1143 bio_put(bio);
1144 }
1145
1146 if (unlikely(error && !quiet)) {
1147 printk_ratelimited(KERN_ERR
1148 "%s: writeback error on sector %llu",
1149 inode->i_sb->s_id, start);
1150 }
1151}
1152
1153void
1154iomap_finish_ioends(struct iomap_ioend *ioend, int error)
1155{
1156 struct list_head tmp;
1157
1158 list_replace_init(&ioend->io_list, &tmp);
1159 iomap_finish_ioend(ioend, error);
1160
1161 while (!list_empty(&tmp)) {
1162 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
1163 list_del_init(&ioend->io_list);
1164 iomap_finish_ioend(ioend, error);
1165 }
1166}
1167EXPORT_SYMBOL_GPL(iomap_finish_ioends);
1168
1169/*
1170 * We can merge two adjacent ioends if they have the same set of work to do.
1171 */
1172static bool
1173iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next)
1174{
1175 if (ioend->io_bio->bi_status != next->io_bio->bi_status)
1176 return false;
1177 if ((ioend->io_flags & IOMAP_F_SHARED) ^
1178 (next->io_flags & IOMAP_F_SHARED))
1179 return false;
1180 if ((ioend->io_type == IOMAP_UNWRITTEN) ^
1181 (next->io_type == IOMAP_UNWRITTEN))
1182 return false;
1183 if (ioend->io_offset + ioend->io_size != next->io_offset)
1184 return false;
1185 return true;
1186}
1187
1188void
1189iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends,
1190 void (*merge_private)(struct iomap_ioend *ioend,
1191 struct iomap_ioend *next))
1192{
1193 struct iomap_ioend *next;
1194
1195 INIT_LIST_HEAD(&ioend->io_list);
1196
1197 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
1198 io_list))) {
1199 if (!iomap_ioend_can_merge(ioend, next))
1200 break;
1201 list_move_tail(&next->io_list, &ioend->io_list);
1202 ioend->io_size += next->io_size;
1203 if (next->io_private && merge_private)
1204 merge_private(ioend, next);
1205 }
1206}
1207EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
1208
1209static int
1210iomap_ioend_compare(void *priv, struct list_head *a, struct list_head *b)
1211{
1212 struct iomap_ioend *ia, *ib;
1213
1214 ia = container_of(a, struct iomap_ioend, io_list);
1215 ib = container_of(b, struct iomap_ioend, io_list);
1216 if (ia->io_offset < ib->io_offset)
1217 return -1;
1218 else if (ia->io_offset > ib->io_offset)
1219 return 1;
1220 return 0;
1221}
1222
1223void
1224iomap_sort_ioends(struct list_head *ioend_list)
1225{
1226 list_sort(NULL, ioend_list, iomap_ioend_compare);
1227}
1228EXPORT_SYMBOL_GPL(iomap_sort_ioends);
1229
1230static void iomap_writepage_end_bio(struct bio *bio)
1231{
1232 struct iomap_ioend *ioend = bio->bi_private;
1233
1234 iomap_finish_ioend(ioend, blk_status_to_errno(bio->bi_status));
1235}
1236
1237/*
1238 * Submit the final bio for an ioend.
1239 *
1240 * If @error is non-zero, it means that we have a situation where some part of
1241 * the submission process has failed after we have marked paged for writeback
1242 * and unlocked them. In this situation, we need to fail the bio instead of
1243 * submitting it. This typically only happens on a filesystem shutdown.
1244 */
1245static int
1246iomap_submit_ioend(struct iomap_writepage_ctx *wpc, struct iomap_ioend *ioend,
1247 int error)
1248{
1249 ioend->io_bio->bi_private = ioend;
1250 ioend->io_bio->bi_end_io = iomap_writepage_end_bio;
1251
1252 if (wpc->ops->prepare_ioend)
1253 error = wpc->ops->prepare_ioend(ioend, error);
1254 if (error) {
1255 /*
1256 * If we are failing the IO now, just mark the ioend with an
1257 * error and finish it. This will run IO completion immediately
1258 * as there is only one reference to the ioend at this point in
1259 * time.
1260 */
1261 ioend->io_bio->bi_status = errno_to_blk_status(error);
1262 bio_endio(ioend->io_bio);
1263 return error;
1264 }
1265
1266 submit_bio(ioend->io_bio);
1267 return 0;
1268}
1269
1270static struct iomap_ioend *
1271iomap_alloc_ioend(struct inode *inode, struct iomap_writepage_ctx *wpc,
1272 loff_t offset, sector_t sector, struct writeback_control *wbc)
1273{
1274 struct iomap_ioend *ioend;
1275 struct bio *bio;
1276
1277 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &iomap_ioend_bioset);
1278 bio_set_dev(bio, wpc->iomap.bdev);
1279 bio->bi_iter.bi_sector = sector;
1280 bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
1281 bio->bi_write_hint = inode->i_write_hint;
1282 wbc_init_bio(wbc, bio);
1283
1284 ioend = container_of(bio, struct iomap_ioend, io_inline_bio);
1285 INIT_LIST_HEAD(&ioend->io_list);
1286 ioend->io_type = wpc->iomap.type;
1287 ioend->io_flags = wpc->iomap.flags;
1288 ioend->io_inode = inode;
1289 ioend->io_size = 0;
1290 ioend->io_offset = offset;
1291 ioend->io_private = NULL;
1292 ioend->io_bio = bio;
1293 return ioend;
1294}
1295
1296/*
1297 * Allocate a new bio, and chain the old bio to the new one.
1298 *
1299 * Note that we have to do perform the chaining in this unintuitive order
1300 * so that the bi_private linkage is set up in the right direction for the
1301 * traversal in iomap_finish_ioend().
1302 */
1303static struct bio *
1304iomap_chain_bio(struct bio *prev)
1305{
1306 struct bio *new;
1307
1308 new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
1309 bio_copy_dev(new, prev);/* also copies over blkcg information */
1310 new->bi_iter.bi_sector = bio_end_sector(prev);
1311 new->bi_opf = prev->bi_opf;
1312 new->bi_write_hint = prev->bi_write_hint;
1313
1314 bio_chain(prev, new);
1315 bio_get(prev); /* for iomap_finish_ioend */
1316 submit_bio(prev);
1317 return new;
1318}
1319
1320static bool
1321iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t offset,
1322 sector_t sector)
1323{
1324 if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
1325 (wpc->ioend->io_flags & IOMAP_F_SHARED))
1326 return false;
1327 if (wpc->iomap.type != wpc->ioend->io_type)
1328 return false;
1329 if (offset != wpc->ioend->io_offset + wpc->ioend->io_size)
1330 return false;
1331 if (sector != bio_end_sector(wpc->ioend->io_bio))
1332 return false;
1333 return true;
1334}
1335
1336/*
1337 * Test to see if we have an existing ioend structure that we could append to
1338 * first, otherwise finish off the current ioend and start another.
1339 */
1340static void
1341iomap_add_to_ioend(struct inode *inode, loff_t offset, struct page *page,
1342 struct iomap_page *iop, struct iomap_writepage_ctx *wpc,
1343 struct writeback_control *wbc, struct list_head *iolist)
1344{
1345 sector_t sector = iomap_sector(&wpc->iomap, offset);
1346 unsigned len = i_blocksize(inode);
1347 unsigned poff = offset & (PAGE_SIZE - 1);
1348 bool merged, same_page = false;
1349
1350 if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, offset, sector)) {
1351 if (wpc->ioend)
1352 list_add(&wpc->ioend->io_list, iolist);
1353 wpc->ioend = iomap_alloc_ioend(inode, wpc, offset, sector, wbc);
1354 }
1355
1356 merged = __bio_try_merge_page(wpc->ioend->io_bio, page, len, poff,
1357 &same_page);
1358 if (iop && !same_page)
1359 atomic_inc(&iop->write_count);
1360
1361 if (!merged) {
1362 if (bio_full(wpc->ioend->io_bio, len)) {
1363 wpc->ioend->io_bio =
1364 iomap_chain_bio(wpc->ioend->io_bio);
1365 }
1366 bio_add_page(wpc->ioend->io_bio, page, len, poff);
1367 }
1368
1369 wpc->ioend->io_size += len;
1370 wbc_account_cgroup_owner(wbc, page, len);
1371}
1372
1373/*
1374 * We implement an immediate ioend submission policy here to avoid needing to
1375 * chain multiple ioends and hence nest mempool allocations which can violate
1376 * forward progress guarantees we need to provide. The current ioend we are
1377 * adding blocks to is cached on the writepage context, and if the new block
1378 * does not append to the cached ioend it will create a new ioend and cache that
1379 * instead.
1380 *
1381 * If a new ioend is created and cached, the old ioend is returned and queued
1382 * locally for submission once the entire page is processed or an error has been
1383 * detected. While ioends are submitted immediately after they are completed,
1384 * batching optimisations are provided by higher level block plugging.
1385 *
1386 * At the end of a writeback pass, there will be a cached ioend remaining on the
1387 * writepage context that the caller will need to submit.
1388 */
1389static int
1390iomap_writepage_map(struct iomap_writepage_ctx *wpc,
1391 struct writeback_control *wbc, struct inode *inode,
1392 struct page *page, u64 end_offset)
1393{
1394 struct iomap_page *iop = to_iomap_page(page);
1395 struct iomap_ioend *ioend, *next;
1396 unsigned len = i_blocksize(inode);
1397 u64 file_offset; /* file offset of page */
1398 int error = 0, count = 0, i;
1399 LIST_HEAD(submit_list);
1400
1401 WARN_ON_ONCE(i_blocksize(inode) < PAGE_SIZE && !iop);
1402 WARN_ON_ONCE(iop && atomic_read(&iop->write_count) != 0);
1403
1404 /*
1405 * Walk through the page to find areas to write back. If we run off the
1406 * end of the current map or find the current map invalid, grab a new
1407 * one.
1408 */
1409 for (i = 0, file_offset = page_offset(page);
1410 i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset;
1411 i++, file_offset += len) {
1412 if (iop && !test_bit(i, iop->uptodate))
1413 continue;
1414
1415 error = wpc->ops->map_blocks(wpc, inode, file_offset);
1416 if (error)
1417 break;
1418 if (wpc->iomap.type == IOMAP_HOLE)
1419 continue;
1420 iomap_add_to_ioend(inode, file_offset, page, iop, wpc, wbc,
1421 &submit_list);
1422 count++;
1423 }
1424
1425 WARN_ON_ONCE(!wpc->ioend && !list_empty(&submit_list));
1426 WARN_ON_ONCE(!PageLocked(page));
1427 WARN_ON_ONCE(PageWriteback(page));
1428
1429 /*
1430 * We cannot cancel the ioend directly here on error. We may have
1431 * already set other pages under writeback and hence we have to run I/O
1432 * completion to mark the error state of the pages under writeback
1433 * appropriately.
1434 */
1435 if (unlikely(error)) {
1436 if (!count) {
1437 /*
1438 * If the current page hasn't been added to ioend, it
1439 * won't be affected by I/O completions and we must
1440 * discard and unlock it right here.
1441 */
1442 if (wpc->ops->discard_page)
1443 wpc->ops->discard_page(page);
1444 ClearPageUptodate(page);
1445 unlock_page(page);
1446 goto done;
1447 }
1448
1449 /*
1450 * If the page was not fully cleaned, we need to ensure that the
1451 * higher layers come back to it correctly. That means we need
1452 * to keep the page dirty, and for WB_SYNC_ALL writeback we need
1453 * to ensure the PAGECACHE_TAG_TOWRITE index mark is not removed
1454 * so another attempt to write this page in this writeback sweep
1455 * will be made.
1456 */
1457 set_page_writeback_keepwrite(page);
1458 } else {
1459 clear_page_dirty_for_io(page);
1460 set_page_writeback(page);
1461 }
1462
1463 unlock_page(page);
1464
1465 /*
1466 * Preserve the original error if there was one, otherwise catch
1467 * submission errors here and propagate into subsequent ioend
1468 * submissions.
1469 */
1470 list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
1471 int error2;
1472
1473 list_del_init(&ioend->io_list);
1474 error2 = iomap_submit_ioend(wpc, ioend, error);
1475 if (error2 && !error)
1476 error = error2;
1477 }
1478
1479 /*
1480 * We can end up here with no error and nothing to write only if we race
1481 * with a partial page truncate on a sub-page block sized filesystem.
1482 */
1483 if (!count)
1484 end_page_writeback(page);
1485done:
1486 mapping_set_error(page->mapping, error);
1487 return error;
1488}
1489
1490/*
1491 * Write out a dirty page.
1492 *
1493 * For delalloc space on the page we need to allocate space and flush it.
1494 * For unwritten space on the page we need to start the conversion to
1495 * regular allocated space.
1496 */
1497static int
1498iomap_do_writepage(struct page *page, struct writeback_control *wbc, void *data)
1499{
1500 struct iomap_writepage_ctx *wpc = data;
1501 struct inode *inode = page->mapping->host;
1502 pgoff_t end_index;
1503 u64 end_offset;
1504 loff_t offset;
1505
1506 trace_iomap_writepage(inode, page, 0, 0);
1507
1508 /*
1509 * Refuse to write the page out if we are called from reclaim context.
1510 *
1511 * This avoids stack overflows when called from deeply used stacks in
1512 * random callers for direct reclaim or memcg reclaim. We explicitly
1513 * allow reclaim from kswapd as the stack usage there is relatively low.
1514 *
1515 * This should never happen except in the case of a VM regression so
1516 * warn about it.
1517 */
1518 if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
1519 PF_MEMALLOC))
1520 goto redirty;
1521
1522 /*
1523 * Given that we do not allow direct reclaim to call us, we should
1524 * never be called in a recursive filesystem reclaim context.
1525 */
1526 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC_NOFS))
1527 goto redirty;
1528
1529 /*
1530 * Is this page beyond the end of the file?
1531 *
1532 * The page index is less than the end_index, adjust the end_offset
1533 * to the highest offset that this page should represent.
1534 * -----------------------------------------------------
1535 * | file mapping | <EOF> |
1536 * -----------------------------------------------------
1537 * | Page ... | Page N-2 | Page N-1 | Page N | |
1538 * ^--------------------------------^----------|--------
1539 * | desired writeback range | see else |
1540 * ---------------------------------^------------------|
1541 */
1542 offset = i_size_read(inode);
1543 end_index = offset >> PAGE_SHIFT;
1544 if (page->index < end_index)
1545 end_offset = (loff_t)(page->index + 1) << PAGE_SHIFT;
1546 else {
1547 /*
1548 * Check whether the page to write out is beyond or straddles
1549 * i_size or not.
1550 * -------------------------------------------------------
1551 * | file mapping | <EOF> |
1552 * -------------------------------------------------------
1553 * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
1554 * ^--------------------------------^-----------|---------
1555 * | | Straddles |
1556 * ---------------------------------^-----------|--------|
1557 */
1558 unsigned offset_into_page = offset & (PAGE_SIZE - 1);
1559
1560 /*
1561 * Skip the page if it is fully outside i_size, e.g. due to a
1562 * truncate operation that is in progress. We must redirty the
1563 * page so that reclaim stops reclaiming it. Otherwise
1564 * iomap_vm_releasepage() is called on it and gets confused.
1565 *
1566 * Note that the end_index is unsigned long, it would overflow
1567 * if the given offset is greater than 16TB on 32-bit system
1568 * and if we do check the page is fully outside i_size or not
1569 * via "if (page->index >= end_index + 1)" as "end_index + 1"
1570 * will be evaluated to 0. Hence this page will be redirtied
1571 * and be written out repeatedly which would result in an
1572 * infinite loop, the user program that perform this operation
1573 * will hang. Instead, we can verify this situation by checking
1574 * if the page to write is totally beyond the i_size or if it's
1575 * offset is just equal to the EOF.
1576 */
1577 if (page->index > end_index ||
1578 (page->index == end_index && offset_into_page == 0))
1579 goto redirty;
1580
1581 /*
1582 * The page straddles i_size. It must be zeroed out on each
1583 * and every writepage invocation because it may be mmapped.
1584 * "A file is mapped in multiples of the page size. For a file
1585 * that is not a multiple of the page size, the remaining
1586 * memory is zeroed when mapped, and writes to that region are
1587 * not written out to the file."
1588 */
1589 zero_user_segment(page, offset_into_page, PAGE_SIZE);
1590
1591 /* Adjust the end_offset to the end of file */
1592 end_offset = offset;
1593 }
1594
1595 return iomap_writepage_map(wpc, wbc, inode, page, end_offset);
1596
1597redirty:
1598 redirty_page_for_writepage(wbc, page);
1599 unlock_page(page);
1600 return 0;
1601}
1602
1603int
1604iomap_writepage(struct page *page, struct writeback_control *wbc,
1605 struct iomap_writepage_ctx *wpc,
1606 const struct iomap_writeback_ops *ops)
1607{
1608 int ret;
1609
1610 wpc->ops = ops;
1611 ret = iomap_do_writepage(page, wbc, wpc);
1612 if (!wpc->ioend)
1613 return ret;
1614 return iomap_submit_ioend(wpc, wpc->ioend, ret);
1615}
1616EXPORT_SYMBOL_GPL(iomap_writepage);
1617
1618int
1619iomap_writepages(struct address_space *mapping, struct writeback_control *wbc,
1620 struct iomap_writepage_ctx *wpc,
1621 const struct iomap_writeback_ops *ops)
1622{
1623 int ret;
1624
1625 wpc->ops = ops;
1626 ret = write_cache_pages(mapping, wbc, iomap_do_writepage, wpc);
1627 if (!wpc->ioend)
1628 return ret;
1629 return iomap_submit_ioend(wpc, wpc->ioend, ret);
1630}
1631EXPORT_SYMBOL_GPL(iomap_writepages);
1632
1633static int __init iomap_init(void)
1634{
1635 return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
1636 offsetof(struct iomap_ioend, io_inline_bio),
1637 BIOSET_NEED_BVECS);
1638}
1639fs_initcall(iomap_init);
fs/iomap/trace.h+1−0
@@ -76,6 +76,7 @@ DEFINE_EVENT(iomap_page_class, name, \ · 75 unmodified lines
76 TP_PROTO(struct inode *inode, struct page *page, unsigned long off, \76 TP_PROTO(struct inode *inode, struct page *page, unsigned long off, \
77 unsigned int len), \77 unsigned int len), \
78 TP_ARGS(inode, page, off, len))78 TP_ARGS(inode, page, off, len))
79DEFINE_PAGE_EVENT(iomap_writepage);
79DEFINE_PAGE_EVENT(iomap_releasepage);80DEFINE_PAGE_EVENT(iomap_releasepage);
80DEFINE_PAGE_EVENT(iomap_invalidatepage);81DEFINE_PAGE_EVENT(iomap_invalidatepage);
81 82
fs/xfs/xfs_aops.c+58−581
@@ -18,16 +18,18 @@ · 17 unmodified lines
18#include "xfs_bmap_util.h"18#include "xfs_bmap_util.h"
19#include "xfs_reflink.h"19#include "xfs_reflink.h"
20 20
21/*
22 * structure owned by writepages passed to individual writepage calls
23 */
24struct xfs_writepage_ctx {21struct xfs_writepage_ctx {
25 struct iomap iomap;22 struct iomap_writepage_ctx ctx;
26 unsigned int data_seq;23 unsigned int data_seq;
27 unsigned int cow_seq;24 unsigned int cow_seq;
28 struct xfs_ioend *ioend;
29};25};
30 26
27static inline struct xfs_writepage_ctx *
28XFS_WPC(struct iomap_writepage_ctx *ctx)
29{
30 return container_of(ctx, struct xfs_writepage_ctx, ctx);
31}
32
31struct block_device *33struct block_device *
32xfs_find_bdev_for_inode(34xfs_find_bdev_for_inode(
33 struct inode *inode)35 struct inode *inode)
@@ -54,87 +56,10 @@ xfs_find_daxdev_for_inode( · 20 unmodified lines
54 return mp->m_ddev_targp->bt_daxdev;56 return mp->m_ddev_targp->bt_daxdev;
55}57}
56 58
57static void
58xfs_finish_page_writeback(
59 struct inode *inode,
60 struct bio_vec *bvec,
61 int error)
62{
63 struct iomap_page *iop = to_iomap_page(bvec->bv_page);
64
65 if (error) {
66 SetPageError(bvec->bv_page);
67 mapping_set_error(inode->i_mapping, -EIO);
68 }
69
70 ASSERT(iop || i_blocksize(inode) == PAGE_SIZE);
71 ASSERT(!iop || atomic_read(&iop->write_count) > 0);
72
73 if (!iop || atomic_dec_and_test(&iop->write_count))
74 end_page_writeback(bvec->bv_page);
75}
76
77/*
78 * We're now finished for good with this ioend structure. Update the page
79 * state, release holds on bios, and finally free up memory. Do not use the
80 * ioend after this.
81 */
82STATIC void
83xfs_destroy_ioend(
84 struct xfs_ioend *ioend,
85 int error)
86{
87 struct inode *inode = ioend->io_inode;
88 struct bio *bio = &ioend->io_inline_bio;
89 struct bio *last = ioend->io_bio, *next;
90 u64 start = bio->bi_iter.bi_sector;
91 bool quiet = bio_flagged(bio, BIO_QUIET);
92
93 for (bio = &ioend->io_inline_bio; bio; bio = next) {
94 struct bio_vec *bvec;
95 struct bvec_iter_all iter_all;
96
97 /*
98 * For the last bio, bi_private points to the ioend, so we
99 * need to explicitly end the iteration here.
100 */
101 if (bio == last)
102 next = NULL;
103 else
104 next = bio->bi_private;
105
106 /* walk each page on bio, ending page IO on them */
107 bio_for_each_segment_all(bvec, bio, iter_all)
108 xfs_finish_page_writeback(inode, bvec, error);
109 bio_put(bio);
110 }
111
112 if (unlikely(error && !quiet)) {
113 xfs_err_ratelimited(XFS_I(inode)->i_mount,
114 "writeback error on sector %llu", start);
115 }
116}
117
118static void
119xfs_destroy_ioends(
120 struct xfs_ioend *ioend,
121 int error)
122{
123 struct list_head tmp;
124
125 list_replace_init(&ioend->io_list, &tmp);
126 xfs_destroy_ioend(ioend, error);
127 while ((ioend = list_first_entry_or_null(&tmp, struct xfs_ioend,
128 io_list))) {
129 list_del_init(&ioend->io_list);
130 xfs_destroy_ioend(ioend, error);
131 }
132}
133
134/*59/*
135 * Fast and loose check if this write could update the on-disk inode size.60 * Fast and loose check if this write could update the on-disk inode size.
136 */61 */
137static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend)62static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
138{63{
139 return ioend->io_offset + ioend->io_size >64 return ioend->io_offset + ioend->io_size >
140 XFS_I(ioend->io_inode)->i_d.di_size;65 XFS_I(ioend->io_inode)->i_d.di_size;
@@ -142,7 +67,7 @@ static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend) · 1 unmodified line
142 67
143STATIC int68STATIC int
144xfs_setfilesize_trans_alloc(69xfs_setfilesize_trans_alloc(
145 struct xfs_ioend *ioend)70 struct iomap_ioend *ioend)
146{71{
147 struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount;72 struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount;
148 struct xfs_trans *tp;73 struct xfs_trans *tp;
@@ -215,7 +140,7 @@ xfs_setfilesize( · 66 unmodified lines
215 140
216STATIC int141STATIC int
217xfs_setfilesize_ioend(142xfs_setfilesize_ioend(
218 struct xfs_ioend *ioend,143 struct iomap_ioend *ioend,
219 int error)144 int error)
220{145{
221 struct xfs_inode *ip = XFS_I(ioend->io_inode);146 struct xfs_inode *ip = XFS_I(ioend->io_inode);
@@ -243,7 +168,7 @@ xfs_setfilesize_ioend( · 21 unmodified lines
243 */168 */
244STATIC void169STATIC void
245xfs_end_ioend(170xfs_end_ioend(
246 struct xfs_ioend *ioend)171 struct iomap_ioend *ioend)
247{172{
248 struct xfs_inode *ip = XFS_I(ioend->io_inode);173 struct xfs_inode *ip = XFS_I(ioend->io_inode);
249 xfs_off_t offset = ioend->io_offset;174 xfs_off_t offset = ioend->io_offset;
@@ -289,31 +214,10 @@ xfs_end_ioend( · 39 unmodified lines
289done:214done:
290 if (ioend->io_private)215 if (ioend->io_private)
291 error = xfs_setfilesize_ioend(ioend, error);216 error = xfs_setfilesize_ioend(ioend, error);
292 xfs_destroy_ioends(ioend, error);217 iomap_finish_ioends(ioend, error);
293 memalloc_nofs_restore(nofs_flag);218 memalloc_nofs_restore(nofs_flag);
294}219}
295 220
296/*
297 * We can merge two adjacent ioends if they have the same set of work to do.
298 */
299static bool
300xfs_ioend_can_merge(
301 struct xfs_ioend *ioend,
302 struct xfs_ioend *next)
303{
304 if (ioend->io_bio->bi_status != next->io_bio->bi_status)
305 return false;
306 if ((ioend->io_flags & IOMAP_F_SHARED) ^
307 (next->io_flags & IOMAP_F_SHARED))
308 return false;
309 if ((ioend->io_type == IOMAP_UNWRITTEN) ^
310 (next->io_type == IOMAP_UNWRITTEN))
311 return false;
312 if (ioend->io_offset + ioend->io_size != next->io_offset)
313 return false;
314 return true;
315}
316
317/*221/*
318 * If the to be merged ioend has a preallocated transaction for file222 * If the to be merged ioend has a preallocated transaction for file
319 * size updates we need to ensure the ioend it is merged into also223 * size updates we need to ensure the ioend it is merged into also
@@ -322,8 +226,8 @@ xfs_ioend_can_merge( · 2 unmodified lines
322 */226 */
323static void227static void
324xfs_ioend_merge_private(228xfs_ioend_merge_private(
325 struct xfs_ioend *ioend,229 struct iomap_ioend *ioend,
326 struct xfs_ioend *next)230 struct iomap_ioend *next)
327{231{
328 if (!ioend->io_private) {232 if (!ioend->io_private) {
329 ioend->io_private = next->io_private;233 ioend->io_private = next->io_private;
@@ -333,53 +237,6 @@ xfs_ioend_merge_private( · 3 unmodified lines
333 }237 }
334}238}
335 239
336/* Try to merge adjacent completions. */
337STATIC void
338xfs_ioend_try_merge(
339 struct xfs_ioend *ioend,
340 struct list_head *more_ioends)
341{
342 struct xfs_ioend *next;
343
344 INIT_LIST_HEAD(&ioend->io_list);
345
346 while ((next = list_first_entry_or_null(more_ioends, struct xfs_ioend,
347 io_list))) {
348 if (!xfs_ioend_can_merge(ioend, next))
349 break;
350 list_move_tail(&next->io_list, &ioend->io_list);
351 ioend->io_size += next->io_size;
352 if (next->io_private)
353 xfs_ioend_merge_private(ioend, next);
354 }
355}
356
357/* list_sort compare function for ioends */
358static int
359xfs_ioend_compare(
360 void *priv,
361 struct list_head *a,
362 struct list_head *b)
363{
364 struct xfs_ioend *ia;
365 struct xfs_ioend *ib;
366
367 ia = container_of(a, struct xfs_ioend, io_list);
368 ib = container_of(b, struct xfs_ioend, io_list);
369 if (ia->io_offset < ib->io_offset)
370 return -1;
371 else if (ia->io_offset > ib->io_offset)
372 return 1;
373 return 0;
374}
375
376static void
377xfs_sort_ioends(
378 struct list_head *ioend_list)
379{
380 list_sort(NULL, ioend_list, xfs_ioend_compare);
381}
382
383/* Finish all pending io completions. */240/* Finish all pending io completions. */
384void241void
385xfs_end_io(242xfs_end_io(
@@ -387,7 +244,7 @@ xfs_end_io( · 1 unmodified line
387{244{
388 struct xfs_inode *ip =245 struct xfs_inode *ip =
389 container_of(work, struct xfs_inode, i_ioend_work);246 container_of(work, struct xfs_inode, i_ioend_work);
390 struct xfs_ioend *ioend;247 struct iomap_ioend *ioend;
391 struct list_head tmp;248 struct list_head tmp;
392 unsigned long flags;249 unsigned long flags;
393 250
@@ -395,16 +252,16 @@ xfs_end_io( · 1 unmodified line
395 list_replace_init(&ip->i_ioend_list, &tmp);252 list_replace_init(&ip->i_ioend_list, &tmp);
396 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);253 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
397 254
398 xfs_sort_ioends(&tmp);255 iomap_sort_ioends(&tmp);
399 while ((ioend = list_first_entry_or_null(&tmp, struct xfs_ioend,256 while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
400 io_list))) {257 io_list))) {
401 list_del_init(&ioend->io_list);258 list_del_init(&ioend->io_list);
402 xfs_ioend_try_merge(ioend, &tmp);259 iomap_ioend_try_merge(ioend, &tmp, xfs_ioend_merge_private);
403 xfs_end_ioend(ioend);260 xfs_end_ioend(ioend);
404 }261 }
405}262}
406 263
407static inline bool xfs_ioend_needs_workqueue(struct xfs_ioend *ioend)264static inline bool xfs_ioend_needs_workqueue(struct iomap_ioend *ioend)
408{265{
409 return ioend->io_private ||266 return ioend->io_private ||
410 ioend->io_type == IOMAP_UNWRITTEN ||267 ioend->io_type == IOMAP_UNWRITTEN ||
@@ -415,20 +272,18 @@ STATIC void · 4 unmodified lines
415xfs_end_bio(272xfs_end_bio(
416 struct bio *bio)273 struct bio *bio)
417{274{
418 struct xfs_ioend *ioend = bio->bi_private;275 struct iomap_ioend *ioend = bio->bi_private;
419 struct xfs_inode *ip = XFS_I(ioend->io_inode);276 struct xfs_inode *ip = XFS_I(ioend->io_inode);
420 struct xfs_mount *mp = ip->i_mount;
421 unsigned long flags;277 unsigned long flags;
422 278
423 if (xfs_ioend_needs_workqueue(ioend)) {279 ASSERT(xfs_ioend_needs_workqueue(ioend));
424 spin_lock_irqsave(&ip->i_ioend_lock, flags);280
425 if (list_empty(&ip->i_ioend_list))281 spin_lock_irqsave(&ip->i_ioend_lock, flags);
426 WARN_ON_ONCE(!queue_work(mp->m_unwritten_workqueue,282 if (list_empty(&ip->i_ioend_list))
427 &ip->i_ioend_work));283 WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
428 list_add_tail(&ioend->io_list, &ip->i_ioend_list);284 &ip->i_ioend_work));
429 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);285 list_add_tail(&ioend->io_list, &ip->i_ioend_list);
430 } else286 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
431 xfs_destroy_ioend(ioend, blk_status_to_errno(bio->bi_status));
432}287}
433 288
434/*289/*
@@ -437,7 +292,7 @@ xfs_end_bio( · 2 unmodified lines
437 */292 */
438static bool293static bool
439xfs_imap_valid(294xfs_imap_valid(
440 struct xfs_writepage_ctx *wpc,295 struct iomap_writepage_ctx *wpc,
441 struct xfs_inode *ip,296 struct xfs_inode *ip,
442 loff_t offset)297 loff_t offset)
443{298{
@@ -459,10 +314,10 @@ xfs_imap_valid( · 15 unmodified lines
459 * checked (and found nothing at this offset) could have added314 * checked (and found nothing at this offset) could have added
460 * overlapping blocks.315 * overlapping blocks.
461 */316 */
462 if (wpc->data_seq != READ_ONCE(ip->i_df.if_seq))317 if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq))
463 return false;318 return false;
464 if (xfs_inode_has_cow_data(ip) &&319 if (xfs_inode_has_cow_data(ip) &&
465 wpc->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))320 XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
466 return false;321 return false;
467 return true;322 return true;
468}323}
@@ -477,12 +332,18 @@ xfs_imap_valid( · 8 unmodified lines
477 */332 */
478static int333static int
479xfs_convert_blocks(334xfs_convert_blocks(
480 struct xfs_writepage_ctx *wpc,335 struct iomap_writepage_ctx *wpc,
481 struct xfs_inode *ip,336 struct xfs_inode *ip,
482 int whichfork,337 int whichfork,
483 loff_t offset)338 loff_t offset)
484{339{
485 int error;340 int error;
341 unsigned *seq;
342
343 if (whichfork == XFS_COW_FORK)
344 seq = &XFS_WPC(wpc)->cow_seq;
345 else
346 seq = &XFS_WPC(wpc)->data_seq;
486 347
487 /*348 /*
488 * Attempt to allocate whatever delalloc extent currently backs offset349 * Attempt to allocate whatever delalloc extent currently backs offset
@@ -492,8 +353,7 @@ xfs_convert_blocks( · 3 unmodified lines
492 */353 */
493 do {354 do {
494 error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,355 error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
495 &wpc->iomap, whichfork == XFS_COW_FORK ?356 &wpc->iomap, seq);
496 &wpc->cow_seq : &wpc->data_seq);
497 if (error)357 if (error)
498 return error;358 return error;
499 } while (wpc->iomap.offset + wpc->iomap.length <= offset);359 } while (wpc->iomap.offset + wpc->iomap.length <= offset);
@@ -501,9 +361,9 @@ xfs_convert_blocks( · 1 unmodified line
501 return 0;361 return 0;
502}362}
503 363
504STATIC int364static int
505xfs_map_blocks(365xfs_map_blocks(
506 struct xfs_writepage_ctx *wpc,366 struct iomap_writepage_ctx *wpc,
507 struct inode *inode,367 struct inode *inode,
508 loff_t offset)368 loff_t offset)
509{369{
@@ -559,7 +419,7 @@ retry: · 49 unmodified lines
559 xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))419 xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
560 cow_fsb = imap.br_startoff;420 cow_fsb = imap.br_startoff;
561 if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {421 if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
562 wpc->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);422 XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
563 xfs_iunlock(ip, XFS_ILOCK_SHARED);423 xfs_iunlock(ip, XFS_ILOCK_SHARED);
564 424
565 whichfork = XFS_COW_FORK;425 whichfork = XFS_COW_FORK;
@@ -582,7 +442,7 @@ retry: · 16 unmodified lines
582 */442 */
583 if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))443 if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
584 imap.br_startoff = end_fsb; /* fake a hole past EOF */444 imap.br_startoff = end_fsb; /* fake a hole past EOF */
585 wpc->data_seq = READ_ONCE(ip->i_df.if_seq);445 XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
586 xfs_iunlock(ip, XFS_ILOCK_SHARED);446 xfs_iunlock(ip, XFS_ILOCK_SHARED);
587 447
588 /* landed in a hole or beyond EOF? */448 /* landed in a hole or beyond EOF? */
@@ -645,24 +505,9 @@ allocate_blocks: · 56 unmodified lines
645 return 0;505 return 0;
646}506}
647 507
648/*508static int
649 * Submit the bio for an ioend. We are passed an ioend with a bio attached to509xfs_prepare_ioend(
650 * it, and we submit that bio. The ioend may be used for multiple bio510 struct iomap_ioend *ioend,
651 * submissions, so we only want to allocate an append transaction for the ioend
652 * once. In the case of multiple bio submission, each bio will take an IO
653 * reference to the ioend to ensure that the ioend completion is only done once
654 * all bios have been submitted and the ioend is really done.
655 *
656 * If @status is non-zero, it means that we have a situation where some part of
657 * the submission process has failed after we have marked paged for writeback
658 * and unlocked them. In this situation, we need to fail the bio and ioend
659 * rather than submit it to IO. This typically only happens on a filesystem
660 * shutdown.
661 */
662STATIC int
663xfs_submit_ioend(
664 struct writeback_control *wbc,
665 struct xfs_ioend *ioend,
666 int status)511 int status)
667{512{
668 unsigned int nofs_flag;513 unsigned int nofs_flag;
@@ -690,137 +535,9 @@ xfs_submit_ioend( · 21 unmodified lines
690 535
691 memalloc_nofs_restore(nofs_flag);536 memalloc_nofs_restore(nofs_flag);
692 537
693 ioend->io_bio->bi_private = ioend;538 if (xfs_ioend_needs_workqueue(ioend))
694 ioend->io_bio->bi_end_io = xfs_end_bio;539 ioend->io_bio->bi_end_io = xfs_end_bio;
695 540 return status;
696 /*
697 * If we are failing the IO now, just mark the ioend with an
698 * error and finish it. This will run IO completion immediately
699 * as there is only one reference to the ioend at this point in
700 * time.
701 */
702 if (status) {
703 ioend->io_bio->bi_status = errno_to_blk_status(status);
704 bio_endio(ioend->io_bio);
705 return status;
706 }
707
708 submit_bio(ioend->io_bio);
709 return 0;
710}
711
712static struct xfs_ioend *
713xfs_alloc_ioend(
714 struct inode *inode,
715 struct xfs_writepage_ctx *wpc,
716 xfs_off_t offset,
717 sector_t sector,
718 struct writeback_control *wbc)
719{
720 struct xfs_ioend *ioend;
721 struct bio *bio;
722
723 bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &xfs_ioend_bioset);
724 bio_set_dev(bio, wpc->iomap.bdev);
725 bio->bi_iter.bi_sector = sector;
726 bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
727 bio->bi_write_hint = inode->i_write_hint;
728 wbc_init_bio(wbc, bio);
729
730 ioend = container_of(bio, struct xfs_ioend, io_inline_bio);
731 INIT_LIST_HEAD(&ioend->io_list);
732 ioend->io_type = wpc->iomap.type;
733 ioend->io_flags = wpc->iomap.flags;
734 ioend->io_inode = inode;
735 ioend->io_size = 0;
736 ioend->io_offset = offset;
737 ioend->io_private = NULL;
738 ioend->io_bio = bio;
739 return ioend;
740}
741
742/*
743 * Allocate a new bio, and chain the old bio to the new one.
744 *
745 * Note that we have to do perform the chaining in this unintuitive order
746 * so that the bi_private linkage is set up in the right direction for the
747 * traversal in xfs_destroy_ioend().
748 */
749static struct bio *
750xfs_chain_bio(
751 struct bio *prev)
752{
753 struct bio *new;
754
755 new = bio_alloc(GFP_NOFS, BIO_MAX_PAGES);
756 bio_copy_dev(new, prev);/* also copies over blkcg information */
757 new->bi_iter.bi_sector = bio_end_sector(prev);
758 new->bi_opf = prev->bi_opf;
759 new->bi_write_hint = prev->bi_write_hint;
760
761 bio_chain(prev, new);
762 bio_get(prev); /* for xfs_destroy_ioend */
763 submit_bio(prev);
764 return new;
765}
766
767static bool
768xfs_can_add_to_ioend(
769 struct xfs_writepage_ctx *wpc,
770 xfs_off_t offset,
771 sector_t sector)
772{
773 if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
774 (wpc->ioend->io_flags & IOMAP_F_SHARED))
775 return false;
776 if (wpc->iomap.type != wpc->ioend->io_type)
777 return false;
778 if (offset != wpc->ioend->io_offset + wpc->ioend->io_size)
779 return false;
780 if (sector != bio_end_sector(wpc->ioend->io_bio))
781 return false;
782 return true;
783}
784
785/*
786 * Test to see if we have an existing ioend structure that we could append to
787 * first, otherwise finish off the current ioend and start another.
788 */
789STATIC void
790xfs_add_to_ioend(
791 struct inode *inode,
792 xfs_off_t offset,
793 struct page *page,
794 struct iomap_page *iop,
795 struct xfs_writepage_ctx *wpc,
796 struct writeback_control *wbc,
797 struct list_head *iolist)
798{
799 sector_t sector = iomap_sector(&wpc->iomap, offset);
800 unsigned len = i_blocksize(inode);
801 unsigned poff = offset & (PAGE_SIZE - 1);
802 bool merged, same_page = false;
803
804 if (!wpc->ioend || !xfs_can_add_to_ioend(wpc, offset, sector)) {
805 if (wpc->ioend)
806 list_add(&wpc->ioend->io_list, iolist);
807 wpc->ioend = xfs_alloc_ioend(inode, wpc, offset, sector, wbc);
808 }
809
810 merged = __bio_try_merge_page(wpc->ioend->io_bio, page, len, poff,
811 &same_page);
812
813 if (iop && !same_page)
814 atomic_inc(&iop->write_count);
815
816 if (!merged) {
817 if (bio_full(wpc->ioend->io_bio, len))
818 wpc->ioend->io_bio = xfs_chain_bio(wpc->ioend->io_bio);
819 bio_add_page(wpc->ioend->io_bio, page, len, poff);
820 }
821
822 wpc->ioend->io_size += len;
823 wbc_account_cgroup_owner(wbc, page, len);
824}541}
825 542
826/*543/*
@@ -834,8 +551,8 @@ xfs_add_to_ioend( · 7 unmodified lines
834 * transaction as there is no space left for block reservation (typically why we551 * transaction as there is no space left for block reservation (typically why we
835 * see a ENOSPC in writeback).552 * see a ENOSPC in writeback).
836 */553 */
837STATIC void554static void
838xfs_aops_discard_page(555xfs_discard_page(
839 struct page *page)556 struct page *page)
840{557{
841 struct inode *inode = page->mapping->host;558 struct inode *inode = page->mapping->host;
@@ -860,243 +577,11 @@ out_invalidate: · 18 unmodified lines
860 iomap_invalidatepage(page, 0, PAGE_SIZE);577 iomap_invalidatepage(page, 0, PAGE_SIZE);
861}578}
862 579
863/*580static const struct iomap_writeback_ops xfs_writeback_ops = {
864 * We implement an immediate ioend submission policy here to avoid needing to581 .map_blocks = xfs_map_blocks,
865 * chain multiple ioends and hence nest mempool allocations which can violate582 .prepare_ioend = xfs_prepare_ioend,
866 * forward progress guarantees we need to provide. The current ioend we are583 .discard_page = xfs_discard_page,
867 * adding blocks to is cached on the writepage context, and if the new block584};
868 * does not append to the cached ioend it will create a new ioend and cache that
869 * instead.
870 *
871 * If a new ioend is created and cached, the old ioend is returned and queued
872 * locally for submission once the entire page is processed or an error has been
873 * detected. While ioends are submitted immediately after they are completed,
874 * batching optimisations are provided by higher level block plugging.
875 *
876 * At the end of a writeback pass, there will be a cached ioend remaining on the
877 * writepage context that the caller will need to submit.
878 */
879static int
880xfs_writepage_map(
881 struct xfs_writepage_ctx *wpc,
882 struct writeback_control *wbc,
883 struct inode *inode,
884 struct page *page,
885 uint64_t end_offset)
886{
887 LIST_HEAD(submit_list);
888 struct iomap_page *iop = to_iomap_page(page);
889 unsigned len = i_blocksize(inode);
890 struct xfs_ioend *ioend, *next;
891 uint64_t file_offset; /* file offset of page */
892 int error = 0, count = 0, i;
893
894 ASSERT(iop || i_blocksize(inode) == PAGE_SIZE);
895 ASSERT(!iop || atomic_read(&iop->write_count) == 0);
896
897 /*
898 * Walk through the page to find areas to write back. If we run off the
899 * end of the current map or find the current map invalid, grab a new
900 * one.
901 */
902 for (i = 0, file_offset = page_offset(page);
903 i < (PAGE_SIZE >> inode->i_blkbits) && file_offset < end_offset;
904 i++, file_offset += len) {
905 if (iop && !test_bit(i, iop->uptodate))
906 continue;
907
908 error = xfs_map_blocks(wpc, inode, file_offset);
909 if (error)
910 break;
911 if (wpc->iomap.type == IOMAP_HOLE)
912 continue;
913 xfs_add_to_ioend(inode, file_offset, page, iop, wpc, wbc,
914 &submit_list);
915 count++;
916 }
917
918 ASSERT(wpc->ioend || list_empty(&submit_list));
919 ASSERT(PageLocked(page));
920 ASSERT(!PageWriteback(page));
921
922 /*
923 * On error, we have to fail the ioend here because we may have set
924 * pages under writeback, we have to make sure we run IO completion to
925 * mark the error state of the IO appropriately, so we can't cancel the
926 * ioend directly here. That means we have to mark this page as under
927 * writeback if we included any blocks from it in the ioend chain so
928 * that completion treats it correctly.
929 *
930 * If we didn't include the page in the ioend, the on error we can
931 * simply discard and unlock it as there are no other users of the page
932 * now. The caller will still need to trigger submission of outstanding
933 * ioends on the writepage context so they are treated correctly on
934 * error.
935 */
936 if (unlikely(error)) {
937 if (!count) {
938 xfs_aops_discard_page(page);
939 ClearPageUptodate(page);
940 unlock_page(page);
941 goto done;
942 }
943
944 /*
945 * If the page was not fully cleaned, we need to ensure that the
946 * higher layers come back to it correctly. That means we need
947 * to keep the page dirty, and for WB_SYNC_ALL writeback we need
948 * to ensure the PAGECACHE_TAG_TOWRITE index mark is not removed
949 * so another attempt to write this page in this writeback sweep
950 * will be made.
951 */
952 set_page_writeback_keepwrite(page);
953 } else {
954 clear_page_dirty_for_io(page);
955 set_page_writeback(page);
956 }
957
958 unlock_page(page);
959
960 /*
961 * Preserve the original error if there was one, otherwise catch
962 * submission errors here and propagate into subsequent ioend
963 * submissions.
964 */
965 list_for_each_entry_safe(ioend, next, &submit_list, io_list) {
966 int error2;
967
968 list_del_init(&ioend->io_list);
969 error2 = xfs_submit_ioend(wbc, ioend, error);
970 if (error2 && !error)
971 error = error2;
972 }
973
974 /*
975 * We can end up here with no error and nothing to write only if we race
976 * with a partial page truncate on a sub-page block sized filesystem.
977 */
978 if (!count)
979 end_page_writeback(page);
980done:
981 mapping_set_error(page->mapping, error);
982 return error;
983}
984
985/*
986 * Write out a dirty page.
987 *
988 * For delalloc space on the page we need to allocate space and flush it.
989 * For unwritten space on the page we need to start the conversion to
990 * regular allocated space.
991 */
992STATIC int
993xfs_do_writepage(
994 struct page *page,
995 struct writeback_control *wbc,
996 void *data)
997{
998 struct xfs_writepage_ctx *wpc = data;
999 struct inode *inode = page->mapping->host;
1000 loff_t offset;
1001 uint64_t end_offset;
1002 pgoff_t end_index;
1003
1004 trace_xfs_writepage(inode, page, 0, 0);
1005
1006 /*
1007 * Refuse to write the page out if we are called from reclaim context.
1008 *
1009 * This avoids stack overflows when called from deeply used stacks in
1010 * random callers for direct reclaim or memcg reclaim. We explicitly
1011 * allow reclaim from kswapd as the stack usage there is relatively low.
1012 *
1013 * This should never happen except in the case of a VM regression so
1014 * warn about it.
1015 */
1016 if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
1017 PF_MEMALLOC))
1018 goto redirty;
1019
1020 /*
1021 * Given that we do not allow direct reclaim to call us, we should
1022 * never be called while in a filesystem transaction.
1023 */
1024 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC_NOFS))
1025 goto redirty;
1026
1027 /*
1028 * Is this page beyond the end of the file?
1029 *
1030 * The page index is less than the end_index, adjust the end_offset
1031 * to the highest offset that this page should represent.
1032 * -----------------------------------------------------
1033 * | file mapping | <EOF> |
1034 * -----------------------------------------------------
1035 * | Page ... | Page N-2 | Page N-1 | Page N | |
1036 * ^--------------------------------^----------|--------
1037 * | desired writeback range | see else |
1038 * ---------------------------------^------------------|
1039 */
1040 offset = i_size_read(inode);
1041 end_index = offset >> PAGE_SHIFT;
1042 if (page->index < end_index)
1043 end_offset = (xfs_off_t)(page->index + 1) << PAGE_SHIFT;
1044 else {
1045 /*
1046 * Check whether the page to write out is beyond or straddles
1047 * i_size or not.
1048 * -------------------------------------------------------
1049 * | file mapping | <EOF> |
1050 * -------------------------------------------------------
1051 * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
1052 * ^--------------------------------^-----------|---------
1053 * | | Straddles |
1054 * ---------------------------------^-----------|--------|
1055 */
1056 unsigned offset_into_page = offset & (PAGE_SIZE - 1);
1057
1058 /*
1059 * Skip the page if it is fully outside i_size, e.g. due to a
1060 * truncate operation that is in progress. We must redirty the
1061 * page so that reclaim stops reclaiming it. Otherwise
1062 * xfs_vm_releasepage() is called on it and gets confused.
1063 *
1064 * Note that the end_index is unsigned long, it would overflow
1065 * if the given offset is greater than 16TB on 32-bit system
1066 * and if we do check the page is fully outside i_size or not
1067 * via "if (page->index >= end_index + 1)" as "end_index + 1"
1068 * will be evaluated to 0. Hence this page will be redirtied
1069 * and be written out repeatedly which would result in an
1070 * infinite loop, the user program that perform this operation
1071 * will hang. Instead, we can verify this situation by checking
1072 * if the page to write is totally beyond the i_size or if it's
1073 * offset is just equal to the EOF.
1074 */
1075 if (page->index > end_index ||
1076 (page->index == end_index && offset_into_page == 0))
1077 goto redirty;
1078
1079 /*
1080 * The page straddles i_size. It must be zeroed out on each
1081 * and every writepage invocation because it may be mmapped.
1082 * "A file is mapped in multiples of the page size. For a file
1083 * that is not a multiple of the page size, the remaining
1084 * memory is zeroed when mapped, and writes to that region are
1085 * not written out to the file."
1086 */
1087 zero_user_segment(page, offset_into_page, PAGE_SIZE);
1088
1089 /* Adjust the end_offset to the end of file */
1090 end_offset = offset;
1091 }
1092
1093 return xfs_writepage_map(wpc, wbc, inode, page, end_offset);
1094
1095redirty:
1096 redirty_page_for_writepage(wbc, page);
1097 unlock_page(page);
1098 return 0;
1099}
1100 585
1101STATIC int586STATIC int
1102xfs_vm_writepage(587xfs_vm_writepage(
@@ -1104,12 +589,8 @@ xfs_vm_writepage( · 1 unmodified line
1104 struct writeback_control *wbc)589 struct writeback_control *wbc)
1105{590{
1106 struct xfs_writepage_ctx wpc = { };591 struct xfs_writepage_ctx wpc = { };
1107 int ret;
1108 592
1109 ret = xfs_do_writepage(page, wbc, &wpc);593 return iomap_writepage(page, wbc, &wpc.ctx, &xfs_writeback_ops);
1110 if (wpc.ioend)
1111 ret = xfs_submit_ioend(wbc, wpc.ioend, ret);
1112 return ret;
1113}594}
1114 595
1115STATIC int596STATIC int
@@ -1118,13 +599,9 @@ xfs_vm_writepages( · 2 unmodified lines
1118 struct writeback_control *wbc)599 struct writeback_control *wbc)
1119{600{
1120 struct xfs_writepage_ctx wpc = { };601 struct xfs_writepage_ctx wpc = { };
1121 int ret;
1122 602
1123 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);603 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
1124 ret = write_cache_pages(mapping, wbc, xfs_do_writepage, &wpc);604 return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
1125 if (wpc.ioend)
1126 ret = xfs_submit_ioend(wbc, wpc.ioend, ret);
1127 return ret;
1128}605}
1129 606
1130STATIC int607STATIC int
fs/xfs/xfs_aops.h+0−17
@@ -6,23 +6,6 @@ · 5 unmodified lines
6#ifndef __XFS_AOPS_H__6#ifndef __XFS_AOPS_H__
7#define __XFS_AOPS_H__7#define __XFS_AOPS_H__
8 8
9extern struct bio_set xfs_ioend_bioset;
10
11/*
12 * Structure for buffered I/O completions.
13 */
14struct xfs_ioend {
15 struct list_head io_list; /* next ioend in chain */
16 u16 io_type;
17 u16 io_flags; /* IOMAP_F_* */
18 struct inode *io_inode; /* file being written to */
19 size_t io_size; /* size of the extent */
20 xfs_off_t io_offset; /* offset in the file */
21 void *io_private; /* file system private data */
22 struct bio *io_bio; /* bio being built */
23 struct bio io_inline_bio; /* MUST BE LAST! */
24};
25
26extern const struct address_space_operations xfs_address_space_operations;9extern const struct address_space_operations xfs_address_space_operations;
27extern const struct address_space_operations xfs_dax_aops;10extern const struct address_space_operations xfs_dax_aops;
28 11
fs/xfs/xfs_super.c+1−10
@@ -40,7 +40,6 @@ · 39 unmodified lines
40#include <linux/parser.h>40#include <linux/parser.h>
41 41
42static const struct super_operations xfs_super_operations;42static const struct super_operations xfs_super_operations;
43struct bio_set xfs_ioend_bioset;
44 43
45static struct kset *xfs_kset; /* top-level xfs sysfs dir */44static struct kset *xfs_kset; /* top-level xfs sysfs dir */
46#ifdef DEBUG45#ifdef DEBUG
@@ -1853,15 +1852,10 @@ MODULE_ALIAS_FS("xfs"); · 1806 unmodified lines
1853STATIC int __init1852STATIC int __init
1854xfs_init_zones(void)1853xfs_init_zones(void)
1855{1854{
1856 if (bioset_init(&xfs_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
1857 offsetof(struct xfs_ioend, io_inline_bio),
1858 BIOSET_NEED_BVECS))
1859 goto out;
1860
1861 xfs_log_ticket_zone = kmem_zone_init(sizeof(xlog_ticket_t),1855 xfs_log_ticket_zone = kmem_zone_init(sizeof(xlog_ticket_t),
1862 "xfs_log_ticket");1856 "xfs_log_ticket");
1863 if (!xfs_log_ticket_zone)1857 if (!xfs_log_ticket_zone)
1864 goto out_free_ioend_bioset;1858 goto out;
1865 1859
1866 xfs_bmap_free_item_zone = kmem_zone_init(1860 xfs_bmap_free_item_zone = kmem_zone_init(
1867 sizeof(struct xfs_extent_free_item),1861 sizeof(struct xfs_extent_free_item),
@@ -1996,8 +1990,6 @@ xfs_init_zones(void) · 128 unmodified lines
1996 kmem_zone_destroy(xfs_bmap_free_item_zone);1990 kmem_zone_destroy(xfs_bmap_free_item_zone);
1997 out_destroy_log_ticket_zone:1991 out_destroy_log_ticket_zone:
1998 kmem_zone_destroy(xfs_log_ticket_zone);1992 kmem_zone_destroy(xfs_log_ticket_zone);
1999 out_free_ioend_bioset:
2000 bioset_exit(&xfs_ioend_bioset);
2001 out:1993 out:
2002 return -ENOMEM;1994 return -ENOMEM;
2003}1995}
@@ -2028,7 +2020,6 @@ xfs_destroy_zones(void) · 24 unmodified lines
2028 kmem_zone_destroy(xfs_btree_cur_zone);2020 kmem_zone_destroy(xfs_btree_cur_zone);
2029 kmem_zone_destroy(xfs_bmap_free_item_zone);2021 kmem_zone_destroy(xfs_bmap_free_item_zone);
2030 kmem_zone_destroy(xfs_log_ticket_zone);2022 kmem_zone_destroy(xfs_log_ticket_zone);
2031 bioset_exit(&xfs_ioend_bioset);
2032}2023}
2033 2024
2034STATIC int __init2025STATIC int __init
fs/xfs/xfs_trace.h+0−39
@@ -1158,45 +1158,6 @@ DEFINE_RW_EVENT(xfs_file_buffered_write); · 1157 unmodified lines
1158DEFINE_RW_EVENT(xfs_file_direct_write);1158DEFINE_RW_EVENT(xfs_file_direct_write);
1159DEFINE_RW_EVENT(xfs_file_dax_write);1159DEFINE_RW_EVENT(xfs_file_dax_write);
1160 1160
1161DECLARE_EVENT_CLASS(xfs_page_class,
1162 TP_PROTO(struct inode *inode, struct page *page, unsigned long off,
1163 unsigned int len),
1164 TP_ARGS(inode, page, off, len),
1165 TP_STRUCT__entry(
1166 __field(dev_t, dev)
1167 __field(xfs_ino_t, ino)
1168 __field(pgoff_t, pgoff)
1169 __field(loff_t, size)
1170 __field(unsigned long, offset)
1171 __field(unsigned int, length)
1172 ),
1173 TP_fast_assign(
1174 __entry->dev = inode->i_sb->s_dev;
1175 __entry->ino = XFS_I(inode)->i_ino;
1176 __entry->pgoff = page_offset(page);
1177 __entry->size = i_size_read(inode);
1178 __entry->offset = off;
1179 __entry->length = len;
1180 ),
1181 TP_printk("dev %d:%d ino 0x%llx pgoff 0x%lx size 0x%llx offset %lx "
1182 "length %x",
1183 MAJOR(__entry->dev), MINOR(__entry->dev),
1184 __entry->ino,
1185 __entry->pgoff,
1186 __entry->size,
1187 __entry->offset,
1188 __entry->length)
1189)
1190
1191#define DEFINE_PAGE_EVENT(name) \
1192DEFINE_EVENT(xfs_page_class, name, \
1193 TP_PROTO(struct inode *inode, struct page *page, unsigned long off, \
1194 unsigned int len), \
1195 TP_ARGS(inode, page, off, len))
1196DEFINE_PAGE_EVENT(xfs_writepage);
1197DEFINE_PAGE_EVENT(xfs_releasepage);
1198DEFINE_PAGE_EVENT(xfs_invalidatepage);
1199
1200DECLARE_EVENT_CLASS(xfs_imap_class,1161DECLARE_EVENT_CLASS(xfs_imap_class,
1201 TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count,1162 TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count,
1202 int whichfork, struct xfs_bmbt_irec *irec),1163 int whichfork, struct xfs_bmbt_irec *irec),
include/linux/iomap.h+59−0
@@ -4,6 +4,7 @@ · 3 unmodified lines
4 4
5#include <linux/atomic.h>5#include <linux/atomic.h>
6#include <linux/bitmap.h>6#include <linux/bitmap.h>
7#include <linux/blk_types.h>
7#include <linux/mm.h>8#include <linux/mm.h>
8#include <linux/types.h>9#include <linux/types.h>
9#include <linux/mm_types.h>10#include <linux/mm_types.h>
@@ -12,6 +13,7 @@ · 2 unmodified lines
12struct address_space;13struct address_space;
13struct fiemap_extent_info;14struct fiemap_extent_info;
14struct inode;15struct inode;
16struct iomap_writepage_ctx;
15struct iov_iter;17struct iov_iter;
16struct kiocb;18struct kiocb;
17struct page;19struct page;
@@ -185,6 +187,63 @@ loff_t iomap_seek_data(struct inode *inode, loff_t offset, · 167 unmodified lines
185sector_t iomap_bmap(struct address_space *mapping, sector_t bno,187sector_t iomap_bmap(struct address_space *mapping, sector_t bno,
186 const struct iomap_ops *ops);188 const struct iomap_ops *ops);
187 189
190/*
191 * Structure for writeback I/O completions.
192 */
193struct iomap_ioend {
194 struct list_head io_list; /* next ioend in chain */
195 u16 io_type;
196 u16 io_flags; /* IOMAP_F_* */
197 struct inode *io_inode; /* file being written to */
198 size_t io_size; /* size of the extent */
199 loff_t io_offset; /* offset in the file */
200 void *io_private; /* file system private data */
201 struct bio *io_bio; /* bio being built */
202 struct bio io_inline_bio; /* MUST BE LAST! */
203};
204
205struct iomap_writeback_ops {
206 /*
207 * Required, maps the blocks so that writeback can be performed on
208 * the range starting at offset.
209 */
210 int (*map_blocks)(struct iomap_writepage_ctx *wpc, struct inode *inode,
211 loff_t offset);
212
213 /*
214 * Optional, allows the file systems to perform actions just before
215 * submitting the bio and/or override the bio end_io handler for complex
216 * operations like copy on write extent manipulation or unwritten extent
217 * conversions.
218 */
219 int (*prepare_ioend)(struct iomap_ioend *ioend, int status);
220
221 /*
222 * Optional, allows the file system to discard state on a page where
223 * we failed to submit any I/O.
224 */
225 void (*discard_page)(struct page *page);
226};
227
228struct iomap_writepage_ctx {
229 struct iomap iomap;
230 struct iomap_ioend *ioend;
231 const struct iomap_writeback_ops *ops;
232};
233
234void iomap_finish_ioends(struct iomap_ioend *ioend, int error);
235void iomap_ioend_try_merge(struct iomap_ioend *ioend,
236 struct list_head *more_ioends,
237 void (*merge_private)(struct iomap_ioend *ioend,
238 struct iomap_ioend *next));
239void iomap_sort_ioends(struct list_head *ioend_list);
240int iomap_writepage(struct page *page, struct writeback_control *wbc,
241 struct iomap_writepage_ctx *wpc,
242 const struct iomap_writeback_ops *ops);
243int iomap_writepages(struct address_space *mapping,
244 struct writeback_control *wbc, struct iomap_writepage_ctx *wpc,
245 const struct iomap_writeback_ops *ops);
246
188/*247/*
189 * Flags for direct I/O ->end_io:248 * Flags for direct I/O ->end_io:
190 */249 */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t