iomap: move the direct IO code into a separate file

Move the direct IO code into a separate file so that we can group related functions in a single file instead of having a single enormous source file.

Darrick J. Wong committed 7 years ago

Commit db074431 parent 56a17894 files · +570 −554Browse files

Unified · Split

4 files

fs/iomap.c +0 −554
fs/iomap/Makefile +1 −0
fs/iomap/direct-io.c +562 −0
include/linux/iomap.h +7 −0

fs/iomap.c+0−554

@@ -90,12 +90,6 @@ iomap_apply(struct inode *inode, loff_t pos, loff_t length, unsigned flags, · 89 unmodified lines
90	90	return written ? written : ret;
91	91	}
92	92
93		−static sector_t
94		−iomap_sector(struct iomap *iomap, loff_t pos)
95		−{
96		− return (iomap->addr + pos - iomap->offset) >> SECTOR_SHIFT;
97		−}
98		−
99	93	static struct iomap_page *
100	94	iomap_page_create(struct inode inode, struct page page)
101	95	{
@@ -1148,551 +1142,3 @@ out_unlock: · 1046 unmodified lines
1148	1142	return block_page_mkwrite_return(ret);
1149	1143	}
1150	1144	EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1151		−
1152		−/*
1153		− * Private flags for iomap_dio, must not overlap with the public ones in
1154		− * iomap.h:
1155		− */
1156		−#define IOMAP_DIO_WRITE_FUA (1 << 28)
1157		−#define IOMAP_DIO_NEED_SYNC (1 << 29)
1158		−#define IOMAP_DIO_WRITE (1 << 30)
1159		−#define IOMAP_DIO_DIRTY (1 << 31)
1160		−
1161		−struct iomap_dio {
1162		− struct kiocb *iocb;
1163		− iomap_dio_end_io_t *end_io;
1164		− loff_t i_size;
1165		− loff_t size;
1166		− atomic_t ref;
1167		− unsigned flags;
1168		− int error;
1169		− bool wait_for_completion;
1170		−
1171		− union {
1172		− /* used during submission and for synchronous completion: */
1173		− struct {
1174		− struct iov_iter *iter;
1175		− struct task_struct *waiter;
1176		− struct request_queue *last_queue;
1177		− blk_qc_t cookie;
1178		− } submit;
1179		−
1180		− /* used for aio completion: */
1181		− struct {
1182		− struct work_struct work;
1183		− } aio;
1184		− };
1185		−};
1186		−
1187		−int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
1188		−{
1189		− struct request_queue *q = READ_ONCE(kiocb->private);
1190		−
1191		− if (!q)
1192		− return 0;
1193		− return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
1194		−}
1195		−EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
1196		−
1197		−static void iomap_dio_submit_bio(struct iomap_dio dio, struct iomap iomap,
1198		− struct bio *bio)
1199		−{
1200		− atomic_inc(&dio->ref);
1201		−
1202		− if (dio->iocb->ki_flags & IOCB_HIPRI)
1203		− bio_set_polled(bio, dio->iocb);
1204		−
1205		− dio->submit.last_queue = bdev_get_queue(iomap->bdev);
1206		− dio->submit.cookie = submit_bio(bio);
1207		−}
1208		−
1209		−static ssize_t iomap_dio_complete(struct iomap_dio *dio)
1210		−{
1211		− struct kiocb *iocb = dio->iocb;
1212		− struct inode *inode = file_inode(iocb->ki_filp);
1213		− loff_t offset = iocb->ki_pos;
1214		− ssize_t ret;
1215		−
1216		− if (dio->end_io) {
1217		− ret = dio->end_io(iocb,
1218		− dio->error ? dio->error : dio->size,
1219		− dio->flags);
1220		− } else {
1221		− ret = dio->error;
1222		− }
1223		−
1224		− if (likely(!ret)) {
1225		− ret = dio->size;
1226		− /* check for short read */
1227		− if (offset + ret > dio->i_size &&
1228		− !(dio->flags & IOMAP_DIO_WRITE))
1229		− ret = dio->i_size - offset;
1230		− iocb->ki_pos += ret;
1231		− }
1232		−
1233		− /*
1234		− * Try again to invalidate clean pages which might have been cached by
1235		− * non-direct readahead, or faulted in by get_user_pages() if the source
1236		− * of the write was an mmap'ed region of the file we're writing. Either
1237		− * one is a pretty crazy thing to do, so we don't support it 100%. If
1238		− * this invalidation fails, tough, the write still worked...
1239		− *
1240		− * And this page cache invalidation has to be after dio->end_io(), as
1241		− * some filesystems convert unwritten extents to real allocations in
1242		− * end_io() when necessary, otherwise a racing buffer read would cache
1243		− * zeros from unwritten extents.
1244		− */
1245		− if (!dio->error &&
1246		− (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {
1247		− int err;
1248		− err = invalidate_inode_pages2_range(inode->i_mapping,
1249		− offset >> PAGE_SHIFT,
1250		− (offset + dio->size - 1) >> PAGE_SHIFT);
1251		− if (err)
1252		− dio_warn_stale_pagecache(iocb->ki_filp);
1253		− }
1254		−
1255		− /*
1256		− * If this is a DSYNC write, make sure we push it to stable storage now
1257		− * that we've written data.
1258		− */
1259		− if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))
1260		− ret = generic_write_sync(iocb, ret);
1261		−
1262		− inode_dio_end(file_inode(iocb->ki_filp));
1263		− kfree(dio);
1264		−
1265		− return ret;
1266		−}
1267		−
1268		−static void iomap_dio_complete_work(struct work_struct *work)
1269		−{
1270		− struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
1271		− struct kiocb *iocb = dio->iocb;
1272		−
1273		− iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);
1274		−}
1275		−
1276		−/*
1277		− * Set an error in the dio if none is set yet. We have to use cmpxchg
1278		− * as the submission context and the completion context(s) can race to
1279		− * update the error.
1280		− */
1281		−static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
1282		−{
1283		− cmpxchg(&dio->error, 0, ret);
1284		−}
1285		−
1286		−static void iomap_dio_bio_end_io(struct bio *bio)
1287		−{
1288		− struct iomap_dio *dio = bio->bi_private;
1289		− bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
1290		−
1291		− if (bio->bi_status)
1292		− iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
1293		−
1294		− if (atomic_dec_and_test(&dio->ref)) {
1295		− if (dio->wait_for_completion) {
1296		− struct task_struct *waiter = dio->submit.waiter;
1297		− WRITE_ONCE(dio->submit.waiter, NULL);
1298		− blk_wake_io_task(waiter);
1299		− } else if (dio->flags & IOMAP_DIO_WRITE) {
1300		− struct inode *inode = file_inode(dio->iocb->ki_filp);
1301		−
1302		− INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
1303		− queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
1304		− } else {
1305		− iomap_dio_complete_work(&dio->aio.work);
1306		− }
1307		− }
1308		−
1309		− if (should_dirty) {
1310		− bio_check_pages_dirty(bio);
1311		− } else {
1312		− bio_release_pages(bio, false);
1313		− bio_put(bio);
1314		− }
1315		−}
1316		−
1317		−static void
1318		−iomap_dio_zero(struct iomap_dio dio, struct iomap iomap, loff_t pos,
1319		− unsigned len)
1320		−{
1321		− struct page *page = ZERO_PAGE(0);
1322		− int flags = REQ_SYNC \| REQ_IDLE;
1323		− struct bio *bio;
1324		−
1325		− bio = bio_alloc(GFP_KERNEL, 1);
1326		− bio_set_dev(bio, iomap->bdev);
1327		− bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
1328		− bio->bi_private = dio;
1329		− bio->bi_end_io = iomap_dio_bio_end_io;
1330		−
1331		− get_page(page);
1332		− __bio_add_page(bio, page, len, 0);
1333		− bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
1334		− iomap_dio_submit_bio(dio, iomap, bio);
1335		−}
1336		−
1337		−static loff_t
1338		−iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
1339		− struct iomap_dio dio, struct iomap iomap)
1340		−{
1341		− unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
1342		− unsigned int fs_block_size = i_blocksize(inode), pad;
1343		− unsigned int align = iov_iter_alignment(dio->submit.iter);
1344		− struct iov_iter iter;
1345		− struct bio *bio;
1346		− bool need_zeroout = false;
1347		− bool use_fua = false;
1348		− int nr_pages, ret = 0;
1349		− size_t copied = 0;
1350		−
1351		− if ((pos \| length \| align) & ((1 << blkbits) - 1))
1352		− return -EINVAL;
1353		−
1354		− if (iomap->type == IOMAP_UNWRITTEN) {
1355		− dio->flags \|= IOMAP_DIO_UNWRITTEN;
1356		− need_zeroout = true;
1357		− }
1358		−
1359		− if (iomap->flags & IOMAP_F_SHARED)
1360		− dio->flags \|= IOMAP_DIO_COW;
1361		−
1362		− if (iomap->flags & IOMAP_F_NEW) {
1363		− need_zeroout = true;
1364		− } else if (iomap->type == IOMAP_MAPPED) {
1365		− /*
1366		− * Use a FUA write if we need datasync semantics, this is a pure
1367		− * data IO that doesn't require any metadata updates (including
1368		− * after IO completion such as unwritten extent conversion) and
1369		− * the underlying device supports FUA. This allows us to avoid
1370		− * cache flushes on IO completion.
1371		− */
1372		− if (!(iomap->flags & (IOMAP_F_SHARED\|IOMAP_F_DIRTY)) &&
1373		− (dio->flags & IOMAP_DIO_WRITE_FUA) &&
1374		− blk_queue_fua(bdev_get_queue(iomap->bdev)))
1375		− use_fua = true;
1376		− }
1377		−
1378		− /*
1379		− * Operate on a partial iter trimmed to the extent we were called for.
1380		− * We'll update the iter in the dio once we're done with this extent.
1381		− */
1382		− iter = *dio->submit.iter;
1383		− iov_iter_truncate(&iter, length);
1384		−
1385		− nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
1386		− if (nr_pages <= 0)
1387		− return nr_pages;
1388		−
1389		− if (need_zeroout) {
1390		− /* zero out from the start of the block to the write offset */
1391		− pad = pos & (fs_block_size - 1);
1392		− if (pad)
1393		− iomap_dio_zero(dio, iomap, pos - pad, pad);
1394		− }
1395		−
1396		− do {
1397		− size_t n;
1398		− if (dio->error) {
1399		− iov_iter_revert(dio->submit.iter, copied);
1400		− return 0;
1401		− }
1402		−
1403		− bio = bio_alloc(GFP_KERNEL, nr_pages);
1404		− bio_set_dev(bio, iomap->bdev);
1405		− bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
1406		− bio->bi_write_hint = dio->iocb->ki_hint;
1407		− bio->bi_ioprio = dio->iocb->ki_ioprio;
1408		− bio->bi_private = dio;
1409		− bio->bi_end_io = iomap_dio_bio_end_io;
1410		−
1411		− ret = bio_iov_iter_get_pages(bio, &iter);
1412		− if (unlikely(ret)) {
1413		− /*
1414		− * We have to stop part way through an IO. We must fall
1415		− * through to the sub-block tail zeroing here, otherwise
1416		− * this short IO may expose stale data in the tail of
1417		− * the block we haven't written data to.
1418		− */
1419		− bio_put(bio);
1420		− goto zero_tail;
1421		− }
1422		−
1423		− n = bio->bi_iter.bi_size;
1424		− if (dio->flags & IOMAP_DIO_WRITE) {
1425		− bio->bi_opf = REQ_OP_WRITE \| REQ_SYNC \| REQ_IDLE;
1426		− if (use_fua)
1427		− bio->bi_opf \|= REQ_FUA;
1428		− else
1429		− dio->flags &= ~IOMAP_DIO_WRITE_FUA;
1430		− task_io_account_write(n);
1431		− } else {
1432		− bio->bi_opf = REQ_OP_READ;
1433		− if (dio->flags & IOMAP_DIO_DIRTY)
1434		− bio_set_pages_dirty(bio);
1435		− }
1436		−
1437		− iov_iter_advance(dio->submit.iter, n);
1438		−
1439		− dio->size += n;
1440		− pos += n;
1441		− copied += n;
1442		−
1443		− nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
1444		− iomap_dio_submit_bio(dio, iomap, bio);
1445		− } while (nr_pages);
1446		−
1447		− /*
1448		− * We need to zeroout the tail of a sub-block write if the extent type
1449		− * requires zeroing or the write extends beyond EOF. If we don't zero
1450		− * the block tail in the latter case, we can expose stale data via mmap
1451		− * reads of the EOF block.
1452		− */
1453		−zero_tail:
1454		− if (need_zeroout \|\|
1455		− ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
1456		− /* zero out from the end of the write to the end of the block */
1457		− pad = pos & (fs_block_size - 1);
1458		− if (pad)
1459		− iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);
1460		− }
1461		− return copied ? copied : ret;
1462		−}
1463		−
1464		−static loff_t
1465		−iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio)
1466		−{
1467		− length = iov_iter_zero(length, dio->submit.iter);
1468		− dio->size += length;
1469		− return length;
1470		−}
1471		−
1472		−static loff_t
1473		−iomap_dio_inline_actor(struct inode *inode, loff_t pos, loff_t length,
1474		− struct iomap_dio dio, struct iomap iomap)
1475		−{
1476		− struct iov_iter *iter = dio->submit.iter;
1477		− size_t copied;
1478		−
1479		− BUG_ON(pos + length > PAGE_SIZE - offset_in_page(iomap->inline_data));
1480		−
1481		− if (dio->flags & IOMAP_DIO_WRITE) {
1482		− loff_t size = inode->i_size;
1483		−
1484		− if (pos > size)
1485		− memset(iomap->inline_data + size, 0, pos - size);
1486		− copied = copy_from_iter(iomap->inline_data + pos, length, iter);
1487		− if (copied) {
1488		− if (pos + copied > size)
1489		− i_size_write(inode, pos + copied);
1490		− mark_inode_dirty(inode);
1491		− }
1492		− } else {
1493		− copied = copy_to_iter(iomap->inline_data + pos, length, iter);
1494		− }
1495		− dio->size += copied;
1496		− return copied;
1497		−}
1498		−
1499		−static loff_t
1500		−iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
1501		− void data, struct iomap iomap)
1502		−{
1503		− struct iomap_dio *dio = data;
1504		−
1505		− switch (iomap->type) {
1506		− case IOMAP_HOLE:
1507		− if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
1508		− return -EIO;
1509		− return iomap_dio_hole_actor(length, dio);
1510		− case IOMAP_UNWRITTEN:
1511		− if (!(dio->flags & IOMAP_DIO_WRITE))
1512		− return iomap_dio_hole_actor(length, dio);
1513		− return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
1514		− case IOMAP_MAPPED:
1515		− return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
1516		− case IOMAP_INLINE:
1517		− return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
1518		− default:
1519		− WARN_ON_ONCE(1);
1520		− return -EIO;
1521		− }
1522		−}
1523		−
1524		−/*
1525		− * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
1526		− * is being issued as AIO or not. This allows us to optimise pure data writes
1527		− * to use REQ_FUA rather than requiring generic_write_sync() to issue a
1528		− * REQ_FLUSH post write. This is slightly tricky because a single request here
1529		− * can be mapped into multiple disjoint IOs and only a subset of the IOs issued
1530		− * may be pure data writes. In that case, we still need to do a full data sync
1531		− * completion.
1532		− */
1533		−ssize_t
1534		−iomap_dio_rw(struct kiocb iocb, struct iov_iter iter,
1535		− const struct iomap_ops *ops, iomap_dio_end_io_t end_io)
1536		−{
1537		− struct address_space *mapping = iocb->ki_filp->f_mapping;
1538		− struct inode *inode = file_inode(iocb->ki_filp);
1539		− size_t count = iov_iter_count(iter);
1540		− loff_t pos = iocb->ki_pos, start = pos;
1541		− loff_t end = iocb->ki_pos + count - 1, ret = 0;
1542		− unsigned int flags = IOMAP_DIRECT;
1543		− bool wait_for_completion = is_sync_kiocb(iocb);
1544		− struct blk_plug plug;
1545		− struct iomap_dio *dio;
1546		−
1547		− lockdep_assert_held(&inode->i_rwsem);
1548		−
1549		− if (!count)
1550		− return 0;
1551		−
1552		− dio = kmalloc(sizeof(*dio), GFP_KERNEL);
1553		− if (!dio)
1554		− return -ENOMEM;
1555		−
1556		− dio->iocb = iocb;
1557		− atomic_set(&dio->ref, 1);
1558		− dio->size = 0;
1559		− dio->i_size = i_size_read(inode);
1560		− dio->end_io = end_io;
1561		− dio->error = 0;
1562		− dio->flags = 0;
1563		−
1564		− dio->submit.iter = iter;
1565		− dio->submit.waiter = current;
1566		− dio->submit.cookie = BLK_QC_T_NONE;
1567		− dio->submit.last_queue = NULL;
1568		−
1569		− if (iov_iter_rw(iter) == READ) {
1570		− if (pos >= dio->i_size)
1571		− goto out_free_dio;
1572		−
1573		− if (iter_is_iovec(iter) && iov_iter_rw(iter) == READ)
1574		− dio->flags \|= IOMAP_DIO_DIRTY;
1575		− } else {
1576		− flags \|= IOMAP_WRITE;
1577		− dio->flags \|= IOMAP_DIO_WRITE;
1578		−
1579		− /* for data sync or sync, we need sync completion processing */
1580		− if (iocb->ki_flags & IOCB_DSYNC)
1581		− dio->flags \|= IOMAP_DIO_NEED_SYNC;
1582		−
1583		− /*
1584		− * For datasync only writes, we optimistically try using FUA for
1585		− * this IO. Any non-FUA write that occurs will clear this flag,
1586		− * hence we know before completion whether a cache flush is
1587		− * necessary.
1588		− */
1589		− if ((iocb->ki_flags & (IOCB_DSYNC \| IOCB_SYNC)) == IOCB_DSYNC)
1590		− dio->flags \|= IOMAP_DIO_WRITE_FUA;
1591		− }
1592		−
1593		− if (iocb->ki_flags & IOCB_NOWAIT) {
1594		− if (filemap_range_has_page(mapping, start, end)) {
1595		− ret = -EAGAIN;
1596		− goto out_free_dio;
1597		− }
1598		− flags \|= IOMAP_NOWAIT;
1599		− }
1600		−
1601		− ret = filemap_write_and_wait_range(mapping, start, end);
1602		− if (ret)
1603		− goto out_free_dio;
1604		−
1605		− /*
1606		− * Try to invalidate cache pages for the range we're direct
1607		− * writing. If this invalidation fails, tough, the write will
1608		− * still work, but racing two incompatible write paths is a
1609		− * pretty crazy thing to do, so we don't support it 100%.
1610		− */
1611		− ret = invalidate_inode_pages2_range(mapping,
1612		− start >> PAGE_SHIFT, end >> PAGE_SHIFT);
1613		− if (ret)
1614		− dio_warn_stale_pagecache(iocb->ki_filp);
1615		− ret = 0;
1616		−
1617		− if (iov_iter_rw(iter) == WRITE && !wait_for_completion &&
1618		− !inode->i_sb->s_dio_done_wq) {
1619		− ret = sb_init_dio_done_wq(inode->i_sb);
1620		− if (ret < 0)
1621		− goto out_free_dio;
1622		− }
1623		−
1624		− inode_dio_begin(inode);
1625		−
1626		− blk_start_plug(&plug);
1627		− do {
1628		− ret = iomap_apply(inode, pos, count, flags, ops, dio,
1629		− iomap_dio_actor);
1630		− if (ret <= 0) {
1631		− /* magic error code to fall back to buffered I/O */
1632		− if (ret == -ENOTBLK) {
1633		− wait_for_completion = true;
1634		− ret = 0;
1635		− }
1636		− break;
1637		− }
1638		− pos += ret;
1639		−
1640		− if (iov_iter_rw(iter) == READ && pos >= dio->i_size)
1641		− break;
1642		− } while ((count = iov_iter_count(iter)) > 0);
1643		− blk_finish_plug(&plug);
1644		−
1645		− if (ret < 0)
1646		− iomap_dio_set_error(dio, ret);
1647		−
1648		− /*
1649		− * If all the writes we issued were FUA, we don't need to flush the
1650		− * cache on IO completion. Clear the sync flag for this case.
1651		− */
1652		− if (dio->flags & IOMAP_DIO_WRITE_FUA)
1653		− dio->flags &= ~IOMAP_DIO_NEED_SYNC;
1654		−
1655		− WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);
1656		− WRITE_ONCE(iocb->private, dio->submit.last_queue);
1657		−
1658		− /*
1659		− * We are about to drop our additional submission reference, which
1660		− * might be the last reference to the dio. There are three three
1661		− * different ways we can progress here:
1662		− *
1663		− * (a) If this is the last reference we will always complete and free
1664		− * the dio ourselves.
1665		− * (b) If this is not the last reference, and we serve an asynchronous
1666		− * iocb, we must never touch the dio after the decrement, the
1667		− * I/O completion handler will complete and free it.
1668		− * (c) If this is not the last reference, but we serve a synchronous
1669		− * iocb, the I/O completion handler will wake us up on the drop
1670		− * of the final reference, and we will complete and free it here
1671		− * after we got woken by the I/O completion handler.
1672		− */
1673		− dio->wait_for_completion = wait_for_completion;
1674		− if (!atomic_dec_and_test(&dio->ref)) {
1675		− if (!wait_for_completion)
1676		− return -EIOCBQUEUED;
1677		−
1678		− for (;;) {
1679		− set_current_state(TASK_UNINTERRUPTIBLE);
1680		− if (!READ_ONCE(dio->submit.waiter))
1681		− break;
1682		−
1683		− if (!(iocb->ki_flags & IOCB_HIPRI) \|\|
1684		− !dio->submit.last_queue \|\|
1685		− !blk_poll(dio->submit.last_queue,
1686		− dio->submit.cookie, true))
1687		− io_schedule();
1688		− }
1689		− __set_current_state(TASK_RUNNING);
1690		− }
1691		−
1692		− return iomap_dio_complete(dio);
1693		−
1694		−out_free_dio:
1695		− kfree(dio);
1696		− return ret;
1697		−}
1698		−EXPORT_SYMBOL_GPL(iomap_dio_rw);

fs/iomap/Makefile+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	obj-$(CONFIG_FS_IOMAP) += iomap.o
7	7
8	8	iomap-y += \
	9	+ direct-io.o \
9	10	fiemap.o \
10	11	seek.o
11	12

fs/iomap/direct-io.c+562−0

@@ -0,0 +1,562 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/*
	3	+ * Copyright (C) 2010 Red Hat, Inc.
	4	+ * Copyright (c) 2016-2018 Christoph Hellwig.
	5	+ */
	6	+#include <linux/module.h>
	7	+#include <linux/compiler.h>
	8	+#include <linux/fs.h>
	9	+#include <linux/iomap.h>
	10	+#include <linux/backing-dev.h>
	11	+#include <linux/uio.h>
	12	+#include <linux/task_io_accounting_ops.h>
	13	+
	14	+#include "../internal.h"
	15	+
	16	+/*
	17	+ * Private flags for iomap_dio, must not overlap with the public ones in
	18	+ * iomap.h:
	19	+ */
	20	+#define IOMAP_DIO_WRITE_FUA (1 << 28)
	21	+#define IOMAP_DIO_NEED_SYNC (1 << 29)
	22	+#define IOMAP_DIO_WRITE (1 << 30)
	23	+#define IOMAP_DIO_DIRTY (1 << 31)
	24	+
	25	+struct iomap_dio {
	26	+ struct kiocb *iocb;
	27	+ iomap_dio_end_io_t *end_io;
	28	+ loff_t i_size;
	29	+ loff_t size;
	30	+ atomic_t ref;
	31	+ unsigned flags;
	32	+ int error;
	33	+ bool wait_for_completion;
	34	+
	35	+ union {
	36	+ /* used during submission and for synchronous completion: */
	37	+ struct {
	38	+ struct iov_iter *iter;
	39	+ struct task_struct *waiter;
	40	+ struct request_queue *last_queue;
	41	+ blk_qc_t cookie;
	42	+ } submit;
	43	+
	44	+ /* used for aio completion: */
	45	+ struct {
	46	+ struct work_struct work;
	47	+ } aio;
	48	+ };
	49	+};
	50	+
	51	+int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
	52	+{
	53	+ struct request_queue *q = READ_ONCE(kiocb->private);
	54	+
	55	+ if (!q)
	56	+ return 0;
	57	+ return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
	58	+}
	59	+EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
	60	+
	61	+static void iomap_dio_submit_bio(struct iomap_dio dio, struct iomap iomap,
	62	+ struct bio *bio)
	63	+{
	64	+ atomic_inc(&dio->ref);
	65	+
	66	+ if (dio->iocb->ki_flags & IOCB_HIPRI)
	67	+ bio_set_polled(bio, dio->iocb);
	68	+
	69	+ dio->submit.last_queue = bdev_get_queue(iomap->bdev);
	70	+ dio->submit.cookie = submit_bio(bio);
	71	+}
	72	+
	73	+static ssize_t iomap_dio_complete(struct iomap_dio *dio)
	74	+{
	75	+ struct kiocb *iocb = dio->iocb;
	76	+ struct inode *inode = file_inode(iocb->ki_filp);
	77	+ loff_t offset = iocb->ki_pos;
	78	+ ssize_t ret;
	79	+
	80	+ if (dio->end_io) {
	81	+ ret = dio->end_io(iocb,
	82	+ dio->error ? dio->error : dio->size,
	83	+ dio->flags);
	84	+ } else {
	85	+ ret = dio->error;
	86	+ }
	87	+
	88	+ if (likely(!ret)) {
	89	+ ret = dio->size;
	90	+ /* check for short read */
	91	+ if (offset + ret > dio->i_size &&
	92	+ !(dio->flags & IOMAP_DIO_WRITE))
	93	+ ret = dio->i_size - offset;
	94	+ iocb->ki_pos += ret;
	95	+ }
	96	+
	97	+ /*
	98	+ * Try again to invalidate clean pages which might have been cached by
	99	+ * non-direct readahead, or faulted in by get_user_pages() if the source
	100	+ * of the write was an mmap'ed region of the file we're writing. Either
	101	+ * one is a pretty crazy thing to do, so we don't support it 100%. If
	102	+ * this invalidation fails, tough, the write still worked...
	103	+ *
	104	+ * And this page cache invalidation has to be after dio->end_io(), as
	105	+ * some filesystems convert unwritten extents to real allocations in
	106	+ * end_io() when necessary, otherwise a racing buffer read would cache
	107	+ * zeros from unwritten extents.
	108	+ */
	109	+ if (!dio->error &&
	110	+ (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {
	111	+ int err;
	112	+ err = invalidate_inode_pages2_range(inode->i_mapping,
	113	+ offset >> PAGE_SHIFT,
	114	+ (offset + dio->size - 1) >> PAGE_SHIFT);
	115	+ if (err)
	116	+ dio_warn_stale_pagecache(iocb->ki_filp);
	117	+ }
	118	+
	119	+ /*
	120	+ * If this is a DSYNC write, make sure we push it to stable storage now
	121	+ * that we've written data.
	122	+ */
	123	+ if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))
	124	+ ret = generic_write_sync(iocb, ret);
	125	+
	126	+ inode_dio_end(file_inode(iocb->ki_filp));
	127	+ kfree(dio);
	128	+
	129	+ return ret;
	130	+}
	131	+
	132	+static void iomap_dio_complete_work(struct work_struct *work)
	133	+{
	134	+ struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
	135	+ struct kiocb *iocb = dio->iocb;
	136	+
	137	+ iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);
	138	+}
	139	+
	140	+/*
	141	+ * Set an error in the dio if none is set yet. We have to use cmpxchg
	142	+ * as the submission context and the completion context(s) can race to
	143	+ * update the error.
	144	+ */
	145	+static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
	146	+{
	147	+ cmpxchg(&dio->error, 0, ret);
	148	+}
	149	+
	150	+static void iomap_dio_bio_end_io(struct bio *bio)
	151	+{
	152	+ struct iomap_dio *dio = bio->bi_private;
	153	+ bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
	154	+
	155	+ if (bio->bi_status)
	156	+ iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
	157	+
	158	+ if (atomic_dec_and_test(&dio->ref)) {
	159	+ if (dio->wait_for_completion) {
	160	+ struct task_struct *waiter = dio->submit.waiter;
	161	+ WRITE_ONCE(dio->submit.waiter, NULL);
	162	+ blk_wake_io_task(waiter);
	163	+ } else if (dio->flags & IOMAP_DIO_WRITE) {
	164	+ struct inode *inode = file_inode(dio->iocb->ki_filp);
	165	+
	166	+ INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
	167	+ queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
	168	+ } else {
	169	+ iomap_dio_complete_work(&dio->aio.work);
	170	+ }
	171	+ }
	172	+
	173	+ if (should_dirty) {
	174	+ bio_check_pages_dirty(bio);
	175	+ } else {
	176	+ bio_release_pages(bio, false);
	177	+ bio_put(bio);
	178	+ }
	179	+}
	180	+
	181	+static void
	182	+iomap_dio_zero(struct iomap_dio dio, struct iomap iomap, loff_t pos,
	183	+ unsigned len)
	184	+{
	185	+ struct page *page = ZERO_PAGE(0);
	186	+ int flags = REQ_SYNC \| REQ_IDLE;
	187	+ struct bio *bio;
	188	+
	189	+ bio = bio_alloc(GFP_KERNEL, 1);
	190	+ bio_set_dev(bio, iomap->bdev);
	191	+ bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
	192	+ bio->bi_private = dio;
	193	+ bio->bi_end_io = iomap_dio_bio_end_io;
	194	+
	195	+ get_page(page);
	196	+ __bio_add_page(bio, page, len, 0);
	197	+ bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
	198	+ iomap_dio_submit_bio(dio, iomap, bio);
	199	+}
	200	+
	201	+static loff_t
	202	+iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
	203	+ struct iomap_dio dio, struct iomap iomap)
	204	+{
	205	+ unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
	206	+ unsigned int fs_block_size = i_blocksize(inode), pad;
	207	+ unsigned int align = iov_iter_alignment(dio->submit.iter);
	208	+ struct iov_iter iter;
	209	+ struct bio *bio;
	210	+ bool need_zeroout = false;
	211	+ bool use_fua = false;
	212	+ int nr_pages, ret = 0;
	213	+ size_t copied = 0;
	214	+
	215	+ if ((pos \| length \| align) & ((1 << blkbits) - 1))
	216	+ return -EINVAL;
	217	+
	218	+ if (iomap->type == IOMAP_UNWRITTEN) {
	219	+ dio->flags \|= IOMAP_DIO_UNWRITTEN;
	220	+ need_zeroout = true;
	221	+ }
	222	+
	223	+ if (iomap->flags & IOMAP_F_SHARED)
	224	+ dio->flags \|= IOMAP_DIO_COW;
	225	+
	226	+ if (iomap->flags & IOMAP_F_NEW) {
	227	+ need_zeroout = true;
	228	+ } else if (iomap->type == IOMAP_MAPPED) {
	229	+ /*
	230	+ * Use a FUA write if we need datasync semantics, this is a pure
	231	+ * data IO that doesn't require any metadata updates (including
	232	+ * after IO completion such as unwritten extent conversion) and
	233	+ * the underlying device supports FUA. This allows us to avoid
	234	+ * cache flushes on IO completion.
	235	+ */
	236	+ if (!(iomap->flags & (IOMAP_F_SHARED\|IOMAP_F_DIRTY)) &&
	237	+ (dio->flags & IOMAP_DIO_WRITE_FUA) &&
	238	+ blk_queue_fua(bdev_get_queue(iomap->bdev)))
	239	+ use_fua = true;
	240	+ }
	241	+
	242	+ /*
	243	+ * Operate on a partial iter trimmed to the extent we were called for.
	244	+ * We'll update the iter in the dio once we're done with this extent.
	245	+ */
	246	+ iter = *dio->submit.iter;
	247	+ iov_iter_truncate(&iter, length);
	248	+
	249	+ nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
	250	+ if (nr_pages <= 0)
	251	+ return nr_pages;
	252	+
	253	+ if (need_zeroout) {
	254	+ /* zero out from the start of the block to the write offset */
	255	+ pad = pos & (fs_block_size - 1);
	256	+ if (pad)
	257	+ iomap_dio_zero(dio, iomap, pos - pad, pad);
	258	+ }
	259	+
	260	+ do {
	261	+ size_t n;
	262	+ if (dio->error) {
	263	+ iov_iter_revert(dio->submit.iter, copied);
	264	+ return 0;
	265	+ }
	266	+
	267	+ bio = bio_alloc(GFP_KERNEL, nr_pages);
	268	+ bio_set_dev(bio, iomap->bdev);
	269	+ bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
	270	+ bio->bi_write_hint = dio->iocb->ki_hint;
	271	+ bio->bi_ioprio = dio->iocb->ki_ioprio;
	272	+ bio->bi_private = dio;
	273	+ bio->bi_end_io = iomap_dio_bio_end_io;
	274	+
	275	+ ret = bio_iov_iter_get_pages(bio, &iter);
	276	+ if (unlikely(ret)) {
	277	+ /*
	278	+ * We have to stop part way through an IO. We must fall
	279	+ * through to the sub-block tail zeroing here, otherwise
	280	+ * this short IO may expose stale data in the tail of
	281	+ * the block we haven't written data to.
	282	+ */
	283	+ bio_put(bio);
	284	+ goto zero_tail;
	285	+ }
	286	+
	287	+ n = bio->bi_iter.bi_size;
	288	+ if (dio->flags & IOMAP_DIO_WRITE) {
	289	+ bio->bi_opf = REQ_OP_WRITE \| REQ_SYNC \| REQ_IDLE;
	290	+ if (use_fua)
	291	+ bio->bi_opf \|= REQ_FUA;
	292	+ else
	293	+ dio->flags &= ~IOMAP_DIO_WRITE_FUA;
	294	+ task_io_account_write(n);
	295	+ } else {
	296	+ bio->bi_opf = REQ_OP_READ;
	297	+ if (dio->flags & IOMAP_DIO_DIRTY)
	298	+ bio_set_pages_dirty(bio);
	299	+ }
	300	+
	301	+ iov_iter_advance(dio->submit.iter, n);
	302	+
	303	+ dio->size += n;
	304	+ pos += n;
	305	+ copied += n;
	306	+
	307	+ nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
	308	+ iomap_dio_submit_bio(dio, iomap, bio);
	309	+ } while (nr_pages);
	310	+
	311	+ /*
	312	+ * We need to zeroout the tail of a sub-block write if the extent type
	313	+ * requires zeroing or the write extends beyond EOF. If we don't zero
	314	+ * the block tail in the latter case, we can expose stale data via mmap
	315	+ * reads of the EOF block.
	316	+ */
	317	+zero_tail:
	318	+ if (need_zeroout \|\|
	319	+ ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
	320	+ /* zero out from the end of the write to the end of the block */
	321	+ pad = pos & (fs_block_size - 1);
	322	+ if (pad)
	323	+ iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);
	324	+ }
	325	+ return copied ? copied : ret;
	326	+}
	327	+
	328	+static loff_t
	329	+iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio)
	330	+{
	331	+ length = iov_iter_zero(length, dio->submit.iter);
	332	+ dio->size += length;
	333	+ return length;
	334	+}
	335	+
	336	+static loff_t
	337	+iomap_dio_inline_actor(struct inode *inode, loff_t pos, loff_t length,
	338	+ struct iomap_dio dio, struct iomap iomap)
	339	+{
	340	+ struct iov_iter *iter = dio->submit.iter;
	341	+ size_t copied;
	342	+
	343	+ BUG_ON(pos + length > PAGE_SIZE - offset_in_page(iomap->inline_data));
	344	+
	345	+ if (dio->flags & IOMAP_DIO_WRITE) {
	346	+ loff_t size = inode->i_size;
	347	+
	348	+ if (pos > size)
	349	+ memset(iomap->inline_data + size, 0, pos - size);
	350	+ copied = copy_from_iter(iomap->inline_data + pos, length, iter);
	351	+ if (copied) {
	352	+ if (pos + copied > size)
	353	+ i_size_write(inode, pos + copied);
	354	+ mark_inode_dirty(inode);
	355	+ }
	356	+ } else {
	357	+ copied = copy_to_iter(iomap->inline_data + pos, length, iter);
	358	+ }
	359	+ dio->size += copied;
	360	+ return copied;
	361	+}
	362	+
	363	+static loff_t
	364	+iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
	365	+ void data, struct iomap iomap)
	366	+{
	367	+ struct iomap_dio *dio = data;
	368	+
	369	+ switch (iomap->type) {
	370	+ case IOMAP_HOLE:
	371	+ if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
	372	+ return -EIO;
	373	+ return iomap_dio_hole_actor(length, dio);
	374	+ case IOMAP_UNWRITTEN:
	375	+ if (!(dio->flags & IOMAP_DIO_WRITE))
	376	+ return iomap_dio_hole_actor(length, dio);
	377	+ return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
	378	+ case IOMAP_MAPPED:
	379	+ return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
	380	+ case IOMAP_INLINE:
	381	+ return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
	382	+ default:
	383	+ WARN_ON_ONCE(1);
	384	+ return -EIO;
	385	+ }
	386	+}
	387	+
	388	+/*
	389	+ * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
	390	+ * is being issued as AIO or not. This allows us to optimise pure data writes
	391	+ * to use REQ_FUA rather than requiring generic_write_sync() to issue a
	392	+ * REQ_FLUSH post write. This is slightly tricky because a single request here
	393	+ * can be mapped into multiple disjoint IOs and only a subset of the IOs issued
	394	+ * may be pure data writes. In that case, we still need to do a full data sync
	395	+ * completion.
	396	+ */
	397	+ssize_t
	398	+iomap_dio_rw(struct kiocb iocb, struct iov_iter iter,
	399	+ const struct iomap_ops *ops, iomap_dio_end_io_t end_io)
	400	+{
	401	+ struct address_space *mapping = iocb->ki_filp->f_mapping;
	402	+ struct inode *inode = file_inode(iocb->ki_filp);
	403	+ size_t count = iov_iter_count(iter);
	404	+ loff_t pos = iocb->ki_pos, start = pos;
	405	+ loff_t end = iocb->ki_pos + count - 1, ret = 0;
	406	+ unsigned int flags = IOMAP_DIRECT;
	407	+ bool wait_for_completion = is_sync_kiocb(iocb);
	408	+ struct blk_plug plug;
	409	+ struct iomap_dio *dio;
	410	+
	411	+ lockdep_assert_held(&inode->i_rwsem);
	412	+
	413	+ if (!count)
	414	+ return 0;
	415	+
	416	+ dio = kmalloc(sizeof(*dio), GFP_KERNEL);
	417	+ if (!dio)
	418	+ return -ENOMEM;
	419	+
	420	+ dio->iocb = iocb;
	421	+ atomic_set(&dio->ref, 1);
	422	+ dio->size = 0;
	423	+ dio->i_size = i_size_read(inode);
	424	+ dio->end_io = end_io;
	425	+ dio->error = 0;
	426	+ dio->flags = 0;
	427	+
	428	+ dio->submit.iter = iter;
	429	+ dio->submit.waiter = current;
	430	+ dio->submit.cookie = BLK_QC_T_NONE;
	431	+ dio->submit.last_queue = NULL;
	432	+
	433	+ if (iov_iter_rw(iter) == READ) {
	434	+ if (pos >= dio->i_size)
	435	+ goto out_free_dio;
	436	+
	437	+ if (iter_is_iovec(iter) && iov_iter_rw(iter) == READ)
	438	+ dio->flags \|= IOMAP_DIO_DIRTY;
	439	+ } else {
	440	+ flags \|= IOMAP_WRITE;
	441	+ dio->flags \|= IOMAP_DIO_WRITE;
	442	+
	443	+ /* for data sync or sync, we need sync completion processing */
	444	+ if (iocb->ki_flags & IOCB_DSYNC)
	445	+ dio->flags \|= IOMAP_DIO_NEED_SYNC;
	446	+
	447	+ /*
	448	+ * For datasync only writes, we optimistically try using FUA for
	449	+ * this IO. Any non-FUA write that occurs will clear this flag,
	450	+ * hence we know before completion whether a cache flush is
	451	+ * necessary.
	452	+ */
	453	+ if ((iocb->ki_flags & (IOCB_DSYNC \| IOCB_SYNC)) == IOCB_DSYNC)
	454	+ dio->flags \|= IOMAP_DIO_WRITE_FUA;
	455	+ }
	456	+
	457	+ if (iocb->ki_flags & IOCB_NOWAIT) {
	458	+ if (filemap_range_has_page(mapping, start, end)) {
	459	+ ret = -EAGAIN;
	460	+ goto out_free_dio;
	461	+ }
	462	+ flags \|= IOMAP_NOWAIT;
	463	+ }
	464	+
	465	+ ret = filemap_write_and_wait_range(mapping, start, end);
	466	+ if (ret)
	467	+ goto out_free_dio;
	468	+
	469	+ /*
	470	+ * Try to invalidate cache pages for the range we're direct
	471	+ * writing. If this invalidation fails, tough, the write will
	472	+ * still work, but racing two incompatible write paths is a
	473	+ * pretty crazy thing to do, so we don't support it 100%.
	474	+ */
	475	+ ret = invalidate_inode_pages2_range(mapping,
	476	+ start >> PAGE_SHIFT, end >> PAGE_SHIFT);
	477	+ if (ret)
	478	+ dio_warn_stale_pagecache(iocb->ki_filp);
	479	+ ret = 0;
	480	+
	481	+ if (iov_iter_rw(iter) == WRITE && !wait_for_completion &&
	482	+ !inode->i_sb->s_dio_done_wq) {
	483	+ ret = sb_init_dio_done_wq(inode->i_sb);
	484	+ if (ret < 0)
	485	+ goto out_free_dio;
	486	+ }
	487	+
	488	+ inode_dio_begin(inode);
	489	+
	490	+ blk_start_plug(&plug);
	491	+ do {
	492	+ ret = iomap_apply(inode, pos, count, flags, ops, dio,
	493	+ iomap_dio_actor);
	494	+ if (ret <= 0) {
	495	+ /* magic error code to fall back to buffered I/O */
	496	+ if (ret == -ENOTBLK) {
	497	+ wait_for_completion = true;
	498	+ ret = 0;
	499	+ }
	500	+ break;
	501	+ }
	502	+ pos += ret;
	503	+
	504	+ if (iov_iter_rw(iter) == READ && pos >= dio->i_size)
	505	+ break;
	506	+ } while ((count = iov_iter_count(iter)) > 0);
	507	+ blk_finish_plug(&plug);
	508	+
	509	+ if (ret < 0)
	510	+ iomap_dio_set_error(dio, ret);
	511	+
	512	+ /*
	513	+ * If all the writes we issued were FUA, we don't need to flush the
	514	+ * cache on IO completion. Clear the sync flag for this case.
	515	+ */
	516	+ if (dio->flags & IOMAP_DIO_WRITE_FUA)
	517	+ dio->flags &= ~IOMAP_DIO_NEED_SYNC;
	518	+
	519	+ WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);
	520	+ WRITE_ONCE(iocb->private, dio->submit.last_queue);
	521	+
	522	+ /*
	523	+ * We are about to drop our additional submission reference, which
	524	+ * might be the last reference to the dio. There are three three
	525	+ * different ways we can progress here:
	526	+ *
	527	+ * (a) If this is the last reference we will always complete and free
	528	+ * the dio ourselves.
	529	+ * (b) If this is not the last reference, and we serve an asynchronous
	530	+ * iocb, we must never touch the dio after the decrement, the
	531	+ * I/O completion handler will complete and free it.
	532	+ * (c) If this is not the last reference, but we serve a synchronous
	533	+ * iocb, the I/O completion handler will wake us up on the drop
	534	+ * of the final reference, and we will complete and free it here
	535	+ * after we got woken by the I/O completion handler.
	536	+ */
	537	+ dio->wait_for_completion = wait_for_completion;
	538	+ if (!atomic_dec_and_test(&dio->ref)) {
	539	+ if (!wait_for_completion)
	540	+ return -EIOCBQUEUED;
	541	+
	542	+ for (;;) {
	543	+ set_current_state(TASK_UNINTERRUPTIBLE);
	544	+ if (!READ_ONCE(dio->submit.waiter))
	545	+ break;
	546	+
	547	+ if (!(iocb->ki_flags & IOCB_HIPRI) \|\|
	548	+ !dio->submit.last_queue \|\|
	549	+ !blk_poll(dio->submit.last_queue,
	550	+ dio->submit.cookie, true))
	551	+ io_schedule();
	552	+ }
	553	+ __set_current_state(TASK_RUNNING);
	554	+ }
	555	+
	556	+ return iomap_dio_complete(dio);
	557	+
	558	+out_free_dio:
	559	+ kfree(dio);
	560	+ return ret;
	561	+}
	562	+EXPORT_SYMBOL_GPL(iomap_dio_rw);

include/linux/iomap.h+7−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/mm.h>
8	8	#include <linux/types.h>
9	9	#include <linux/mm_types.h>
	10	+#include <linux/blkdev.h>
10	11
11	12	struct address_space;
12	13	struct fiemap_extent_info;
@@ -69,6 +70,12 @@ struct iomap { · 56 unmodified lines
69	70	const struct iomap_page_ops *page_ops;
70	71	};
71	72
	73	+static inline sector_t
	74	+iomap_sector(struct iomap *iomap, loff_t pos)
	75	+{
	76	+ return (iomap->addr + pos - iomap->offset) >> SECTOR_SHIFT;
	77	+}
	78	+
72	79	/*
73	80	* When a filesystem sets page_ops in an iomap mapping it returns, page_prepare
74	81	* and page_done will be called for each page written to. This only applies to

iomap: move the direct IO code into a separate file

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code