iomap: move the direct IO code into a separate file

Move the direct IO code into a separate file so that we can group related functions in a single file instead of having a single enormous source file.

Darrick J. Wong committed 7 years ago

Commit db074431 parent 56a17894 files · +570 −554Browse files

Unified · Split

4 files

fs/iomap.c +0 −554
fs/iomap/Makefile +1 −0
fs/iomap/direct-io.c +562 −0
include/linux/iomap.h +7 −0

fs/iomap.c+0−554

@@ -90,12 +90,6 @@ iomap_apply(struct inode *inode, loff_t pos, loff_t length, unsigned flags, · 89 unmodified lines
90	return written ? written : ret;	90	return written ? written : ret;
91	}	91	}
92		92
93	static sector_t
94	iomap_sector(struct iomap *iomap, loff_t pos)
95	{
96	return (iomap->addr + pos - iomap->offset) >> SECTOR_SHIFT;
97	}
98
99	static struct iomap_page *	93	static struct iomap_page *
100	iomap_page_create(struct inode inode, struct page page)	94	iomap_page_create(struct inode inode, struct page page)
101	{	95	{
@@ -1148,551 +1142,3 @@ out_unlock: · 1046 unmodified lines
1148	return block_page_mkwrite_return(ret);	1142	return block_page_mkwrite_return(ret);
1149	}	1143	}
1150	EXPORT_SYMBOL_GPL(iomap_page_mkwrite);	1144	EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1151
1152	/*
1153	* Private flags for iomap_dio, must not overlap with the public ones in
1154	* iomap.h:
1155	*/
1156	#define IOMAP_DIO_WRITE_FUA (1 << 28)
1157	#define IOMAP_DIO_NEED_SYNC (1 << 29)
1158	#define IOMAP_DIO_WRITE (1 << 30)
1159	#define IOMAP_DIO_DIRTY (1 << 31)
1160
1161	struct iomap_dio {
1162	struct kiocb *iocb;
1163	iomap_dio_end_io_t *end_io;
1164	loff_t i_size;
1165	loff_t size;
1166	atomic_t ref;
1167	unsigned flags;
1168	int error;
1169	bool wait_for_completion;
1170
1171	union {
1172	/* used during submission and for synchronous completion: */
1173	struct {
1174	struct iov_iter *iter;
1175	struct task_struct *waiter;
1176	struct request_queue *last_queue;
1177	blk_qc_t cookie;
1178	} submit;
1179
1180	/* used for aio completion: */
1181	struct {
1182	struct work_struct work;
1183	} aio;
1184	};
1185	};
1186
1187	int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
1188	{
1189	struct request_queue *q = READ_ONCE(kiocb->private);
1190
1191	if (!q)
1192	return 0;
1193	return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
1194	}
1195	EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
1196
1197	static void iomap_dio_submit_bio(struct iomap_dio dio, struct iomap iomap,
1198	struct bio *bio)
1199	{
1200	atomic_inc(&dio->ref);
1201
1202	if (dio->iocb->ki_flags & IOCB_HIPRI)
1203	bio_set_polled(bio, dio->iocb);
1204
1205	dio->submit.last_queue = bdev_get_queue(iomap->bdev);
1206	dio->submit.cookie = submit_bio(bio);
1207	}
1208
1209	static ssize_t iomap_dio_complete(struct iomap_dio *dio)
1210	{
1211	struct kiocb *iocb = dio->iocb;
1212	struct inode *inode = file_inode(iocb->ki_filp);
1213	loff_t offset = iocb->ki_pos;
1214	ssize_t ret;
1215
1216	if (dio->end_io) {
1217	ret = dio->end_io(iocb,
1218	dio->error ? dio->error : dio->size,
1219	dio->flags);
1220	} else {
1221	ret = dio->error;
1222	}
1223
1224	if (likely(!ret)) {
1225	ret = dio->size;
1226	/* check for short read */
1227	if (offset + ret > dio->i_size &&
1228	!(dio->flags & IOMAP_DIO_WRITE))
1229	ret = dio->i_size - offset;
1230	iocb->ki_pos += ret;
1231	}
1232
1233	/*
1234	* Try again to invalidate clean pages which might have been cached by
1235	* non-direct readahead, or faulted in by get_user_pages() if the source
1236	* of the write was an mmap'ed region of the file we're writing. Either
1237	* one is a pretty crazy thing to do, so we don't support it 100%. If
1238	* this invalidation fails, tough, the write still worked...
1239	*
1240	* And this page cache invalidation has to be after dio->end_io(), as
1241	* some filesystems convert unwritten extents to real allocations in
1242	* end_io() when necessary, otherwise a racing buffer read would cache
1243	* zeros from unwritten extents.
1244	*/
1245	if (!dio->error &&
1246	(dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {
1247	int err;
1248	err = invalidate_inode_pages2_range(inode->i_mapping,
1249	offset >> PAGE_SHIFT,
1250	(offset + dio->size - 1) >> PAGE_SHIFT);
1251	if (err)
1252	dio_warn_stale_pagecache(iocb->ki_filp);
1253	}
1254
1255	/*
1256	* If this is a DSYNC write, make sure we push it to stable storage now
1257	* that we've written data.
1258	*/
1259	if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))
1260	ret = generic_write_sync(iocb, ret);
1261
1262	inode_dio_end(file_inode(iocb->ki_filp));
1263	kfree(dio);
1264
1265	return ret;
1266	}
1267
1268	static void iomap_dio_complete_work(struct work_struct *work)
1269	{
1270	struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
1271	struct kiocb *iocb = dio->iocb;
1272
1273	iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);
1274	}
1275
1276	/*
1277	* Set an error in the dio if none is set yet. We have to use cmpxchg
1278	* as the submission context and the completion context(s) can race to
1279	* update the error.
1280	*/
1281	static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
1282	{
1283	cmpxchg(&dio->error, 0, ret);
1284	}
1285
1286	static void iomap_dio_bio_end_io(struct bio *bio)
1287	{
1288	struct iomap_dio *dio = bio->bi_private;
1289	bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
1290
1291	if (bio->bi_status)
1292	iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
1293
1294	if (atomic_dec_and_test(&dio->ref)) {
1295	if (dio->wait_for_completion) {
1296	struct task_struct *waiter = dio->submit.waiter;
1297	WRITE_ONCE(dio->submit.waiter, NULL);
1298	blk_wake_io_task(waiter);
1299	} else if (dio->flags & IOMAP_DIO_WRITE) {
1300	struct inode *inode = file_inode(dio->iocb->ki_filp);
1301
1302	INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
1303	queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
1304	} else {
1305	iomap_dio_complete_work(&dio->aio.work);
1306	}
1307	}
1308
1309	if (should_dirty) {
1310	bio_check_pages_dirty(bio);
1311	} else {
1312	bio_release_pages(bio, false);
1313	bio_put(bio);
1314	}
1315	}
1316
1317	static void
1318	iomap_dio_zero(struct iomap_dio dio, struct iomap iomap, loff_t pos,
1319	unsigned len)
1320	{
1321	struct page *page = ZERO_PAGE(0);
1322	int flags = REQ_SYNC \| REQ_IDLE;
1323	struct bio *bio;
1324
1325	bio = bio_alloc(GFP_KERNEL, 1);
1326	bio_set_dev(bio, iomap->bdev);
1327	bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
1328	bio->bi_private = dio;
1329	bio->bi_end_io = iomap_dio_bio_end_io;
1330
1331	get_page(page);
1332	__bio_add_page(bio, page, len, 0);
1333	bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
1334	iomap_dio_submit_bio(dio, iomap, bio);
1335	}
1336
1337	static loff_t
1338	iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
1339	struct iomap_dio dio, struct iomap iomap)
1340	{
1341	unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
1342	unsigned int fs_block_size = i_blocksize(inode), pad;
1343	unsigned int align = iov_iter_alignment(dio->submit.iter);
1344	struct iov_iter iter;
1345	struct bio *bio;
1346	bool need_zeroout = false;
1347	bool use_fua = false;
1348	int nr_pages, ret = 0;
1349	size_t copied = 0;
1350
1351	if ((pos \| length \| align) & ((1 << blkbits) - 1))
1352	return -EINVAL;
1353
1354	if (iomap->type == IOMAP_UNWRITTEN) {
1355	dio->flags \|= IOMAP_DIO_UNWRITTEN;
1356	need_zeroout = true;
1357	}
1358
1359	if (iomap->flags & IOMAP_F_SHARED)
1360	dio->flags \|= IOMAP_DIO_COW;
1361
1362	if (iomap->flags & IOMAP_F_NEW) {
1363	need_zeroout = true;
1364	} else if (iomap->type == IOMAP_MAPPED) {
1365	/*
1366	* Use a FUA write if we need datasync semantics, this is a pure
1367	* data IO that doesn't require any metadata updates (including
1368	* after IO completion such as unwritten extent conversion) and
1369	* the underlying device supports FUA. This allows us to avoid
1370	* cache flushes on IO completion.
1371	*/
1372	if (!(iomap->flags & (IOMAP_F_SHARED\|IOMAP_F_DIRTY)) &&
1373	(dio->flags & IOMAP_DIO_WRITE_FUA) &&
1374	blk_queue_fua(bdev_get_queue(iomap->bdev)))
1375	use_fua = true;
1376	}
1377
1378	/*
1379	* Operate on a partial iter trimmed to the extent we were called for.
1380	* We'll update the iter in the dio once we're done with this extent.
1381	*/
1382	iter = *dio->submit.iter;
1383	iov_iter_truncate(&iter, length);
1384
1385	nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
1386	if (nr_pages <= 0)
1387	return nr_pages;
1388
1389	if (need_zeroout) {
1390	/* zero out from the start of the block to the write offset */
1391	pad = pos & (fs_block_size - 1);
1392	if (pad)
1393	iomap_dio_zero(dio, iomap, pos - pad, pad);
1394	}
1395
1396	do {
1397	size_t n;
1398	if (dio->error) {
1399	iov_iter_revert(dio->submit.iter, copied);
1400	return 0;
1401	}
1402
1403	bio = bio_alloc(GFP_KERNEL, nr_pages);
1404	bio_set_dev(bio, iomap->bdev);
1405	bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
1406	bio->bi_write_hint = dio->iocb->ki_hint;
1407	bio->bi_ioprio = dio->iocb->ki_ioprio;
1408	bio->bi_private = dio;
1409	bio->bi_end_io = iomap_dio_bio_end_io;
1410
1411	ret = bio_iov_iter_get_pages(bio, &iter);
1412	if (unlikely(ret)) {
1413	/*
1414	* We have to stop part way through an IO. We must fall
1415	* through to the sub-block tail zeroing here, otherwise
1416	* this short IO may expose stale data in the tail of
1417	* the block we haven't written data to.
1418	*/
1419	bio_put(bio);
1420	goto zero_tail;
1421	}
1422
1423	n = bio->bi_iter.bi_size;
1424	if (dio->flags & IOMAP_DIO_WRITE) {
1425	bio->bi_opf = REQ_OP_WRITE \| REQ_SYNC \| REQ_IDLE;
1426	if (use_fua)
1427	bio->bi_opf \|= REQ_FUA;
1428	else
1429	dio->flags &= ~IOMAP_DIO_WRITE_FUA;
1430	task_io_account_write(n);
1431	} else {
1432	bio->bi_opf = REQ_OP_READ;
1433	if (dio->flags & IOMAP_DIO_DIRTY)
1434	bio_set_pages_dirty(bio);
1435	}
1436
1437	iov_iter_advance(dio->submit.iter, n);
1438
1439	dio->size += n;
1440	pos += n;
1441	copied += n;
1442
1443	nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
1444	iomap_dio_submit_bio(dio, iomap, bio);
1445	} while (nr_pages);
1446
1447	/*
1448	* We need to zeroout the tail of a sub-block write if the extent type
1449	* requires zeroing or the write extends beyond EOF. If we don't zero
1450	* the block tail in the latter case, we can expose stale data via mmap
1451	* reads of the EOF block.
1452	*/
1453	zero_tail:
1454	if (need_zeroout \|\|
1455	((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
1456	/* zero out from the end of the write to the end of the block */
1457	pad = pos & (fs_block_size - 1);
1458	if (pad)
1459	iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);
1460	}
1461	return copied ? copied : ret;
1462	}
1463
1464	static loff_t
1465	iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio)
1466	{
1467	length = iov_iter_zero(length, dio->submit.iter);
1468	dio->size += length;
1469	return length;
1470	}
1471
1472	static loff_t
1473	iomap_dio_inline_actor(struct inode *inode, loff_t pos, loff_t length,
1474	struct iomap_dio dio, struct iomap iomap)
1475	{
1476	struct iov_iter *iter = dio->submit.iter;
1477	size_t copied;
1478
1479	BUG_ON(pos + length > PAGE_SIZE - offset_in_page(iomap->inline_data));
1480
1481	if (dio->flags & IOMAP_DIO_WRITE) {
1482	loff_t size = inode->i_size;
1483
1484	if (pos > size)
1485	memset(iomap->inline_data + size, 0, pos - size);
1486	copied = copy_from_iter(iomap->inline_data + pos, length, iter);
1487	if (copied) {
1488	if (pos + copied > size)
1489	i_size_write(inode, pos + copied);
1490	mark_inode_dirty(inode);
1491	}
1492	} else {
1493	copied = copy_to_iter(iomap->inline_data + pos, length, iter);
1494	}
1495	dio->size += copied;
1496	return copied;
1497	}
1498
1499	static loff_t
1500	iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
1501	void data, struct iomap iomap)
1502	{
1503	struct iomap_dio *dio = data;
1504
1505	switch (iomap->type) {
1506	case IOMAP_HOLE:
1507	if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
1508	return -EIO;
1509	return iomap_dio_hole_actor(length, dio);
1510	case IOMAP_UNWRITTEN:
1511	if (!(dio->flags & IOMAP_DIO_WRITE))
1512	return iomap_dio_hole_actor(length, dio);
1513	return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
1514	case IOMAP_MAPPED:
1515	return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
1516	case IOMAP_INLINE:
1517	return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
1518	default:
1519	WARN_ON_ONCE(1);
1520	return -EIO;
1521	}
1522	}
1523
1524	/*
1525	* iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
1526	* is being issued as AIO or not. This allows us to optimise pure data writes
1527	* to use REQ_FUA rather than requiring generic_write_sync() to issue a
1528	* REQ_FLUSH post write. This is slightly tricky because a single request here
1529	* can be mapped into multiple disjoint IOs and only a subset of the IOs issued
1530	* may be pure data writes. In that case, we still need to do a full data sync
1531	* completion.
1532	*/
1533	ssize_t
1534	iomap_dio_rw(struct kiocb iocb, struct iov_iter iter,
1535	const struct iomap_ops *ops, iomap_dio_end_io_t end_io)
1536	{
1537	struct address_space *mapping = iocb->ki_filp->f_mapping;
1538	struct inode *inode = file_inode(iocb->ki_filp);
1539	size_t count = iov_iter_count(iter);
1540	loff_t pos = iocb->ki_pos, start = pos;
1541	loff_t end = iocb->ki_pos + count - 1, ret = 0;
1542	unsigned int flags = IOMAP_DIRECT;
1543	bool wait_for_completion = is_sync_kiocb(iocb);
1544	struct blk_plug plug;
1545	struct iomap_dio *dio;
1546
1547	lockdep_assert_held(&inode->i_rwsem);
1548
1549	if (!count)
1550	return 0;
1551
1552	dio = kmalloc(sizeof(*dio), GFP_KERNEL);
1553	if (!dio)
1554	return -ENOMEM;
1555
1556	dio->iocb = iocb;
1557	atomic_set(&dio->ref, 1);
1558	dio->size = 0;
1559	dio->i_size = i_size_read(inode);
1560	dio->end_io = end_io;
1561	dio->error = 0;
1562	dio->flags = 0;
1563
1564	dio->submit.iter = iter;
1565	dio->submit.waiter = current;
1566	dio->submit.cookie = BLK_QC_T_NONE;
1567	dio->submit.last_queue = NULL;
1568
1569	if (iov_iter_rw(iter) == READ) {
1570	if (pos >= dio->i_size)
1571	goto out_free_dio;
1572
1573	if (iter_is_iovec(iter) && iov_iter_rw(iter) == READ)
1574	dio->flags \|= IOMAP_DIO_DIRTY;
1575	} else {
1576	flags \|= IOMAP_WRITE;
1577	dio->flags \|= IOMAP_DIO_WRITE;
1578
1579	/* for data sync or sync, we need sync completion processing */
1580	if (iocb->ki_flags & IOCB_DSYNC)
1581	dio->flags \|= IOMAP_DIO_NEED_SYNC;
1582
1583	/*
1584	* For datasync only writes, we optimistically try using FUA for
1585	* this IO. Any non-FUA write that occurs will clear this flag,
1586	* hence we know before completion whether a cache flush is
1587	* necessary.
1588	*/
1589	if ((iocb->ki_flags & (IOCB_DSYNC \| IOCB_SYNC)) == IOCB_DSYNC)
1590	dio->flags \|= IOMAP_DIO_WRITE_FUA;
1591	}
1592
1593	if (iocb->ki_flags & IOCB_NOWAIT) {
1594	if (filemap_range_has_page(mapping, start, end)) {
1595	ret = -EAGAIN;
1596	goto out_free_dio;
1597	}
1598	flags \|= IOMAP_NOWAIT;
1599	}
1600
1601	ret = filemap_write_and_wait_range(mapping, start, end);
1602	if (ret)
1603	goto out_free_dio;
1604
1605	/*
1606	* Try to invalidate cache pages for the range we're direct
1607	* writing. If this invalidation fails, tough, the write will
1608	* still work, but racing two incompatible write paths is a
1609	* pretty crazy thing to do, so we don't support it 100%.
1610	*/
1611	ret = invalidate_inode_pages2_range(mapping,
1612	start >> PAGE_SHIFT, end >> PAGE_SHIFT);
1613	if (ret)
1614	dio_warn_stale_pagecache(iocb->ki_filp);
1615	ret = 0;
1616
1617	if (iov_iter_rw(iter) == WRITE && !wait_for_completion &&
1618	!inode->i_sb->s_dio_done_wq) {
1619	ret = sb_init_dio_done_wq(inode->i_sb);
1620	if (ret < 0)
1621	goto out_free_dio;
1622	}
1623
1624	inode_dio_begin(inode);
1625
1626	blk_start_plug(&plug);
1627	do {
1628	ret = iomap_apply(inode, pos, count, flags, ops, dio,
1629	iomap_dio_actor);
1630	if (ret <= 0) {
1631	/* magic error code to fall back to buffered I/O */
1632	if (ret == -ENOTBLK) {
1633	wait_for_completion = true;
1634	ret = 0;
1635	}
1636	break;
1637	}
1638	pos += ret;
1639
1640	if (iov_iter_rw(iter) == READ && pos >= dio->i_size)
1641	break;
1642	} while ((count = iov_iter_count(iter)) > 0);
1643	blk_finish_plug(&plug);
1644
1645	if (ret < 0)
1646	iomap_dio_set_error(dio, ret);
1647
1648	/*
1649	* If all the writes we issued were FUA, we don't need to flush the
1650	* cache on IO completion. Clear the sync flag for this case.
1651	*/
1652	if (dio->flags & IOMAP_DIO_WRITE_FUA)
1653	dio->flags &= ~IOMAP_DIO_NEED_SYNC;
1654
1655	WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);
1656	WRITE_ONCE(iocb->private, dio->submit.last_queue);
1657
1658	/*
1659	* We are about to drop our additional submission reference, which
1660	* might be the last reference to the dio. There are three three
1661	* different ways we can progress here:
1662	*
1663	* (a) If this is the last reference we will always complete and free
1664	* the dio ourselves.
1665	* (b) If this is not the last reference, and we serve an asynchronous
1666	* iocb, we must never touch the dio after the decrement, the
1667	* I/O completion handler will complete and free it.
1668	* (c) If this is not the last reference, but we serve a synchronous
1669	* iocb, the I/O completion handler will wake us up on the drop
1670	* of the final reference, and we will complete and free it here
1671	* after we got woken by the I/O completion handler.
1672	*/
1673	dio->wait_for_completion = wait_for_completion;
1674	if (!atomic_dec_and_test(&dio->ref)) {
1675	if (!wait_for_completion)
1676	return -EIOCBQUEUED;
1677
1678	for (;;) {
1679	set_current_state(TASK_UNINTERRUPTIBLE);
1680	if (!READ_ONCE(dio->submit.waiter))
1681	break;
1682
1683	if (!(iocb->ki_flags & IOCB_HIPRI) \|\|
1684	!dio->submit.last_queue \|\|
1685	!blk_poll(dio->submit.last_queue,
1686	dio->submit.cookie, true))
1687	io_schedule();
1688	}
1689	__set_current_state(TASK_RUNNING);
1690	}
1691
1692	return iomap_dio_complete(dio);
1693
1694	out_free_dio:
1695	kfree(dio);
1696	return ret;
1697	}
1698	EXPORT_SYMBOL_GPL(iomap_dio_rw);

fs/iomap/Makefile+1−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	obj-$(CONFIG_FS_IOMAP) += iomap.o	6	obj-$(CONFIG_FS_IOMAP) += iomap.o
7		7
8	iomap-y += \	8	iomap-y += \
		9	direct-io.o \
9	fiemap.o \	10	fiemap.o \
10	seek.o	11	seek.o
11		12

fs/iomap/direct-io.c+562−0

@@ -0,0 +1,562 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (C) 2010 Red Hat, Inc.
	4	* Copyright (c) 2016-2018 Christoph Hellwig.
	5	*/
	6	#include <linux/module.h>
	7	#include <linux/compiler.h>
	8	#include <linux/fs.h>
	9	#include <linux/iomap.h>
	10	#include <linux/backing-dev.h>
	11	#include <linux/uio.h>
	12	#include <linux/task_io_accounting_ops.h>
	13
	14	#include "../internal.h"
	15
	16	/*
	17	* Private flags for iomap_dio, must not overlap with the public ones in
	18	* iomap.h:
	19	*/
	20	#define IOMAP_DIO_WRITE_FUA (1 << 28)
	21	#define IOMAP_DIO_NEED_SYNC (1 << 29)
	22	#define IOMAP_DIO_WRITE (1 << 30)
	23	#define IOMAP_DIO_DIRTY (1 << 31)
	24
	25	struct iomap_dio {
	26	struct kiocb *iocb;
	27	iomap_dio_end_io_t *end_io;
	28	loff_t i_size;
	29	loff_t size;
	30	atomic_t ref;
	31	unsigned flags;
	32	int error;
	33	bool wait_for_completion;
	34
	35	union {
	36	/* used during submission and for synchronous completion: */
	37	struct {
	38	struct iov_iter *iter;
	39	struct task_struct *waiter;
	40	struct request_queue *last_queue;
	41	blk_qc_t cookie;
	42	} submit;
	43
	44	/* used for aio completion: */
	45	struct {
	46	struct work_struct work;
	47	} aio;
	48	};
	49	};
	50
	51	int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
	52	{
	53	struct request_queue *q = READ_ONCE(kiocb->private);
	54
	55	if (!q)
	56	return 0;
	57	return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
	58	}
	59	EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
	60
	61	static void iomap_dio_submit_bio(struct iomap_dio dio, struct iomap iomap,
	62	struct bio *bio)
	63	{
	64	atomic_inc(&dio->ref);
	65
	66	if (dio->iocb->ki_flags & IOCB_HIPRI)
	67	bio_set_polled(bio, dio->iocb);
	68
	69	dio->submit.last_queue = bdev_get_queue(iomap->bdev);
	70	dio->submit.cookie = submit_bio(bio);
	71	}
	72
	73	static ssize_t iomap_dio_complete(struct iomap_dio *dio)
	74	{
	75	struct kiocb *iocb = dio->iocb;
	76	struct inode *inode = file_inode(iocb->ki_filp);
	77	loff_t offset = iocb->ki_pos;
	78	ssize_t ret;
	79
	80	if (dio->end_io) {
	81	ret = dio->end_io(iocb,
	82	dio->error ? dio->error : dio->size,
	83	dio->flags);
	84	} else {
	85	ret = dio->error;
	86	}
	87
	88	if (likely(!ret)) {
	89	ret = dio->size;
	90	/* check for short read */
	91	if (offset + ret > dio->i_size &&
	92	!(dio->flags & IOMAP_DIO_WRITE))
	93	ret = dio->i_size - offset;
	94	iocb->ki_pos += ret;
	95	}
	96
	97	/*
	98	* Try again to invalidate clean pages which might have been cached by
	99	* non-direct readahead, or faulted in by get_user_pages() if the source
	100	* of the write was an mmap'ed region of the file we're writing. Either
	101	* one is a pretty crazy thing to do, so we don't support it 100%. If
	102	* this invalidation fails, tough, the write still worked...
	103	*
	104	* And this page cache invalidation has to be after dio->end_io(), as
	105	* some filesystems convert unwritten extents to real allocations in
	106	* end_io() when necessary, otherwise a racing buffer read would cache
	107	* zeros from unwritten extents.
	108	*/
	109	if (!dio->error &&
	110	(dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {
	111	int err;
	112	err = invalidate_inode_pages2_range(inode->i_mapping,
	113	offset >> PAGE_SHIFT,
	114	(offset + dio->size - 1) >> PAGE_SHIFT);
	115	if (err)
	116	dio_warn_stale_pagecache(iocb->ki_filp);
	117	}
	118
	119	/*
	120	* If this is a DSYNC write, make sure we push it to stable storage now
	121	* that we've written data.
	122	*/
	123	if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))
	124	ret = generic_write_sync(iocb, ret);
	125
	126	inode_dio_end(file_inode(iocb->ki_filp));
	127	kfree(dio);
	128
	129	return ret;
	130	}
	131
	132	static void iomap_dio_complete_work(struct work_struct *work)
	133	{
	134	struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
	135	struct kiocb *iocb = dio->iocb;
	136
	137	iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);
	138	}
	139
	140	/*
	141	* Set an error in the dio if none is set yet. We have to use cmpxchg
	142	* as the submission context and the completion context(s) can race to
	143	* update the error.
	144	*/
	145	static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
	146	{
	147	cmpxchg(&dio->error, 0, ret);
	148	}
	149
	150	static void iomap_dio_bio_end_io(struct bio *bio)
	151	{
	152	struct iomap_dio *dio = bio->bi_private;
	153	bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
	154
	155	if (bio->bi_status)
	156	iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
	157
	158	if (atomic_dec_and_test(&dio->ref)) {
	159	if (dio->wait_for_completion) {
	160	struct task_struct *waiter = dio->submit.waiter;
	161	WRITE_ONCE(dio->submit.waiter, NULL);
	162	blk_wake_io_task(waiter);
	163	} else if (dio->flags & IOMAP_DIO_WRITE) {
	164	struct inode *inode = file_inode(dio->iocb->ki_filp);
	165
	166	INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
	167	queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
	168	} else {
	169	iomap_dio_complete_work(&dio->aio.work);
	170	}
	171	}
	172
	173	if (should_dirty) {
	174	bio_check_pages_dirty(bio);
	175	} else {
	176	bio_release_pages(bio, false);
	177	bio_put(bio);
	178	}
	179	}
	180
	181	static void
	182	iomap_dio_zero(struct iomap_dio dio, struct iomap iomap, loff_t pos,
	183	unsigned len)
	184	{
	185	struct page *page = ZERO_PAGE(0);
	186	int flags = REQ_SYNC \| REQ_IDLE;
	187	struct bio *bio;
	188
	189	bio = bio_alloc(GFP_KERNEL, 1);
	190	bio_set_dev(bio, iomap->bdev);
	191	bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
	192	bio->bi_private = dio;
	193	bio->bi_end_io = iomap_dio_bio_end_io;
	194
	195	get_page(page);
	196	__bio_add_page(bio, page, len, 0);
	197	bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
	198	iomap_dio_submit_bio(dio, iomap, bio);
	199	}
	200
	201	static loff_t
	202	iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
	203	struct iomap_dio dio, struct iomap iomap)
	204	{
	205	unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
	206	unsigned int fs_block_size = i_blocksize(inode), pad;
	207	unsigned int align = iov_iter_alignment(dio->submit.iter);
	208	struct iov_iter iter;
	209	struct bio *bio;
	210	bool need_zeroout = false;
	211	bool use_fua = false;
	212	int nr_pages, ret = 0;
	213	size_t copied = 0;
	214
	215	if ((pos \| length \| align) & ((1 << blkbits) - 1))
	216	return -EINVAL;
	217
	218	if (iomap->type == IOMAP_UNWRITTEN) {
	219	dio->flags \|= IOMAP_DIO_UNWRITTEN;
	220	need_zeroout = true;
	221	}
	222
	223	if (iomap->flags & IOMAP_F_SHARED)
	224	dio->flags \|= IOMAP_DIO_COW;
	225
	226	if (iomap->flags & IOMAP_F_NEW) {
	227	need_zeroout = true;
	228	} else if (iomap->type == IOMAP_MAPPED) {
	229	/*
	230	* Use a FUA write if we need datasync semantics, this is a pure
	231	* data IO that doesn't require any metadata updates (including
	232	* after IO completion such as unwritten extent conversion) and
	233	* the underlying device supports FUA. This allows us to avoid
	234	* cache flushes on IO completion.
	235	*/
	236	if (!(iomap->flags & (IOMAP_F_SHARED\|IOMAP_F_DIRTY)) &&
	237	(dio->flags & IOMAP_DIO_WRITE_FUA) &&
	238	blk_queue_fua(bdev_get_queue(iomap->bdev)))
	239	use_fua = true;
	240	}
	241
	242	/*
	243	* Operate on a partial iter trimmed to the extent we were called for.
	244	* We'll update the iter in the dio once we're done with this extent.
	245	*/
	246	iter = *dio->submit.iter;
	247	iov_iter_truncate(&iter, length);
	248
	249	nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
	250	if (nr_pages <= 0)
	251	return nr_pages;
	252
	253	if (need_zeroout) {
	254	/* zero out from the start of the block to the write offset */
	255	pad = pos & (fs_block_size - 1);
	256	if (pad)
	257	iomap_dio_zero(dio, iomap, pos - pad, pad);
	258	}
	259
	260	do {
	261	size_t n;
	262	if (dio->error) {
	263	iov_iter_revert(dio->submit.iter, copied);
	264	return 0;
	265	}
	266
	267	bio = bio_alloc(GFP_KERNEL, nr_pages);
	268	bio_set_dev(bio, iomap->bdev);
	269	bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
	270	bio->bi_write_hint = dio->iocb->ki_hint;
	271	bio->bi_ioprio = dio->iocb->ki_ioprio;
	272	bio->bi_private = dio;
	273	bio->bi_end_io = iomap_dio_bio_end_io;
	274
	275	ret = bio_iov_iter_get_pages(bio, &iter);
	276	if (unlikely(ret)) {
	277	/*
	278	* We have to stop part way through an IO. We must fall
	279	* through to the sub-block tail zeroing here, otherwise
	280	* this short IO may expose stale data in the tail of
	281	* the block we haven't written data to.
	282	*/
	283	bio_put(bio);
	284	goto zero_tail;
	285	}
	286
	287	n = bio->bi_iter.bi_size;
	288	if (dio->flags & IOMAP_DIO_WRITE) {
	289	bio->bi_opf = REQ_OP_WRITE \| REQ_SYNC \| REQ_IDLE;
	290	if (use_fua)
	291	bio->bi_opf \|= REQ_FUA;
	292	else
	293	dio->flags &= ~IOMAP_DIO_WRITE_FUA;
	294	task_io_account_write(n);
	295	} else {
	296	bio->bi_opf = REQ_OP_READ;
	297	if (dio->flags & IOMAP_DIO_DIRTY)
	298	bio_set_pages_dirty(bio);
	299	}
	300
	301	iov_iter_advance(dio->submit.iter, n);
	302
	303	dio->size += n;
	304	pos += n;
	305	copied += n;
	306
	307	nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
	308	iomap_dio_submit_bio(dio, iomap, bio);
	309	} while (nr_pages);
	310
	311	/*
	312	* We need to zeroout the tail of a sub-block write if the extent type
	313	* requires zeroing or the write extends beyond EOF. If we don't zero
	314	* the block tail in the latter case, we can expose stale data via mmap
	315	* reads of the EOF block.
	316	*/
	317	zero_tail:
	318	if (need_zeroout \|\|
	319	((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
	320	/* zero out from the end of the write to the end of the block */
	321	pad = pos & (fs_block_size - 1);
	322	if (pad)
	323	iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);
	324	}
	325	return copied ? copied : ret;
	326	}
	327
	328	static loff_t
	329	iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio)
	330	{
	331	length = iov_iter_zero(length, dio->submit.iter);
	332	dio->size += length;
	333	return length;
	334	}
	335
	336	static loff_t
	337	iomap_dio_inline_actor(struct inode *inode, loff_t pos, loff_t length,
	338	struct iomap_dio dio, struct iomap iomap)
	339	{
	340	struct iov_iter *iter = dio->submit.iter;
	341	size_t copied;
	342
	343	BUG_ON(pos + length > PAGE_SIZE - offset_in_page(iomap->inline_data));
	344
	345	if (dio->flags & IOMAP_DIO_WRITE) {
	346	loff_t size = inode->i_size;
	347
	348	if (pos > size)
	349	memset(iomap->inline_data + size, 0, pos - size);
	350	copied = copy_from_iter(iomap->inline_data + pos, length, iter);
	351	if (copied) {
	352	if (pos + copied > size)
	353	i_size_write(inode, pos + copied);
	354	mark_inode_dirty(inode);
	355	}
	356	} else {
	357	copied = copy_to_iter(iomap->inline_data + pos, length, iter);
	358	}
	359	dio->size += copied;
	360	return copied;
	361	}
	362
	363	static loff_t
	364	iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
	365	void data, struct iomap iomap)
	366	{
	367	struct iomap_dio *dio = data;
	368
	369	switch (iomap->type) {
	370	case IOMAP_HOLE:
	371	if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
	372	return -EIO;
	373	return iomap_dio_hole_actor(length, dio);
	374	case IOMAP_UNWRITTEN:
	375	if (!(dio->flags & IOMAP_DIO_WRITE))
	376	return iomap_dio_hole_actor(length, dio);
	377	return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
	378	case IOMAP_MAPPED:
	379	return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
	380	case IOMAP_INLINE:
	381	return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
	382	default:
	383	WARN_ON_ONCE(1);
	384	return -EIO;
	385	}
	386	}
	387
	388	/*
	389	* iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
	390	* is being issued as AIO or not. This allows us to optimise pure data writes
	391	* to use REQ_FUA rather than requiring generic_write_sync() to issue a
	392	* REQ_FLUSH post write. This is slightly tricky because a single request here
	393	* can be mapped into multiple disjoint IOs and only a subset of the IOs issued
	394	* may be pure data writes. In that case, we still need to do a full data sync
	395	* completion.
	396	*/
	397	ssize_t
	398	iomap_dio_rw(struct kiocb iocb, struct iov_iter iter,
	399	const struct iomap_ops *ops, iomap_dio_end_io_t end_io)
	400	{
	401	struct address_space *mapping = iocb->ki_filp->f_mapping;
	402	struct inode *inode = file_inode(iocb->ki_filp);
	403	size_t count = iov_iter_count(iter);
	404	loff_t pos = iocb->ki_pos, start = pos;
	405	loff_t end = iocb->ki_pos + count - 1, ret = 0;
	406	unsigned int flags = IOMAP_DIRECT;
	407	bool wait_for_completion = is_sync_kiocb(iocb);
	408	struct blk_plug plug;
	409	struct iomap_dio *dio;
	410
	411	lockdep_assert_held(&inode->i_rwsem);
	412
	413	if (!count)
	414	return 0;
	415
	416	dio = kmalloc(sizeof(*dio), GFP_KERNEL);
	417	if (!dio)
	418	return -ENOMEM;
	419
	420	dio->iocb = iocb;
	421	atomic_set(&dio->ref, 1);
	422	dio->size = 0;
	423	dio->i_size = i_size_read(inode);
	424	dio->end_io = end_io;
	425	dio->error = 0;
	426	dio->flags = 0;
	427
	428	dio->submit.iter = iter;
	429	dio->submit.waiter = current;
	430	dio->submit.cookie = BLK_QC_T_NONE;
	431	dio->submit.last_queue = NULL;
	432
	433	if (iov_iter_rw(iter) == READ) {
	434	if (pos >= dio->i_size)
	435	goto out_free_dio;
	436
	437	if (iter_is_iovec(iter) && iov_iter_rw(iter) == READ)
	438	dio->flags \|= IOMAP_DIO_DIRTY;
	439	} else {
	440	flags \|= IOMAP_WRITE;
	441	dio->flags \|= IOMAP_DIO_WRITE;
	442
	443	/* for data sync or sync, we need sync completion processing */
	444	if (iocb->ki_flags & IOCB_DSYNC)
	445	dio->flags \|= IOMAP_DIO_NEED_SYNC;
	446
	447	/*
	448	* For datasync only writes, we optimistically try using FUA for
	449	* this IO. Any non-FUA write that occurs will clear this flag,
	450	* hence we know before completion whether a cache flush is
	451	* necessary.
	452	*/
	453	if ((iocb->ki_flags & (IOCB_DSYNC \| IOCB_SYNC)) == IOCB_DSYNC)
	454	dio->flags \|= IOMAP_DIO_WRITE_FUA;
	455	}
	456
	457	if (iocb->ki_flags & IOCB_NOWAIT) {
	458	if (filemap_range_has_page(mapping, start, end)) {
	459	ret = -EAGAIN;
	460	goto out_free_dio;
	461	}
	462	flags \|= IOMAP_NOWAIT;
	463	}
	464
	465	ret = filemap_write_and_wait_range(mapping, start, end);
	466	if (ret)
	467	goto out_free_dio;
	468
	469	/*
	470	* Try to invalidate cache pages for the range we're direct
	471	* writing. If this invalidation fails, tough, the write will
	472	* still work, but racing two incompatible write paths is a
	473	* pretty crazy thing to do, so we don't support it 100%.
	474	*/
	475	ret = invalidate_inode_pages2_range(mapping,
	476	start >> PAGE_SHIFT, end >> PAGE_SHIFT);
	477	if (ret)
	478	dio_warn_stale_pagecache(iocb->ki_filp);
	479	ret = 0;
	480
	481	if (iov_iter_rw(iter) == WRITE && !wait_for_completion &&
	482	!inode->i_sb->s_dio_done_wq) {
	483	ret = sb_init_dio_done_wq(inode->i_sb);
	484	if (ret < 0)
	485	goto out_free_dio;
	486	}
	487
	488	inode_dio_begin(inode);
	489
	490	blk_start_plug(&plug);
	491	do {
	492	ret = iomap_apply(inode, pos, count, flags, ops, dio,
	493	iomap_dio_actor);
	494	if (ret <= 0) {
	495	/* magic error code to fall back to buffered I/O */
	496	if (ret == -ENOTBLK) {
	497	wait_for_completion = true;
	498	ret = 0;
	499	}
	500	break;
	501	}
	502	pos += ret;
	503
	504	if (iov_iter_rw(iter) == READ && pos >= dio->i_size)
	505	break;
	506	} while ((count = iov_iter_count(iter)) > 0);
	507	blk_finish_plug(&plug);
	508
	509	if (ret < 0)
	510	iomap_dio_set_error(dio, ret);
	511
	512	/*
	513	* If all the writes we issued were FUA, we don't need to flush the
	514	* cache on IO completion. Clear the sync flag for this case.
	515	*/
	516	if (dio->flags & IOMAP_DIO_WRITE_FUA)
	517	dio->flags &= ~IOMAP_DIO_NEED_SYNC;
	518
	519	WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);
	520	WRITE_ONCE(iocb->private, dio->submit.last_queue);
	521
	522	/*
	523	* We are about to drop our additional submission reference, which
	524	* might be the last reference to the dio. There are three three
	525	* different ways we can progress here:
	526	*
	527	* (a) If this is the last reference we will always complete and free
	528	* the dio ourselves.
	529	* (b) If this is not the last reference, and we serve an asynchronous
	530	* iocb, we must never touch the dio after the decrement, the
	531	* I/O completion handler will complete and free it.
	532	* (c) If this is not the last reference, but we serve a synchronous
	533	* iocb, the I/O completion handler will wake us up on the drop
	534	* of the final reference, and we will complete and free it here
	535	* after we got woken by the I/O completion handler.
	536	*/
	537	dio->wait_for_completion = wait_for_completion;
	538	if (!atomic_dec_and_test(&dio->ref)) {
	539	if (!wait_for_completion)
	540	return -EIOCBQUEUED;
	541
	542	for (;;) {
	543	set_current_state(TASK_UNINTERRUPTIBLE);
	544	if (!READ_ONCE(dio->submit.waiter))
	545	break;
	546
	547	if (!(iocb->ki_flags & IOCB_HIPRI) \|\|
	548	!dio->submit.last_queue \|\|
	549	!blk_poll(dio->submit.last_queue,
	550	dio->submit.cookie, true))
	551	io_schedule();
	552	}
	553	__set_current_state(TASK_RUNNING);
	554	}
	555
	556	return iomap_dio_complete(dio);
	557
	558	out_free_dio:
	559	kfree(dio);
	560	return ret;
	561	}
	562	EXPORT_SYMBOL_GPL(iomap_dio_rw);

include/linux/iomap.h+7−0

 #include <linux/mm.h>
 #include <linux/types.h>
 #include <linux/mm_types.h>
+#include <linux/blkdev.h>
 struct address_space;
 struct fiemap_extent_info;
 	const struct iomap_page_ops *page_ops;
 };
+static inline sector_t
+iomap_sector(struct iomap *iomap, loff_t pos)
+{
+	return (iomap->addr + pos - iomap->offset) >> SECTOR_SHIFT;
+}
 /*
  * When a filesystem sets page_ops in an iomap mapping it returns, page_prepare
  * and page_done will be called for each page written to.  This only applies to

iomap: move the direct IO code into a separate file

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code