direct-io: only inc/dec inode->i_dio_count for file systems

do_blockdev_direct_IO() increments and decrements the inode ->i_dio_count for each IO operation. It does this to protect against truncate of a file. Block devices don't need this sort of protection. For a capable multiqueue setup, this atomic int is the only shared state between applications accessing the device for O_DIRECT, and it presents a scaling wall for that. In my testing, as much as 30% of system time is spent incrementing and decrementing this value. A mixed read/write workload improved from ~2.5M IOPS to ~9.6M IOPS, with better latencies too. Before: clat percentiles (usec): | 1.00th=[ 33], 5.00th=[ 34], 10.00th=[ 34], 20.00th=[ 34], | 30.00th=[ 34], 40.00th=[ 34], 50.00th=[ 35], 60.00th=[ 35], | 70.00th=[ 35], 80.00th=[ 35], 90.00th=[ 37], 95.00th=[ 80], | 99.00th=[ 98], 99.50th=[ 151], 99.90th=[ 155], 99.95th=[ 155], | 99.99th=[ 165] After: clat percentiles (usec): | 1.00th=[ 95], 5.00th=[ 108], 10.00th=[ 129], 20.00th=[ 149], | 30.00th=[ 155], 40.00th=[ 161], 50.00th=[ 167], 60.00th=[ 171], | 70.00th=[ 177], 80.00th=[ 185], 90.00th=[ 201], 95.00th=[ 270], | 99.00th=[ 390], 99.50th=[ 398], 99.90th=[ 418], 99.95th=[ 422], | 99.99th=[ 438] In other setups, Robert Elliott reported seeing good performance improvements: https://lkml.org/lkml/2015/4/3/557 The more applications accessing the device, the worse it gets. Add a new direct-io flags, DIO_SKIP_DIO_COUNT, which tells do_blockdev_direct_IO() that it need not worry about incrementing or decrementing the inode i_dio_count for this caller.

Jens Axboe committed 11 years ago

Commit fe0f07d1 parent 8e3c5009 files · +50 −33Browse files

Unified · Split

9 files

fs/block_dev.c +2 −1
fs/btrfs/inode.c +3 −3
fs/dax.c +2 −2
fs/direct-io.c +5 −2
fs/ext4/indirect.c +3 −3
fs/ext4/inode.c +2 −2
fs/inode.c +0 −14
fs/nfs/direct.c +5 −5
include/linux/fs.h +28 −1

fs/block_dev.c+2−1

@@ -152,7 +152,8 @@ blkdev_direct_IO(struct kiocb iocb, struct iov_iter iter, loff_t offset) · 151 unmodified lines
152	152	struct inode *inode = file->f_mapping->host;
153	153
154	154	return __blockdev_direct_IO(iocb, inode, I_BDEV(inode), iter, offset,
155		− blkdev_get_block, NULL, NULL, 0);
	155	+ blkdev_get_block, NULL, NULL,
	156	+ DIO_SKIP_DIO_COUNT);
156	157	}
157	158
158	159	int __sync_blockdev(struct block_device *bdev, int wait)

fs/btrfs/inode.c+3−3

@@ -8129,7 +8129,7 @@ static ssize_t btrfs_direct_IO(struct kiocb iocb, struct iov_iter iter, · 8128 unmodified lines
8129	8129	if (check_direct_IO(BTRFS_I(inode)->root, iocb, iter, offset))
8130	8130	return 0;
8131	8131
8132		− atomic_inc(&inode->i_dio_count);
	8132	+ inode_dio_begin(inode);
8133	8133	smp_mb__after_atomic();
8134	8134
8135	8135	/*
@@ -8169,7 +8169,7 @@ static ssize_t btrfs_direct_IO(struct kiocb iocb, struct iov_iter iter, · 33 unmodified lines
8169	8169	current->journal_info = &outstanding_extents;
8170	8170	} else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8171	8171	&BTRFS_I(inode)->runtime_flags)) {
8172		− inode_dio_done(inode);
	8172	+ inode_dio_end(inode);
8173	8173	flags = DIO_LOCKING \| DIO_SKIP_HOLES;
8174	8174	wakeup = false;
8175	8175	}
@@ -8188,7 +8188,7 @@ static ssize_t btrfs_direct_IO(struct kiocb iocb, struct iov_iter iter, · 12 unmodified lines
8188	8188	}
8189	8189	out:
8190	8190	if (wakeup)
8191		− inode_dio_done(inode);
	8191	+ inode_dio_end(inode);
8192	8192	if (relock)
8193	8193	mutex_lock(&inode->i_mutex);
8194	8194

fs/dax.c+2−2

@@ -209,7 +209,7 @@ ssize_t dax_do_io(struct kiocb iocb, struct inode inode, · 208 unmodified lines
209	209	}
210	210
211	211	/* Protects against truncate */
212		− atomic_inc(&inode->i_dio_count);
	212	+ inode_dio_begin(inode);
213	213
214	214	retval = dax_io(inode, iter, pos, end, get_block, &bh);
215	215
@@ -219,7 +219,7 @@ ssize_t dax_do_io(struct kiocb iocb, struct inode inode, · 3 unmodified lines
219	219	if ((retval > 0) && end_io)
220	220	end_io(iocb, pos, retval, bh.b_private);
221	221
222		− inode_dio_done(inode);
	222	+ inode_dio_end(inode);
223	223	out:
224	224	return retval;
225	225	}

fs/direct-io.c+5−2

@@ -253,7 +253,9 @@ static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret, · 252 unmodified lines
253	253	if (dio->end_io && dio->result)
254	254	dio->end_io(dio->iocb, offset, transferred, dio->private);
255	255
256		− inode_dio_done(dio->inode);
	256	+ if (!(dio->flags & DIO_SKIP_DIO_COUNT))
	257	+ inode_dio_end(dio->inode);
	258	+
257	259	if (is_async) {
258	260	if (dio->rw & WRITE) {
259	261	int err;
@@ -1195,7 +1197,8 @@ do_blockdev_direct_IO(struct kiocb iocb, struct inode inode, · 935 unmodified lines
1195	1197	/*
1196	1198	* Will be decremented at I/O completion time.
1197	1199	*/
1198		− atomic_inc(&inode->i_dio_count);
	1200	+ if (!(dio->flags & DIO_SKIP_DIO_COUNT))
	1201	+ inode_dio_begin(inode);
1199	1202
1200	1203	retval = 0;
1201	1204	sdio.blkbits = blkbits;

fs/ext4/indirect.c+3−3

@@ -682,11 +682,11 @@ retry: · 681 unmodified lines
682	682	* via ext4_inode_block_unlocked_dio(). Check inode's state
683	683	* while holding extra i_dio_count ref.
684	684	*/
685		− atomic_inc(&inode->i_dio_count);
	685	+ inode_dio_begin(inode);
686	686	smp_mb();
687	687	if (unlikely(ext4_test_inode_state(inode,
688	688	EXT4_STATE_DIOREAD_LOCK))) {
689		− inode_dio_done(inode);
	689	+ inode_dio_end(inode);
690	690	goto locked;
691	691	}
692	692	if (IS_DAX(inode))
@@ -697,7 +697,7 @@ retry: · 4 unmodified lines
697	697	inode->i_sb->s_bdev, iter,
698	698	offset, ext4_get_block, NULL,
699	699	NULL, 0);
700		− inode_dio_done(inode);
	700	+ inode_dio_end(inode);
701	701	} else {
702	702	locked:
703	703	if (IS_DAX(inode))

fs/ext4/inode.c+2−2

@@ -2977,7 +2977,7 @@ static ssize_t ext4_ext_direct_IO(struct kiocb iocb, struct iov_iter iter, · 2976 unmodified lines
2977	2977	* overwrite DIO as i_dio_count needs to be incremented under i_mutex.
2978	2978	*/
2979	2979	if (iov_iter_rw(iter) == WRITE)
2980		− atomic_inc(&inode->i_dio_count);
	2980	+ inode_dio_begin(inode);
2981	2981
2982	2982	/* If we do a overwrite dio, i_mutex locking can be released */
2983	2983	overwrite = ((int )iocb->private);
@@ -3079,7 +3079,7 @@ static ssize_t ext4_ext_direct_IO(struct kiocb iocb, struct iov_iter iter, · 95 unmodified lines
3079	3079
3080	3080	retake_lock:
3081	3081	if (iov_iter_rw(iter) == WRITE)
3082		− inode_dio_done(inode);
	3082	+ inode_dio_end(inode);
3083	3083	/* take i_mutex locking again if we do a ovewrite dio */
3084	3084	if (overwrite) {
3085	3085	up_read(&EXT4_I(inode)->i_data_sem);

fs/inode.c+0−14

@@ -1945,20 +1945,6 @@ void inode_dio_wait(struct inode *inode) · 1944 unmodified lines
1945	1945	}
1946	1946	EXPORT_SYMBOL(inode_dio_wait);
1947	1947
1948		−/*
1949		− * inode_dio_done - signal finish of a direct I/O requests
1950		− * @inode: inode the direct I/O happens on
1951		− *
1952		− * This is called once we've finished processing a direct I/O request,
1953		− * and is used to wake up callers waiting for direct I/O to be quiesced.
1954		− */
1955		−void inode_dio_done(struct inode *inode)
1956		−{
1957		− if (atomic_dec_and_test(&inode->i_dio_count))
1958		− wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
1959		−}
1960		−EXPORT_SYMBOL(inode_dio_done);
1961		−
1962	1948	/*
1963	1949	* inode_set_flags - atomically set some inode flags
1964	1950	*

fs/nfs/direct.c+5−5

@@ -386,7 +386,7 @@ static void nfs_direct_complete(struct nfs_direct_req *dreq, bool write) · 385 unmodified lines
386	386	if (write)
387	387	nfs_zap_mapping(inode, inode->i_mapping);
388	388
389		− inode_dio_done(inode);
	389	+ inode_dio_end(inode);
390	390
391	391	if (dreq->iocb) {
392	392	long res = (long) dreq->error;
@@ -486,7 +486,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, · 93 unmodified lines
486	486	&nfs_direct_read_completion_ops);
487	487	get_dreq(dreq);
488	488	desc.pg_dreq = dreq;
489		− atomic_inc(&inode->i_dio_count);
	489	+ inode_dio_begin(inode);
490	490
491	491	while (iov_iter_count(iter)) {
492	492	struct page **pagevec;
@@ -538,7 +538,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, · 45 unmodified lines
538	538	* generic layer handle the completion.
539	539	*/
540	540	if (requested_bytes == 0) {
541		− inode_dio_done(inode);
	541	+ inode_dio_end(inode);
542	542	nfs_direct_req_release(dreq);
543	543	return result < 0 ? result : -EIO;
544	544	}
@@ -872,7 +872,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, · 327 unmodified lines
872	872	&nfs_direct_write_completion_ops);
873	873	desc.pg_dreq = dreq;
874	874	get_dreq(dreq);
875		− atomic_inc(&inode->i_dio_count);
	875	+ inode_dio_begin(inode);
876	876
877	877	NFS_I(inode)->write_io += iov_iter_count(iter);
878	878	while (iov_iter_count(iter)) {
@@ -928,7 +928,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, · 49 unmodified lines
928	928	* generic layer handle the completion.
929	929	*/
930	930	if (requested_bytes == 0) {
931		− inode_dio_done(inode);
	931	+ inode_dio_end(inode);
932	932	nfs_direct_req_release(dreq);
933	933	return result < 0 ? result : -EIO;
934	934	}

include/linux/fs.h+28−1

@@ -2635,6 +2635,9 @@ enum { · 2634 unmodified lines
2635	2635
2636	2636	/* filesystem can handle aio writes beyond i_size */
2637	2637	DIO_ASYNC_EXTEND = 0x04,
	2638	+
	2639	+ /* inode/fs/bdev does not need truncate protection */
	2640	+ DIO_SKIP_DIO_COUNT = 0x08,
2638	2641	};
2639	2642
2640	2643	void dio_end_io(struct bio *bio, int error);
@@ -2657,7 +2660,31 @@ static inline ssize_t blockdev_direct_IO(struct kiocb *iocb, · 16 unmodified lines
2657	2660	#endif
2658	2661
2659	2662	void inode_dio_wait(struct inode *inode);
2660		−void inode_dio_done(struct inode *inode);
	2663	+
	2664	+/*
	2665	+ * inode_dio_begin - signal start of a direct I/O requests
	2666	+ * @inode: inode the direct I/O happens on
	2667	+ *
	2668	+ * This is called once we've finished processing a direct I/O request,
	2669	+ * and is used to wake up callers waiting for direct I/O to be quiesced.
	2670	+ */
	2671	+static inline void inode_dio_begin(struct inode *inode)
	2672	+{
	2673	+ atomic_inc(&inode->i_dio_count);
	2674	+}
	2675	+
	2676	+/*
	2677	+ * inode_dio_end - signal finish of a direct I/O requests
	2678	+ * @inode: inode the direct I/O happens on
	2679	+ *
	2680	+ * This is called once we've finished processing a direct I/O request,
	2681	+ * and is used to wake up callers waiting for direct I/O to be quiesced.
	2682	+ */
	2683	+static inline void inode_dio_end(struct inode *inode)
	2684	+{
	2685	+ if (atomic_dec_and_test(&inode->i_dio_count))
	2686	+ wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
	2687	+}
2661	2688
2662	2689	extern void inode_set_flags(struct inode *inode, unsigned int flags,
2663	2690	unsigned int mask);