History

direct-io: only inc/dec inode->i_dio_count for file systems

do_blockdev_direct_IO() increments and decrements the inode ->i_dio_count for each IO operation. It does this to protect against truncate of a file. Block devices don't need this sort of protection. For a capable multiqueue setup, this atomic int is the only shared state between applications accessing the device for O_DIRECT, and it presents a scaling wall for that. In my testing, as much as 30% of system time is spent incrementing and decrementing this value. A mixed read/write workload improved from ~2.5M IOPS to ~9.6M IOPS, with better latencies too. Before: clat percentiles (usec): | 1.00th=[ 33], 5.00th=[ 34], 10.00th=[ 34], 20.00th=[ 34], | 30.00th=[ 34], 40.00th=[ 34], 50.00th=[ 35], 60.00th=[ 35], | 70.00th=[ 35], 80.00th=[ 35], 90.00th=[ 37], 95.00th=[ 80], | 99.00th=[ 98], 99.50th=[ 151], 99.90th=[ 155], 99.95th=[ 155], | 99.99th=[ 165] After: clat percentiles (usec): | 1.00th=[ 95], 5.00th=[ 108], 10.00th=[ 129], 20.00th=[ 149], | 30.00th=[ 155], 40.00th=[ 161], 50.00th=[ 167], 60.00th=[ 171], | 70.00th=[ 177], 80.00th=[ 185], 90.00th=[ 201], 95.00th=[ 270], | 99.00th=[ 390], 99.50th=[ 398], 99.90th=[ 418], 99.95th=[ 422], | 99.99th=[ 438] In other setups, Robert Elliott reported seeing good performance improvements: https://lkml.org/lkml/2015/4/3/557 The more applications accessing the device, the worse it gets. Add a new direct-io flags, DIO_SKIP_DIO_COUNT, which tells do_blockdev_direct_IO() that it need not worry about incrementing or decrementing the inode i_dio_count for this caller.

Jens Axboe committed 11 years ago

Commit fe0f07d1 parent 8e3c5009 files · +50 −33Browse files

Unified · Split

9 files
  1. fs/block_dev.c +2 −1
  2. fs/btrfs/inode.c +3 −3
  3. fs/dax.c +2 −2
  4. fs/direct-io.c +5 −2
  5. fs/ext4/indirect.c +3 −3
  6. fs/ext4/inode.c +2 −2
  7. fs/inode.c +0 −14
  8. fs/nfs/direct.c +5 −5
  9. include/linux/fs.h +28 −1
fs/block_dev.c+2−1
@@ -152,7 +152,8 @@ blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t offset) · 151 unmodified lines
152 struct inode *inode = file->f_mapping->host;152 struct inode *inode = file->f_mapping->host;
153 153
154 return __blockdev_direct_IO(iocb, inode, I_BDEV(inode), iter, offset,154 return __blockdev_direct_IO(iocb, inode, I_BDEV(inode), iter, offset,
155 blkdev_get_block, NULL, NULL, 0);155 blkdev_get_block, NULL, NULL,
156 DIO_SKIP_DIO_COUNT);
156}157}
157 158
158int __sync_blockdev(struct block_device *bdev, int wait)159int __sync_blockdev(struct block_device *bdev, int wait)
fs/btrfs/inode.c+3−3
@@ -8129,7 +8129,7 @@ static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter, · 8128 unmodified lines
8129 if (check_direct_IO(BTRFS_I(inode)->root, iocb, iter, offset))8129 if (check_direct_IO(BTRFS_I(inode)->root, iocb, iter, offset))
8130 return 0;8130 return 0;
8131 8131
8132 atomic_inc(&inode->i_dio_count);8132 inode_dio_begin(inode);
8133 smp_mb__after_atomic();8133 smp_mb__after_atomic();
8134 8134
8135 /*8135 /*
@@ -8169,7 +8169,7 @@ static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter, · 33 unmodified lines
8169 current->journal_info = &outstanding_extents;8169 current->journal_info = &outstanding_extents;
8170 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,8170 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8171 &BTRFS_I(inode)->runtime_flags)) {8171 &BTRFS_I(inode)->runtime_flags)) {
8172 inode_dio_done(inode);8172 inode_dio_end(inode);
8173 flags = DIO_LOCKING | DIO_SKIP_HOLES;8173 flags = DIO_LOCKING | DIO_SKIP_HOLES;
8174 wakeup = false;8174 wakeup = false;
8175 }8175 }
@@ -8188,7 +8188,7 @@ static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter, · 12 unmodified lines
8188 }8188 }
8189out:8189out:
8190 if (wakeup)8190 if (wakeup)
8191 inode_dio_done(inode);8191 inode_dio_end(inode);
8192 if (relock)8192 if (relock)
8193 mutex_lock(&inode->i_mutex);8193 mutex_lock(&inode->i_mutex);
8194 8194
fs/dax.c+2−2
@@ -209,7 +209,7 @@ ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode, · 208 unmodified lines
209 }209 }
210 210
211 /* Protects against truncate */211 /* Protects against truncate */
212 atomic_inc(&inode->i_dio_count);212 inode_dio_begin(inode);
213 213
214 retval = dax_io(inode, iter, pos, end, get_block, &bh);214 retval = dax_io(inode, iter, pos, end, get_block, &bh);
215 215
@@ -219,7 +219,7 @@ ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode, · 3 unmodified lines
219 if ((retval > 0) && end_io)219 if ((retval > 0) && end_io)
220 end_io(iocb, pos, retval, bh.b_private);220 end_io(iocb, pos, retval, bh.b_private);
221 221
222 inode_dio_done(inode);222 inode_dio_end(inode);
223 out:223 out:
224 return retval;224 return retval;
225}225}
fs/direct-io.c+5−2
@@ -253,7 +253,9 @@ static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret, · 252 unmodified lines
253 if (dio->end_io && dio->result)253 if (dio->end_io && dio->result)
254 dio->end_io(dio->iocb, offset, transferred, dio->private);254 dio->end_io(dio->iocb, offset, transferred, dio->private);
255 255
256 inode_dio_done(dio->inode);256 if (!(dio->flags & DIO_SKIP_DIO_COUNT))
257 inode_dio_end(dio->inode);
258
257 if (is_async) {259 if (is_async) {
258 if (dio->rw & WRITE) {260 if (dio->rw & WRITE) {
259 int err;261 int err;
@@ -1195,7 +1197,8 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, · 935 unmodified lines
1195 /*1197 /*
1196 * Will be decremented at I/O completion time.1198 * Will be decremented at I/O completion time.
1197 */1199 */
1198 atomic_inc(&inode->i_dio_count);1200 if (!(dio->flags & DIO_SKIP_DIO_COUNT))
1201 inode_dio_begin(inode);
1199 1202
1200 retval = 0;1203 retval = 0;
1201 sdio.blkbits = blkbits;1204 sdio.blkbits = blkbits;
fs/ext4/indirect.c+3−3
@@ -682,11 +682,11 @@ retry: · 681 unmodified lines
682 * via ext4_inode_block_unlocked_dio(). Check inode's state682 * via ext4_inode_block_unlocked_dio(). Check inode's state
683 * while holding extra i_dio_count ref.683 * while holding extra i_dio_count ref.
684 */684 */
685 atomic_inc(&inode->i_dio_count);685 inode_dio_begin(inode);
686 smp_mb();686 smp_mb();
687 if (unlikely(ext4_test_inode_state(inode,687 if (unlikely(ext4_test_inode_state(inode,
688 EXT4_STATE_DIOREAD_LOCK))) {688 EXT4_STATE_DIOREAD_LOCK))) {
689 inode_dio_done(inode);689 inode_dio_end(inode);
690 goto locked;690 goto locked;
691 }691 }
692 if (IS_DAX(inode))692 if (IS_DAX(inode))
@@ -697,7 +697,7 @@ retry: · 4 unmodified lines
697 inode->i_sb->s_bdev, iter,697 inode->i_sb->s_bdev, iter,
698 offset, ext4_get_block, NULL,698 offset, ext4_get_block, NULL,
699 NULL, 0);699 NULL, 0);
700 inode_dio_done(inode);700 inode_dio_end(inode);
701 } else {701 } else {
702locked:702locked:
703 if (IS_DAX(inode))703 if (IS_DAX(inode))
fs/ext4/inode.c+2−2
@@ -2977,7 +2977,7 @@ static ssize_t ext4_ext_direct_IO(struct kiocb *iocb, struct iov_iter *iter, · 2976 unmodified lines
2977 * overwrite DIO as i_dio_count needs to be incremented under i_mutex.2977 * overwrite DIO as i_dio_count needs to be incremented under i_mutex.
2978 */2978 */
2979 if (iov_iter_rw(iter) == WRITE)2979 if (iov_iter_rw(iter) == WRITE)
2980 atomic_inc(&inode->i_dio_count);2980 inode_dio_begin(inode);
2981 2981
2982 /* If we do a overwrite dio, i_mutex locking can be released */2982 /* If we do a overwrite dio, i_mutex locking can be released */
2983 overwrite = *((int *)iocb->private);2983 overwrite = *((int *)iocb->private);
@@ -3079,7 +3079,7 @@ static ssize_t ext4_ext_direct_IO(struct kiocb *iocb, struct iov_iter *iter, · 95 unmodified lines
3079 3079
3080retake_lock:3080retake_lock:
3081 if (iov_iter_rw(iter) == WRITE)3081 if (iov_iter_rw(iter) == WRITE)
3082 inode_dio_done(inode);3082 inode_dio_end(inode);
3083 /* take i_mutex locking again if we do a ovewrite dio */3083 /* take i_mutex locking again if we do a ovewrite dio */
3084 if (overwrite) {3084 if (overwrite) {
3085 up_read(&EXT4_I(inode)->i_data_sem);3085 up_read(&EXT4_I(inode)->i_data_sem);
fs/inode.c+0−14
@@ -1945,20 +1945,6 @@ void inode_dio_wait(struct inode *inode) · 1944 unmodified lines
1945}1945}
1946EXPORT_SYMBOL(inode_dio_wait);1946EXPORT_SYMBOL(inode_dio_wait);
1947 1947
1948/*
1949 * inode_dio_done - signal finish of a direct I/O requests
1950 * @inode: inode the direct I/O happens on
1951 *
1952 * This is called once we've finished processing a direct I/O request,
1953 * and is used to wake up callers waiting for direct I/O to be quiesced.
1954 */
1955void inode_dio_done(struct inode *inode)
1956{
1957 if (atomic_dec_and_test(&inode->i_dio_count))
1958 wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
1959}
1960EXPORT_SYMBOL(inode_dio_done);
1961
1962/*1948/*
1963 * inode_set_flags - atomically set some inode flags1949 * inode_set_flags - atomically set some inode flags
1964 *1950 *
fs/nfs/direct.c+5−5
@@ -386,7 +386,7 @@ static void nfs_direct_complete(struct nfs_direct_req *dreq, bool write) · 385 unmodified lines
386 if (write)386 if (write)
387 nfs_zap_mapping(inode, inode->i_mapping);387 nfs_zap_mapping(inode, inode->i_mapping);
388 388
389 inode_dio_done(inode);389 inode_dio_end(inode);
390 390
391 if (dreq->iocb) {391 if (dreq->iocb) {
392 long res = (long) dreq->error;392 long res = (long) dreq->error;
@@ -486,7 +486,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, · 93 unmodified lines
486 &nfs_direct_read_completion_ops);486 &nfs_direct_read_completion_ops);
487 get_dreq(dreq);487 get_dreq(dreq);
488 desc.pg_dreq = dreq;488 desc.pg_dreq = dreq;
489 atomic_inc(&inode->i_dio_count);489 inode_dio_begin(inode);
490 490
491 while (iov_iter_count(iter)) {491 while (iov_iter_count(iter)) {
492 struct page **pagevec;492 struct page **pagevec;
@@ -538,7 +538,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, · 45 unmodified lines
538 * generic layer handle the completion.538 * generic layer handle the completion.
539 */539 */
540 if (requested_bytes == 0) {540 if (requested_bytes == 0) {
541 inode_dio_done(inode);541 inode_dio_end(inode);
542 nfs_direct_req_release(dreq);542 nfs_direct_req_release(dreq);
543 return result < 0 ? result : -EIO;543 return result < 0 ? result : -EIO;
544 }544 }
@@ -872,7 +872,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, · 327 unmodified lines
872 &nfs_direct_write_completion_ops);872 &nfs_direct_write_completion_ops);
873 desc.pg_dreq = dreq;873 desc.pg_dreq = dreq;
874 get_dreq(dreq);874 get_dreq(dreq);
875 atomic_inc(&inode->i_dio_count);875 inode_dio_begin(inode);
876 876
877 NFS_I(inode)->write_io += iov_iter_count(iter);877 NFS_I(inode)->write_io += iov_iter_count(iter);
878 while (iov_iter_count(iter)) {878 while (iov_iter_count(iter)) {
@@ -928,7 +928,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, · 49 unmodified lines
928 * generic layer handle the completion.928 * generic layer handle the completion.
929 */929 */
930 if (requested_bytes == 0) {930 if (requested_bytes == 0) {
931 inode_dio_done(inode);931 inode_dio_end(inode);
932 nfs_direct_req_release(dreq);932 nfs_direct_req_release(dreq);
933 return result < 0 ? result : -EIO;933 return result < 0 ? result : -EIO;
934 }934 }
include/linux/fs.h+28−1
@@ -2635,6 +2635,9 @@ enum { · 2634 unmodified lines
2635 2635
2636 /* filesystem can handle aio writes beyond i_size */2636 /* filesystem can handle aio writes beyond i_size */
2637 DIO_ASYNC_EXTEND = 0x04,2637 DIO_ASYNC_EXTEND = 0x04,
2638
2639 /* inode/fs/bdev does not need truncate protection */
2640 DIO_SKIP_DIO_COUNT = 0x08,
2638};2641};
2639 2642
2640void dio_end_io(struct bio *bio, int error);2643void dio_end_io(struct bio *bio, int error);
@@ -2657,7 +2660,31 @@ static inline ssize_t blockdev_direct_IO(struct kiocb *iocb, · 16 unmodified lines
2657#endif2660#endif
2658 2661
2659void inode_dio_wait(struct inode *inode);2662void inode_dio_wait(struct inode *inode);
2660void inode_dio_done(struct inode *inode);2663
2664/*
2665 * inode_dio_begin - signal start of a direct I/O requests
2666 * @inode: inode the direct I/O happens on
2667 *
2668 * This is called once we've finished processing a direct I/O request,
2669 * and is used to wake up callers waiting for direct I/O to be quiesced.
2670 */
2671static inline void inode_dio_begin(struct inode *inode)
2672{
2673 atomic_inc(&inode->i_dio_count);
2674}
2675
2676/*
2677 * inode_dio_end - signal finish of a direct I/O requests
2678 * @inode: inode the direct I/O happens on
2679 *
2680 * This is called once we've finished processing a direct I/O request,
2681 * and is used to wake up callers waiting for direct I/O to be quiesced.
2682 */
2683static inline void inode_dio_end(struct inode *inode)
2684{
2685 if (atomic_dec_and_test(&inode->i_dio_count))
2686 wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
2687}
2661 2688
2662extern void inode_set_flags(struct inode *inode, unsigned int flags,2689extern void inode_set_flags(struct inode *inode, unsigned int flags,
2663 unsigned int mask);2690 unsigned int mask);