block: Add flag for telling the IO schedulers NOT to anticipate more IO

By default, CFQ will anticipate more IO from a given io context if the previously completed IO was sync. This used to be fine, since the only sync IO was reads and O_DIRECT writes. But with more "normal" sync writes being used now, we don't want to anticipate for those. Add a bio/request flag that informs the IO scheduler that this is a sync request that we should not idle for. Introduce WRITE_ODIRECT specifically for O_DIRECT writes, and make sure that the other sync writes set this flag.

Jens Axboe committed 17 years ago

Commit aeb6faf1 parent 644b2d96 files · +25 −14Browse files

Unified · Split

6 files

block/blk-core.c +2 −0
block/cfq-iosched.c +3 −1
fs/direct-io.c +1 −1
include/linux/bio.h +11 −8
include/linux/blkdev.h +3 −0
include/linux/fs.h +5 −4

block/blk-core.c+2−0

@@ -1128,6 +1128,8 @@ void init_request_from_bio(struct request req, struct bio bio) · 1127 unmodified lines
1128	req->cmd_flags \|= REQ_UNPLUG;	1128	req->cmd_flags \|= REQ_UNPLUG;
1129	if (bio_rw_meta(bio))	1129	if (bio_rw_meta(bio))
1130	req->cmd_flags \|= REQ_RW_META;	1130	req->cmd_flags \|= REQ_RW_META;
		1131	if (bio_noidle(bio))
		1132	req->cmd_flags \|= REQ_NOIDLE;
1131		1133
1132	req->errors = 0;	1134	req->errors = 0;
1133	req->hard_sector = req->sector = bio->bi_sector;	1135	req->hard_sector = req->sector = bio->bi_sector;

block/cfq-iosched.c+3−1

 		}
 		if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq))
 			cfq_slice_expired(cfqd, 1);
-		else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list))
+		else if (sync && !rq_noidle(rq) &&
+			 RB_EMPTY_ROOT(&cfqq->sort_list)) {
 			cfq_arm_slice_timer(cfqd);
+		}
 	}
 	if (!cfqd->rq_in_driver)

fs/direct-io.c+1−1

@@ -1126,7 +1126,7 @@ __blockdev_direct_IO(int rw, struct kiocb iocb, struct inode inode, · 1125 unmodified lines
1126	int acquire_i_mutex = 0;	1126	int acquire_i_mutex = 0;
1127		1127
1128	if (rw & WRITE)	1128	if (rw & WRITE)
1129	rw = WRITE_SYNC;	1129	rw = WRITE_ODIRECT;
1130		1130
1131	if (bdev)	1131	if (bdev)
1132	bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev));	1132	bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev));

include/linux/bio.h+11−8

  * bit 2 -- barrier
  *	Insert a serialization point in the IO queue, forcing previously
  *	submitted IO to be completed before this one is issued.
- * bit 3 -- synchronous I/O hint: the block layer will unplug immediately
+ * bit 3 -- synchronous I/O hint.
- *	Note that this does NOT indicate that the IO itself is sync, just
+ * bit 4 -- Unplug the device immediately after submitting this bio.
- *	that the block layer will not postpone issue of this IO by plugging.
+ * bit 5 -- metadata request
- * bit 4 -- metadata request
  *	Used for tracing to differentiate metadata and data IO. May also
  *	get some preferential treatment in the IO scheduler
- * bit 5 -- discard sectors
+ * bit 6 -- discard sectors
  *	Informs the lower level device that this range of sectors is no longer
  *	used by the file system and may thus be freed by the device. Used
  *	for flash based storage.
- * bit 6 -- fail fast device errors
+ * bit 7 -- fail fast device errors
- * bit 7 -- fail fast transport errors
+ * bit 8 -- fail fast transport errors
- * bit 8 -- fail fast driver errors
+ * bit 9 -- fail fast driver errors
  *	Don't want driver retries for any fast fail whatever the reason.
+ * bit 10 -- Tell the IO scheduler not to wait for more requests after this
+	one has been submitted, even if it is a SYNC request.
  */
 #define BIO_RW		0	/* Must match RW in req flags (blkdev.h) */
 #define BIO_RW_AHEAD	1	/* Must match FAILFAST in req flags */
 #define BIO_RW_FAILFAST_DEV		7
 #define BIO_RW_FAILFAST_TRANSPORT	8
 #define BIO_RW_FAILFAST_DRIVER		9
+#define BIO_RW_NOIDLE	10
 #define bio_rw_flagged(bio, flag)	((bio)->bi_rw & (1 << (flag)))
 #define bio_rw_ahead(bio)	bio_rw_flagged(bio, BIO_RW_AHEAD)
 #define bio_rw_meta(bio)	bio_rw_flagged(bio, BIO_RW_META)
 #define bio_discard(bio)	bio_rw_flagged(bio, BIO_RW_DISCARD)
+#define bio_noidle(bio)		bio_rw_flagged(bio, BIO_RW_NOIDLE)
 /*
  * upper 16 bits of bi_rw define the io priority of this bio

include/linux/blkdev.h+3−0

 	__REQ_COPY_USER,	/* contains copies of user pages */
 	__REQ_INTEGRITY,	/* integrity metadata has been remapped */
 	__REQ_UNPLUG,		/* unplug queue on submission */
+	__REQ_NOIDLE,		/* Don't anticipate more IO after this one */
 	__REQ_NR_BITS,		/* stops here */
 };
 #define REQ_COPY_USER	(1 << __REQ_COPY_USER)
 #define REQ_INTEGRITY	(1 << __REQ_INTEGRITY)
 #define REQ_UNPLUG	(1 << __REQ_UNPLUG)
+#define REQ_NOIDLE	(1 << __REQ_NOIDLE)
 #define BLK_MAX_CDB	16
 }
 #define rq_is_meta(rq)		((rq)->cmd_flags & REQ_RW_META)
+#define rq_noidle(rq)		((rq)->cmd_flags & REQ_NOIDLE)
 static inline int blk_queue_full(struct request_queue *q, int sync)
 {

include/linux/fs.h+5−4

 #define SWRITE 3	/* for ll_rw_block() - wait for buffer lock */
 #define READ_SYNC	(READ | (1 << BIO_RW_SYNCIO) | (1 << BIO_RW_UNPLUG))
 #define READ_META	(READ | (1 << BIO_RW_META))
-#define WRITE_SYNC	(WRITE | (1 << BIO_RW_SYNCIO) | (1 << BIO_RW_UNPLUG))
+#define WRITE_SYNC_PLUG	(WRITE | (1 << BIO_RW_SYNCIO) | (1 << BIO_RW_NOIDLE))
-#define WRITE_SYNC_PLUG	(WRITE | (1 << BIO_RW_SYNCIO))
+#define WRITE_SYNC	(WRITE_SYNC_PLUG | (1 << BIO_RW_UNPLUG))
-#define SWRITE_SYNC	(SWRITE | (1 << BIO_RW_SYNCIO) | (1 << BIO_RW_UNPLUG))
+#define WRITE_ODIRECT	(WRITE | (1 << BIO_RW_SYNCIO) | (1 << BIO_RW_UNPLUG))
 #define SWRITE_SYNC_PLUG	\
-	(SWRITE | (1 << BIO_RW_SYNCIO))
+			(SWRITE | (1 << BIO_RW_SYNCIO) | (1 << BIO_RW_NOIDLE))
+#define SWRITE_SYNC	(SWRITE_SYNC_PLUG | (1 << BIO_RW_UNPLUG))
 #define WRITE_BARRIER	(WRITE | (1 << BIO_RW_BARRIER))
 #define DISCARD_NOBARRIER (1 << BIO_RW_DISCARD)
 #define DISCARD_BARRIER ((1 << BIO_RW_DISCARD) | (1 << BIO_RW_BARRIER))