block: Add flag for telling the IO schedulers NOT to anticipate more IO

By default, CFQ will anticipate more IO from a given io context if the previously completed IO was sync. This used to be fine, since the only sync IO was reads and O_DIRECT writes. But with more "normal" sync writes being used now, we don't want to anticipate for those. Add a bio/request flag that informs the IO scheduler that this is a sync request that we should not idle for. Introduce WRITE_ODIRECT specifically for O_DIRECT writes, and make sure that the other sync writes set this flag.

Jens Axboe committed 17 years ago

Commit aeb6faf1 parent 644b2d96 files · +25 −14Browse files

Unified · Split

6 files

block/blk-core.c +2 −0
block/cfq-iosched.c +3 −1
fs/direct-io.c +1 −1
include/linux/bio.h +11 −8
include/linux/blkdev.h +3 −0
include/linux/fs.h +5 −4

block/blk-core.c+2−0

@@ -1128,6 +1128,8 @@ void init_request_from_bio(struct request req, struct bio bio) · 1127 unmodified lines
1128	1128	req->cmd_flags \|= REQ_UNPLUG;
1129	1129	if (bio_rw_meta(bio))
1130	1130	req->cmd_flags \|= REQ_RW_META;
	1131	+ if (bio_noidle(bio))
	1132	+ req->cmd_flags \|= REQ_NOIDLE;
1131	1133
1132	1134	req->errors = 0;
1133	1135	req->hard_sector = req->sector = bio->bi_sector;

block/cfq-iosched.c+3−1

@@ -1992,8 +1992,10 @@ static void cfq_completed_request(struct request_queue q, struct request rq) · 1991 unmodified lines
1992	1992	}
1993	1993	if (cfq_slice_used(cfqq) \|\| cfq_class_idle(cfqq))
1994	1994	cfq_slice_expired(cfqd, 1);
1995		− else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list))
	1995	+ else if (sync && !rq_noidle(rq) &&
	1996	+ RB_EMPTY_ROOT(&cfqq->sort_list)) {
1996	1997	cfq_arm_slice_timer(cfqd);
	1998	+ }
1997	1999	}
1998	2000
1999	2001	if (!cfqd->rq_in_driver)

fs/direct-io.c+1−1

@@ -1126,7 +1126,7 @@ __blockdev_direct_IO(int rw, struct kiocb iocb, struct inode inode, · 1125 unmodified lines
1126	1126	int acquire_i_mutex = 0;
1127	1127
1128	1128	if (rw & WRITE)
1129		− rw = WRITE_SYNC;
	1129	+ rw = WRITE_ODIRECT;
1130	1130
1131	1131	if (bdev)
1132	1132	bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev));

include/linux/bio.h+11−8

@@ -145,20 +145,21 @@ struct bio { · 144 unmodified lines
145	145	* bit 2 -- barrier
146	146	* Insert a serialization point in the IO queue, forcing previously
147	147	* submitted IO to be completed before this one is issued.
148		− * bit 3 -- synchronous I/O hint: the block layer will unplug immediately
149		− * Note that this does NOT indicate that the IO itself is sync, just
150		− * that the block layer will not postpone issue of this IO by plugging.
151		− * bit 4 -- metadata request
	148	+ * bit 3 -- synchronous I/O hint.
	149	+ * bit 4 -- Unplug the device immediately after submitting this bio.
	150	+ * bit 5 -- metadata request
152	151	* Used for tracing to differentiate metadata and data IO. May also
153	152	* get some preferential treatment in the IO scheduler
154		− * bit 5 -- discard sectors
	153	+ * bit 6 -- discard sectors
155	154	* Informs the lower level device that this range of sectors is no longer
156	155	* used by the file system and may thus be freed by the device. Used
157	156	* for flash based storage.
158		− * bit 6 -- fail fast device errors
159		− * bit 7 -- fail fast transport errors
160		− * bit 8 -- fail fast driver errors
	157	+ * bit 7 -- fail fast device errors
	158	+ * bit 8 -- fail fast transport errors
	159	+ * bit 9 -- fail fast driver errors
161	160	* Don't want driver retries for any fast fail whatever the reason.
	161	+ * bit 10 -- Tell the IO scheduler not to wait for more requests after this
	162	+ one has been submitted, even if it is a SYNC request.
162	163	*/
163	164	#define BIO_RW 0 /* Must match RW in req flags (blkdev.h) */
164	165	#define BIO_RW_AHEAD 1 /* Must match FAILFAST in req flags */
@@ -170,6 +171,7 @@ struct bio { · 5 unmodified lines
170	171	#define BIO_RW_FAILFAST_DEV 7
171	172	#define BIO_RW_FAILFAST_TRANSPORT 8
172	173	#define BIO_RW_FAILFAST_DRIVER 9
	174	+#define BIO_RW_NOIDLE 10
173	175
174	176	#define bio_rw_flagged(bio, flag) ((bio)->bi_rw & (1 << (flag)))
175	177
@@ -188,6 +190,7 @@ struct bio { · 12 unmodified lines
188	190	#define bio_rw_ahead(bio) bio_rw_flagged(bio, BIO_RW_AHEAD)
189	191	#define bio_rw_meta(bio) bio_rw_flagged(bio, BIO_RW_META)
190	192	#define bio_discard(bio) bio_rw_flagged(bio, BIO_RW_DISCARD)
	193	+#define bio_noidle(bio) bio_rw_flagged(bio, BIO_RW_NOIDLE)
191	194
192	195	/*
193	196	* upper 16 bits of bi_rw define the io priority of this bio

include/linux/blkdev.h+3−0

@@ -118,6 +118,7 @@ enum rq_flag_bits { · 117 unmodified lines
118	118	__REQ_COPY_USER, /* contains copies of user pages */
119	119	__REQ_INTEGRITY, /* integrity metadata has been remapped */
120	120	__REQ_UNPLUG, /* unplug queue on submission */
	121	+ __REQ_NOIDLE, /* Don't anticipate more IO after this one */
121	122	__REQ_NR_BITS, /* stops here */
122	123	};
123	124
@@ -145,6 +146,7 @@ enum rq_flag_bits { · 21 unmodified lines
145	146	#define REQ_COPY_USER (1 << __REQ_COPY_USER)
146	147	#define REQ_INTEGRITY (1 << __REQ_INTEGRITY)
147	148	#define REQ_UNPLUG (1 << __REQ_UNPLUG)
	149	+#define REQ_NOIDLE (1 << __REQ_NOIDLE)
148	150
149	151	#define BLK_MAX_CDB 16
150	152
@@ -633,6 +635,7 @@ static inline bool rq_is_sync(struct request *rq) · 482 unmodified lines
633	635	}
634	636
635	637	#define rq_is_meta(rq) ((rq)->cmd_flags & REQ_RW_META)
	638	+#define rq_noidle(rq) ((rq)->cmd_flags & REQ_NOIDLE)
636	639
637	640	static inline int blk_queue_full(struct request_queue *q, int sync)
638	641	{

include/linux/fs.h+5−4

@@ -95,11 +95,12 @@ struct inodes_stat_t { · 94 unmodified lines
95	95	#define SWRITE 3 /* for ll_rw_block() - wait for buffer lock */
96	96	#define READ_SYNC (READ \| (1 << BIO_RW_SYNCIO) \| (1 << BIO_RW_UNPLUG))
97	97	#define READ_META (READ \| (1 << BIO_RW_META))
98		−#define WRITE_SYNC (WRITE \| (1 << BIO_RW_SYNCIO) \| (1 << BIO_RW_UNPLUG))
99		−#define WRITE_SYNC_PLUG (WRITE \| (1 << BIO_RW_SYNCIO))
100		−#define SWRITE_SYNC (SWRITE \| (1 << BIO_RW_SYNCIO) \| (1 << BIO_RW_UNPLUG))
	98	+#define WRITE_SYNC_PLUG (WRITE \| (1 << BIO_RW_SYNCIO) \| (1 << BIO_RW_NOIDLE))
	99	+#define WRITE_SYNC (WRITE_SYNC_PLUG \| (1 << BIO_RW_UNPLUG))
	100	+#define WRITE_ODIRECT (WRITE \| (1 << BIO_RW_SYNCIO) \| (1 << BIO_RW_UNPLUG))
101	101	#define SWRITE_SYNC_PLUG \
102		− (SWRITE \| (1 << BIO_RW_SYNCIO))
	102	+ (SWRITE \| (1 << BIO_RW_SYNCIO) \| (1 << BIO_RW_NOIDLE))
	103	+#define SWRITE_SYNC (SWRITE_SYNC_PLUG \| (1 << BIO_RW_UNPLUG))
103	104	#define WRITE_BARRIER (WRITE \| (1 << BIO_RW_BARRIER))
104	105	#define DISCARD_NOBARRIER (1 << BIO_RW_DISCARD)
105	106	#define DISCARD_BARRIER ((1 << BIO_RW_DISCARD) \| (1 << BIO_RW_BARRIER))