lael-labs/linux

MAINTAINERS+1−0

@@ -2075,6 +2075,7 @@ M: Jens Axboe <axboe@kernel.dk> · 2074 unmodified lines
2075	T: git git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git	2075	T: git git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git
2076	S: Maintained	2076	S: Maintained
2077	F: block/	2077	F: block/
		2078	F: kernel/trace/blktrace.c
2078		2079
2079	BLOCK2MTD DRIVER	2080	BLOCK2MTD DRIVER
2080	M: Joern Engel <joern@lazybastard.org>	2081	M: Joern Engel <joern@lazybastard.org>

block/bio-integrity.c+2−2

@@ -361,7 +361,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) · 360 unmodified lines
361		361
362	/* Restore original bio completion handler */	362	/* Restore original bio completion handler */
363	bio->bi_end_io = bip->bip_end_io;	363	bio->bi_end_io = bip->bip_end_io;
364	bio_endio_nodec(bio, error);	364	bio_endio(bio, error);
365	}	365	}
366		366
367	/**	367	/**
@@ -388,7 +388,7 @@ void bio_integrity_endio(struct bio *bio, int error) · 20 unmodified lines
388	*/	388	*/
389	if (error) {	389	if (error) {
390	bio->bi_end_io = bip->bip_end_io;	390	bio->bi_end_io = bip->bip_end_io;
391	bio_endio_nodec(bio, error);	391	bio_endio(bio, error);
392		392
393	return;	393	return;
394	}	394	}

block/bio.c+47−30

 {
 	memset(bio, 0, sizeof(*bio));
 	bio->bi_flags = 1 << BIO_UPTODATE;
-	atomic_set(&bio->bi_remaining, 1);
+	atomic_set(&bio->__bi_remaining, 1);
-	atomic_set(&bio->bi_cnt, 1);
+	atomic_set(&bio->__bi_cnt, 1);
 }
 EXPORT_SYMBOL(bio_init);
 	__bio_free(bio);
 	memset(bio, 0, BIO_RESET_BYTES);
-	bio->bi_flags = flags|(1 << BIO_UPTODATE);
+	bio->bi_flags = flags | (1 << BIO_UPTODATE);
-	atomic_set(&bio->bi_remaining, 1);
+	atomic_set(&bio->__bi_remaining, 1);
 }
 EXPORT_SYMBOL(bio_reset);
 	bio_put(bio);
 }
+/*
+ * Increment chain count for the bio. Make sure the CHAIN flag update
+ * is visible before the raised count.
+ */
+static inline void bio_inc_remaining(struct bio *bio)
+{
+	bio->bi_flags |= (1 << BIO_CHAIN);
+	smp_mb__before_atomic();
+	atomic_inc(&bio->__bi_remaining);
+}
 /**
  * bio_chain - chain bio completions
  * @bio: the target bio
 	bio->bi_private = parent;
 	bio->bi_end_io	= bio_chain_endio;
-	atomic_inc(&parent->bi_remaining);
+	bio_inc_remaining(parent);
 }
 EXPORT_SYMBOL(bio_chain);
  **/
 void bio_put(struct bio *bio)
 {
-	BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
+	if (!bio_flagged(bio, BIO_REFFED))
-	/*
-	 * last put frees it
-	 */
-	if (atomic_dec_and_test(&bio->bi_cnt))
 		bio_free(bio);
+	else {
+		BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
+		/*
+		 * last put frees it
+		 */
+		if (atomic_dec_and_test(&bio->__bi_cnt))
+			bio_free(bio);
+	}
 }
 EXPORT_SYMBOL(bio_put);
 EXPORT_SYMBOL(bio_flush_dcache_pages);
 #endif
+static inline bool bio_remaining_done(struct bio *bio)
+{
+	/*
+	 * If we're not chaining, then ->__bi_remaining is always 1 and
+	 * we always end io on the first invocation.
+	 */
+	if (!bio_flagged(bio, BIO_CHAIN))
+		return true;
+	BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
+	if (atomic_dec_and_test(&bio->__bi_remaining)) {
+		clear_bit(BIO_CHAIN, &bio->bi_flags);
+		return true;
+	}
+	return false;
+}
 /**
  * bio_endio - end I/O on a bio
  * @bio:	bio
 void bio_endio(struct bio *bio, int error)
 {
 	while (bio) {
-		BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
 		if (error)
 			clear_bit(BIO_UPTODATE, &bio->bi_flags);
 		else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
 			error = -EIO;
-		if (!atomic_dec_and_test(&bio->bi_remaining))
+		if (unlikely(!bio_remaining_done(bio)))
-			return;
+			break;
 		/*
 		 * Need to have a real endio function for chained bios,
 }
 EXPORT_SYMBOL(bio_endio);
-/**
- * bio_endio_nodec - end I/O on a bio, without decrementing bi_remaining
- * @bio:	bio
- * @error:	error, if any
- *
- * For code that has saved and restored bi_end_io; thing hard before using this
- * function, probably you should've cloned the entire bio.
- **/
-void bio_endio_nodec(struct bio *bio, int error)
-{
-	atomic_inc(&bio->bi_remaining);
-	bio_endio(bio, error);
-}
-EXPORT_SYMBOL(bio_endio_nodec);
 /**
  * bio_split - split a bio
  * @bio:	bio to split

block/blk-cgroup.c+81−11

  *
  * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  * 	              Nauman Rafique <nauman@google.com>
+ *
+ * For policy-specific per-blkcg data:
+ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
+ *                    Arianna Avanzini <avanzini.arianna@gmail.com>
  */
 #include <linux/ioprio.h>
 #include <linux/kdev_t.h>
 static DEFINE_MUTEX(blkcg_pol_mutex);
-struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT,
+struct blkcg blkcg_root;
-			    .cfq_leaf_weight = 2 * CFQ_WEIGHT_DEFAULT, };
 EXPORT_SYMBOL_GPL(blkcg_root);
 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
 blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
 {
 	struct blkcg *blkcg;
+	struct cgroup_subsys_state *ret;
+	int i;
 	if (!parent_css) {
 		blkcg = &blkcg_root;
 	}
 	blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
-	if (!blkcg)
+	if (!blkcg) {
-		return ERR_PTR(-ENOMEM);
+		ret = ERR_PTR(-ENOMEM);
+		goto free_blkcg;
+	}
+	for (i = 0; i < BLKCG_MAX_POLS ; i++) {
+		struct blkcg_policy *pol = blkcg_policy[i];
+		struct blkcg_policy_data *cpd;
+		/*
+		 * If the policy hasn't been attached yet, wait for it
+		 * to be attached before doing anything else. Otherwise,
+		 * check if the policy requires any specific per-cgroup
+		 * data: if it does, allocate and initialize it.
+		 */
+		if (!pol || !pol->cpd_size)
+			continue;
+		BUG_ON(blkcg->pd[i]);
+		cpd = kzalloc(pol->cpd_size, GFP_KERNEL);
+		if (!cpd) {
+			ret = ERR_PTR(-ENOMEM);
+			goto free_pd_blkcg;
+		}
+		blkcg->pd[i] = cpd;
+		cpd->plid = i;
+		pol->cpd_init_fn(blkcg);
+	}
-	blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;
-	blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT;
 done:
 	spin_lock_init(&blkcg->lock);
 	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);
 	INIT_HLIST_HEAD(&blkcg->blkg_list);
 	return &blkcg->css;
+free_pd_blkcg:
+	for (i--; i >= 0; i--)
+		kfree(blkcg->pd[i]);
+free_blkcg:
+	kfree(blkcg);
+	return ret;
 }
 /**
 			  const struct blkcg_policy *pol)
 {
 	LIST_HEAD(pds);
+	LIST_HEAD(cpds);
 	struct blkcg_gq *blkg, *new_blkg;
-	struct blkg_policy_data *pd, *n;
+	struct blkg_policy_data *pd, *nd;
+	struct blkcg_policy_data *cpd, *cnd;
 	int cnt = 0, ret;
 	bool preloaded;
 	spin_unlock_irq(q->queue_lock);
-	/* allocate policy_data for all existing blkgs */
+	/*
+	 * Allocate per-blkg and per-blkcg policy data
+	 * for all existing blkgs.
+	 */
 	while (cnt--) {
 		pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
 		if (!pd) {
 			goto out_free;
 		}
 		list_add_tail(&pd->alloc_node, &pds);
+		if (!pol->cpd_size)
+			continue;
+		cpd = kzalloc_node(pol->cpd_size, GFP_KERNEL, q->node);
+		if (!cpd) {
+			ret = -ENOMEM;
+			goto out_free;
+		}
+		list_add_tail(&cpd->alloc_node, &cpds);
 	}
 	/*
-	 * Install the allocated pds.  With @q bypassing, no new blkg
+	 * Install the allocated pds and cpds. With @q bypassing, no new blkg
 	 * should have been created while the queue lock was dropped.
 	 */
 	spin_lock_irq(q->queue_lock);
 	list_for_each_entry(blkg, &q->blkg_list, q_node) {
-		if (WARN_ON(list_empty(&pds))) {
+		if (WARN_ON(list_empty(&pds)) ||
+		    WARN_ON(pol->cpd_size && list_empty(&cpds))) {
 			/* umm... this shouldn't happen, just abort */
 			ret = -ENOMEM;
 			goto out_unlock;
 		}
+		cpd = list_first_entry(&cpds, struct blkcg_policy_data,
+				       alloc_node);
+		list_del_init(&cpd->alloc_node);
 		pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node);
 		list_del_init(&pd->alloc_node);
 		/* grab blkcg lock too while installing @pd on @blkg */
 		spin_lock(&blkg->blkcg->lock);
+		if (!pol->cpd_size)
+			goto no_cpd;
+		if (!blkg->blkcg->pd[pol->plid]) {
+			/* Per-policy per-blkcg data */
+			blkg->blkcg->pd[pol->plid] = cpd;
+			cpd->plid = pol->plid;
+			pol->cpd_init_fn(blkg->blkcg);
+		} else { /* must free it as it has already been extracted */
+			kfree(cpd);
+		}
+no_cpd:
 		blkg->pd[pol->plid] = pd;
 		pd->blkg = blkg;
 		pd->plid = pol->plid;
 	spin_unlock_irq(q->queue_lock);
 out_free:
 	blk_queue_bypass_end(q);
-	list_for_each_entry_safe(pd, n, &pds, alloc_node)
+	list_for_each_entry_safe(pd, nd, &pds, alloc_node)
 		kfree(pd);
+	list_for_each_entry_safe(cpd, cnd, &cpds, alloc_node)
+		kfree(cpd);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(blkcg_activate_policy);
 		kfree(blkg->pd[pol->plid]);
 		blkg->pd[pol->plid] = NULL;
+		kfree(blkg->blkcg->pd[pol->plid]);
+		blkg->blkcg->pd[pol->plid] = NULL;
 		spin_unlock(&blkg->blkcg->lock);
 	}

block/blk-cgroup.h+32−8

 /* Max limits for throttle policy */
 #define THROTL_IOPS_MAX		UINT_MAX
-/* CFQ specific, out here for blkcg->cfq_weight */
-#define CFQ_WEIGHT_MIN		10
-#define CFQ_WEIGHT_MAX		1000
-#define CFQ_WEIGHT_DEFAULT	500
 #ifdef CONFIG_BLK_CGROUP
 enum blkg_rwstat_type {
 	struct blkcg_gq			*blkg_hint;
 	struct hlist_head		blkg_list;
-	/* TODO: per-policy storage in blkcg */
+	struct blkcg_policy_data	*pd[BLKCG_MAX_POLS];
-	unsigned int			cfq_weight;	/* belongs to cfq */
-	unsigned int			cfq_leaf_weight;
 };
 struct blkg_stat {
 	struct list_head		alloc_node;
 };
+/*
+ * Policies that need to keep per-blkcg data which is independent
+ * from any request_queue associated to it must specify its size
+ * with the cpd_size field of the blkcg_policy structure and
+ * embed a blkcg_policy_data in it. blkcg core allocates
+ * policy-specific per-blkcg structures lazily the first time
+ * they are actually needed, so it handles them together with
+ * blkgs. cpd_init() is invoked to let each policy handle
+ * per-blkcg data.
+ */
+struct blkcg_policy_data {
+	/* the policy id this per-policy data belongs to */
+	int				plid;
+	/* used during policy activation */
+	struct list_head		alloc_node;
+};
 /* association between a blk cgroup and a request queue */
 struct blkcg_gq {
 	/* Pointer to the associated request_queue */
 	struct rcu_head			rcu_head;
 };
+typedef void (blkcg_pol_init_cpd_fn)(const struct blkcg *blkcg);
 typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg);
 typedef void (blkcg_pol_online_pd_fn)(struct blkcg_gq *blkg);
 typedef void (blkcg_pol_offline_pd_fn)(struct blkcg_gq *blkg);
 	int				plid;
 	/* policy specific private data size */
 	size_t				pd_size;
+	/* policy specific per-blkcg data size */
+	size_t				cpd_size;
 	/* cgroup files for the policy */
 	struct cftype			*cftypes;
 	/* operations */
+	blkcg_pol_init_cpd_fn		*cpd_init_fn;
 	blkcg_pol_init_pd_fn		*pd_init_fn;
 	blkcg_pol_online_pd_fn		*pd_online_fn;
 	blkcg_pol_offline_pd_fn		*pd_offline_fn;
 	return blkg ? blkg->pd[pol->plid] : NULL;
 }
+static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
+						     struct blkcg_policy *pol)
+{
+	return blkcg ? blkcg->pd[pol->plid] : NULL;
+}
 /**
  * pdata_to_blkg - get blkg associated with policy private data
  * @pd: policy private data of interest
 struct blkg_policy_data {
 };
+struct blkcg_policy_data {
+};
 struct blkcg_gq {
 };

block/blk-core.c+36−100

@@ -117,7 +117,7 @@ EXPORT_SYMBOL(blk_rq_init); · 116 unmodified lines
117	static void req_bio_endio(struct request rq, struct bio bio,	117	static void req_bio_endio(struct request rq, struct bio bio,
118	unsigned int nbytes, int error)	118	unsigned int nbytes, int error)
119	{	119	{
120	if (error)	120	if (error && !(rq->cmd_flags & REQ_CLONE))
121	clear_bit(BIO_UPTODATE, &bio->bi_flags);	121	clear_bit(BIO_UPTODATE, &bio->bi_flags);
122	else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))	122	else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
123	error = -EIO;	123	error = -EIO;
@@ -128,7 +128,8 @@ static void req_bio_endio(struct request rq, struct bio bio, · 4 unmodified lines
128	bio_advance(bio, nbytes);	128	bio_advance(bio, nbytes);
129		129
130	/* don't actually finish bio if it's part of flush sequence */	130	/* don't actually finish bio if it's part of flush sequence */
131	if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))	131	if (bio->bi_iter.bi_size == 0 &&
		132	!(rq->cmd_flags & (REQ_FLUSH_SEQ\|REQ_CLONE)))
132	bio_endio(bio, error);	133	bio_endio(bio, error);
133	}	134	}
134		135
@@ -285,6 +286,7 @@ inline void __blk_run_queue_uncond(struct request_queue *q) · 150 unmodified lines
285	q->request_fn(q);	286	q->request_fn(q);
286	q->request_fn_active--;	287	q->request_fn_active--;
287	}	288	}
		289	EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
288		290
289	/**	291	/**
290	* __blk_run_queue - run a single device queue	292	* __blk_run_queue - run a single device queue
@@ -1525,7 +1527,8 @@ bool bio_attempt_front_merge(struct request_queue q, struct request req, · 1234 unmodified lines
1525	* Caller must ensure !blk_queue_nomerges(q) beforehand.	1527	* Caller must ensure !blk_queue_nomerges(q) beforehand.
1526	*/	1528	*/
1527	bool blk_attempt_plug_merge(struct request_queue q, struct bio bio,	1529	bool blk_attempt_plug_merge(struct request_queue q, struct bio bio,
1528	unsigned int *request_count)	1530	unsigned int *request_count,
		1531	struct request **same_queue_rq)
1529	{	1532	{
1530	struct blk_plug *plug;	1533	struct blk_plug *plug;
1531	struct request *rq;	1534	struct request *rq;
@@ -1545,8 +1548,16 @@ bool blk_attempt_plug_merge(struct request_queue q, struct bio bio, · 13 unmodified lines
1545	list_for_each_entry_reverse(rq, plug_list, queuelist) {	1548	list_for_each_entry_reverse(rq, plug_list, queuelist) {
1546	int el_ret;	1549	int el_ret;
1547		1550
1548	if (rq->q == q)	1551	if (rq->q == q) {
1549	(*request_count)++;	1552	(*request_count)++;
		1553	/*
		1554	* Only blk-mq multiple hardware queues case checks the
		1555	* rq in the same queue, there should be only one such
		1556	* rq in a queue
		1557	**/
		1558	if (same_queue_rq)
		1559	*same_queue_rq = rq;
		1560	}
1550		1561
1551	if (rq->q != q \|\| !blk_rq_merge_ok(rq, bio))	1562	if (rq->q != q \|\| !blk_rq_merge_ok(rq, bio))
1552	continue;	1563	continue;
@@ -1611,7 +1622,7 @@ static void blk_queue_bio(struct request_queue q, struct bio bio) · 58 unmodified lines
1611	* any locks.	1622	* any locks.
1612	*/	1623	*/
1613	if (!blk_queue_nomerges(q) &&	1624	if (!blk_queue_nomerges(q) &&
1614	blk_attempt_plug_merge(q, bio, &request_count))	1625	blk_attempt_plug_merge(q, bio, &request_count, NULL))
1615	return;	1626	return;
1616		1627
1617	spin_lock_irq(q->queue_lock);	1628	spin_lock_irq(q->queue_lock);
@@ -1718,8 +1729,6 @@ static void handle_bad_sector(struct bio *bio) · 100 unmodified lines
1718	bio->bi_rw,	1729	bio->bi_rw,
1719	(unsigned long long)bio_end_sector(bio),	1730	(unsigned long long)bio_end_sector(bio),
1720	(long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));	1731	(long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));
1721
1722	set_bit(BIO_EOF, &bio->bi_flags);
1723	}	1732	}
1724		1733
1725	#ifdef CONFIG_FAIL_MAKE_REQUEST	1734	#ifdef CONFIG_FAIL_MAKE_REQUEST
@@ -2904,95 +2913,22 @@ int blk_lld_busy(struct request_queue *q) · 1178 unmodified lines
2904	}	2913	}
2905	EXPORT_SYMBOL_GPL(blk_lld_busy);	2914	EXPORT_SYMBOL_GPL(blk_lld_busy);
2906		2915
2907	/**	2916	void blk_rq_prep_clone(struct request dst, struct request src)
2908	* blk_rq_unprep_clone - Helper function to free all bios in a cloned request
2909	* @rq: the clone request to be cleaned up
2910	*
2911	* Description:
2912	* Free all bios in @rq for a cloned request.
2913	*/
2914	void blk_rq_unprep_clone(struct request *rq)
2915	{
2916	struct bio *bio;
2917
2918	while ((bio = rq->bio) != NULL) {
2919	rq->bio = bio->bi_next;
2920
2921	bio_put(bio);
2922	}
2923	}
2924	EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
2925
2926	/*
2927	* Copy attributes of the original request to the clone request.
2928	* The actual data parts (e.g. ->cmd, ->sense) are not copied.
2929	*/
2930	static void __blk_rq_prep_clone(struct request dst, struct request src)
2931	{	2917	{
2932	dst->cpu = src->cpu;	2918	dst->cpu = src->cpu;
2933	dst->cmd_flags \|= (src->cmd_flags & REQ_CLONE_MASK) \| REQ_NOMERGE;	2919	dst->cmd_flags \|= (src->cmd_flags & REQ_CLONE_MASK);
		2920	dst->cmd_flags \|= REQ_NOMERGE \| REQ_CLONE;
2934	dst->cmd_type = src->cmd_type;	2921	dst->cmd_type = src->cmd_type;
2935	dst->__sector = blk_rq_pos(src);	2922	dst->__sector = blk_rq_pos(src);
2936	dst->__data_len = blk_rq_bytes(src);	2923	dst->__data_len = blk_rq_bytes(src);
2937	dst->nr_phys_segments = src->nr_phys_segments;	2924	dst->nr_phys_segments = src->nr_phys_segments;
2938	dst->ioprio = src->ioprio;	2925	dst->ioprio = src->ioprio;
2939	dst->extra_len = src->extra_len;	2926	dst->extra_len = src->extra_len;
2940	}	2927	dst->bio = src->bio;
2941		2928	dst->biotail = src->biotail;
2942	/**	2929	dst->cmd = src->cmd;
2943	* blk_rq_prep_clone - Helper function to setup clone request	2930	dst->cmd_len = src->cmd_len;
2944	* @rq: the request to be setup	2931	dst->sense = src->sense;
2945	* @rq_src: original request to be cloned
2946	* @bs: bio_set that bios for clone are allocated from
2947	* @gfp_mask: memory allocation mask for bio
2948	* @bio_ctr: setup function to be called for each clone bio.
2949	* Returns %0 for success, non %0 for failure.
2950	* @data: private data to be passed to @bio_ctr
2951	*
2952	* Description:
2953	* Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
2954	* The actual data parts of @rq_src (e.g. ->cmd, ->sense)
2955	* are not copied, and copying such parts is the caller's responsibility.
2956	* Also, pages which the original bios are pointing to are not copied
2957	* and the cloned bios just point same pages.
2958	* So cloned bios must be completed before original bios, which means
2959	* the caller must complete @rq before @rq_src.
2960	*/
2961	int blk_rq_prep_clone(struct request rq, struct request rq_src,
2962	struct bio_set *bs, gfp_t gfp_mask,
2963	int (bio_ctr)(struct bio , struct bio , void ),
2964	void *data)
2965	{
2966	struct bio bio, bio_src;
2967
2968	if (!bs)
2969	bs = fs_bio_set;
2970
2971	__rq_for_each_bio(bio_src, rq_src) {
2972	bio = bio_clone_fast(bio_src, gfp_mask, bs);
2973	if (!bio)
2974	goto free_and_out;
2975
2976	if (bio_ctr && bio_ctr(bio, bio_src, data))
2977	goto free_and_out;
2978
2979	if (rq->bio) {
2980	rq->biotail->bi_next = bio;
2981	rq->biotail = bio;
2982	} else
2983	rq->bio = rq->biotail = bio;
2984	}
2985
2986	__blk_rq_prep_clone(rq, rq_src);
2987
2988	return 0;
2989
2990	free_and_out:
2991	if (bio)
2992	bio_put(bio);
2993	blk_rq_unprep_clone(rq);
2994
2995	return -ENOMEM;
2996	}	2932	}
2997	EXPORT_SYMBOL_GPL(blk_rq_prep_clone);	2933	EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
2998		2934
@@ -3034,21 +2970,20 @@ void blk_start_plug(struct blk_plug *plug) · 35 unmodified lines
3034	{	2970	{
3035	struct task_struct *tsk = current;	2971	struct task_struct *tsk = current;
3036		2972
		2973	/*
		2974	* If this is a nested plug, don't actually assign it.
		2975	*/
		2976	if (tsk->plug)
		2977	return;
		2978
3037	INIT_LIST_HEAD(&plug->list);	2979	INIT_LIST_HEAD(&plug->list);
3038	INIT_LIST_HEAD(&plug->mq_list);	2980	INIT_LIST_HEAD(&plug->mq_list);
3039	INIT_LIST_HEAD(&plug->cb_list);	2981	INIT_LIST_HEAD(&plug->cb_list);
3040
3041	/*	2982	/*
3042	* If this is a nested plug, don't actually assign it. It will be	2983	* Store ordering should not be needed here, since a potential
3043	* flushed on its own.	2984	* preempt will imply a full memory barrier
3044	*/	2985	*/
3045	if (!tsk->plug) {	2986	tsk->plug = plug;
3046	/*
3047	* Store ordering should not be needed here, since a potential
3048	* preempt will imply a full memory barrier
3049	*/
3050	tsk->plug = plug;
3051	}
3052	}	2987	}
3053	EXPORT_SYMBOL(blk_start_plug);	2988	EXPORT_SYMBOL(blk_start_plug);
3054		2989
@@ -3195,10 +3130,11 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) · 140 unmodified lines
3195		3130
3196	void blk_finish_plug(struct blk_plug *plug)	3131	void blk_finish_plug(struct blk_plug *plug)
3197	{	3132	{
		3133	if (plug != current->plug)
		3134	return;
3198	blk_flush_plug_list(plug, false);	3135	blk_flush_plug_list(plug, false);
3199		3136
3200	if (plug == current->plug)	3137	current->plug = NULL;
3201	current->plug = NULL;
3202	}	3138	}
3203	EXPORT_SYMBOL(blk_finish_plug);	3139	EXPORT_SYMBOL(blk_finish_plug);
3204		3140

block/blk-exec.c+0−10

 			   rq_end_io_fn *done)
 {
 	int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
-	bool is_pm_resume;
 	WARN_ON(irqs_disabled());
 	WARN_ON(rq->cmd_type == REQ_TYPE_FS);
 		return;
 	}
-	/*
-	 * need to check this before __blk_run_queue(), because rq can
-	 * be freed before that returns.
-	 */
-	is_pm_resume = rq->cmd_type == REQ_TYPE_PM_RESUME;
 	spin_lock_irq(q->queue_lock);
 	if (unlikely(blk_queue_dying(q))) {
 	__elv_add_request(q, rq, where);
 	__blk_run_queue(q);
-	/* the queue is stopped so it won't be run */
-	if (is_pm_resume)
-		__blk_run_queue_uncond(q);
 	spin_unlock_irq(q->queue_lock);
 }
 EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);

block/blk-merge.c+2−1

@@ -589,7 +589,8 @@ bool blk_rq_merge_ok(struct request rq, struct bio bio) · 588 unmodified lines
589	!blk_write_same_mergeable(rq->bio, bio))	589	!blk_write_same_mergeable(rq->bio, bio))
590	return false;	590	return false;
591		591
592	if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS)) {	592	/* Only check gaps if the bio carries data */
		593	if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) && bio_has_data(bio)) {
593	struct bio_vec *bprev;	594	struct bio_vec *bprev;
594		595
595	bprev = &rq->biotail->bi_io_vec[rq->biotail->bi_vcnt - 1];	596	bprev = &rq->biotail->bi_io_vec[rq->biotail->bi_vcnt - 1];

block/blk-mq-tag.c+38−0

@@ -438,6 +438,39 @@ static void bt_for_each(struct blk_mq_hw_ctx *hctx, · 437 unmodified lines
438	}	438	}
439	}	439	}
440		440
		441	static void bt_tags_for_each(struct blk_mq_tags *tags,
		442	struct blk_mq_bitmap_tags *bt, unsigned int off,
		443	busy_tag_iter_fn fn, void data, bool reserved)
		444	{
		445	struct request *rq;
		446	int bit, i;
		447
		448	if (!tags->rqs)
		449	return;
		450	for (i = 0; i < bt->map_nr; i++) {
		451	struct blk_align_bitmap *bm = &bt->map[i];
		452
		453	for (bit = find_first_bit(&bm->word, bm->depth);
		454	bit < bm->depth;
		455	bit = find_next_bit(&bm->word, bm->depth, bit + 1)) {
		456	rq = blk_mq_tag_to_rq(tags, off + bit);
		457	fn(rq, data, reserved);
		458	}
		459
		460	off += (1 << bt->bits_per_word);
		461	}
		462	}
		463
		464	void blk_mq_all_tag_busy_iter(struct blk_mq_tags tags, busy_tag_iter_fn fn,
		465	void *priv)
		466	{
		467	if (tags->nr_reserved_tags)
		468	bt_tags_for_each(tags, &tags->breserved_tags, 0, fn, priv, true);
		469	bt_tags_for_each(tags, &tags->bitmap_tags, tags->nr_reserved_tags, fn, priv,
		470	false);
		471	}
		472	EXPORT_SYMBOL(blk_mq_all_tag_busy_iter);
		473
441	void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx hctx, busy_iter_fn fn,	474	void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx hctx, busy_iter_fn fn,
442	void *priv)	475	void *priv)
443	{	476	{
@@ -580,6 +613,11 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, · 136 unmodified lines
580	if (!tags)	613	if (!tags)
581	return NULL;	614	return NULL;
582		615
		616	if (!zalloc_cpumask_var(&tags->cpumask, GFP_KERNEL)) {
		617	kfree(tags);
		618	return NULL;
		619	}
		620
583	tags->nr_tags = total_tags;	621	tags->nr_tags = total_tags;
584	tags->nr_reserved_tags = reserved_tags;	622	tags->nr_reserved_tags = reserved_tags;
585		623

block/blk-mq-tag.h+1−0

@@ -44,6 +44,7 @@ struct blk_mq_tags { · 43 unmodified lines
44	struct list_head page_list;	44	struct list_head page_list;
45		45
46	int alloc_policy;	46	int alloc_policy;
		47	cpumask_var_t cpumask;
47	};	48	};
48		49
49		50

block/blk-mq.c+99−61

@@ -89,7 +89,8 @@ static int blk_mq_queue_enter(struct request_queue *q, gfp_t gfp) · 88 unmodified lines
89	return -EBUSY;	89	return -EBUSY;
90		90
91	ret = wait_event_interruptible(q->mq_freeze_wq,	91	ret = wait_event_interruptible(q->mq_freeze_wq,
92	!q->mq_freeze_depth \|\| blk_queue_dying(q));	92	!atomic_read(&q->mq_freeze_depth) \|\|
		93	blk_queue_dying(q));
93	if (blk_queue_dying(q))	94	if (blk_queue_dying(q))
94	return -ENODEV;	95	return -ENODEV;
95	if (ret)	96	if (ret)
@@ -112,13 +113,10 @@ static void blk_mq_usage_counter_release(struct percpu_ref *ref) · 16 unmodified lines
112		113
113	void blk_mq_freeze_queue_start(struct request_queue *q)	114	void blk_mq_freeze_queue_start(struct request_queue *q)
114	{	115	{
115	bool freeze;	116	int freeze_depth;
116		117
117	spin_lock_irq(q->queue_lock);	118	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
118	freeze = !q->mq_freeze_depth++;	119	if (freeze_depth == 1) {
119	spin_unlock_irq(q->queue_lock);
120
121	if (freeze) {
122	percpu_ref_kill(&q->mq_usage_counter);	120	percpu_ref_kill(&q->mq_usage_counter);
123	blk_mq_run_hw_queues(q, false);	121	blk_mq_run_hw_queues(q, false);
124	}	122	}
@@ -143,13 +141,11 @@ EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); · 18 unmodified lines
143		141
144	void blk_mq_unfreeze_queue(struct request_queue *q)	142	void blk_mq_unfreeze_queue(struct request_queue *q)
145	{	143	{
146	bool wake;	144	int freeze_depth;
147		145
148	spin_lock_irq(q->queue_lock);	146	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
149	wake = !--q->mq_freeze_depth;	147	WARN_ON_ONCE(freeze_depth < 0);
150	WARN_ON_ONCE(q->mq_freeze_depth < 0);	148	if (!freeze_depth) {
151	spin_unlock_irq(q->queue_lock);
152	if (wake) {
153	percpu_ref_reinit(&q->mq_usage_counter);	149	percpu_ref_reinit(&q->mq_usage_counter);
154	wake_up_all(&q->mq_freeze_wq);	150	wake_up_all(&q->mq_freeze_wq);
155	}	151	}
@@ -1237,6 +1233,38 @@ static struct request blk_mq_map_request(struct request_queue q, · 1081 unmodified lines
1237	return rq;	1233	return rq;
1238	}	1234	}
1239		1235
		1236	static int blk_mq_direct_issue_request(struct request *rq)
		1237	{
		1238	int ret;
		1239	struct request_queue *q = rq->q;
		1240	struct blk_mq_hw_ctx *hctx = q->mq_ops->map_queue(q,
		1241	rq->mq_ctx->cpu);
		1242	struct blk_mq_queue_data bd = {
		1243	.rq = rq,
		1244	.list = NULL,
		1245	.last = 1
		1246	};
		1247
		1248	/*
		1249	* For OK queue, we are done. For error, kill it. Any other
		1250	* error (busy), just add it to our list as we previously
		1251	* would have done
		1252	*/
		1253	ret = q->mq_ops->queue_rq(hctx, &bd);
		1254	if (ret == BLK_MQ_RQ_QUEUE_OK)
		1255	return 0;
		1256	else {
		1257	__blk_mq_requeue_request(rq);
		1258
		1259	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
		1260	rq->errors = -EIO;
		1261	blk_mq_end_request(rq, rq->errors);
		1262	return 0;
		1263	}
		1264	return -1;
		1265	}
		1266	}
		1267
1240	/*	1268	/*
1241	* Multiple hardware queue variant. This will not use per-process plugs,	1269	* Multiple hardware queue variant. This will not use per-process plugs,
1242	* but will attempt to bypass the hctx queueing if we can go straight to	1270	* but will attempt to bypass the hctx queueing if we can go straight to
@@ -1248,6 +1276,9 @@ static void blk_mq_make_request(struct request_queue q, struct bio bio) · 5 unmodified lines
1248	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH \| REQ_FUA);	1276	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH \| REQ_FUA);
1249	struct blk_map_ctx data;	1277	struct blk_map_ctx data;
1250	struct request *rq;	1278	struct request *rq;
		1279	unsigned int request_count = 0;
		1280	struct blk_plug *plug;
		1281	struct request *same_queue_rq = NULL;
1251		1282
1252	blk_queue_bounce(q, &bio);	1283	blk_queue_bounce(q, &bio);
1253		1284
@@ -1256,6 +1287,10 @@ static void blk_mq_make_request(struct request_queue q, struct bio bio) · 2 unmodified lines
1256	return;	1287	return;
1257	}	1288	}
1258		1289
		1290	if (!is_flush_fua && !blk_queue_nomerges(q) &&
		1291	blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
		1292	return;
		1293
1259	rq = blk_mq_map_request(q, bio, &data);	1294	rq = blk_mq_map_request(q, bio, &data);
1260	if (unlikely(!rq))	1295	if (unlikely(!rq))
1261	return;	1296	return;
@@ -1266,38 +1301,42 @@ static void blk_mq_make_request(struct request_queue q, struct bio bio) · 4 unmodified lines
1266	goto run_queue;	1301	goto run_queue;
1267	}	1302	}
1268		1303
		1304	plug = current->plug;
1269	/*	1305	/*
1270	* If the driver supports defer issued based on 'last', then	1306	* If the driver supports defer issued based on 'last', then
1271	* queue it up like normal since we can potentially save some	1307	* queue it up like normal since we can potentially save some
1272	* CPU this way.	1308	* CPU this way.
1273	*/	1309	*/
1274	if (is_sync && !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {	1310	if (((plug && !blk_queue_nomerges(q)) \|\| is_sync) &&
1275	struct blk_mq_queue_data bd = {	1311	!(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
1276	.rq = rq,	1312	struct request *old_rq = NULL;
1277	.list = NULL,
1278	.last = 1
1279	};
1280	int ret;
1281		1313
1282	blk_mq_bio_to_request(rq, bio);	1314	blk_mq_bio_to_request(rq, bio);
1283		1315
1284	/*	1316	/*
1285	* For OK queue, we are done. For error, kill it. Any other	1317	* we do limited pluging. If bio can be merged, do merge.
1286	* error (busy), just add it to our list as we previously	1318	* Otherwise the existing request in the plug list will be
1287	* would have done	1319	* issued. So the plug list will have one request at most
1288	*/	1320	*/
1289	ret = q->mq_ops->queue_rq(data.hctx, &bd);	1321	if (plug) {
1290	if (ret == BLK_MQ_RQ_QUEUE_OK)	1322	/*
1291	goto done;	1323	* The plug list might get flushed before this. If that
1292	else {	1324	* happens, same_queue_rq is invalid and plug list is empty
1293	__blk_mq_requeue_request(rq);	1325	**/
1294		1326	if (same_queue_rq && !list_empty(&plug->mq_list)) {
1295	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {	1327	old_rq = same_queue_rq;
1296	rq->errors = -EIO;	1328	list_del_init(&old_rq->queuelist);
1297	blk_mq_end_request(rq, rq->errors);
1298	goto done;
1299	}	1329	}
1300	}	1330	list_add_tail(&rq->queuelist, &plug->mq_list);
		1331	} else /* is_sync */
		1332	old_rq = rq;
		1333	blk_mq_put_ctx(data.ctx);
		1334	if (!old_rq)
		1335	return;
		1336	if (!blk_mq_direct_issue_request(old_rq))
		1337	return;
		1338	blk_mq_insert_request(old_rq, false, true, true);
		1339	return;
1301	}	1340	}
1302		1341
1303	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {	1342	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
@@ -1310,7 +1349,6 @@ static void blk_mq_make_request(struct request_queue q, struct bio bio) · 6 unmodified lines
1310	run_queue:	1349	run_queue:
1311	blk_mq_run_hw_queue(data.hctx, !is_sync \|\| is_flush_fua);	1350	blk_mq_run_hw_queue(data.hctx, !is_sync \|\| is_flush_fua);
1312	}	1351	}
1313	done:
1314	blk_mq_put_ctx(data.ctx);	1352	blk_mq_put_ctx(data.ctx);
1315	}	1353	}
1316		1354
@@ -1322,16 +1360,11 @@ static void blk_sq_make_request(struct request_queue q, struct bio bio) · 5 unmodified lines
1322	{	1360	{
1323	const int is_sync = rw_is_sync(bio->bi_rw);	1361	const int is_sync = rw_is_sync(bio->bi_rw);
1324	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH \| REQ_FUA);	1362	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH \| REQ_FUA);
1325	unsigned int use_plug, request_count = 0;	1363	struct blk_plug *plug;
		1364	unsigned int request_count = 0;
1326	struct blk_map_ctx data;	1365	struct blk_map_ctx data;
1327	struct request *rq;	1366	struct request *rq;
1328		1367
1329	/*
1330	* If we have multiple hardware queues, just go directly to
1331	* one of those for sync IO.
1332	*/
1333	use_plug = !is_flush_fua && !is_sync;
1334
1335	blk_queue_bounce(q, &bio);	1368	blk_queue_bounce(q, &bio);
1336		1369
1337	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {	1370	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
@@ -1339,8 +1372,8 @@ static void blk_sq_make_request(struct request_queue q, struct bio bio) · 1 unmodified line
1339	return;	1372	return;
1340	}	1373	}
1341		1374
1342	if (use_plug && !blk_queue_nomerges(q) &&	1375	if (!is_flush_fua && !blk_queue_nomerges(q) &&
1343	blk_attempt_plug_merge(q, bio, &request_count))	1376	blk_attempt_plug_merge(q, bio, &request_count, NULL))
1344	return;	1377	return;
1345		1378
1346	rq = blk_mq_map_request(q, bio, &data);	1379	rq = blk_mq_map_request(q, bio, &data);
@@ -1358,21 +1391,18 @@ static void blk_sq_make_request(struct request_queue q, struct bio bio) · 11 unmodified lines
1358	* utilize that to temporarily store requests until the task is	1391	* utilize that to temporarily store requests until the task is
1359	* either done or scheduled away.	1392	* either done or scheduled away.
1360	*/	1393	*/
1361	if (use_plug) {	1394	plug = current->plug;
1362	struct blk_plug *plug = current->plug;	1395	if (plug) {
1363		1396	blk_mq_bio_to_request(rq, bio);
1364	if (plug) {	1397	if (list_empty(&plug->mq_list))
1365	blk_mq_bio_to_request(rq, bio);	1398	trace_block_plug(q);
1366	if (list_empty(&plug->mq_list))	1399	else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1367	trace_block_plug(q);	1400	blk_flush_plug_list(plug, false);
1368	else if (request_count >= BLK_MAX_REQUEST_COUNT) {	1401	trace_block_plug(q);
1369	blk_flush_plug_list(plug, false);
1370	trace_block_plug(q);
1371	}
1372	list_add_tail(&rq->queuelist, &plug->mq_list);
1373	blk_mq_put_ctx(data.ctx);
1374	return;
1375	}	1402	}
		1403	list_add_tail(&rq->queuelist, &plug->mq_list);
		1404	blk_mq_put_ctx(data.ctx);
		1405	return;
1376	}	1406	}
1377		1407
1378	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {	1408	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
@@ -1508,7 +1538,6 @@ static struct blk_mq_tags blk_mq_init_rq_map(struct blk_mq_tag_set set, · 129 unmodified lines
1508	i++;	1538	i++;
1509	}	1539	}
1510	}	1540	}
1511
1512	return tags;	1541	return tags;
1513		1542
1514	fail:	1543	fail:
@@ -1792,6 +1821,7 @@ static void blk_mq_map_swqueue(struct request_queue *q) · 277 unmodified lines
1792		1821
1793	hctx = q->mq_ops->map_queue(q, i);	1822	hctx = q->mq_ops->map_queue(q, i);
1794	cpumask_set_cpu(i, hctx->cpumask);	1823	cpumask_set_cpu(i, hctx->cpumask);
		1824	cpumask_set_cpu(i, hctx->tags->cpumask);
1795	ctx->index_hw = hctx->nr_ctx;	1825	ctx->index_hw = hctx->nr_ctx;
1796	hctx->ctxs[hctx->nr_ctx++] = ctx;	1826	hctx->ctxs[hctx->nr_ctx++] = ctx;
1797	}	1827	}
@@ -2056,7 +2086,7 @@ void blk_mq_free_queue(struct request_queue *q) · 258 unmodified lines
2056	/* Basically redo blk_mq_init_queue with queue frozen */	2086	/* Basically redo blk_mq_init_queue with queue frozen */
2057	static void blk_mq_queue_reinit(struct request_queue *q)	2087	static void blk_mq_queue_reinit(struct request_queue *q)
2058	{	2088	{
2059	WARN_ON_ONCE(!q->mq_freeze_depth);	2089	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2060		2090
2061	blk_mq_sysfs_unregister(q);	2091	blk_mq_sysfs_unregister(q);
2062		2092
@@ -2173,6 +2203,12 @@ static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) · 110 unmodified lines
2173	return 0;	2203	return 0;
2174	}	2204	}
2175		2205
		2206	struct cpumask blk_mq_tags_cpumask(struct blk_mq_tags tags)
		2207	{
		2208	return tags->cpumask;
		2209	}
		2210	EXPORT_SYMBOL_GPL(blk_mq_tags_cpumask);
		2211
2176	/*	2212	/*
2177	* Alloc a tag set to be associated with one or more request queues.	2213	* Alloc a tag set to be associated with one or more request queues.
2178	* May fail with EINVAL for various error conditions. May adjust the	2214	* May fail with EINVAL for various error conditions. May adjust the
@@ -2234,8 +2270,10 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) · 55 unmodified lines
2234	int i;	2270	int i;
2235		2271
2236	for (i = 0; i < set->nr_hw_queues; i++) {	2272	for (i = 0; i < set->nr_hw_queues; i++) {
2237	if (set->tags[i])	2273	if (set->tags[i]) {
2238	blk_mq_free_rq_map(set, set->tags[i], i);	2274	blk_mq_free_rq_map(set, set->tags[i], i);
		2275	free_cpumask_var(set->tags[i]->cpumask);
		2276	}
2239	}	2277	}
2240		2278
2241	kfree(set->tags);	2279	kfree(set->tags);

block/blk.h+2−3

 bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
 			    struct bio *bio);
 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
-			    unsigned int *request_count);
+			    unsigned int *request_count,
+			    struct request **same_queue_rq);
 void blk_account_io_start(struct request *req, bool new_io);
 void blk_account_io_completion(struct request *req, unsigned int bytes);
 void blk_queue_congestion_threshold(struct request_queue *q);
-void __blk_run_queue_uncond(struct request_queue *q);
 int blk_dev_init(void);

block/bounce.c+0−3

@@ -128,9 +128,6 @@ static void bounce_end_io(struct bio bio, mempool_t pool, int err) · 127 unmodified lines
128	struct bio_vec bvec, org_vec;	128	struct bio_vec bvec, org_vec;
129	int i;	129	int i;
130		130
131	if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
132	set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
133
134	/*	131	/*
135	* free up bounce indirect pages used	132	* free up bounce indirect pages used
136	*/	133	*/

block/cfq-iosched.c+104−21

 #define sample_valid(samples)	((samples) > 80)
 #define rb_entry_cfqg(node)	rb_entry((node), struct cfq_group, rb_node)
+/* blkio-related constants */
+#define CFQ_WEIGHT_MIN          10
+#define CFQ_WEIGHT_MAX          1000
+#define CFQ_WEIGHT_DEFAULT      500
 struct cfq_ttime {
 	unsigned long last_end_request;
 #endif	/* CONFIG_CFQ_GROUP_IOSCHED */
 };
+/* Per-cgroup data */
+struct cfq_group_data {
+	/* must be the first member */
+	struct blkcg_policy_data pd;
+	unsigned int weight;
+	unsigned int leaf_weight;
+};
 /* This is per cgroup per device grouping structure */
 struct cfq_group {
 	/* must be the first member */
 CFQ_CFQQ_FNS(wait_busy);
 #undef CFQ_CFQQ_FNS
-static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
-{
-	return pd ? container_of(pd, struct cfq_group, pd) : NULL;
-}
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
-{
-	return pd_to_blkg(&cfqg->pd);
-}
 #if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
 /* cfqg stats flags */
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
+static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
+{
+	return pd ? container_of(pd, struct cfq_group, pd) : NULL;
+}
+static struct cfq_group_data
+*cpd_to_cfqgd(struct blkcg_policy_data *cpd)
+{
+	return cpd ? container_of(cpd, struct cfq_group_data, pd) : NULL;
+}
+static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
+{
+	return pd_to_blkg(&cfqg->pd);
+}
 static struct blkcg_policy blkcg_policy_cfq;
 static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
 	return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
 }
+static struct cfq_group_data *blkcg_to_cfqgd(struct blkcg *blkcg)
+{
+	return cpd_to_cfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_cfq));
+}
 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)
 {
 	struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent;
 #endif
 }
+static void cfq_cpd_init(const struct blkcg *blkcg)
+{
+	struct cfq_group_data *cgd =
+		cpd_to_cfqgd(blkcg->pd[blkcg_policy_cfq.plid]);
+	if (blkcg == &blkcg_root) {
+		cgd->weight = 2 * CFQ_WEIGHT_DEFAULT;
+		cgd->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT;
+	} else {
+		cgd->weight = CFQ_WEIGHT_DEFAULT;
+		cgd->leaf_weight = CFQ_WEIGHT_DEFAULT;
+	}
+}
 static void cfq_pd_init(struct blkcg_gq *blkg)
 {
 	struct cfq_group *cfqg = blkg_to_cfqg(blkg);
+	struct cfq_group_data *cgd = blkcg_to_cfqgd(blkg->blkcg);
 	cfq_init_cfqg_base(cfqg);
-	cfqg->weight = blkg->blkcg->cfq_weight;
+	cfqg->weight = cgd->weight;
-	cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight;
+	cfqg->leaf_weight = cgd->leaf_weight;
 	cfqg_stats_init(&cfqg->stats);
 	cfqg_stats_init(&cfqg->dead_stats);
 }
 static int cfq_print_weight(struct seq_file *sf, void *v)
 {
-	seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_weight);
+	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+	struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+	unsigned int val = 0;
+	if (cgd)
+		val = cgd->weight;
+	seq_printf(sf, "%u\n", val);
 	return 0;
 }
 static int cfq_print_leaf_weight(struct seq_file *sf, void *v)
 {
-	seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_leaf_weight);
+	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+	struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+	unsigned int val = 0;
+	if (cgd)
+		val = cgd->leaf_weight;
+	seq_printf(sf, "%u\n", val);
 	return 0;
 }
 	struct blkcg *blkcg = css_to_blkcg(of_css(of));
 	struct blkg_conf_ctx ctx;
 	struct cfq_group *cfqg;
+	struct cfq_group_data *cfqgd;
 	int ret;
 	ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx);
 	ret = -EINVAL;
 	cfqg = blkg_to_cfqg(ctx.blkg);
+	cfqgd = blkcg_to_cfqgd(blkcg);
+	if (!cfqg || !cfqgd)
+		goto err;
 	if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) {
 		if (!is_leaf_weight) {
 			cfqg->dev_weight = ctx.v;
-			cfqg->new_weight = ctx.v ?: blkcg->cfq_weight;
+			cfqg->new_weight = ctx.v ?: cfqgd->weight;
 		} else {
 			cfqg->dev_leaf_weight = ctx.v;
-			cfqg->new_leaf_weight = ctx.v ?: blkcg->cfq_leaf_weight;
+			cfqg->new_leaf_weight = ctx.v ?: cfqgd->leaf_weight;
 		}
 		ret = 0;
 	}
+err:
 	blkg_conf_finish(&ctx);
 	return ret ?: nbytes;
 }
 {
 	struct blkcg *blkcg = css_to_blkcg(css);
 	struct blkcg_gq *blkg;
+	struct cfq_group_data *cfqgd;
+	int ret = 0;
 	if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX)
 		return -EINVAL;
 	spin_lock_irq(&blkcg->lock);
+	cfqgd = blkcg_to_cfqgd(blkcg);
+	if (!cfqgd) {
+		ret = -EINVAL;
+		goto out;
+	}
 	if (!is_leaf_weight)
-		blkcg->cfq_weight = val;
+		cfqgd->weight = val;
 	else
-		blkcg->cfq_leaf_weight = val;
+		cfqgd->leaf_weight = val;
 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
 		struct cfq_group *cfqg = blkg_to_cfqg(blkg);
 		if (!is_leaf_weight) {
 			if (!cfqg->dev_weight)
-				cfqg->new_weight = blkcg->cfq_weight;
+				cfqg->new_weight = cfqgd->weight;
 		} else {
 			if (!cfqg->dev_leaf_weight)
-				cfqg->new_leaf_weight = blkcg->cfq_leaf_weight;
+				cfqg->new_leaf_weight = cfqgd->leaf_weight;
 		}
 	}
+out:
 	spin_unlock_irq(&blkcg->lock);
-	return 0;
+	return ret;
 }
 static int cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
 	return ret;
 }
+static void cfq_registered_queue(struct request_queue *q)
+{
+	struct elevator_queue *e = q->elevator;
+	struct cfq_data *cfqd = e->elevator_data;
+	/*
+	 * Default to IOPS mode with no idling for SSDs
+	 */
+	if (blk_queue_nonrot(q))
+		cfqd->cfq_slice_idle = 0;
+}
 /*
  * sysfs parts below -->
  */
 		.elevator_may_queue_fn =	cfq_may_queue,
 		.elevator_init_fn =		cfq_init_queue,
 		.elevator_exit_fn =		cfq_exit_queue,
+		.elevator_registered_fn =	cfq_registered_queue,
 	},
 	.icq_size	=	sizeof(struct cfq_io_cq),
 	.icq_align	=	__alignof__(struct cfq_io_cq),
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
 static struct blkcg_policy blkcg_policy_cfq = {
 	.pd_size		= sizeof(struct cfq_group),
+	.cpd_size		= sizeof(struct cfq_group_data),
 	.cftypes		= cfq_blkcg_files,
+	.cpd_init_fn		= cfq_cpd_init,
 	.pd_init_fn		= cfq_pd_init,
 	.pd_offline_fn		= cfq_pd_offline,
 	.pd_reset_stats_fn	= cfq_pd_reset_stats,

block/elevator.c+2−0

 		}
 		kobject_uevent(&e->kobj, KOBJ_ADD);
 		e->registered = 1;
+		if (e->type->ops.elevator_registered_fn)
+			e->type->ops.elevator_registered_fn(q);
 	}
 	return error;
 }

block/ioctl.c+32−5

@@ -150,21 +150,48 @@ static int blkpg_ioctl(struct block_device *bdev, struct blkpg_ioctl_arg __user · 149 unmodified lines
150	}	150	}
151	}	151	}
152		152
153	static int blkdev_reread_part(struct block_device *bdev)	153	/*
		154	* This is an exported API for the block driver, and will not
		155	* acquire bd_mutex. This API should be used in case that
		156	* caller has held bd_mutex already.
		157	*/
		158	int __blkdev_reread_part(struct block_device *bdev)
154	{	159	{
155	struct gendisk *disk = bdev->bd_disk;	160	struct gendisk *disk = bdev->bd_disk;
156	int res;
157		161
158	if (!disk_part_scan_enabled(disk) \|\| bdev != bdev->bd_contains)	162	if (!disk_part_scan_enabled(disk) \|\| bdev != bdev->bd_contains)
159	return -EINVAL;	163	return -EINVAL;
160	if (!capable(CAP_SYS_ADMIN))	164	if (!capable(CAP_SYS_ADMIN))
161	return -EACCES;	165	return -EACCES;
162	if (!mutex_trylock(&bdev->bd_mutex))	166
163	return -EBUSY;	167	lockdep_assert_held(&bdev->bd_mutex);
164	res = rescan_partitions(disk, bdev);	168
		169	return rescan_partitions(disk, bdev);
		170	}
		171	EXPORT_SYMBOL(__blkdev_reread_part);
		172
		173	/*
		174	* This is an exported API for the block driver, and will
		175	* try to acquire bd_mutex. If bd_mutex has been held already
		176	* in current context, please call __blkdev_reread_part().
		177	*
		178	* Make sure the held locks in current context aren't required
		179	* in open()/close() handler and I/O path for avoiding ABBA deadlock:
		180	* - bd_mutex is held before calling block driver's open/close
		181	* handler
		182	* - reading partition table may submit I/O to the block device
		183	*/
		184	int blkdev_reread_part(struct block_device *bdev)
		185	{
		186	int res;
		187
		188	mutex_lock(&bdev->bd_mutex);
		189	res = __blkdev_reread_part(bdev);
165	mutex_unlock(&bdev->bd_mutex);	190	mutex_unlock(&bdev->bd_mutex);
		191
166	return res;	192	return res;
167	}	193	}
		194	EXPORT_SYMBOL(blkdev_reread_part);
168		195
169	static int blk_ioctl_discard(struct block_device *bdev, uint64_t start,	196	static int blk_ioctl_discard(struct block_device *bdev, uint64_t start,
170	uint64_t len, int secure)	197	uint64_t len, int secure)

drivers/block/nbd.c+24−26

@@ -230,29 +230,40 @@ static int nbd_send_req(struct nbd_device nbd, struct request req) · 229 unmodified lines
230	int result, flags;	230	int result, flags;
231	struct nbd_request request;	231	struct nbd_request request;
232	unsigned long size = blk_rq_bytes(req);	232	unsigned long size = blk_rq_bytes(req);
		233	u32 type;
		234
		235	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
		236	type = NBD_CMD_DISC;
		237	else if (req->cmd_flags & REQ_DISCARD)
		238	type = NBD_CMD_TRIM;
		239	else if (req->cmd_flags & REQ_FLUSH)
		240	type = NBD_CMD_FLUSH;
		241	else if (rq_data_dir(req) == WRITE)
		242	type = NBD_CMD_WRITE;
		243	else
		244	type = NBD_CMD_READ;
233		245
234	memset(&request, 0, sizeof(request));	246	memset(&request, 0, sizeof(request));
235	request.magic = htonl(NBD_REQUEST_MAGIC);	247	request.magic = htonl(NBD_REQUEST_MAGIC);
236	request.type = htonl(nbd_cmd(req));	248	request.type = htonl(type);
237		249	if (type != NBD_CMD_FLUSH && type != NBD_CMD_DISC) {
238	if (nbd_cmd(req) != NBD_CMD_FLUSH && nbd_cmd(req) != NBD_CMD_DISC) {
239	request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);	250	request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9);
240	request.len = htonl(size);	251	request.len = htonl(size);
241	}	252	}
242	memcpy(request.handle, &req, sizeof(req));	253	memcpy(request.handle, &req, sizeof(req));
243		254
244	dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",	255	dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n",
245	req, nbdcmd_to_ascii(nbd_cmd(req)),	256	req, nbdcmd_to_ascii(type),
246	(unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));	257	(unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req));
247	result = sock_xmit(nbd, 1, &request, sizeof(request),	258	result = sock_xmit(nbd, 1, &request, sizeof(request),
248	(nbd_cmd(req) == NBD_CMD_WRITE) ? MSG_MORE : 0);	259	(type == NBD_CMD_WRITE) ? MSG_MORE : 0);
249	if (result <= 0) {	260	if (result <= 0) {
250	dev_err(disk_to_dev(nbd->disk),	261	dev_err(disk_to_dev(nbd->disk),
251	"Send control failed (result %d)\n", result);	262	"Send control failed (result %d)\n", result);
252	return -EIO;	263	return -EIO;
253	}	264	}
254		265
255	if (nbd_cmd(req) == NBD_CMD_WRITE) {	266	if (type == NBD_CMD_WRITE) {
256	struct req_iterator iter;	267	struct req_iterator iter;
257	struct bio_vec bvec;	268	struct bio_vec bvec;
258	/*	269	/*
@@ -352,7 +363,7 @@ static struct request nbd_read_stat(struct nbd_device nbd) · 93 unmodified lines
352	}	363	}
353		364
354	dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", req);	365	dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", req);
355	if (nbd_cmd(req) == NBD_CMD_READ) {	366	if (rq_data_dir(req) != WRITE) {
356	struct req_iterator iter;	367	struct req_iterator iter;
357	struct bio_vec bvec;	368	struct bio_vec bvec;
358		369
@@ -452,23 +463,11 @@ static void nbd_handle_req(struct nbd_device nbd, struct request req) · 93 unmodified lines
452	if (req->cmd_type != REQ_TYPE_FS)	463	if (req->cmd_type != REQ_TYPE_FS)
453	goto error_out;	464	goto error_out;
454		465
455	nbd_cmd(req) = NBD_CMD_READ;	466	if (rq_data_dir(req) == WRITE &&
456	if (rq_data_dir(req) == WRITE) {	467	(nbd->flags & NBD_FLAG_READ_ONLY)) {
457	if ((req->cmd_flags & REQ_DISCARD)) {	468	dev_err(disk_to_dev(nbd->disk),
458	WARN_ON(!(nbd->flags & NBD_FLAG_SEND_TRIM));	469	"Write on read-only\n");
459	nbd_cmd(req) = NBD_CMD_TRIM;	470	goto error_out;
460	} else
461	nbd_cmd(req) = NBD_CMD_WRITE;
462	if (nbd->flags & NBD_FLAG_READ_ONLY) {
463	dev_err(disk_to_dev(nbd->disk),
464	"Write on read-only\n");
465	goto error_out;
466	}
467	}
468
469	if (req->cmd_flags & REQ_FLUSH) {
470	BUG_ON(unlikely(blk_rq_sectors(req)));
471	nbd_cmd(req) = NBD_CMD_FLUSH;
472	}	471	}
473		472
474	req->errors = 0;	473	req->errors = 0;
@@ -592,8 +591,7 @@ static int __nbd_ioctl(struct block_device bdev, struct nbd_device nbd, · 117 unmodified lines
592	fsync_bdev(bdev);	591	fsync_bdev(bdev);
593	mutex_lock(&nbd->tx_lock);	592	mutex_lock(&nbd->tx_lock);
594	blk_rq_init(NULL, &sreq);	593	blk_rq_init(NULL, &sreq);
595	sreq.cmd_type = REQ_TYPE_SPECIAL;	594	sreq.cmd_type = REQ_TYPE_DRV_PRIV;
596	nbd_cmd(&sreq) = NBD_CMD_DISC;
597		595
598	/* Check again after getting mutex back. */	596	/* Check again after getting mutex back. */
599	if (!nbd->sock)	597	if (!nbd->sock)

drivers/block/paride/pd.c+2−2

@@ -442,7 +442,7 @@ static char pd_buf; / buffer for request in progress */ · 441 unmodified lines
442		442
443	static enum action do_pd_io_start(void)	443	static enum action do_pd_io_start(void)
444	{	444	{
445	if (pd_req->cmd_type == REQ_TYPE_SPECIAL) {	445	if (pd_req->cmd_type == REQ_TYPE_DRV_PRIV) {
446	phase = pd_special;	446	phase = pd_special;
447	return pd_special();	447	return pd_special();
448	}	448	}
@@ -725,7 +725,7 @@ static int pd_special_command(struct pd_unit *disk, · 276 unmodified lines
725	if (IS_ERR(rq))	725	if (IS_ERR(rq))
726	return PTR_ERR(rq);	726	return PTR_ERR(rq);
727		727
728	rq->cmd_type = REQ_TYPE_SPECIAL;	728	rq->cmd_type = REQ_TYPE_DRV_PRIV;
729	rq->special = func;	729	rq->special = func;
730		730
731	err = blk_execute_rq(disk->gd->queue, disk->gd, rq, 0);	731	err = blk_execute_rq(disk->gd->queue, disk->gd, rq, 0);

drivers/block/sx8.c+2−2

@@ -620,7 +620,7 @@ static int carm_array_info (struct carm_host *host, unsigned int array_idx) · 619 unmodified lines
620	spin_unlock_irq(&host->lock);	620	spin_unlock_irq(&host->lock);
621		621
622	DPRINTK("blk_execute_rq_nowait, tag == %u\n", idx);	622	DPRINTK("blk_execute_rq_nowait, tag == %u\n", idx);
623	crq->rq->cmd_type = REQ_TYPE_SPECIAL;	623	crq->rq->cmd_type = REQ_TYPE_DRV_PRIV;
624	crq->rq->special = crq;	624	crq->rq->special = crq;
625	blk_execute_rq_nowait(host->oob_q, NULL, crq->rq, true, NULL);	625	blk_execute_rq_nowait(host->oob_q, NULL, crq->rq, true, NULL);
626		626
@@ -661,7 +661,7 @@ static int carm_send_special (struct carm_host *host, carm_sspc_t func) · 34 unmodified lines
661	crq->msg_bucket = (u32) rc;	661	crq->msg_bucket = (u32) rc;
662		662
663	DPRINTK("blk_execute_rq_nowait, tag == %u\n", idx);	663	DPRINTK("blk_execute_rq_nowait, tag == %u\n", idx);
664	crq->rq->cmd_type = REQ_TYPE_SPECIAL;	664	crq->rq->cmd_type = REQ_TYPE_DRV_PRIV;
665	crq->rq->special = crq;	665	crq->rq->special = crq;
666	blk_execute_rq_nowait(host->oob_q, NULL, crq->rq, true, NULL);	666	blk_execute_rq_nowait(host->oob_q, NULL, crq->rq, true, NULL);
667		667

drivers/block/virtio_blk.c+3−3

@@ -124,7 +124,7 @@ static inline void virtblk_request_done(struct request *req) · 123 unmodified lines
124	req->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual);	124	req->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual);
125	req->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len);	125	req->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len);
126	req->errors = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors);	126	req->errors = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors);
127	} else if (req->cmd_type == REQ_TYPE_SPECIAL) {	127	} else if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
128	req->errors = (error != 0);	128	req->errors = (error != 0);
129	}	129	}
130		130
@@ -188,7 +188,7 @@ static int virtio_queue_rq(struct blk_mq_hw_ctx *hctx, · 57 unmodified lines
188	vbr->out_hdr.sector = 0;	188	vbr->out_hdr.sector = 0;
189	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));	189	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));
190	break;	190	break;
191	case REQ_TYPE_SPECIAL:	191	case REQ_TYPE_DRV_PRIV:
192	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_GET_ID);	192	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_GET_ID);
193	vbr->out_hdr.sector = 0;	193	vbr->out_hdr.sector = 0;
194	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));	194	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(vbr->req));
@@ -251,7 +251,7 @@ static int virtblk_get_id(struct gendisk disk, char id_str) · 56 unmodified lines
251	return PTR_ERR(req);	251	return PTR_ERR(req);
252	}	252	}
253		253
254	req->cmd_type = REQ_TYPE_SPECIAL;	254	req->cmd_type = REQ_TYPE_DRV_PRIV;
255	err = blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);	255	err = blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);
256	blk_put_request(req);	256	blk_put_request(req);
257		257

drivers/ide/ide-atapi.c+5−5

@@ -93,7 +93,7 @@ int ide_queue_pc_tail(ide_drive_t drive, struct gendisk disk, · 92 unmodified lines
93	int error;	93	int error;
94		94
95	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);	95	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);
96	rq->cmd_type = REQ_TYPE_SPECIAL;	96	rq->cmd_type = REQ_TYPE_DRV_PRIV;
97	rq->special = (char *)pc;	97	rq->special = (char *)pc;
98		98
99	if (buf && bufflen) {	99	if (buf && bufflen) {
@@ -191,7 +191,7 @@ void ide_prep_sense(ide_drive_t drive, struct request rq) · 91 unmodified lines
191		191
192	BUG_ON(sense_len > sizeof(*sense));	192	BUG_ON(sense_len > sizeof(*sense));
193		193
194	if (rq->cmd_type == REQ_TYPE_SENSE \|\| drive->sense_rq_armed)	194	if (rq->cmd_type == REQ_TYPE_ATA_SENSE \|\| drive->sense_rq_armed)
195	return;	195	return;
196		196
197	memset(sense, 0, sizeof(*sense));	197	memset(sense, 0, sizeof(*sense));
@@ -210,7 +210,7 @@ void ide_prep_sense(ide_drive_t drive, struct request rq) · 12 unmodified lines
210	sense_rq->rq_disk = rq->rq_disk;	210	sense_rq->rq_disk = rq->rq_disk;
211	sense_rq->cmd[0] = GPCMD_REQUEST_SENSE;	211	sense_rq->cmd[0] = GPCMD_REQUEST_SENSE;
212	sense_rq->cmd[4] = cmd_len;	212	sense_rq->cmd[4] = cmd_len;
213	sense_rq->cmd_type = REQ_TYPE_SENSE;	213	sense_rq->cmd_type = REQ_TYPE_ATA_SENSE;
214	sense_rq->cmd_flags \|= REQ_PREEMPT;	214	sense_rq->cmd_flags \|= REQ_PREEMPT;
215		215
216	if (drive->media == ide_tape)	216	if (drive->media == ide_tape)
@@ -310,7 +310,7 @@ int ide_cd_get_xferlen(struct request *rq) · 93 unmodified lines
310	switch (rq->cmd_type) {	310	switch (rq->cmd_type) {
311	case REQ_TYPE_FS:	311	case REQ_TYPE_FS:
312	return 32768;	312	return 32768;
313	case REQ_TYPE_SENSE:	313	case REQ_TYPE_ATA_SENSE:
314	case REQ_TYPE_BLOCK_PC:	314	case REQ_TYPE_BLOCK_PC:
315	case REQ_TYPE_ATA_PC:	315	case REQ_TYPE_ATA_PC:
316	return blk_rq_bytes(rq);	316	return blk_rq_bytes(rq);
@@ -477,7 +477,7 @@ static ide_startstop_t ide_pc_intr(ide_drive_t *drive) · 160 unmodified lines
477	if (uptodate == 0)	477	if (uptodate == 0)
478	drive->failed_pc = NULL;	478	drive->failed_pc = NULL;
479		479
480	if (rq->cmd_type == REQ_TYPE_SPECIAL) {	480	if (rq->cmd_type == REQ_TYPE_DRV_PRIV) {
481	rq->errors = 0;	481	rq->errors = 0;
482	error = 0;	482	error = 0;
483	} else {	483	} else {

drivers/ide/ide-cd.c+5−5

@@ -210,7 +210,7 @@ static void cdrom_analyze_sense_data(ide_drive_t *drive, · 209 unmodified lines
210	static void ide_cd_complete_failed_rq(ide_drive_t drive, struct request rq)	210	static void ide_cd_complete_failed_rq(ide_drive_t drive, struct request rq)
211	{	211	{
212	/*	212	/*
213	* For REQ_TYPE_SENSE, "rq->special" points to the original	213	* For REQ_TYPE_ATA_SENSE, "rq->special" points to the original
214	* failed request. Also, the sense data should be read	214	* failed request. Also, the sense data should be read
215	* directly from rq which might be different from the original	215	* directly from rq which might be different from the original
216	* sense buffer if it got copied during mapping.	216	* sense buffer if it got copied during mapping.
@@ -285,7 +285,7 @@ static int cdrom_decode_status(ide_drive_t *drive, u8 stat) · 68 unmodified lines
285	"stat 0x%x",	285	"stat 0x%x",
286	rq->cmd[0], rq->cmd_type, err, stat);	286	rq->cmd[0], rq->cmd_type, err, stat);
287		287
288	if (rq->cmd_type == REQ_TYPE_SENSE) {	288	if (rq->cmd_type == REQ_TYPE_ATA_SENSE) {
289	/*	289	/*
290	* We got an error trying to get sense info from the drive	290	* We got an error trying to get sense info from the drive
291	* (probably while trying to recover from a former error).	291	* (probably while trying to recover from a former error).
@@ -526,7 +526,7 @@ static ide_startstop_t cdrom_newpc_intr(ide_drive_t *drive) · 234 unmodified lines
526	ide_expiry_t *expiry = NULL;	526	ide_expiry_t *expiry = NULL;
527	int dma_error = 0, dma, thislen, uptodate = 0;	527	int dma_error = 0, dma, thislen, uptodate = 0;
528	int write = (rq_data_dir(rq) == WRITE) ? 1 : 0, rc = 0;	528	int write = (rq_data_dir(rq) == WRITE) ? 1 : 0, rc = 0;
529	int sense = (rq->cmd_type == REQ_TYPE_SENSE);	529	int sense = (rq->cmd_type == REQ_TYPE_ATA_SENSE);
530	unsigned int timeout;	530	unsigned int timeout;
531	u16 len;	531	u16 len;
532	u8 ireason, stat;	532	u8 ireason, stat;
@@ -791,7 +791,7 @@ static ide_startstop_t ide_cd_do_request(ide_drive_t drive, struct request rq, · 258 unmodified lines
791	if (cdrom_start_rw(drive, rq) == ide_stopped)	791	if (cdrom_start_rw(drive, rq) == ide_stopped)
792	goto out_end;	792	goto out_end;
793	break;	793	break;
794	case REQ_TYPE_SENSE:	794	case REQ_TYPE_ATA_SENSE:
795	case REQ_TYPE_BLOCK_PC:	795	case REQ_TYPE_BLOCK_PC:
796	case REQ_TYPE_ATA_PC:	796	case REQ_TYPE_ATA_PC:
797	if (!rq->timeout)	797	if (!rq->timeout)
@@ -799,7 +799,7 @@ static ide_startstop_t ide_cd_do_request(ide_drive_t drive, struct request rq, · 1 unmodified line
799		799
800	cdrom_do_block_pc(drive, rq);	800	cdrom_do_block_pc(drive, rq);
801	break;	801	break;
802	case REQ_TYPE_SPECIAL:	802	case REQ_TYPE_DRV_PRIV:
803	/* right now this can only be a reset... */	803	/* right now this can only be a reset... */
804	uptodate = 1;	804	uptodate = 1;
805	goto out_end;	805	goto out_end;

drivers/ide/ide-cd_ioctl.c+1−1

@@ -304,7 +304,7 @@ int ide_cdrom_reset(struct cdrom_device_info *cdi) · 303 unmodified lines
304	int ret;	304	int ret;
305		305
306	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);	306	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);
307	rq->cmd_type = REQ_TYPE_SPECIAL;	307	rq->cmd_type = REQ_TYPE_DRV_PRIV;
308	rq->cmd_flags = REQ_QUIET;	308	rq->cmd_flags = REQ_QUIET;
309	ret = blk_execute_rq(drive->queue, cd->disk, rq, 0);	309	ret = blk_execute_rq(drive->queue, cd->disk, rq, 0);
310	blk_put_request(rq);	310	blk_put_request(rq);

drivers/ide/ide-devsets.c+1−1

@@ -166,7 +166,7 @@ int ide_devset_execute(ide_drive_t drive, const struct ide_devset setting, · 165 unmodified lines
166	return setting->set(drive, arg);	166	return setting->set(drive, arg);
167		167
168	rq = blk_get_request(q, READ, __GFP_WAIT);	168	rq = blk_get_request(q, READ, __GFP_WAIT);
169	rq->cmd_type = REQ_TYPE_SPECIAL;	169	rq->cmd_type = REQ_TYPE_DRV_PRIV;
170	rq->cmd_len = 5;	170	rq->cmd_len = 5;
171	rq->cmd[0] = REQ_DEVSET_EXEC;	171	rq->cmd[0] = REQ_DEVSET_EXEC;
172	(int )&rq->cmd[1] = arg;	172	(int )&rq->cmd[1] = arg;

drivers/ide/ide-eh.c+2−2

@@ -129,7 +129,7 @@ ide_startstop_t ide_error(ide_drive_t drive, const char msg, u8 stat) · 128 unmodified lines
129		129
130	if (cmd)	130	if (cmd)
131	ide_complete_cmd(drive, cmd, stat, err);	131	ide_complete_cmd(drive, cmd, stat, err);
132	} else if (blk_pm_request(rq)) {	132	} else if (ata_pm_request(rq)) {
133	rq->errors = 1;	133	rq->errors = 1;
134	ide_complete_pm_rq(drive, rq);	134	ide_complete_pm_rq(drive, rq);
135	return ide_stopped;	135	return ide_stopped;
@@ -147,7 +147,7 @@ static inline void ide_complete_drive_reset(ide_drive_t *drive, int err) · 11 unmodified lines
147	{	147	{
148	struct request *rq = drive->hwif->rq;	148	struct request *rq = drive->hwif->rq;
149		149
150	if (rq && rq->cmd_type == REQ_TYPE_SPECIAL &&	150	if (rq && rq->cmd_type == REQ_TYPE_DRV_PRIV &&
151	rq->cmd[0] == REQ_DRIVE_RESET) {	151	rq->cmd[0] == REQ_DRIVE_RESET) {
152	if (err <= 0 && rq->errors == 0)	152	if (err <= 0 && rq->errors == 0)
153	rq->errors = -EIO;	153	rq->errors = -EIO;

drivers/ide/ide-floppy.c+4−4

@@ -97,7 +97,7 @@ static int ide_floppy_callback(ide_drive_t *drive, int dsc) · 96 unmodified lines
97	"Aborting request!\n");	97	"Aborting request!\n");
98	}	98	}
99		99
100	if (rq->cmd_type == REQ_TYPE_SPECIAL)	100	if (rq->cmd_type == REQ_TYPE_DRV_PRIV)
101	rq->errors = uptodate ? 0 : IDE_DRV_ERROR_GENERAL;	101	rq->errors = uptodate ? 0 : IDE_DRV_ERROR_GENERAL;
102		102
103	return uptodate;	103	return uptodate;
@@ -246,7 +246,7 @@ static ide_startstop_t ide_floppy_do_request(ide_drive_t *drive, · 142 unmodified lines
246	} else	246	} else
247	printk(KERN_ERR PFX "%s: I/O error\n", drive->name);	247	printk(KERN_ERR PFX "%s: I/O error\n", drive->name);
248		248
249	if (rq->cmd_type == REQ_TYPE_SPECIAL) {	249	if (rq->cmd_type == REQ_TYPE_DRV_PRIV) {
250	rq->errors = 0;	250	rq->errors = 0;
251	ide_complete_rq(drive, 0, blk_rq_bytes(rq));	251	ide_complete_rq(drive, 0, blk_rq_bytes(rq));
252	return ide_stopped;	252	return ide_stopped;
@@ -265,8 +265,8 @@ static ide_startstop_t ide_floppy_do_request(ide_drive_t *drive, · 12 unmodified lines
265	pc = &floppy->queued_pc;	265	pc = &floppy->queued_pc;
266	idefloppy_create_rw_cmd(drive, pc, rq, (unsigned long)block);	266	idefloppy_create_rw_cmd(drive, pc, rq, (unsigned long)block);
267	break;	267	break;
268	case REQ_TYPE_SPECIAL:	268	case REQ_TYPE_DRV_PRIV:
269	case REQ_TYPE_SENSE:	269	case REQ_TYPE_ATA_SENSE:
270	pc = (struct ide_atapi_pc *)rq->special;	270	pc = (struct ide_atapi_pc *)rq->special;
271	break;	271	break;
272	case REQ_TYPE_BLOCK_PC:	272	case REQ_TYPE_BLOCK_PC:

drivers/ide/ide-io.c+6−6

@@ -135,7 +135,7 @@ EXPORT_SYMBOL(ide_complete_rq); · 134 unmodified lines
135		135
136	void ide_kill_rq(ide_drive_t drive, struct request rq)	136	void ide_kill_rq(ide_drive_t drive, struct request rq)
137	{	137	{
138	u8 drv_req = (rq->cmd_type == REQ_TYPE_SPECIAL) && rq->rq_disk;	138	u8 drv_req = (rq->cmd_type == REQ_TYPE_DRV_PRIV) && rq->rq_disk;
139	u8 media = drive->media;	139	u8 media = drive->media;
140		140
141	drive->failed_pc = NULL;	141	drive->failed_pc = NULL;
@@ -320,7 +320,7 @@ static ide_startstop_t start_request (ide_drive_t drive, struct request rq) · 178 unmodified lines
320	goto kill_rq;	320	goto kill_rq;
321	}	321	}
322		322
323	if (blk_pm_request(rq))	323	if (ata_pm_request(rq))
324	ide_check_pm_state(drive, rq);	324	ide_check_pm_state(drive, rq);
325		325
326	drive->hwif->tp_ops->dev_select(drive);	326	drive->hwif->tp_ops->dev_select(drive);
@@ -342,8 +342,8 @@ static ide_startstop_t start_request (ide_drive_t drive, struct request rq) · 15 unmodified lines
342		342
343	if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE)	343	if (rq->cmd_type == REQ_TYPE_ATA_TASKFILE)
344	return execute_drive_cmd(drive, rq);	344	return execute_drive_cmd(drive, rq);
345	else if (blk_pm_request(rq)) {	345	else if (ata_pm_request(rq)) {
346	struct request_pm_state *pm = rq->special;	346	struct ide_pm_state *pm = rq->special;
347	#ifdef DEBUG_PM	347	#ifdef DEBUG_PM
348	printk("%s: start_power_step(step: %d)\n",	348	printk("%s: start_power_step(step: %d)\n",
349	drive->name, pm->pm_step);	349	drive->name, pm->pm_step);
@@ -353,7 +353,7 @@ static ide_startstop_t start_request (ide_drive_t drive, struct request rq) · 3 unmodified lines
353	pm->pm_step == IDE_PM_COMPLETED)	353	pm->pm_step == IDE_PM_COMPLETED)
354	ide_complete_pm_rq(drive, rq);	354	ide_complete_pm_rq(drive, rq);
355	return startstop;	355	return startstop;
356	} else if (!rq->rq_disk && rq->cmd_type == REQ_TYPE_SPECIAL)	356	} else if (!rq->rq_disk && rq->cmd_type == REQ_TYPE_DRV_PRIV)
357	/*	357	/*
358	* TODO: Once all ULDs have been modified to	358	* TODO: Once all ULDs have been modified to
359	* check for specific op codes rather than	359	* check for specific op codes rather than
@@ -538,7 +538,7 @@ repeat: · 178 unmodified lines
538	* state machine.	538	* state machine.
539	*/	539	*/
540	if ((drive->dev_flags & IDE_DFLAG_BLOCKED) &&	540	if ((drive->dev_flags & IDE_DFLAG_BLOCKED) &&
541	blk_pm_request(rq) == 0 &&	541	ata_pm_request(rq) == 0 &&
542	(rq->cmd_flags & REQ_PREEMPT) == 0) {	542	(rq->cmd_flags & REQ_PREEMPT) == 0) {
543	/* there should be no pending command at this point */	543	/* there should be no pending command at this point */
544	ide_unlock_port(hwif);	544	ide_unlock_port(hwif);

drivers/ide/ide-ioctls.c+1−1

@@ -222,7 +222,7 @@ static int generic_drive_reset(ide_drive_t *drive) · 221 unmodified lines
222	int ret = 0;	222	int ret = 0;
223		223
224	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);	224	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);
225	rq->cmd_type = REQ_TYPE_SPECIAL;	225	rq->cmd_type = REQ_TYPE_DRV_PRIV;
226	rq->cmd_len = 1;	226	rq->cmd_len = 1;
227	rq->cmd[0] = REQ_DRIVE_RESET;	227	rq->cmd[0] = REQ_DRIVE_RESET;
228	if (blk_execute_rq(drive->queue, NULL, rq, 1))	228	if (blk_execute_rq(drive->queue, NULL, rq, 1))

drivers/ide/ide-park.c+2−2

@@ -34,7 +34,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) · 33 unmodified lines
34	rq = blk_get_request(q, READ, __GFP_WAIT);	34	rq = blk_get_request(q, READ, __GFP_WAIT);
35	rq->cmd[0] = REQ_PARK_HEADS;	35	rq->cmd[0] = REQ_PARK_HEADS;
36	rq->cmd_len = 1;	36	rq->cmd_len = 1;
37	rq->cmd_type = REQ_TYPE_SPECIAL;	37	rq->cmd_type = REQ_TYPE_DRV_PRIV;
38	rq->special = &timeout;	38	rq->special = &timeout;
39	rc = blk_execute_rq(q, NULL, rq, 1);	39	rc = blk_execute_rq(q, NULL, rq, 1);
40	blk_put_request(rq);	40	blk_put_request(rq);
@@ -51,7 +51,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) · 10 unmodified lines
51		51
52	rq->cmd[0] = REQ_UNPARK_HEADS;	52	rq->cmd[0] = REQ_UNPARK_HEADS;
53	rq->cmd_len = 1;	53	rq->cmd_len = 1;
54	rq->cmd_type = REQ_TYPE_SPECIAL;	54	rq->cmd_type = REQ_TYPE_DRV_PRIV;
55	elv_add_request(q, rq, ELEVATOR_INSERT_FRONT);	55	elv_add_request(q, rq, ELEVATOR_INSERT_FRONT);
56		56
57	out:	57	out:

drivers/ide/ide-pm.c+43−13

 	ide_drive_t *pair = ide_get_pair_dev(drive);
 	ide_hwif_t *hwif = drive->hwif;
 	struct request *rq;
-	struct request_pm_state rqpm;
+	struct ide_pm_state rqpm;
 	int ret;
 	if (ide_port_acpi(hwif)) {
 	memset(&rqpm, 0, sizeof(rqpm));
 	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);
-	rq->cmd_type = REQ_TYPE_PM_SUSPEND;
+	rq->cmd_type = REQ_TYPE_ATA_PM_SUSPEND;
 	rq->special = &rqpm;
 	rqpm.pm_step = IDE_PM_START_SUSPEND;
 	if (mesg.event == PM_EVENT_PRETHAW)
 	return ret;
 }
+static void ide_end_sync_rq(struct request *rq, int error)
+{
+	complete(rq->end_io_data);
+}
+static int ide_pm_execute_rq(struct request *rq)
+{
+	struct request_queue *q = rq->q;
+	DECLARE_COMPLETION_ONSTACK(wait);
+	rq->end_io_data = &wait;
+	rq->end_io = ide_end_sync_rq;
+	spin_lock_irq(q->queue_lock);
+	if (unlikely(blk_queue_dying(q))) {
+		rq->cmd_flags |= REQ_QUIET;
+		rq->errors = -ENXIO;
+		__blk_end_request_all(rq, rq->errors);
+		spin_unlock_irq(q->queue_lock);
+		return -ENXIO;
+	}
+	__elv_add_request(q, rq, ELEVATOR_INSERT_FRONT);
+	__blk_run_queue_uncond(q);
+	spin_unlock_irq(q->queue_lock);
+	wait_for_completion_io(&wait);
+	return rq->errors ? -EIO : 0;
+}
 int generic_ide_resume(struct device *dev)
 {
 	ide_drive_t *drive = to_ide_device(dev);
 	ide_drive_t *pair = ide_get_pair_dev(drive);
 	ide_hwif_t *hwif = drive->hwif;
 	struct request *rq;
-	struct request_pm_state rqpm;
+	struct ide_pm_state rqpm;
 	int err;
 	if (ide_port_acpi(hwif)) {
 	memset(&rqpm, 0, sizeof(rqpm));
 	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);
-	rq->cmd_type = REQ_TYPE_PM_RESUME;
+	rq->cmd_type = REQ_TYPE_ATA_PM_RESUME;
 	rq->cmd_flags |= REQ_PREEMPT;
 	rq->special = &rqpm;
 	rqpm.pm_step = IDE_PM_START_RESUME;
 	rqpm.pm_state = PM_EVENT_ON;
-	err = blk_execute_rq(drive->queue, NULL, rq, 1);
+	err = ide_pm_execute_rq(rq);
 	blk_put_request(rq);
 	if (err == 0 && dev->driver) {
 void ide_complete_power_step(ide_drive_t *drive, struct request *rq)
 {
-	struct request_pm_state *pm = rq->special;
+	struct ide_pm_state *pm = rq->special;
 #ifdef DEBUG_PM
 	printk(KERN_INFO "%s: complete_power_step(step: %d)\n",
 ide_startstop_t ide_start_power_step(ide_drive_t *drive, struct request *rq)
 {
-	struct request_pm_state *pm = rq->special;
+	struct ide_pm_state *pm = rq->special;
 	struct ide_cmd cmd = { };
 	switch (pm->pm_step) {
 void ide_complete_pm_rq(ide_drive_t *drive, struct request *rq)
 {
 	struct request_queue *q = drive->queue;
-	struct request_pm_state *pm = rq->special;
+	struct ide_pm_state *pm = rq->special;
 	unsigned long flags;
 	ide_complete_power_step(drive, rq);
 #ifdef DEBUG_PM
 	printk("%s: completing PM request, %s\n", drive->name,
-	       (rq->cmd_type == REQ_TYPE_PM_SUSPEND) ? "suspend" : "resume");
+	       (rq->cmd_type == REQ_TYPE_ATA_PM_SUSPEND) ? "suspend" : "resume");
 #endif
 	spin_lock_irqsave(q->queue_lock, flags);
-	if (rq->cmd_type == REQ_TYPE_PM_SUSPEND)
+	if (rq->cmd_type == REQ_TYPE_ATA_PM_SUSPEND)
 		blk_stop_queue(q);
 	else
 		drive->dev_flags &= ~IDE_DFLAG_BLOCKED;
 void ide_check_pm_state(ide_drive_t *drive, struct request *rq)
 {
-	struct request_pm_state *pm = rq->special;
+	struct ide_pm_state *pm = rq->special;
-	if (rq->cmd_type == REQ_TYPE_PM_SUSPEND &&
+	if (rq->cmd_type == REQ_TYPE_ATA_PM_SUSPEND &&
 	    pm->pm_step == IDE_PM_START_SUSPEND)
 		/* Mark drive blocked when starting the suspend sequence. */
 		drive->dev_flags |= IDE_DFLAG_BLOCKED;
-	else if (rq->cmd_type == REQ_TYPE_PM_RESUME &&
+	else if (rq->cmd_type == REQ_TYPE_ATA_PM_RESUME &&
 		 pm->pm_step == IDE_PM_START_RESUME) {
 		/*
 		 * The first thing we do on wakeup is to wait for BSY bit to

drivers/ide/ide-tape.c+3−3

@@ -576,8 +576,8 @@ static ide_startstop_t idetape_do_request(ide_drive_t *drive, · 575 unmodified lines
576	rq->cmd[0], (unsigned long long)blk_rq_pos(rq),	576	rq->cmd[0], (unsigned long long)blk_rq_pos(rq),
577	blk_rq_sectors(rq));	577	blk_rq_sectors(rq));
578		578
579	BUG_ON(!(rq->cmd_type == REQ_TYPE_SPECIAL \|\|	579	BUG_ON(!(rq->cmd_type == REQ_TYPE_DRV_PRIV \|\|
580	rq->cmd_type == REQ_TYPE_SENSE));	580	rq->cmd_type == REQ_TYPE_ATA_SENSE));
581		581
582	/* Retry a failed packet command */	582	/* Retry a failed packet command */
583	if (drive->failed_pc && drive->pc->c[0] == REQUEST_SENSE) {	583	if (drive->failed_pc && drive->pc->c[0] == REQUEST_SENSE) {
@@ -853,7 +853,7 @@ static int idetape_queue_rw_tail(ide_drive_t *drive, int cmd, int size) · 269 unmodified lines
853	BUG_ON(size < 0 \|\| size % tape->blk_size);	853	BUG_ON(size < 0 \|\| size % tape->blk_size);
854		854
855	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);	855	rq = blk_get_request(drive->queue, READ, __GFP_WAIT);
856	rq->cmd_type = REQ_TYPE_SPECIAL;	856	rq->cmd_type = REQ_TYPE_DRV_PRIV;
857	rq->cmd[13] = cmd;	857	rq->cmd[13] = cmd;
858	rq->rq_disk = tape->disk;	858	rq->rq_disk = tape->disk;
859	rq->__sector = tape->first_frame;	859	rq->__sector = tape->first_frame;

drivers/ide/ide-taskfile.c+1−1

@@ -186,7 +186,7 @@ static ide_startstop_t task_no_data_intr(ide_drive_t *drive) · 185 unmodified lines
186	tf->command == ATA_CMD_CHK_POWER) {	186	tf->command == ATA_CMD_CHK_POWER) {
187	struct request *rq = hwif->rq;	187	struct request *rq = hwif->rq;
188		188
189	if (blk_pm_request(rq))	189	if (ata_pm_request(rq))
190	ide_complete_pm_rq(drive, rq);	190	ide_complete_pm_rq(drive, rq);
191	else	191	else
192	ide_finish_cmd(drive, cmd, stat);	192	ide_finish_cmd(drive, cmd, stat);

drivers/md/bcache/io.c+1−1

@@ -55,7 +55,7 @@ static void bch_bio_submit_split_done(struct closure *cl) · 54 unmodified lines
55		55
56	s->bio->bi_end_io = s->bi_end_io;	56	s->bio->bi_end_io = s->bi_end_io;
57	s->bio->bi_private = s->bi_private;	57	s->bio->bi_private = s->bi_private;
58	bio_endio_nodec(s->bio, 0);	58	bio_endio(s->bio, 0);
59		59
60	closure_debug_destroy(&s->cl);	60	closure_debug_destroy(&s->cl);
61	mempool_free(s, s->p->bio_split_hook);	61	mempool_free(s, s->p->bio_split_hook);

drivers/md/bcache/request.c+1−1

@@ -619,7 +619,7 @@ static void do_bio_hook(struct search s, struct bio orig_bio) · 618 unmodified lines
619	bio->bi_end_io = request_endio;	619	bio->bi_end_io = request_endio;
620	bio->bi_private = &s->cl;	620	bio->bi_private = &s->cl;
621		621
622	atomic_set(&bio->bi_cnt, 3);	622	bio_cnt_set(bio, 3);
623	}	623	}
624		624
625	static void search_free(struct closure *cl)	625	static void search_free(struct closure *cl)

drivers/md/dm-cache-target.c+0−6

 {
 	bio->bi_end_io = h->bi_end_io;
 	bio->bi_private = h->bi_private;
-	/*
-	 * Must bump bi_remaining to allow bio to complete with
-	 * restored bi_end_io.
-	 */
-	atomic_inc(&bio->bi_remaining);
 }
 /*----------------------------------------------------------------*/

drivers/md/dm-raid1.c+0−2

@@ -1254,8 +1254,6 @@ static int mirror_end_io(struct dm_target ti, struct bio bio, int error) · 1253 unmodified lines
1254	dm_bio_restore(bd, bio);	1254	dm_bio_restore(bd, bio);
1255	bio_record->details.bi_bdev = NULL;	1255	bio_record->details.bi_bdev = NULL;
1256		1256
1257	atomic_inc(&bio->bi_remaining);
1258
1259	queue_bio(ms, bio, rw);	1257	queue_bio(ms, bio, rw);
1260	return DM_ENDIO_INCOMPLETE;	1258	return DM_ENDIO_INCOMPLETE;
1261	}	1259	}

drivers/md/dm-snap.c+0−1

@@ -1478,7 +1478,6 @@ out: · 1477 unmodified lines
1478	if (full_bio) {	1478	if (full_bio) {
1479	full_bio->bi_end_io = pe->full_bio_end_io;	1479	full_bio->bi_end_io = pe->full_bio_end_io;
1480	full_bio->bi_private = pe->full_bio_private;	1480	full_bio->bi_private = pe->full_bio_private;
1481	atomic_inc(&full_bio->bi_remaining);
1482	}	1481	}
1483	increment_pending_exceptions_done_count();	1482	increment_pending_exceptions_done_count();
1484		1483

drivers/md/dm-table.c+16−9

 {
 	unsigned type = dm_table_get_type(t);
 	unsigned per_bio_data_size = 0;
-	struct dm_target *tgt;
 	unsigned i;
-	if (unlikely(type == DM_TYPE_NONE)) {
+	switch (type) {
+	case DM_TYPE_BIO_BASED:
+		for (i = 0; i < t->num_targets; i++) {
+			struct dm_target *tgt = t->targets + i;
+			per_bio_data_size = max(per_bio_data_size,
+						tgt->per_bio_data_size);
+		}
+		t->mempools = dm_alloc_bio_mempools(t->integrity_supported,
+						    per_bio_data_size);
+		break;
+	case DM_TYPE_REQUEST_BASED:
+	case DM_TYPE_MQ_REQUEST_BASED:
+		t->mempools = dm_alloc_rq_mempools(md, type);
+		break;
+	default:
 		DMWARN("no table type is set, can't allocate mempools");
 		return -EINVAL;
 	}
-	if (type == DM_TYPE_BIO_BASED)
-		for (i = 0; i < t->num_targets; i++) {
-			tgt = t->targets + i;
-			per_bio_data_size = max(per_bio_data_size, tgt->per_bio_data_size);
-		}
-	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported, per_bio_data_size);
 	if (!t->mempools)
 		return -ENOMEM;

drivers/md/dm-thin.c+3−6

@@ -793,10 +793,9 @@ static void inc_remap_and_issue_cell(struct thin_c *tc, · 792 unmodified lines
793		793
794	static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)	794	static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
795	{	795	{
796	if (m->bio) {	796	if (m->bio)
797	m->bio->bi_end_io = m->saved_bi_end_io;	797	m->bio->bi_end_io = m->saved_bi_end_io;
798	atomic_inc(&m->bio->bi_remaining);	798
799	}
800	cell_error(m->tc->pool, m->cell);	799	cell_error(m->tc->pool, m->cell);
801	list_del(&m->list);	800	list_del(&m->list);
802	mempool_free(m, m->tc->pool->mapping_pool);	801	mempool_free(m, m->tc->pool->mapping_pool);
@@ -810,10 +809,8 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) · 7 unmodified lines
810	int r;	809	int r;
811		810
812	bio = m->bio;	811	bio = m->bio;
813	if (bio) {	812	if (bio)
814	bio->bi_end_io = m->saved_bi_end_io;	813	bio->bi_end_io = m->saved_bi_end_io;
815	atomic_inc(&bio->bi_remaining);
816	}
817		814
818	if (m->err) {	815	if (m->err) {
819	cell_error(pool, m->cell);	816	cell_error(pool, m->cell);

drivers/md/dm-verity.c+1−1

@@ -459,7 +459,7 @@ static void verity_finish_io(struct dm_verity_io *io, int error) · 458 unmodified lines
459	bio->bi_end_io = io->orig_bi_end_io;	459	bio->bi_end_io = io->orig_bi_end_io;
460	bio->bi_private = io->orig_bi_private;	460	bio->bi_private = io->orig_bi_private;
461		461
462	bio_endio_nodec(bio, error);	462	bio_endio(bio, error);
463	}	463	}
464		464
465	static void verity_work(struct work_struct *w)	465	static void verity_work(struct work_struct *w)

drivers/md/dm.c+40−131

@@ -990,57 +990,6 @@ static void clone_endio(struct bio *bio, int error) · 989 unmodified lines
990	dec_pending(io, error);	990	dec_pending(io, error);
991	}	991	}
992		992
993	/*
994	* Partial completion handling for request-based dm
995	*/
996	static void end_clone_bio(struct bio *clone, int error)
997	{
998	struct dm_rq_clone_bio_info *info =
999	container_of(clone, struct dm_rq_clone_bio_info, clone);
1000	struct dm_rq_target_io *tio = info->tio;
1001	struct bio *bio = info->orig;
1002	unsigned int nr_bytes = info->orig->bi_iter.bi_size;
1003
1004	bio_put(clone);
1005
1006	if (tio->error)
1007	/*
1008	* An error has already been detected on the request.
1009	* Once error occurred, just let clone->end_io() handle
1010	* the remainder.
1011	*/
1012	return;
1013	else if (error) {
1014	/*
1015	* Don't notice the error to the upper layer yet.
1016	* The error handling decision is made by the target driver,
1017	* when the request is completed.
1018	*/
1019	tio->error = error;
1020	return;
1021	}
1022
1023	/*
1024	* I/O for the bio successfully completed.
1025	* Notice the data completion to the upper layer.
1026	*/
1027
1028	/*
1029	* bios are processed from the head of the list.
1030	* So the completing bio should always be rq->bio.
1031	* If it's not, something wrong is happening.
1032	*/
1033	if (tio->orig->bio != bio)
1034	DMERR("bio completion is going in the middle of the request");
1035
1036	/*
1037	* Update the original request.
1038	* Do not use blk_end_request() here, because it may complete
1039	* the original request before the clone, and break the ordering.
1040	*/
1041	blk_update_request(tio->orig, 0, nr_bytes);
1042	}
1043
1044	static struct dm_rq_target_io tio_from_request(struct request rq)	993	static struct dm_rq_target_io tio_from_request(struct request rq)
1045	{	994	{
1046	return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special);	995	return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special);
@@ -1087,8 +1036,6 @@ static void free_rq_clone(struct request *clone) · 40 unmodified lines
1087	struct dm_rq_target_io *tio = clone->end_io_data;	1036	struct dm_rq_target_io *tio = clone->end_io_data;
1088	struct mapped_device *md = tio->md;	1037	struct mapped_device *md = tio->md;
1089		1038
1090	blk_rq_unprep_clone(clone);
1091
1092	if (md->type == DM_TYPE_MQ_REQUEST_BASED)	1039	if (md->type == DM_TYPE_MQ_REQUEST_BASED)
1093	/* stacked on blk-mq queue(s) */	1040	/* stacked on blk-mq queue(s) */
1094	tio->ti->type->release_clone_rq(clone);	1041	tio->ti->type->release_clone_rq(clone);
@@ -1827,39 +1774,13 @@ static void dm_dispatch_clone_request(struct request clone, struct request rq) · 732 unmodified lines
1827	dm_complete_request(rq, r);	1774	dm_complete_request(rq, r);
1828	}	1775	}
1829		1776
1830	static int dm_rq_bio_constructor(struct bio bio, struct bio bio_orig,	1777	static void setup_clone(struct request clone, struct request rq,
1831	void *data)	1778	struct dm_rq_target_io *tio)
1832	{	1779	{
1833	struct dm_rq_target_io *tio = data;	1780	blk_rq_prep_clone(clone, rq);
1834	struct dm_rq_clone_bio_info *info =
1835	container_of(bio, struct dm_rq_clone_bio_info, clone);
1836
1837	info->orig = bio_orig;
1838	info->tio = tio;
1839	bio->bi_end_io = end_clone_bio;
1840
1841	return 0;
1842	}
1843
1844	static int setup_clone(struct request clone, struct request rq,
1845	struct dm_rq_target_io *tio, gfp_t gfp_mask)
1846	{
1847	int r;
1848
1849	r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,
1850	dm_rq_bio_constructor, tio);
1851	if (r)
1852	return r;
1853
1854	clone->cmd = rq->cmd;
1855	clone->cmd_len = rq->cmd_len;
1856	clone->sense = rq->sense;
1857	clone->end_io = end_clone_request;	1781	clone->end_io = end_clone_request;
1858	clone->end_io_data = tio;	1782	clone->end_io_data = tio;
1859
1860	tio->clone = clone;	1783	tio->clone = clone;
1861
1862	return 0;
1863	}	1784	}
1864		1785
1865	static struct request clone_rq(struct request rq, struct mapped_device *md,	1786	static struct request clone_rq(struct request rq, struct mapped_device *md,
@@ -1880,12 +1801,7 @@ static struct request clone_rq(struct request rq, struct mapped_device *md, · 14 unmodified lines
1880	clone = tio->clone;	1801	clone = tio->clone;
1881		1802
1882	blk_rq_init(NULL, clone);	1803	blk_rq_init(NULL, clone);
1883	if (setup_clone(clone, rq, tio, gfp_mask)) {	1804	setup_clone(clone, rq, tio);
1884	/* -ENOMEM */
1885	if (alloc_clone)
1886	free_clone_request(md, clone);
1887	return NULL;
1888	}
1889		1805
1890	return clone;	1806	return clone;
1891	}	1807	}
@@ -1979,11 +1895,7 @@ static int map_request(struct dm_rq_target_io tio, struct request rq, · 87 unmodified lines
1979	}	1895	}
1980	if (r != DM_MAPIO_REMAPPED)	1896	if (r != DM_MAPIO_REMAPPED)
1981	return r;	1897	return r;
1982	if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {	1898	setup_clone(clone, rq, tio);
1983	/* -ENOMEM */
1984	ti->type->release_clone_rq(clone);
1985	return DM_MAPIO_REQUEUE;
1986	}
1987	}	1899	}
1988		1900
1989	switch (r) {	1901	switch (r) {
@@ -2437,8 +2349,6 @@ static void __bind_mempools(struct mapped_device md, struct dm_table t) · 447 unmodified lines
2437	goto out;	2349	goto out;
2438	}	2350	}
2439		2351
2440	BUG_ON(!p \|\| md->io_pool \|\| md->rq_pool \|\| md->bs);
2441
2442	md->io_pool = p->io_pool;	2352	md->io_pool = p->io_pool;
2443	p->io_pool = NULL;	2353	p->io_pool = NULL;
2444	md->rq_pool = p->rq_pool;	2354	md->rq_pool = p->rq_pool;
@@ -3544,48 +3454,23 @@ int dm_noflush_suspending(struct dm_target *ti) · 1099 unmodified lines
3544	}	3454	}
3545	EXPORT_SYMBOL_GPL(dm_noflush_suspending);	3455	EXPORT_SYMBOL_GPL(dm_noflush_suspending);
3546		3456
3547	struct dm_md_mempools dm_alloc_md_mempools(struct mapped_device md, unsigned type,	3457	struct dm_md_mempools *dm_alloc_bio_mempools(unsigned integrity,
3548	unsigned integrity, unsigned per_bio_data_size)	3458	unsigned per_bio_data_size)
3549	{	3459	{
3550	struct dm_md_mempools pools = kzalloc(sizeof(pools), GFP_KERNEL);	3460	struct dm_md_mempools *pools;
3551	struct kmem_cache *cachep = NULL;	3461	unsigned int pool_size = dm_get_reserved_bio_based_ios();
3552	unsigned int pool_size = 0;
3553	unsigned int front_pad;	3462	unsigned int front_pad;
3554		3463
		3464	pools = kzalloc(sizeof(*pools), GFP_KERNEL);
3555	if (!pools)	3465	if (!pools)
3556	return NULL;	3466	return NULL;
3557		3467
3558	type = filter_md_type(type, md);	3468	front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) +
		3469	offsetof(struct dm_target_io, clone);
3559		3470
3560	switch (type) {	3471	pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
3561	case DM_TYPE_BIO_BASED:	3472	if (!pools->io_pool)
3562	cachep = _io_cache;	3473	goto out;
3563	pool_size = dm_get_reserved_bio_based_ios();
3564	front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
3565	break;
3566	case DM_TYPE_REQUEST_BASED:
3567	cachep = _rq_tio_cache;
3568	pool_size = dm_get_reserved_rq_based_ios();
3569	pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
3570	if (!pools->rq_pool)
3571	goto out;
3572	/* fall through to setup remaining rq-based pools */
3573	case DM_TYPE_MQ_REQUEST_BASED:
3574	if (!pool_size)
3575	pool_size = dm_get_reserved_rq_based_ios();
3576	front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
3577	/* per_bio_data_size is not used. See __bind_mempools(). */
3578	WARN_ON(per_bio_data_size != 0);
3579	break;
3580	default:
3581	BUG();
3582	}
3583
3584	if (cachep) {
3585	pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
3586	if (!pools->io_pool)
3587	goto out;
3588	}
3589		3474
3590	pools->bs = bioset_create_nobvec(pool_size, front_pad);	3475	pools->bs = bioset_create_nobvec(pool_size, front_pad);
3591	if (!pools->bs)	3476	if (!pools->bs)
@@ -3595,10 +3480,34 @@ struct dm_md_mempools dm_alloc_md_mempools(struct mapped_device md, unsigned t · 3 unmodified lines
3595	goto out;	3480	goto out;
3596		3481
3597	return pools;	3482	return pools;
3598
3599	out:	3483	out:
3600	dm_free_md_mempools(pools);	3484	dm_free_md_mempools(pools);
		3485	return NULL;
		3486	}
		3487
		3488	struct dm_md_mempools dm_alloc_rq_mempools(struct mapped_device md,
		3489	unsigned type)
		3490	{
		3491	unsigned int pool_size = dm_get_reserved_rq_based_ios();
		3492	struct dm_md_mempools *pools;
		3493
		3494	pools = kzalloc(sizeof(*pools), GFP_KERNEL);
		3495	if (!pools)
		3496	return NULL;
		3497
		3498	if (filter_md_type(type, md) == DM_TYPE_REQUEST_BASED) {
		3499	pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
		3500	if (!pools->rq_pool)
		3501	goto out;
		3502	}
3601		3503
		3504	pools->io_pool = mempool_create_slab_pool(pool_size, _rq_tio_cache);
		3505	if (!pools->io_pool)
		3506	goto out;
		3507
		3508	return pools;
		3509	out:
		3510	dm_free_md_mempools(pools);
3602	return NULL;	3511	return NULL;
3603	}	3512	}
3604		3513

drivers/md/dm.h+3−2

 /*
  * Mempool operations
  */
-struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
+struct dm_md_mempools *dm_alloc_bio_mempools(unsigned integrity,
-					    unsigned integrity, unsigned per_bio_data_size);
+					     unsigned per_bio_data_size);
+struct dm_md_mempools *dm_alloc_rq_mempools(struct mapped_device *md, unsigned type);
 void dm_free_md_mempools(struct dm_md_mempools *pools);
 /*

fs/btrfs/disk-io.c+3−10

 	bio->bi_private = end_io_wq->private;
 	bio->bi_end_io = end_io_wq->end_io;
 	kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq);
-	bio_endio_nodec(bio, error);
+	bio_endio(bio, error);
 }
 static int cleaner_kthread(void *arg)
  */
 static void btrfs_end_empty_barrier(struct bio *bio, int err)
 {
-	if (err) {
+	if (err)
-		if (err == -EOPNOTSUPP)
-			set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
 		clear_bit(BIO_UPTODATE, &bio->bi_flags);
-	}
 	if (bio->bi_private)
 		complete(bio->bi_private);
 	bio_put(bio);
 		wait_for_completion(&device->flush_wait);
-		if (bio_flagged(bio, BIO_EOPNOTSUPP)) {
+		if (!bio_flagged(bio, BIO_UPTODATE)) {
-			printk_in_rcu("BTRFS: disabling barriers on dev %s\n",
-				      rcu_str_deref(device->name));
-			device->nobarriers = 1;
-		} else if (!bio_flagged(bio, BIO_UPTODATE)) {
 			ret = -EIO;
 			btrfs_dev_stat_inc_and_print(device,
 				BTRFS_DEV_STAT_FLUSH_ERRS);

fs/btrfs/extent_io.c+0−2

@@ -2767,8 +2767,6 @@ static int __must_check submit_one_bio(int rw, struct bio *bio, · 2766 unmodified lines
2767	else	2767	else
2768	btrfsic_submit_bio(rw, bio);	2768	btrfsic_submit_bio(rw, bio);
2769		2769
2770	if (bio_flagged(bio, BIO_EOPNOTSUPP))
2771	ret = -EOPNOTSUPP;
2772	bio_put(bio);	2770	bio_put(bio);
2773	return ret;	2771	return ret;
2774	}	2772	}

fs/btrfs/volumes.c+6−12

@@ -345,7 +345,7 @@ loop_lock: · 344 unmodified lines
345	waitqueue_active(&fs_info->async_submit_wait))	345	waitqueue_active(&fs_info->async_submit_wait))
346	wake_up(&fs_info->async_submit_wait);	346	wake_up(&fs_info->async_submit_wait);
347		347
348	BUG_ON(atomic_read(&cur->bi_cnt) == 0);	348	BUG_ON(atomic_read(&cur->__bi_cnt) == 0);
349		349
350	/*	350	/*
351	* if we're doing the sync list, record that our	351	* if we're doing the sync list, record that our
@@ -5586,10 +5586,10 @@ int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree, · 5234 unmodified lines
5586		5586
5587	static inline void btrfs_end_bbio(struct btrfs_bio bbio, struct bio bio, int err)	5587	static inline void btrfs_end_bbio(struct btrfs_bio bbio, struct bio bio, int err)
5588	{	5588	{
5589	if (likely(bbio->flags & BTRFS_BIO_ORIG_BIO_SUBMITTED))	5589	bio->bi_private = bbio->private;
5590	bio_endio_nodec(bio, err);	5590	bio->bi_end_io = bbio->end_io;
5591	else	5591	bio_endio(bio, err);
5592	bio_endio(bio, err);	5592
5593	btrfs_put_bbio(bbio);	5593	btrfs_put_bbio(bbio);
5594	}	5594	}
5595		5595
@@ -5633,8 +5633,6 @@ static void btrfs_end_bio(struct bio *bio, int err) · 37 unmodified lines
5633	bio = bbio->orig_bio;	5633	bio = bbio->orig_bio;
5634	}	5634	}
5635		5635
5636	bio->bi_private = bbio->private;
5637	bio->bi_end_io = bbio->end_io;
5638	btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;	5636	btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
5639	/* only send an error to the higher layers if it is	5637	/* only send an error to the higher layers if it is
5640	* beyond the tolerance of the btrfs bio	5638	* beyond the tolerance of the btrfs bio
@@ -5816,8 +5814,6 @@ static void bbio_error(struct btrfs_bio bbio, struct bio bio, u64 logical) · 175 unmodified lines
5816	/* Shoud be the original bio. */	5814	/* Shoud be the original bio. */
5817	WARN_ON(bio != bbio->orig_bio);	5815	WARN_ON(bio != bbio->orig_bio);
5818		5816
5819	bio->bi_private = bbio->private;
5820	bio->bi_end_io = bbio->end_io;
5821	btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;	5817	btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
5822	bio->bi_iter.bi_sector = logical >> 9;	5818	bio->bi_iter.bi_sector = logical >> 9;
5823		5819
@@ -5898,10 +5894,8 @@ int btrfs_map_bio(struct btrfs_root root, int rw, struct bio bio, · 74 unmodified lines
5898	if (dev_nr < total_devs - 1) {	5894	if (dev_nr < total_devs - 1) {
5899	bio = btrfs_bio_clone(first_bio, GFP_NOFS);	5895	bio = btrfs_bio_clone(first_bio, GFP_NOFS);
5900	BUG_ON(!bio); /* -ENOMEM */	5896	BUG_ON(!bio); /* -ENOMEM */
5901	} else {	5897	} else
5902	bio = first_bio;	5898	bio = first_bio;
5903	bbio->flags \|= BTRFS_BIO_ORIG_BIO_SUBMITTED;
5904	}
5905		5899
5906	submit_stripe_bio(root, bbio, bio,	5900	submit_stripe_bio(root, bbio, bio,
5907	bbio->stripes[dev_nr].physical, dev_nr, rw,	5901	bbio->stripes[dev_nr].physical, dev_nr, rw,

fs/btrfs/volumes.h+0−2

@@ -292,8 +292,6 @@ struct btrfs_bio_stripe { · 291 unmodified lines
292	struct btrfs_bio;	292	struct btrfs_bio;
293	typedef void (btrfs_bio_end_io_t) (struct btrfs_bio *bio, int err);	293	typedef void (btrfs_bio_end_io_t) (struct btrfs_bio *bio, int err);
294		294
295	#define BTRFS_BIO_ORIG_BIO_SUBMITTED (1 << 0)
296
297	struct btrfs_bio {	295	struct btrfs_bio {
298	atomic_t refs;	296	atomic_t refs;
299	atomic_t stripes_pending;	297	atomic_t stripes_pending;

fs/buffer.c+1−12

@@ -2938,10 +2938,6 @@ static void end_bio_bh_io_sync(struct bio *bio, int err) · 2937 unmodified lines
2938	{	2938	{
2939	struct buffer_head *bh = bio->bi_private;	2939	struct buffer_head *bh = bio->bi_private;
2940		2940
2941	if (err == -EOPNOTSUPP) {
2942	set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2943	}
2944
2945	if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))	2941	if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2946	set_bit(BH_Quiet, &bh->b_state);	2942	set_bit(BH_Quiet, &bh->b_state);
2947		2943
@@ -3000,7 +2996,6 @@ void guard_bio_eod(int rw, struct bio *bio) · 52 unmodified lines
3000	int _submit_bh(int rw, struct buffer_head *bh, unsigned long bio_flags)	2996	int _submit_bh(int rw, struct buffer_head *bh, unsigned long bio_flags)
3001	{	2997	{
3002	struct bio *bio;	2998	struct bio *bio;
3003	int ret = 0;
3004		2999
3005	BUG_ON(!buffer_locked(bh));	3000	BUG_ON(!buffer_locked(bh));
3006	BUG_ON(!buffer_mapped(bh));	3001	BUG_ON(!buffer_mapped(bh));
@@ -3041,14 +3036,8 @@ int _submit_bh(int rw, struct buffer_head *bh, unsigned long bio_flags) · 34 unmodified lines
3041	if (buffer_prio(bh))	3036	if (buffer_prio(bh))
3042	rw \|= REQ_PRIO;	3037	rw \|= REQ_PRIO;
3043		3038
3044	bio_get(bio);
3045	submit_bio(rw, bio);	3039	submit_bio(rw, bio);
3046		3040	return 0;
3047	if (bio_flagged(bio, BIO_EOPNOTSUPP))
3048	ret = -EOPNOTSUPP;
3049
3050	bio_put(bio);
3051	return ret;
3052	}	3041	}
3053	EXPORT_SYMBOL_GPL(_submit_bh);	3042	EXPORT_SYMBOL_GPL(_submit_bh);
3054		3043

fs/ext4/page-io.c+0−1

@@ -359,7 +359,6 @@ void ext4_io_submit(struct ext4_io_submit *io) · 358 unmodified lines
359	if (bio) {	359	if (bio) {
360	bio_get(io->io_bio);	360	bio_get(io->io_bio);
361	submit_bio(io->io_op, io->io_bio);	361	submit_bio(io->io_op, io->io_bio);
362	BUG_ON(bio_flagged(io->io_bio, BIO_EOPNOTSUPP));
363	bio_put(io->io_bio);	362	bio_put(io->io_bio);
364	}	363	}
365	io->io_bio = NULL;	364	io->io_bio = NULL;

fs/nilfs2/segbuf.c+0−12

 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 	struct nilfs_segment_buffer *segbuf = bio->bi_private;
-	if (err == -EOPNOTSUPP) {
-		set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
-		/* to be detected by nilfs_segbuf_submit_bio() */
-	}
 	if (!uptodate)
 		atomic_inc(&segbuf->sb_err);
 	bio->bi_end_io = nilfs_end_bio_write;
 	bio->bi_private = segbuf;
-	bio_get(bio);
 	submit_bio(mode, bio);
 	segbuf->sb_nbio++;
-	if (bio_flagged(bio, BIO_EOPNOTSUPP)) {
-		bio_put(bio);
-		err = -EOPNOTSUPP;
-		goto failed;
-	}
-	bio_put(bio);
 	wi->bio = NULL;
 	wi->rest_blocks -= wi->end - wi->start;

fs/xfs/xfs_aops.c+0−1

@@ -356,7 +356,6 @@ xfs_end_bio( · 355 unmodified lines
356	{	356	{
357	xfs_ioend_t *ioend = bio->bi_private;	357	xfs_ioend_t *ioend = bio->bi_private;
358		358
359	ASSERT(atomic_read(&bio->bi_cnt) >= 1);
360	ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;	359	ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
361		360
362	/* Toss bio and pass work off to an xfsdatad thread */	361	/* Toss bio and pass work off to an xfsdatad thread */

include/linux/bio.h+15−2

  * returns. and then bio would be freed memory when if (bio->bi_flags ...)
  * runs
  */
-#define bio_get(bio)	atomic_inc(&(bio)->bi_cnt)
+static inline void bio_get(struct bio *bio)
+{
+	bio->bi_flags |= (1 << BIO_REFFED);
+	smp_mb__before_atomic();
+	atomic_inc(&bio->__bi_cnt);
+}
+static inline void bio_cnt_set(struct bio *bio, unsigned int count)
+{
+	if (count != 1) {
+		bio->bi_flags |= (1 << BIO_REFFED);
+		smp_mb__before_atomic();
+	}
+	atomic_set(&bio->__bi_cnt, count);
+}
 enum bip_flags {
 	BIP_BLOCK_INTEGRITY	= 1 << 0, /* block layer owns integrity data */
 }
 extern void bio_endio(struct bio *, int);
-extern void bio_endio_nodec(struct bio *, int);
 struct request_queue;
 extern int bio_phys_segments(struct request_queue *, struct bio *);

include/linux/blk-mq.h+4−0

 typedef void (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
 		bool);
+typedef void (busy_tag_iter_fn)(struct request *, void *, bool);
 struct blk_mq_ops {
 	/*
 struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
 		gfp_t gfp, bool reserved);
 struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag);
+struct cpumask *blk_mq_tags_cpumask(struct blk_mq_tags *tags);
 enum {
 	BLK_MQ_UNIQUE_TAG_BITS = 16,
 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
 void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx *hctx, busy_iter_fn *fn,
 		void *priv);
+void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn,
+		void *priv);
 void blk_mq_freeze_queue(struct request_queue *q);
 void blk_mq_unfreeze_queue(struct request_queue *q);
 void blk_mq_freeze_queue_start(struct request_queue *q);

include/linux/blk_types.h+13−12

 	unsigned int		bi_seg_front_size;
 	unsigned int		bi_seg_back_size;
-	atomic_t		bi_remaining;
+	atomic_t		__bi_remaining;
 	bio_end_io_t		*bi_end_io;
 	unsigned short		bi_max_vecs;	/* max bvl_vecs we can hold */
-	atomic_t		bi_cnt;		/* pin count */
+	atomic_t		__bi_cnt;	/* pin count */
 	struct bio_vec		*bi_io_vec;	/* the actual vec list */
  * bio flags
  */
 #define BIO_UPTODATE	0	/* ok after I/O completion */
-#define BIO_RW_BLOCK	1	/* RW_AHEAD set, and read/write would block */
+#define BIO_SEG_VALID	1	/* bi_phys_segments valid */
-#define BIO_EOF		2	/* out-out-bounds error */
+#define BIO_CLONED	2	/* doesn't own data */
-#define BIO_SEG_VALID	3	/* bi_phys_segments valid */
+#define BIO_BOUNCED	3	/* bio is a bounce bio */
-#define BIO_CLONED	4	/* doesn't own data */
+#define BIO_USER_MAPPED 4	/* contains user pages */
-#define BIO_BOUNCED	5	/* bio is a bounce bio */
+#define BIO_NULL_MAPPED 5	/* contains invalid user pages */
-#define BIO_USER_MAPPED 6	/* contains user pages */
+#define BIO_QUIET	6	/* Make BIO Quiet */
-#define BIO_EOPNOTSUPP	7	/* not supported */
+#define BIO_SNAP_STABLE	7	/* bio data must be snapshotted during write */
-#define BIO_NULL_MAPPED 8	/* contains invalid user pages */
+#define BIO_CHAIN	8	/* chained bio, ->bi_remaining in effect */
-#define BIO_QUIET	9	/* Make BIO Quiet */
+#define BIO_REFFED	9	/* bio has elevated ->bi_cnt */
-#define BIO_SNAP_STABLE	10	/* bio data must be snapshotted during write */
 /*
  * Flags starting here get preserved by bio_reset() - this includes
 	__REQ_HASHED,		/* on IO scheduler merge hash */
 	__REQ_MQ_INFLIGHT,	/* track inflight for MQ */
 	__REQ_NO_TIMEOUT,	/* requests may never expire */
+	__REQ_CLONE,		/* cloned bios */
 	__REQ_NR_BITS,		/* stops here */
 };
 #define REQ_HASHED		(1ULL << __REQ_HASHED)
 #define REQ_MQ_INFLIGHT		(1ULL << __REQ_MQ_INFLIGHT)
 #define REQ_NO_TIMEOUT		(1ULL << __REQ_NO_TIMEOUT)
+#define REQ_CLONE		(1ULL << __REQ_CLONE)
 #endif /* __LINUX_BLK_TYPES_H */

include/linux/blkdev.h+7−38

 struct request_queue;
 struct elevator_queue;
-struct request_pm_state;
 struct blk_trace;
 struct request;
 struct sg_io_hdr;
 enum rq_cmd_type_bits {
 	REQ_TYPE_FS		= 1,	/* fs request */
 	REQ_TYPE_BLOCK_PC,		/* scsi command */
-	REQ_TYPE_SENSE,			/* sense request */
+	REQ_TYPE_DRV_PRIV,		/* driver defined types from here */
-	REQ_TYPE_PM_SUSPEND,		/* suspend request */
-	REQ_TYPE_PM_RESUME,		/* resume request */
-	REQ_TYPE_PM_SHUTDOWN,		/* shutdown request */
-	REQ_TYPE_SPECIAL,		/* driver defined type */
-	/*
-	 * for ATA/ATAPI devices. this really doesn't belong here, ide should
-	 * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver
-	 * private REQ_LB opcodes to differentiate what type of request this is
-	 */
-	REQ_TYPE_ATA_TASKFILE,
-	REQ_TYPE_ATA_PC,
 };
 #define BLK_MAX_CDB	16
 	struct blk_mq_ctx *mq_ctx;
 	u64 cmd_flags;
-	enum rq_cmd_type_bits cmd_type;
+	unsigned cmd_type;
 	unsigned long atomic_flags;
 	int cpu;
 	return req->ioprio;
 }
-/*
- * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME
- * requests. Some step values could eventually be made generic.
- */
-struct request_pm_state
-{
-	/* PM state machine step value, currently driver specific */
-	int	pm_step;
-	/* requested PM state value (S1, S2, S3, S4, ...) */
-	u32	pm_state;
-	void*	data;		/* for driver use */
-};
 #include <linux/elevator.h>
 struct blk_queue_ctx;
 	struct mutex		sysfs_lock;
 	int			bypass_depth;
-	int			mq_freeze_depth;
+	atomic_t		mq_freeze_depth;
 #if defined(CONFIG_BLK_DEV_BSG)
 	bsg_job_fn		*bsg_job_fn;
 	(((rq)->cmd_flags & REQ_STARTED) && \
 	 ((rq)->cmd_type == REQ_TYPE_FS))
-#define blk_pm_request(rq)	\
-	((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \
-	 (rq)->cmd_type == REQ_TYPE_PM_RESUME)
 #define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
 #define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
 /* rq->queuelist of dequeued request must be list_empty() */
 		unsigned int len);
 extern int blk_rq_check_limits(struct request_queue *q, struct request *rq);
 extern int blk_lld_busy(struct request_queue *q);
-extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
+extern void blk_rq_prep_clone(struct request *rq, struct request *rq_src);
-			     struct bio_set *bs, gfp_t gfp_mask,
-			     int (*bio_ctr)(struct bio *, struct bio *, void *),
-			     void *data);
-extern void blk_rq_unprep_clone(struct request *rq);
 extern int blk_insert_cloned_request(struct request_queue *q,
 				     struct request *rq);
 extern void blk_delay_queue(struct request_queue *, unsigned long);
 extern void blk_sync_queue(struct request_queue *q);
 extern void __blk_stop_queue(struct request_queue *q);
 extern void __blk_run_queue(struct request_queue *q);
+extern void __blk_run_queue_uncond(struct request_queue *q);
 extern void blk_run_queue(struct request_queue *);
 extern void blk_run_queue_async(struct request_queue *q);
 extern int blk_rq_map_user(struct request_queue *, struct request *,
 	if (unlikely(rq->cmd_type == REQ_TYPE_BLOCK_PC))
 		return q->limits.max_hw_sectors;
-	if (!q->limits.chunk_sectors)
+	if (!q->limits.chunk_sectors || (rq->cmd_flags & REQ_DISCARD))
 		return blk_queue_get_max_sectors(q, rq->cmd_flags);
 	return min(blk_max_size_offset(q, blk_rq_pos(rq)),
 struct request_queue *blk_alloc_queue(gfp_t);
 struct request_queue *blk_alloc_queue_node(gfp_t, int);
 extern void blk_put_queue(struct request_queue *);
+extern void blk_set_queue_dying(struct request_queue *);
 /*
  * block layer runtime pm functions

include/linux/elevator.h+2−0

 typedef int (elevator_init_fn) (struct request_queue *,
 				struct elevator_type *e);
 typedef void (elevator_exit_fn) (struct elevator_queue *);
+typedef void (elevator_registered_fn) (struct request_queue *);
 struct elevator_ops
 {
 	elevator_init_fn *elevator_init_fn;
 	elevator_exit_fn *elevator_exit_fn;
+	elevator_registered_fn *elevator_registered_fn;
 };
 #define ELV_NAME_MAX	(16)

include/linux/fs.h+3−0

 extern struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode,
 					      void *holder);
 extern void blkdev_put(struct block_device *bdev, fmode_t mode);
+extern int __blkdev_reread_part(struct block_device *bdev);
+extern int blkdev_reread_part(struct block_device *bdev);
 #ifdef CONFIG_SYSFS
 extern int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk);
 extern void bd_unlink_disk_holder(struct block_device *bdev,

include/linux/ide.h+27−0

@@ -39,6 +39,19 @@ · 38 unmodified lines
39		39
40	struct device;	40	struct device;
41		41
		42	/* IDE-specific values for req->cmd_type */
		43	enum ata_cmd_type_bits {
		44	REQ_TYPE_ATA_TASKFILE = REQ_TYPE_DRV_PRIV + 1,
		45	REQ_TYPE_ATA_PC,
		46	REQ_TYPE_ATA_SENSE, /* sense request */
		47	REQ_TYPE_ATA_PM_SUSPEND,/* suspend request */
		48	REQ_TYPE_ATA_PM_RESUME, /* resume request */
		49	};
		50
		51	#define ata_pm_request(rq) \
		52	((rq)->cmd_type == REQ_TYPE_ATA_PM_SUSPEND \|\| \
		53	(rq)->cmd_type == REQ_TYPE_ATA_PM_RESUME)
		54
42	/* Error codes returned in rq->errors to the higher part of the driver. */	55	/* Error codes returned in rq->errors to the higher part of the driver. */
43	enum {	56	enum {
44	IDE_DRV_ERROR_GENERAL = 101,	57	IDE_DRV_ERROR_GENERAL = 101,
@@ -1314,6 +1327,19 @@ struct ide_port_info { · 1269 unmodified lines
1314	u8 udma_mask;	1327	u8 udma_mask;
1315	};	1328	};
1316		1329
		1330	/*
		1331	* State information carried for REQ_TYPE_ATA_PM_SUSPEND and REQ_TYPE_ATA_PM_RESUME
		1332	* requests.
		1333	*/
		1334	struct ide_pm_state {
		1335	/* PM state machine step value, currently driver specific */
		1336	int pm_step;
		1337	/* requested PM state value (S1, S2, S3, S4, ...) */
		1338	u32 pm_state;
		1339	void* data; /* for driver use */
		1340	};
		1341
		1342
1317	int ide_pci_init_one(struct pci_dev , const struct ide_port_info , void *);	1343	int ide_pci_init_one(struct pci_dev , const struct ide_port_info , void *);
1318	int ide_pci_init_two(struct pci_dev , struct pci_dev ,	1344	int ide_pci_init_two(struct pci_dev , struct pci_dev ,
1319	const struct ide_port_info , void );	1345	const struct ide_port_info , void );
@@ -1551,4 +1577,5 @@ static inline void ide_set_drivedata(ide_drive_t drive, void data) · 231 unmodified lines
1551	#define ide_host_for_each_port(i, port, host) \	1577	#define ide_host_for_each_port(i, port, host) \
1552	for ((i) = 0; ((port) = (host)->ports[i]) \|\| (i) < MAX_HOST_PORTS; (i)++)	1578	for ((i) = 0; ((port) = (host)->ports[i]) \|\| (i) < MAX_HOST_PORTS; (i)++)
1553		1579
		1580
1554	#endif /* _IDE_H */	1581	#endif /* _IDE_H */

include/linux/swap.h+0−1

@@ -377,7 +377,6 @@ extern void end_swap_bio_write(struct bio *bio, int err); · 376 unmodified lines
377	extern int __swap_writepage(struct page page, struct writeback_control wbc,	377	extern int __swap_writepage(struct page page, struct writeback_control wbc,
378	void (end_write_func)(struct bio , int));	378	void (end_write_func)(struct bio , int));
379	extern int swap_set_page_dirty(struct page *page);	379	extern int swap_set_page_dirty(struct page *page);
380	extern void end_swap_bio_read(struct bio *bio, int err);
381		380
382	int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,	381	int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
383	unsigned long nr_pages, sector_t start_block);	382	unsigned long nr_pages, sector_t start_block);

include/uapi/linux/nbd.h+0−2

@@ -44,8 +44,6 @@ enum { · 43 unmodified lines
44	/* there is a gap here to match userspace */	44	/* there is a gap here to match userspace */
45	#define NBD_FLAG_SEND_TRIM (1 << 5) /* send trim/discard */	45	#define NBD_FLAG_SEND_TRIM (1 << 5) /* send trim/discard */
46		46
47	#define nbd_cmd(req) ((req)->cmd[0])
48
49	/* userspace doesn't need the nbd_device structure */	47	/* userspace doesn't need the nbd_device structure */
50		48
51	/* These are sent over the network in the request/reply magic fields */	49	/* These are sent over the network in the request/reply magic fields */

kernel/power/Makefile+1−2

@@ -7,8 +7,7 @@ obj-$(CONFIG_VT_CONSOLE_SLEEP) += console.o · 6 unmodified lines
7	obj-$(CONFIG_FREEZER) += process.o	7	obj-$(CONFIG_FREEZER) += process.o
8	obj-$(CONFIG_SUSPEND) += suspend.o	8	obj-$(CONFIG_SUSPEND) += suspend.o
9	obj-$(CONFIG_PM_TEST_SUSPEND) += suspend_test.o	9	obj-$(CONFIG_PM_TEST_SUSPEND) += suspend_test.o
10	obj-$(CONFIG_HIBERNATION) += hibernate.o snapshot.o swap.o user.o \	10	obj-$(CONFIG_HIBERNATION) += hibernate.o snapshot.o swap.o user.o
11	block_io.o
12	obj-$(CONFIG_PM_AUTOSLEEP) += autosleep.o	11	obj-$(CONFIG_PM_AUTOSLEEP) += autosleep.o
13	obj-$(CONFIG_PM_WAKELOCKS) += wakelock.o	12	obj-$(CONFIG_PM_WAKELOCKS) += wakelock.o
14		13

kernel/power/block_io.c+0−103

@@ -1,103 +0,0 @@
1	/*
2	* This file provides functions for block I/O operations on swap/file.
3	*
4	* Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
5	* Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
6	*
7	* This file is released under the GPLv2.
8	*/
9
10	#include <linux/bio.h>
11	#include <linux/kernel.h>
12	#include <linux/pagemap.h>
13	#include <linux/swap.h>
14
15	#include "power.h"
16
17	/**
18	* submit - submit BIO request.
19	* @rw: READ or WRITE.
20	* @off physical offset of page.
21	* @page: page we're reading or writing.
22	* @bio_chain: list of pending biod (for async reading)
23	*
24	* Straight from the textbook - allocate and initialize the bio.
25	* If we're reading, make sure the page is marked as dirty.
26	* Then submit it and, if @bio_chain == NULL, wait.
27	*/
28	static int submit(int rw, struct block_device *bdev, sector_t sector,
29	struct page page, struct bio *bio_chain)
30	{
31	const int bio_rw = rw \| REQ_SYNC;
32	struct bio *bio;
33
34	bio = bio_alloc(__GFP_WAIT \| __GFP_HIGH, 1);
35	bio->bi_iter.bi_sector = sector;
36	bio->bi_bdev = bdev;
37	bio->bi_end_io = end_swap_bio_read;
38
39	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
40	printk(KERN_ERR "PM: Adding page to bio failed at %llu\n",
41	(unsigned long long)sector);
42	bio_put(bio);
43	return -EFAULT;
44	}
45
46	lock_page(page);
47	bio_get(bio);
48
49	if (bio_chain == NULL) {
50	submit_bio(bio_rw, bio);
51	wait_on_page_locked(page);
52	if (rw == READ)
53	bio_set_pages_dirty(bio);
54	bio_put(bio);
55	} else {
56	if (rw == READ)
57	get_page(page); /* These pages are freed later */
58	bio->bi_private = *bio_chain;
59	*bio_chain = bio;
60	submit_bio(bio_rw, bio);
61	}
62	return 0;
63	}
64
65	int hib_bio_read_page(pgoff_t page_off, void addr, struct bio *bio_chain)
66	{
67	return submit(READ, hib_resume_bdev, page_off * (PAGE_SIZE >> 9),
68	virt_to_page(addr), bio_chain);
69	}
70
71	int hib_bio_write_page(pgoff_t page_off, void addr, struct bio *bio_chain)
72	{
73	return submit(WRITE, hib_resume_bdev, page_off * (PAGE_SIZE >> 9),
74	virt_to_page(addr), bio_chain);
75	}
76
77	int hib_wait_on_bio_chain(struct bio **bio_chain)
78	{
79	struct bio *bio;
80	struct bio *next_bio;
81	int ret = 0;
82
83	if (bio_chain == NULL)
84	return 0;
85
86	bio = *bio_chain;
87	if (bio == NULL)
88	return 0;
89	while (bio) {
90	struct page *page;
91
92	next_bio = bio->bi_private;
93	page = bio->bi_io_vec[0].bv_page;
94	wait_on_page_locked(page);
95	if (!PageUptodate(page) \|\| PageError(page))
96	ret = -EIO;
97	put_page(page);
98	bio_put(bio);
99	bio = next_bio;
100	}
101	*bio_chain = NULL;
102	return ret;
103	}

kernel/power/power.h+0−9

@@ -163,15 +163,6 @@ extern void swsusp_close(fmode_t); · 162 unmodified lines
163	extern int swsusp_unmark(void);	163	extern int swsusp_unmark(void);
164	#endif	164	#endif
165		165
166	/* kernel/power/block_io.c */
167	extern struct block_device *hib_resume_bdev;
168
169	extern int hib_bio_read_page(pgoff_t page_off, void *addr,
170	struct bio **bio_chain);
171	extern int hib_bio_write_page(pgoff_t page_off, void *addr,
172	struct bio **bio_chain);
173	extern int hib_wait_on_bio_chain(struct bio **bio_chain);
174
175	struct timeval;	166	struct timeval;
176	/* kernel/power/swsusp.c */	167	/* kernel/power/swsusp.c */
177	extern void swsusp_show_speed(ktime_t, ktime_t, unsigned int, char *);	168	extern void swsusp_show_speed(ktime_t, ktime_t, unsigned int, char *);

kernel/power/swap.c+120−39

  */
 static unsigned short root_swap = 0xffff;
-struct block_device *hib_resume_bdev;
+static struct block_device *hib_resume_bdev;
+struct hib_bio_batch {
+	atomic_t		count;
+	wait_queue_head_t	wait;
+	int			error;
+};
+static void hib_init_batch(struct hib_bio_batch *hb)
+{
+	atomic_set(&hb->count, 0);
+	init_waitqueue_head(&hb->wait);
+	hb->error = 0;
+}
+static void hib_end_io(struct bio *bio, int error)
+{
+	struct hib_bio_batch *hb = bio->bi_private;
+	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+	struct page *page = bio->bi_io_vec[0].bv_page;
+	if (!uptodate || error) {
+		printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n",
+				imajor(bio->bi_bdev->bd_inode),
+				iminor(bio->bi_bdev->bd_inode),
+				(unsigned long long)bio->bi_iter.bi_sector);
+		if (!error)
+			error = -EIO;
+	}
+	if (bio_data_dir(bio) == WRITE)
+		put_page(page);
+	if (error && !hb->error)
+		hb->error = error;
+	if (atomic_dec_and_test(&hb->count))
+		wake_up(&hb->wait);
+	bio_put(bio);
+}
+static int hib_submit_io(int rw, pgoff_t page_off, void *addr,
+		struct hib_bio_batch *hb)
+{
+	struct page *page = virt_to_page(addr);
+	struct bio *bio;
+	int error = 0;
+	bio = bio_alloc(__GFP_WAIT | __GFP_HIGH, 1);
+	bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9);
+	bio->bi_bdev = hib_resume_bdev;
+	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
+		printk(KERN_ERR "PM: Adding page to bio failed at %llu\n",
+			(unsigned long long)bio->bi_iter.bi_sector);
+		bio_put(bio);
+		return -EFAULT;
+	}
+	if (hb) {
+		bio->bi_end_io = hib_end_io;
+		bio->bi_private = hb;
+		atomic_inc(&hb->count);
+		submit_bio(rw, bio);
+	} else {
+		error = submit_bio_wait(rw, bio);
+		bio_put(bio);
+	}
+	return error;
+}
+static int hib_wait_io(struct hib_bio_batch *hb)
+{
+	wait_event(hb->wait, atomic_read(&hb->count) == 0);
+	return hb->error;
+}
 /*
  * Saving part
 {
 	int error;
-	hib_bio_read_page(swsusp_resume_block, swsusp_header, NULL);
+	hib_submit_io(READ_SYNC, swsusp_resume_block, swsusp_header, NULL);
 	if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
 	    !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
 		memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
 		swsusp_header->flags = flags;
 		if (flags & SF_CRC32_MODE)
 			swsusp_header->crc32 = handle->crc32;
-		error = hib_bio_write_page(swsusp_resume_block,
+		error = hib_submit_io(WRITE_SYNC, swsusp_resume_block,
 					swsusp_header, NULL);
 	} else {
 		printk(KERN_ERR "PM: Swap header not found!\n");
  *	write_page - Write one page to given swap location.
  *	@buf:		Address we're writing.
  *	@offset:	Offset of the swap page we're writing to.
- *	@bio_chain:	Link the next write BIO here
+ *	@hb:		bio completion batch
  */
-static int write_page(void *buf, sector_t offset, struct bio **bio_chain)
+static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb)
 {
 	void *src;
 	int ret;
 	if (!offset)
 		return -ENOSPC;
-	if (bio_chain) {
+	if (hb) {
 		src = (void *)__get_free_page(__GFP_WAIT | __GFP_NOWARN |
 		                              __GFP_NORETRY);
 		if (src) {
 			copy_page(src, buf);
 		} else {
-			ret = hib_wait_on_bio_chain(bio_chain); /* Free pages */
+			ret = hib_wait_io(hb); /* Free pages */
 			if (ret)
 				return ret;
 			src = (void *)__get_free_page(__GFP_WAIT |
 				copy_page(src, buf);
 			} else {
 				WARN_ON_ONCE(1);
-				bio_chain = NULL;	/* Go synchronous */
+				hb = NULL;	/* Go synchronous */
 				src = buf;
 			}
 		}
 	} else {
 		src = buf;
 	}
-	return hib_bio_write_page(offset, src, bio_chain);
+	return hib_submit_io(WRITE_SYNC, offset, src, hb);
 }
 static void release_swap_writer(struct swap_map_handle *handle)
 }
 static int swap_write_page(struct swap_map_handle *handle, void *buf,
-				struct bio **bio_chain)
+		struct hib_bio_batch *hb)
 {
 	int error = 0;
 	sector_t offset;
 	if (!handle->cur)
 		return -EINVAL;
 	offset = alloc_swapdev_block(root_swap);
-	error = write_page(buf, offset, bio_chain);
+	error = write_page(buf, offset, hb);
 	if (error)
 		return error;
 	handle->cur->entries[handle->k++] = offset;
 		if (!offset)
 			return -ENOSPC;
 		handle->cur->next_swap = offset;
-		error = write_page(handle->cur, handle->cur_swap, bio_chain);
+		error = write_page(handle->cur, handle->cur_swap, hb);
 		if (error)
 			goto out;
 		clear_page(handle->cur);
 		handle->cur_swap = offset;
 		handle->k = 0;
-		if (bio_chain && low_free_pages() <= handle->reqd_free_pages) {
+		if (hb && low_free_pages() <= handle->reqd_free_pages) {
-			error = hib_wait_on_bio_chain(bio_chain);
+			error = hib_wait_io(hb);
 			if (error)
 				goto out;
 			/*
 	int ret;
 	int nr_pages;
 	int err2;
-	struct bio *bio;
+	struct hib_bio_batch hb;
 	ktime_t start;
 	ktime_t stop;
+	hib_init_batch(&hb);
 	printk(KERN_INFO "PM: Saving image data pages (%u pages)...\n",
 		nr_to_write);
 	m = nr_to_write / 10;
 	if (!m)
 		m = 1;
 	nr_pages = 0;
-	bio = NULL;
 	start = ktime_get();
 	while (1) {
 		ret = snapshot_read_next(snapshot);
 		if (ret <= 0)
 			break;
-		ret = swap_write_page(handle, data_of(*snapshot), &bio);
+		ret = swap_write_page(handle, data_of(*snapshot), &hb);
 		if (ret)
 			break;
 		if (!(nr_pages % m))
 			       nr_pages / m * 10);
 		nr_pages++;
 	}
-	err2 = hib_wait_on_bio_chain(&bio);
+	err2 = hib_wait_io(&hb);
 	stop = ktime_get();
 	if (!ret)
 		ret = err2;
 	int ret = 0;
 	int nr_pages;
 	int err2;
-	struct bio *bio;
+	struct hib_bio_batch hb;
 	ktime_t start;
 	ktime_t stop;
 	size_t off;
 	struct cmp_data *data = NULL;
 	struct crc_data *crc = NULL;
+	hib_init_batch(&hb);
 	/*
 	 * We'll limit the number of threads for compression to limit memory
 	 * footprint.
 	if (!m)
 		m = 1;
 	nr_pages = 0;
-	bio = NULL;
 	start = ktime_get();
 	for (;;) {
 		for (thr = 0; thr < nr_threads; thr++) {
 			     off += PAGE_SIZE) {
 				memcpy(page, data[thr].cmp + off, PAGE_SIZE);
-				ret = swap_write_page(handle, page, &bio);
+				ret = swap_write_page(handle, page, &hb);
 				if (ret)
 					goto out_finish;
 			}
 	}
 out_finish:
-	err2 = hib_wait_on_bio_chain(&bio);
+	err2 = hib_wait_io(&hb);
 	stop = ktime_get();
 	if (!ret)
 		ret = err2;
 			return -ENOMEM;
 		}
-		error = hib_bio_read_page(offset, tmp->map, NULL);
+		error = hib_submit_io(READ_SYNC, offset, tmp->map, NULL);
 		if (error) {
 			release_swap_reader(handle);
 			return error;
 }
 static int swap_read_page(struct swap_map_handle *handle, void *buf,
-				struct bio **bio_chain)
+		struct hib_bio_batch *hb)
 {
 	sector_t offset;
 	int error;
 	offset = handle->cur->entries[handle->k];
 	if (!offset)
 		return -EFAULT;
-	error = hib_bio_read_page(offset, buf, bio_chain);
+	error = hib_submit_io(READ_SYNC, offset, buf, hb);
 	if (error)
 		return error;
 	if (++handle->k >= MAP_PAGE_ENTRIES) {
 	int ret = 0;
 	ktime_t start;
 	ktime_t stop;
-	struct bio *bio;
+	struct hib_bio_batch hb;
 	int err2;
 	unsigned nr_pages;
+	hib_init_batch(&hb);
 	printk(KERN_INFO "PM: Loading image data pages (%u pages)...\n",
 		nr_to_read);
 	m = nr_to_read / 10;
 	if (!m)
 		m = 1;
 	nr_pages = 0;
-	bio = NULL;
 	start = ktime_get();
 	for ( ; ; ) {
 		ret = snapshot_write_next(snapshot);
 		if (ret <= 0)
 			break;
-		ret = swap_read_page(handle, data_of(*snapshot), &bio);
+		ret = swap_read_page(handle, data_of(*snapshot), &hb);
 		if (ret)
 			break;
 		if (snapshot->sync_read)
-			ret = hib_wait_on_bio_chain(&bio);
+			ret = hib_wait_io(&hb);
 		if (ret)
 			break;
 		if (!(nr_pages % m))
 			       nr_pages / m * 10);
 		nr_pages++;
 	}
-	err2 = hib_wait_on_bio_chain(&bio);
+	err2 = hib_wait_io(&hb);
 	stop = ktime_get();
 	if (!ret)
 		ret = err2;
 	unsigned int m;
 	int ret = 0;
 	int eof = 0;
-	struct bio *bio;
+	struct hib_bio_batch hb;
 	ktime_t start;
 	ktime_t stop;
 	unsigned nr_pages;
 	struct dec_data *data = NULL;
 	struct crc_data *crc = NULL;
+	hib_init_batch(&hb);
 	/*
 	 * We'll limit the number of threads for decompression to limit memory
 	 * footprint.
 	if (!m)
 		m = 1;
 	nr_pages = 0;
-	bio = NULL;
 	start = ktime_get();
 	ret = snapshot_write_next(snapshot);
 	for(;;) {
 		for (i = 0; !eof && i < want; i++) {
-			ret = swap_read_page(handle, page[ring], &bio);
+			ret = swap_read_page(handle, page[ring], &hb);
 			if (ret) {
 				/*
 				 * On real read error, finish. On end of data,
 			if (!asked)
 				break;
-			ret = hib_wait_on_bio_chain(&bio);
+			ret = hib_wait_io(&hb);
 			if (ret)
 				goto out_finish;
 			have += asked;
 		 * Wait for more data while we are decompressing.
 		 */
 		if (have < LZO_CMP_PAGES && asked) {
-			ret = hib_wait_on_bio_chain(&bio);
+			ret = hib_wait_io(&hb);
 			if (ret)
 				goto out_finish;
 			have += asked;
 	if (!IS_ERR(hib_resume_bdev)) {
 		set_blocksize(hib_resume_bdev, PAGE_SIZE);
 		clear_page(swsusp_header);
-		error = hib_bio_read_page(swsusp_resume_block,
+		error = hib_submit_io(READ_SYNC, swsusp_resume_block,
 					swsusp_header, NULL);
 		if (error)
 			goto put;
 		if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
 			memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
 			/* Reset swap signature now */
-			error = hib_bio_write_page(swsusp_resume_block,
+			error = hib_submit_io(WRITE_SYNC, swsusp_resume_block,
 						swsusp_header, NULL);
 		} else {
 			error = -EINVAL;
 {
 	int error;
-	hib_bio_read_page(swsusp_resume_block, swsusp_header, NULL);
+	hib_submit_io(READ_SYNC, swsusp_resume_block, swsusp_header, NULL);
 	if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) {
 		memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10);
-		error = hib_bio_write_page(swsusp_resume_block,
+		error = hib_submit_io(WRITE_SYNC, swsusp_resume_block,
 					swsusp_header, NULL);
 	} else {
 		printk(KERN_ERR "PM: Cannot find swsusp signature!\n");

mm/page_io.c+1−1

@@ -69,7 +69,7 @@ void end_swap_bio_write(struct bio *bio, int err) · 68 unmodified lines
69	bio_put(bio);	69	bio_put(bio);
70	}	70	}
71		71
72	void end_swap_bio_read(struct bio *bio, int err)	72	static void end_swap_bio_read(struct bio *bio, int err)
73	{	73	{
74	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);	74	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
75	struct page *page = bio->bi_io_vec[0].bv_page;	75	struct page *page = bio->bi_io_vec[0].bv_page;