lael-labs/linux

Documentation/block/biovecs.rst+2−0

@@ -40,6 +40,8 @@ normal code doesn't have to deal with bi_bvec_done. · 39 unmodified lines
40	There is a lower level advance function - bvec_iter_advance() - which takes	40	There is a lower level advance function - bvec_iter_advance() - which takes
41	a pointer to a biovec, not a bio; this is used by the bio integrity code.	41	a pointer to a biovec, not a bio; this is used by the bio integrity code.
42		42
		43	As of 5.12 bvec segments with zero bv_len are not supported.
		44
43	What's all this get us?	45	What's all this get us?
44	=======================	46	=======================
45		47

Documentation/block/queue-sysfs.rst+13−0

@@ -261,6 +261,12 @@ For block drivers that support REQ_OP_WRITE_ZEROES, the maximum number of · 260 unmodified lines
261	bytes that can be zeroed at once. The value 0 means that REQ_OP_WRITE_ZEROES	261	bytes that can be zeroed at once. The value 0 means that REQ_OP_WRITE_ZEROES
262	is not supported.	262	is not supported.
263		263
		264	zone_append_max_bytes (RO)
		265	--------------------------
		266	This is the maximum number of bytes that can be written to a sequential
		267	zone of a zoned block device using a zone append write operation
		268	(REQ_OP_ZONE_APPEND). This value is always 0 for regular block devices.
		269
264	zoned (RO)	270	zoned (RO)
265	----------	271	----------
266	This indicates if the device is a zoned block device and the zone model of the	272	This indicates if the device is a zoned block device and the zone model of the
@@ -273,4 +279,11 @@ devices are described in the ZBC (Zoned Block Commands) and ZAC · 6 unmodified lines
273	do not support zone commands, they will be treated as regular block devices	279	do not support zone commands, they will be treated as regular block devices
274	and zoned will report "none".	280	and zoned will report "none".
275		281
		282	zone_write_granularity (RO)
		283	---------------------------
		284	This indicates the alignment constraint, in bytes, for write operations in
		285	sequential zones of zoned block devices (devices with a zoned attributed
		286	that reports "host-managed" or "host-aware"). This value is always 0 for
		287	regular block devices.
		288
276	Jens Axboe <jens.axboe@oracle.com>, February 2009	289	Jens Axboe <jens.axboe@oracle.com>, February 2009

Documentation/filesystems/f2fs.rst+0−1

@@ -179,7 +179,6 @@ fault_type=%d Support configuring fault injection type, should be · 178 unmodified lines
179	FAULT_KVMALLOC 0x000000002	179	FAULT_KVMALLOC 0x000000002
180	FAULT_PAGE_ALLOC 0x000000004	180	FAULT_PAGE_ALLOC 0x000000004
181	FAULT_PAGE_GET 0x000000008	181	FAULT_PAGE_GET 0x000000008
182	FAULT_ALLOC_BIO 0x000000010
183	FAULT_ALLOC_NID 0x000000020	182	FAULT_ALLOC_NID 0x000000020
184	FAULT_ORPHAN 0x000000040	183	FAULT_ORPHAN 0x000000040
185	FAULT_BLOCK 0x000000080	184	FAULT_BLOCK 0x000000080

Documentation/filesystems/porting.rst+16−0

@@ -865,3 +865,19 @@ no matter what. Everything is handled by the caller. · 864 unmodified lines
865		865
866	clone_private_mount() returns a longterm mount now, so the proper destructor of	866	clone_private_mount() returns a longterm mount now, so the proper destructor of
867	its result is kern_unmount() or kern_unmount_array().	867	its result is kern_unmount() or kern_unmount_array().
		868
		869	---
		870
		871	mandatory
		872
		873	zero-length bvec segments are disallowed, they must be filtered out before
		874	passed on to an iterator.
		875
		876	---
		877
		878	mandatory
		879
		880	For bvec based itererators bio_iov_iter_get_pages() now doesn't copy bvecs but
		881	uses the one provided. Anyone issuing kiocb-I/O should ensure that the bvec and
		882	page references stay until I/O has completed, i.e. until ->ki_complete() has
		883	been called or returned with non -EIOCBQUEUED code.

arch/m68k/emu/nfblock.c+1−1

@@ -61,7 +61,7 @@ struct nfhd_device { · 60 unmodified lines
61		61
62	static blk_qc_t nfhd_submit_bio(struct bio *bio)	62	static blk_qc_t nfhd_submit_bio(struct bio *bio)
63	{	63	{
64	struct nfhd_device *dev = bio->bi_disk->private_data;	64	struct nfhd_device *dev = bio->bi_bdev->bd_disk->private_data;
65	struct bio_vec bvec;	65	struct bio_vec bvec;
66	struct bvec_iter iter;	66	struct bvec_iter iter;
67	int dir, len, shift;	67	int dir, len, shift;

arch/xtensa/platforms/iss/simdisk.c+1−1

@@ -103,7 +103,7 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector, · 102 unmodified lines
103		103
104	static blk_qc_t simdisk_submit_bio(struct bio *bio)	104	static blk_qc_t simdisk_submit_bio(struct bio *bio)
105	{	105	{
106	struct simdisk *dev = bio->bi_disk->private_data;	106	struct simdisk *dev = bio->bi_bdev->bd_disk->private_data;
107	struct bio_vec bvec;	107	struct bio_vec bvec;
108	struct bvec_iter iter;	108	struct bvec_iter iter;
109	sector_t sector = bio->bi_iter.bi_sector;	109	sector_t sector = bio->bi_iter.bi_sector;

block/bfq-iosched.c+276−169

 BFQ_BFQQ_FNS(coop);
 BFQ_BFQQ_FNS(split_coop);
 BFQ_BFQQ_FNS(softrt_update);
-BFQ_BFQQ_FNS(has_waker);
 #undef BFQ_BFQQ_FNS						\
 /* Expiration time of sync (0) and async (1) requests, in ns. */
 	else
 		bfq_clear_bfqq_IO_bound(bfqq);
+	bfqq->last_serv_time_ns = bic->saved_last_serv_time_ns;
+	bfqq->inject_limit = bic->saved_inject_limit;
+	bfqq->decrease_time_jif = bic->saved_decrease_time_jif;
 	bfqq->entity.new_weight = bic->saved_weight;
 	bfqq->ttime = bic->saved_ttime;
+	bfqq->io_start_time = bic->saved_io_start_time;
+	bfqq->tot_idle_time = bic->saved_tot_idle_time;
 	bfqq->wr_coeff = bic->saved_wr_coeff;
+	bfqq->service_from_wr = bic->saved_service_from_wr;
 	bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt;
 	bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish;
 	bfqq->wr_cur_max_time = bic->saved_wr_cur_max_time;
 	return bfqq_weight > in_serv_weight;
 }
+static bool bfq_better_to_idle(struct bfq_queue *bfqq);
 static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd,
 					     struct bfq_queue *bfqq,
 					     int old_wr_coeff,
 	 * - it is sync,
 	 * - it does not belong to a large burst,
 	 * - it has been idle for enough time or is soft real-time,
-	 * - is linked to a bfq_io_cq (it is not shared in any sense).
+	 * - is linked to a bfq_io_cq (it is not shared in any sense),
+	 * - has a default weight (otherwise we assume the user wanted
+	 *   to control its weight explicitly)
 	 */
 	in_burst = bfq_bfqq_in_large_burst(bfqq);
 	soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 &&
 		!BFQQ_TOTALLY_SEEKY(bfqq) &&
 		!in_burst &&
 		time_is_before_jiffies(bfqq->soft_rt_next_start) &&
-		bfqq->dispatched == 0;
+		bfqq->dispatched == 0 &&
-	*interactive = !in_burst && idle_for_long_time;
+		bfqq->entity.new_weight == 40;
+	*interactive = !in_burst && idle_for_long_time &&
+		bfqq->entity.new_weight == 40;
 	wr_or_deserves_wr = bfqd->low_latency &&
 		(bfqq->wr_coeff > 1 ||
 		 (bfq_bfqq_sync(bfqq) &&
 	bfq_clear_bfqq_just_created(bfqq);
-	if (!bfq_bfqq_IO_bound(bfqq)) {
-		if (arrived_in_time) {
-			bfqq->requests_within_timer++;
-			if (bfqq->requests_within_timer >=
-			    bfqd->bfq_requests_within_timer)
-				bfq_mark_bfqq_IO_bound(bfqq);
-		} else
-			bfqq->requests_within_timer = 0;
-	}
 	if (bfqd->low_latency) {
 		if (unlikely(time_is_after_jiffies(bfqq->split_time)))
 			/* wraparound */
 	bfq_add_bfqq_busy(bfqd, bfqq);
 	/*
-	 * Expire in-service queue only if preemption may be needed
+	 * Expire in-service queue if preemption may be needed for
-	 * for guarantees. In particular, we care only about two
+	 * guarantees or throughput. As for guarantees, we care
-	 * cases. The first is that bfqq has to recover a service
+	 * explicitly about two cases. The first is that bfqq has to
-	 * hole, as explained in the comments on
+	 * recover a service hole, as explained in the comments on
 	 * bfq_bfqq_update_budg_for_activation(), i.e., that
 	 * bfqq_wants_to_preempt is true. However, if bfqq does not
 	 * carry time-critical I/O, then bfqq's bandwidth is less
 	 * timestamps of the in-service queue would need to be
 	 * updated, and this operation is quite costly (see the
 	 * comments on bfq_bfqq_update_budg_for_activation()).
+	 *
+	 * As for throughput, we ask bfq_better_to_idle() whether we
+	 * still need to plug I/O dispatching. If bfq_better_to_idle()
+	 * says no, then plugging is not needed any longer, either to
+	 * boost throughput or to perserve service guarantees. Then
+	 * the best option is to stop plugging I/O, as not doing so
+	 * would certainly lower throughput. We may end up in this
+	 * case if: (1) upon a dispatch attempt, we detected that it
+	 * was better to plug I/O dispatch, and to wait for a new
+	 * request to arrive for the currently in-service queue, but
+	 * (2) this switch of bfqq to busy changes the scenario.
 	 */
 	if (bfqd->in_service_queue &&
 	    ((bfqq_wants_to_preempt &&
 	      bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) ||
-	     bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) &&
+	     bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue) ||
+	     !bfq_better_to_idle(bfqd->in_service_queue)) &&
 	    next_queue_may_preempt(bfqd))
 		bfq_bfqq_expire(bfqd, bfqd->in_service_queue,
 				false, BFQQE_PREEMPTED);
 	bfqq->decrease_time_jif = jiffies;
 }
+static void bfq_update_io_intensity(struct bfq_queue *bfqq, u64 now_ns)
+{
+	u64 tot_io_time = now_ns - bfqq->io_start_time;
+	if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq->dispatched == 0)
+		bfqq->tot_idle_time +=
+			now_ns - bfqq->ttime.last_end_request;
+	if (unlikely(bfq_bfqq_just_created(bfqq)))
+		return;
+	/*
+	 * Must be busy for at least about 80% of the time to be
+	 * considered I/O bound.
+	 */
+	if (bfqq->tot_idle_time * 5 > tot_io_time)
+		bfq_clear_bfqq_IO_bound(bfqq);
+	else
+		bfq_mark_bfqq_IO_bound(bfqq);
+	/*
+	 * Keep an observation window of at most 200 ms in the past
+	 * from now.
+	 */
+	if (tot_io_time > 200 * NSEC_PER_MSEC) {
+		bfqq->io_start_time = now_ns - (tot_io_time>>1);
+		bfqq->tot_idle_time >>= 1;
+	}
+}
+/*
+ * Detect whether bfqq's I/O seems synchronized with that of some
+ * other queue, i.e., whether bfqq, after remaining empty, happens to
+ * receive new I/O only right after some I/O request of the other
+ * queue has been completed. We call waker queue the other queue, and
+ * we assume, for simplicity, that bfqq may have at most one waker
+ * queue.
+ *
+ * A remarkable throughput boost can be reached by unconditionally
+ * injecting the I/O of the waker queue, every time a new
+ * bfq_dispatch_request happens to be invoked while I/O is being
+ * plugged for bfqq.  In addition to boosting throughput, this
+ * unblocks bfqq's I/O, thereby improving bandwidth and latency for
+ * bfqq. Note that these same results may be achieved with the general
+ * injection mechanism, but less effectively. For details on this
+ * aspect, see the comments on the choice of the queue for injection
+ * in bfq_select_queue().
+ *
+ * Turning back to the detection of a waker queue, a queue Q is deemed
+ * as a waker queue for bfqq if, for three consecutive times, bfqq
+ * happens to become non empty right after a request of Q has been
+ * completed. In particular, on the first time, Q is tentatively set
+ * as a candidate waker queue, while on the third consecutive time
+ * that Q is detected, the field waker_bfqq is set to Q, to confirm
+ * that Q is a waker queue for bfqq. These detection steps are
+ * performed only if bfqq has a long think time, so as to make it more
+ * likely that bfqq's I/O is actually being blocked by a
+ * synchronization. This last filter, plus the above three-times
+ * requirement, make false positives less likely.
+ *
+ * NOTE
+ *
+ * The sooner a waker queue is detected, the sooner throughput can be
+ * boosted by injecting I/O from the waker queue. Fortunately,
+ * detection is likely to be actually fast, for the following
+ * reasons. While blocked by synchronization, bfqq has a long think
+ * time. This implies that bfqq's inject limit is at least equal to 1
+ * (see the comments in bfq_update_inject_limit()). So, thanks to
+ * injection, the waker queue is likely to be served during the very
+ * first I/O-plugging time interval for bfqq. This triggers the first
+ * step of the detection mechanism. Thanks again to injection, the
+ * candidate waker queue is then likely to be confirmed no later than
+ * during the next I/O-plugging interval for bfqq.
+ *
+ * ISSUE
+ *
+ * On queue merging all waker information is lost.
+ */
+static void bfq_check_waker(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+			    u64 now_ns)
+{
+	if (!bfqd->last_completed_rq_bfqq ||
+	    bfqd->last_completed_rq_bfqq == bfqq ||
+	    bfq_bfqq_has_short_ttime(bfqq) ||
+	    now_ns - bfqd->last_completion >= 4 * NSEC_PER_MSEC ||
+	    bfqd->last_completed_rq_bfqq == bfqq->waker_bfqq)
+		return;
+	if (bfqd->last_completed_rq_bfqq !=
+	    bfqq->tentative_waker_bfqq) {
+		/*
+		 * First synchronization detected with a
+		 * candidate waker queue, or with a different
+		 * candidate waker queue from the current one.
+		 */
+		bfqq->tentative_waker_bfqq =
+			bfqd->last_completed_rq_bfqq;
+		bfqq->num_waker_detections = 1;
+	} else /* Same tentative waker queue detected again */
+		bfqq->num_waker_detections++;
+	if (bfqq->num_waker_detections == 3) {
+		bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq;
+		bfqq->tentative_waker_bfqq = NULL;
+		/*
+		 * If the waker queue disappears, then
+		 * bfqq->waker_bfqq must be reset. To
+		 * this goal, we maintain in each
+		 * waker queue a list, woken_list, of
+		 * all the queues that reference the
+		 * waker queue through their
+		 * waker_bfqq pointer. When the waker
+		 * queue exits, the waker_bfqq pointer
+		 * of all the queues in the woken_list
+		 * is reset.
+		 *
+		 * In addition, if bfqq is already in
+		 * the woken_list of a waker queue,
+		 * then, before being inserted into
+		 * the woken_list of a new waker
+		 * queue, bfqq must be removed from
+		 * the woken_list of the old waker
+		 * queue.
+		 */
+		if (!hlist_unhashed(&bfqq->woken_list_node))
+			hlist_del_init(&bfqq->woken_list_node);
+		hlist_add_head(&bfqq->woken_list_node,
+			       &bfqd->last_completed_rq_bfqq->woken_list);
+	}
+}
 static void bfq_add_request(struct request *rq)
 {
 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
 	struct request *next_rq, *prev;
 	unsigned int old_wr_coeff = bfqq->wr_coeff;
 	bool interactive = false;
+	u64 now_ns = ktime_get_ns();
 	bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq));
 	bfqq->queued[rq_is_sync(rq)]++;
 	bfqd->queued++;
 	if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) {
-		/*
+		bfq_check_waker(bfqd, bfqq, now_ns);
-		 * Detect whether bfqq's I/O seems synchronized with
-		 * that of some other queue, i.e., whether bfqq, after
-		 * remaining empty, happens to receive new I/O only
-		 * right after some I/O request of the other queue has
-		 * been completed. We call waker queue the other
-		 * queue, and we assume, for simplicity, that bfqq may
-		 * have at most one waker queue.
-		 *
-		 * A remarkable throughput boost can be reached by
-		 * unconditionally injecting the I/O of the waker
-		 * queue, every time a new bfq_dispatch_request
-		 * happens to be invoked while I/O is being plugged
-		 * for bfqq.  In addition to boosting throughput, this
-		 * unblocks bfqq's I/O, thereby improving bandwidth
-		 * and latency for bfqq. Note that these same results
-		 * may be achieved with the general injection
-		 * mechanism, but less effectively. For details on
-		 * this aspect, see the comments on the choice of the
-		 * queue for injection in bfq_select_queue().
-		 *
-		 * Turning back to the detection of a waker queue, a
-		 * queue Q is deemed as a waker queue for bfqq if, for
-		 * two consecutive times, bfqq happens to become non
-		 * empty right after a request of Q has been
-		 * completed. In particular, on the first time, Q is
-		 * tentatively set as a candidate waker queue, while
-		 * on the second time, the flag
-		 * bfq_bfqq_has_waker(bfqq) is set to confirm that Q
-		 * is a waker queue for bfqq. These detection steps
-		 * are performed only if bfqq has a long think time,
-		 * so as to make it more likely that bfqq's I/O is
-		 * actually being blocked by a synchronization. This
-		 * last filter, plus the above two-times requirement,
-		 * make false positives less likely.
-		 *
-		 * NOTE
-		 *
-		 * The sooner a waker queue is detected, the sooner
-		 * throughput can be boosted by injecting I/O from the
-		 * waker queue. Fortunately, detection is likely to be
-		 * actually fast, for the following reasons. While
-		 * blocked by synchronization, bfqq has a long think
-		 * time. This implies that bfqq's inject limit is at
-		 * least equal to 1 (see the comments in
-		 * bfq_update_inject_limit()). So, thanks to
-		 * injection, the waker queue is likely to be served
-		 * during the very first I/O-plugging time interval
-		 * for bfqq. This triggers the first step of the
-		 * detection mechanism. Thanks again to injection, the
-		 * candidate waker queue is then likely to be
-		 * confirmed no later than during the next
-		 * I/O-plugging interval for bfqq.
-		 */
-		if (bfqd->last_completed_rq_bfqq &&
-		    !bfq_bfqq_has_short_ttime(bfqq) &&
-		    ktime_get_ns() - bfqd->last_completion <
-		    200 * NSEC_PER_USEC) {
-			if (bfqd->last_completed_rq_bfqq != bfqq &&
-			    bfqd->last_completed_rq_bfqq !=
-			    bfqq->waker_bfqq) {
-				/*
-				 * First synchronization detected with
-				 * a candidate waker queue, or with a
-				 * different candidate waker queue
-				 * from the current one.
-				 */
-				bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq;
-				/*
-				 * If the waker queue disappears, then
-				 * bfqq->waker_bfqq must be reset. To
-				 * this goal, we maintain in each
-				 * waker queue a list, woken_list, of
-				 * all the queues that reference the
-				 * waker queue through their
-				 * waker_bfqq pointer. When the waker
-				 * queue exits, the waker_bfqq pointer
-				 * of all the queues in the woken_list
-				 * is reset.
-				 *
-				 * In addition, if bfqq is already in
-				 * the woken_list of a waker queue,
-				 * then, before being inserted into
-				 * the woken_list of a new waker
-				 * queue, bfqq must be removed from
-				 * the woken_list of the old waker
-				 * queue.
-				 */
-				if (!hlist_unhashed(&bfqq->woken_list_node))
-					hlist_del_init(&bfqq->woken_list_node);
-				hlist_add_head(&bfqq->woken_list_node,
-				    &bfqd->last_completed_rq_bfqq->woken_list);
-				bfq_clear_bfqq_has_waker(bfqq);
-			} else if (bfqd->last_completed_rq_bfqq ==
-				   bfqq->waker_bfqq &&
-				   !bfq_bfqq_has_waker(bfqq)) {
-				/*
-				 * synchronization with waker_bfqq
-				 * seen for the second time
-				 */
-				bfq_mark_bfqq_has_waker(bfqq);
-			}
-		}
 		/*
 		 * Periodically reset inject limit, to make sure that
 		}
 	}
+	if (bfq_bfqq_sync(bfqq))
+		bfq_update_io_intensity(bfqq, now_ns);
 	elv_rb_add(&bfqq->sort_list, rq);
 	/*
 /* Must be called with bfqq != NULL */
 static void bfq_bfqq_end_wr(struct bfq_queue *bfqq)
 {
+	/*
+	 * If bfqq has been enjoying interactive weight-raising, then
+	 * reset soft_rt_next_start. We do it for the following
+	 * reason. bfqq may have been conveying the I/O needed to load
+	 * a soft real-time application. Such an application actually
+	 * exhibits a soft real-time I/O pattern after it finishes
+	 * loading, and finally starts doing its job. But, if bfqq has
+	 * been receiving a lot of bandwidth so far (likely to happen
+	 * on a fast device), then soft_rt_next_start now contains a
+	 * high value that. So, without this reset, bfqq would be
+	 * prevented from being possibly considered as soft_rt for a
+	 * very long time.
+	 */
+	if (bfqq->wr_cur_max_time !=
+	    bfqq->bfqd->bfq_wr_rt_max_time)
+		bfqq->soft_rt_next_start = jiffies;
 	if (bfq_bfqq_busy(bfqq))
 		bfqq->bfqd->wr_busy_queues--;
 	bfqq->wr_coeff = 1;
 	if (!bic)
 		return;
+	bic->saved_last_serv_time_ns = bfqq->last_serv_time_ns;
+	bic->saved_inject_limit = bfqq->inject_limit;
+	bic->saved_decrease_time_jif = bfqq->decrease_time_jif;
 	bic->saved_weight = bfqq->entity.orig_weight;
 	bic->saved_ttime = bfqq->ttime;
 	bic->saved_has_short_ttime = bfq_bfqq_has_short_ttime(bfqq);
 	bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq);
+	bic->saved_io_start_time = bfqq->io_start_time;
+	bic->saved_tot_idle_time = bfqq->tot_idle_time;
 	bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
 	bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
 	if (unlikely(bfq_bfqq_just_created(bfqq) &&
 		bic->saved_wr_coeff = bfqq->wr_coeff;
 		bic->saved_wr_start_at_switch_to_srt =
 			bfqq->wr_start_at_switch_to_srt;
+		bic->saved_service_from_wr = bfqq->service_from_wr;
 		bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish;
 		bic->saved_wr_cur_max_time = bfqq->wr_cur_max_time;
 	}
 	}
 	bfqd->in_service_queue = bfqq;
+	bfqd->in_serv_last_pos = 0;
 }
 /*
  * order until all the requests already queued in the device have been
  * served. The last sub-condition commented above somewhat mitigates
  * this problem for weight-raised queues.
+ *
+ * However, as an additional mitigation for this problem, we preserve
+ * plugging for a special symmetric case that may suddenly turn into
+ * asymmetric: the case where only bfqq is busy. In this case, not
+ * expiring bfqq does not cause any harm to any other queues in terms
+ * of service guarantees. In contrast, it avoids the following unlucky
+ * sequence of events: (1) bfqq is expired, (2) a new queue with a
+ * lower weight than bfqq becomes busy (or more queues), (3) the new
+ * queue is served until a new request arrives for bfqq, (4) when bfqq
+ * is finally served, there are so many requests of the new queue in
+ * the drive that the pending requests for bfqq take a lot of time to
+ * be served. In particular, event (2) may case even already
+ * dispatched requests of bfqq to be delayed, inside the drive. So, to
+ * avoid this series of events, the scenario is preventively declared
+ * as asymmetric also if bfqq is the only busy queues
  */
 static bool idling_needed_for_service_guarantees(struct bfq_data *bfqd,
 						 struct bfq_queue *bfqq)
 {
+	int tot_busy_queues = bfq_tot_busy_queues(bfqd);
 	/* No point in idling for bfqq if it won't get requests any longer */
 	if (unlikely(!bfqq_process_refs(bfqq)))
 		return false;
 	return (bfqq->wr_coeff > 1 &&
 		(bfqd->wr_busy_queues <
-		 bfq_tot_busy_queues(bfqd) ||
+		 tot_busy_queues ||
 		 bfqd->rq_in_driver >=
 		 bfqq->dispatched + 4)) ||
-		bfq_asymmetric_scenario(bfqd, bfqq);
+		bfq_asymmetric_scenario(bfqd, bfqq) ||
+		tot_busy_queues == 1;
 }
 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 	      bfq_bfqq_budget_left(bfqq) >=  entity->budget / 3)))
 		bfq_bfqq_charge_time(bfqd, bfqq, delta);
-	if (reason == BFQQE_TOO_IDLE &&
-	    entity->service <= 2 * entity->budget / 10)
-		bfq_clear_bfqq_IO_bound(bfqq);
 	if (bfqd->low_latency && bfqq->wr_coeff == 1)
 		bfqq->last_wr_start_finish = jiffies;
 		 * If we get here, and there are no outstanding
 		 * requests, then the request pattern is isochronous
 		 * (see the comments on the function
-		 * bfq_bfqq_softrt_next_start()). Thus we can compute
+		 * bfq_bfqq_softrt_next_start()). Therefore we can
-		 * soft_rt_next_start. And we do it, unless bfqq is in
+		 * compute soft_rt_next_start.
-		 * interactive weight raising. We do not do it in the
-		 * latter subcase, for the following reason. bfqq may
-		 * be conveying the I/O needed to load a soft
-		 * real-time application. Such an application will
-		 * actually exhibit a soft real-time I/O pattern after
-		 * it finally starts doing its job. But, if
-		 * soft_rt_next_start is computed here for an
-		 * interactive bfqq, and bfqq had received a lot of
-		 * service before remaining with no outstanding
-		 * request (likely to happen on a fast device), then
-		 * soft_rt_next_start would be assigned such a high
-		 * value that, for a very long time, bfqq would be
-		 * prevented from being possibly considered as soft
-		 * real time.
 		 *
 		 * If, instead, the queue still has outstanding
 		 * requests, then we have to wait for the completion
 		 * of all the outstanding requests to discover whether
 		 * the request pattern is actually isochronous.
 		 */
-		if (bfqq->dispatched == 0 &&
+		if (bfqq->dispatched == 0)
-		    bfqq->wr_coeff != bfqd->bfq_wr_coeff)
 			bfqq->soft_rt_next_start =
 				bfq_bfqq_softrt_next_start(bfqd, bfqq);
 		else if (bfqq->dispatched > 0) {
 		    bfq_serv_to_charge(async_bfqq->next_rq, async_bfqq) <=
 		    bfq_bfqq_budget_left(async_bfqq))
 			bfqq = bfqq->bic->bfqq[0];
-		else if (bfq_bfqq_has_waker(bfqq) &&
+		else if (bfqq->waker_bfqq &&
 			   bfq_bfqq_busy(bfqq->waker_bfqq) &&
-			   bfqq->next_rq &&
+			   bfqq->waker_bfqq->next_rq &&
 			   bfq_serv_to_charge(bfqq->waker_bfqq->next_rq,
 					      bfqq->waker_bfqq) <=
 			   bfq_bfqq_budget_left(bfqq->waker_bfqq)
 						bfqq->wr_cur_max_time)) {
 			if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time ||
 			time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +
-					       bfq_wr_duration(bfqd)))
+					       bfq_wr_duration(bfqd))) {
+				/*
+				 * Either in interactive weight
+				 * raising, or in soft_rt weight
+				 * raising with the
+				 * interactive-weight-raising period
+				 * elapsed (so no switch back to
+				 * interactive weight raising).
+				 */
 				bfq_bfqq_end_wr(bfqq);
-			else {
+			} else { /*
+				  * soft_rt finishing while still in
+				  * interactive period, switch back to
+				  * interactive weight raising
+				  */
 				switch_back_to_interactive_wr(bfqq, bfqd);
 				bfqq->entity.prio_changed = 1;
 			}
 {
 	struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
-	if (!atomic_read(&hctx->elevator_queued))
-		return false;
 	/*
 	 * Avoiding lock: a race on bfqd->busy_queues should cause at
 	 * most a call to dispatch for nothing
 	hlist_for_each_entry_safe(item, n, &bfqq->woken_list,
 				  woken_list_node) {
 		item->waker_bfqq = NULL;
-		bfq_clear_bfqq_has_waker(item);
 		hlist_del_init(&item->woken_list_node);
 	}
 	}
 	bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio);
+	bfq_log_bfqq(bfqd, bfqq, "new_ioprio %d new_weight %d",
+		     bfqq->new_ioprio, bfqq->entity.new_weight);
 	bfqq->entity.prio_changed = 1;
 }
 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 			  struct bfq_io_cq *bic, pid_t pid, int is_sync)
 {
+	u64 now_ns = ktime_get_ns();
 	RB_CLEAR_NODE(&bfqq->entity.rb_node);
 	INIT_LIST_HEAD(&bfqq->fifo);
 	INIT_HLIST_NODE(&bfqq->burst_list_node);
 		bfq_clear_bfqq_sync(bfqq);
 	/* set end request to minus infinity from now */
-	bfqq->ttime.last_end_request = ktime_get_ns() + 1;
+	bfqq->ttime.last_end_request = now_ns + 1;
+	bfqq->io_start_time = now_ns;
 	bfq_mark_bfqq_IO_bound(bfqq);
 				    struct bfq_queue *bfqq)
 {
 	struct bfq_ttime *ttime = &bfqq->ttime;
-	u64 elapsed = ktime_get_ns() - bfqq->ttime.last_end_request;
+	u64 elapsed;
+	/*
+	 * We are really interested in how long it takes for the queue to
+	 * become busy when there is no outstanding IO for this queue. So
+	 * ignore cases when the bfq queue has already IO queued.
+	 */
+	if (bfqq->dispatched || bfq_bfqq_busy(bfqq))
+		return;
+	elapsed = ktime_get_ns() - bfqq->ttime.last_end_request;
 	elapsed = min_t(u64, elapsed, 2ULL * bfqd->bfq_slice_idle);
-	ttime->ttime_samples = (7*bfqq->ttime.ttime_samples + 256) / 8;
+	ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8;
 	ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed,  8);
 	ttime->ttime_mean = div64_ul(ttime->ttime_total + 128,
 				     ttime->ttime_samples);
 	if (bfqq->wr_coeff > 1 &&
 	    bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time &&
-	    BFQQ_TOTALLY_SEEKY(bfqq))
+	    BFQQ_TOTALLY_SEEKY(bfqq)) {
-		bfq_bfqq_end_wr(bfqq);
+		if (time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt +
+					   bfq_wr_duration(bfqd))) {
+			/*
+			 * In soft_rt weight raising with the
+			 * interactive-weight-raising period
+			 * elapsed (so no switch back to
+			 * interactive weight raising).
+			 */
+			bfq_bfqq_end_wr(bfqq);
+		} else { /*
+			  * stopping soft_rt weight raising
+			  * while still in interactive period,
+			  * switch back to interactive weight
+			  * raising
+			  */
+			switch_back_to_interactive_wr(bfqq, bfqd);
+			bfqq->entity.prio_changed = 1;
+		}
+	}
 }
 static void bfq_update_has_short_ttime(struct bfq_data *bfqd,
 		return;
 	/* Think time is infinite if no process is linked to
-	 * bfqq. Otherwise check average think time to
+	 * bfqq. Otherwise check average think time to decide whether
-	 * decide whether to mark as has_short_ttime
+	 * to mark as has_short_ttime. To this goal, compare average
+	 * think time with half the I/O-plugging timeout.
 	 */
 	if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||
 	    (bfq_sample_valid(bfqq->ttime.ttime_samples) &&
-	     bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle))
+	     bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle>>1))
 		has_short_ttime = false;
 	state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq);
 		rq = list_first_entry(list, struct request, queuelist);
 		list_del_init(&rq->queuelist);
 		bfq_insert_request(hctx, rq, at_head);
-		atomic_inc(&hctx->elevator_queued);
 	}
 }
 		bfq_completed_request(bfqq, bfqd);
 		bfq_finish_requeue_request_body(bfqq);
-		atomic_dec(&rq->mq_hctx->elevator_queued);
 		spin_unlock_irqrestore(&bfqd->lock, flags);
 	} else {
 	bfqd->bfq_slice_idle = bfq_slice_idle;
 	bfqd->bfq_timeout = bfq_timeout;
-	bfqd->bfq_requests_within_timer = 120;
 	bfqd->bfq_large_burst_thresh = 8;
 	bfqd->bfq_burst_interval = msecs_to_jiffies(180);

block/bfq-iosched.h+19−10

 	/* associated @bfq_ttime struct */
 	struct bfq_ttime ttime;
+	/* when bfqq started to do I/O within the last observation window */
+	u64 io_start_time;
+	/* how long bfqq has remained empty during the last observ. window */
+	u64 tot_idle_time;
 	/* bit vector: a 1 for each seeky requests in history */
 	u32 seek_history;
 	 * bfq_select_queue().
 	 */
 	struct bfq_queue *waker_bfqq;
+	/* pointer to the curr. tentative waker queue, see bfq_check_waker() */
+	struct bfq_queue *tentative_waker_bfqq;
+	/* number of times the same tentative waker has been detected */
+	unsigned int num_waker_detections;
 	/* node for woken_list, see below */
 	struct hlist_node woken_list_node;
 	/*
 	 */
 	bool saved_IO_bound;
+	u64 saved_io_start_time;
+	u64 saved_tot_idle_time;
 	/*
 	 * Same purpose as the previous fields for the value of the
 	 * field keeping the queue's belonging to a large burst
 	 */
 	unsigned long saved_wr_coeff;
 	unsigned long saved_last_wr_start_finish;
+	unsigned long saved_service_from_wr;
 	unsigned long saved_wr_start_at_switch_to_srt;
 	unsigned int saved_wr_cur_max_time;
 	struct bfq_ttime saved_ttime;
+	/* Save also injection state */
+	u64 saved_last_serv_time_ns;
+	unsigned int saved_inject_limit;
+	unsigned long saved_decrease_time_jif;
 };
 /**
 	 */
 	unsigned int bfq_timeout;
-	/*
-	 * Number of consecutive requests that must be issued within
-	 * the idle time slice to set again idling to a queue which
-	 * was marked as non-I/O-bound (see the definition of the
-	 * IO_bound flag for further details).
-	 */
-	unsigned int bfq_requests_within_timer;
 	/*
 	 * Force device idling whenever needed to provide accurate
 	 * service guarantees, without caring about throughput
 				 */
 	BFQQF_coop,		/* bfqq is shared */
 	BFQQF_split_coop,	/* shared bfqq will be split */
-	BFQQF_has_waker		/* bfqq has a waker queue */
 };
 #define BFQ_BFQQ_FNS(name)						\
 BFQ_BFQQ_FNS(coop);
 BFQ_BFQQ_FNS(split_coop);
 BFQ_BFQQ_FNS(softrt_update);
-BFQ_BFQQ_FNS(has_waker);
 #undef BFQ_BFQQ_FNS
 /* Expiration reasons. */

block/bfq-wf2q.c+0−3

@@ -137,9 +137,6 @@ static bool bfq_update_next_in_service(struct bfq_sched_data *sd, · 136 unmodified lines
137		137
138	sd->next_in_service = next_in_service;	138	sd->next_in_service = next_in_service;
139		139
140	if (!next_in_service)
141	return parent_sched_may_change;
142
143	return parent_sched_may_change;	140	return parent_sched_may_change;
144	}	141	}
145		142

block/bio-integrity.c+15−20

@@ -14,8 +14,6 @@ · 13 unmodified lines
14	#include <linux/slab.h>	14	#include <linux/slab.h>
15	#include "blk.h"	15	#include "blk.h"
16		16
17	#define BIP_INLINE_VECS 4
18
19	static struct kmem_cache *bip_slab;	17	static struct kmem_cache *bip_slab;
20	static struct workqueue_struct *kintegrityd_wq;	18	static struct workqueue_struct *kintegrityd_wq;
21		19
@@ -30,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs, · 8 unmodified lines
30	if (bs && mempool_initialized(&bs->bio_integrity_pool)) {	28	if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
31	if (bip->bip_vec)	29	if (bip->bip_vec)
32	bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,	30	bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
33	bip->bip_slab);	31	bip->bip_max_vcnt);
34	mempool_free(bip, &bs->bio_integrity_pool);	32	mempool_free(bip, &bs->bio_integrity_pool);
35	} else {	33	} else {
36	kfree(bip);	34	kfree(bip);
@@ -63,7 +61,7 @@ struct bio_integrity_payload bio_integrity_alloc(struct bio bio, · 26 unmodified lines
63	inline_vecs = nr_vecs;	61	inline_vecs = nr_vecs;
64	} else {	62	} else {
65	bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);	63	bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
66	inline_vecs = BIP_INLINE_VECS;	64	inline_vecs = BIO_INLINE_VECS;
67	}	65	}
68		66
69	if (unlikely(!bip))	67	if (unlikely(!bip))
@@ -72,14 +70,11 @@ struct bio_integrity_payload bio_integrity_alloc(struct bio bio, · 2 unmodified lines
72	memset(bip, 0, sizeof(*bip));	70	memset(bip, 0, sizeof(*bip));
73		71
74	if (nr_vecs > inline_vecs) {	72	if (nr_vecs > inline_vecs) {
75	unsigned long idx = 0;	73	bip->bip_max_vcnt = nr_vecs;
76		74	bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
77	bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,	75	&bip->bip_max_vcnt, gfp_mask);
78	&bs->bvec_integrity_pool);
79	if (!bip->bip_vec)	76	if (!bip->bip_vec)
80	goto err;	77	goto err;
81	bip->bip_max_vcnt = bvec_nr_vecs(idx);
82	bip->bip_slab = idx;
83	} else {	78	} else {
84	bip->bip_vec = bip->bip_inline_vecs;	79	bip->bip_vec = bip->bip_inline_vecs;
85	bip->bip_max_vcnt = inline_vecs;	80	bip->bip_max_vcnt = inline_vecs;
@@ -140,7 +135,7 @@ int bio_integrity_add_page(struct bio bio, struct page page, · 54 unmodified lines
140	iv = bip->bip_vec + bip->bip_vcnt;	135	iv = bip->bip_vec + bip->bip_vcnt;
141		136
142	if (bip->bip_vcnt &&	137	if (bip->bip_vcnt &&
143	bvec_gap_to_prev(bio->bi_disk->queue,	138	bvec_gap_to_prev(bio->bi_bdev->bd_disk->queue,
144	&bip->bip_vec[bip->bip_vcnt - 1], offset))	139	&bip->bip_vec[bip->bip_vcnt - 1], offset))
145	return 0;	140	return 0;
146		141
@@ -162,7 +157,7 @@ EXPORT_SYMBOL(bio_integrity_add_page); · 15 unmodified lines
162	static blk_status_t bio_integrity_process(struct bio *bio,	157	static blk_status_t bio_integrity_process(struct bio *bio,
163	struct bvec_iter proc_iter, integrity_processing_fn proc_fn)	158	struct bvec_iter proc_iter, integrity_processing_fn proc_fn)
164	{	159	{
165	struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);	160	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
166	struct blk_integrity_iter iter;	161	struct blk_integrity_iter iter;
167	struct bvec_iter bviter;	162	struct bvec_iter bviter;
168	struct bio_vec bv;	163	struct bio_vec bv;
@@ -171,7 +166,7 @@ static blk_status_t bio_integrity_process(struct bio *bio, · 2 unmodified lines
171	void *prot_buf = page_address(bip->bip_vec->bv_page) +	166	void *prot_buf = page_address(bip->bip_vec->bv_page) +
172	bip->bip_vec->bv_offset;	167	bip->bip_vec->bv_offset;
173		168
174	iter.disk_name = bio->bi_disk->disk_name;	169	iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
175	iter.interval = 1 << bi->interval_exp;	170	iter.interval = 1 << bi->interval_exp;
176	iter.seed = proc_iter->bi_sector;	171	iter.seed = proc_iter->bi_sector;
177	iter.prot_buf = prot_buf;	172	iter.prot_buf = prot_buf;
@@ -208,8 +203,8 @@ static blk_status_t bio_integrity_process(struct bio *bio, · 30 unmodified lines
208	bool bio_integrity_prep(struct bio *bio)	203	bool bio_integrity_prep(struct bio *bio)
209	{	204	{
210	struct bio_integrity_payload *bip;	205	struct bio_integrity_payload *bip;
211	struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);	206	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
212	struct request_queue *q = bio->bi_disk->queue;	207	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
213	void *buf;	208	void *buf;
214	unsigned long start, end;	209	unsigned long start, end;
215	unsigned int len, nr_pages;	210	unsigned int len, nr_pages;
@@ -329,7 +324,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) · 113 unmodified lines
329	struct bio_integrity_payload *bip =	324	struct bio_integrity_payload *bip =
330	container_of(work, struct bio_integrity_payload, bip_work);	325	container_of(work, struct bio_integrity_payload, bip_work);
331	struct bio *bio = bip->bip_bio;	326	struct bio *bio = bip->bip_bio;
332	struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);	327	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
333		328
334	/*	329	/*
335	* At the moment verify is called bio's iterator was advanced	330	* At the moment verify is called bio's iterator was advanced
@@ -355,7 +350,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) · 19 unmodified lines
355	*/	350	*/
356	bool __bio_integrity_endio(struct bio *bio)	351	bool __bio_integrity_endio(struct bio *bio)
357	{	352	{
358	struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);	353	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
359	struct bio_integrity_payload *bip = bio_integrity(bio);	354	struct bio_integrity_payload *bip = bio_integrity(bio);
360		355
361	if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&	356	if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
@@ -381,7 +376,7 @@ bool __bio_integrity_endio(struct bio *bio) · 19 unmodified lines
381	void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)	376	void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
382	{	377	{
383	struct bio_integrity_payload *bip = bio_integrity(bio);	378	struct bio_integrity_payload *bip = bio_integrity(bio);
384	struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);	379	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
385	unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);	380	unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
386		381
387	bip->bip_iter.bi_sector += bytes_done >> 9;	382	bip->bip_iter.bi_sector += bytes_done >> 9;
@@ -397,7 +392,7 @@ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) · 9 unmodified lines
397	void bio_integrity_trim(struct bio *bio)	392	void bio_integrity_trim(struct bio *bio)
398	{	393	{
399	struct bio_integrity_payload *bip = bio_integrity(bio);	394	struct bio_integrity_payload *bip = bio_integrity(bio);
400	struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);	395	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
401		396
402	bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));	397	bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
403	}	398	}
@@ -470,6 +465,6 @@ void __init bio_integrity_init(void) · 66 unmodified lines
470		465
471	bip_slab = kmem_cache_create("bio_integrity_payload",	466	bip_slab = kmem_cache_create("bio_integrity_payload",
472	sizeof(struct bio_integrity_payload) +	467	sizeof(struct bio_integrity_payload) +
473	sizeof(struct bio_vec) * BIP_INLINE_VECS,	468	sizeof(struct bio_vec) * BIO_INLINE_VECS,
474	0, SLAB_HWCACHE_ALIGN\|SLAB_PANIC, NULL);	469	0, SLAB_HWCACHE_ALIGN\|SLAB_PANIC, NULL);
475	}	470	}

block/bio.c+253−318

@@ -19,27 +19,40 @@ · 18 unmodified lines
19	#include <linux/highmem.h>	19	#include <linux/highmem.h>
20	#include <linux/sched/sysctl.h>	20	#include <linux/sched/sysctl.h>
21	#include <linux/blk-crypto.h>	21	#include <linux/blk-crypto.h>
		22	#include <linux/xarray.h>
22		23
23	#include <trace/events/block.h>	24	#include <trace/events/block.h>
24	#include "blk.h"	25	#include "blk.h"
25	#include "blk-rq-qos.h"	26	#include "blk-rq-qos.h"
26		27
27	/*	28	static struct biovec_slab {
28	* Test patch to inline a certain number of bi_io_vec's inside the bio	29	int nr_vecs;
29	* itself, to shrink a bio data allocation from two mempool calls to one	30	char *name;
30	*/	31	struct kmem_cache *slab;
31	#define BIO_INLINE_VECS 4	32	} bvec_slabs[] __read_mostly = {
32		33	{ .nr_vecs = 16, .name = "biovec-16" },
33	/*	34	{ .nr_vecs = 64, .name = "biovec-64" },
34	* if you change this list, also change bvec_alloc or things will	35	{ .nr_vecs = 128, .name = "biovec-128" },
35	* break badly! cannot be bigger than what you can fit into an	36	{ .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },
36	* unsigned short
37	*/
38	#define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
39	static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
40	BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
41	};	37	};
42	#undef BV	38
		39	static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
		40	{
		41	switch (nr_vecs) {
		42	/* smaller bios use inline vecs */
		43	case 5 ... 16:
		44	return &bvec_slabs[0];
		45	case 17 ... 64:
		46	return &bvec_slabs[1];
		47	case 65 ... 128:
		48	return &bvec_slabs[2];
		49	case 129 ... BIO_MAX_PAGES:
		50	return &bvec_slabs[3];
		51	default:
		52	BUG();
		53	return NULL;
		54	}
		55	}
43		56
44	/*	57	/*
45	* fs_bio_set is the bio_set containing bio and iovec memory pools used by	58	* fs_bio_set is the bio_set containing bio and iovec memory pools used by
@@ -58,178 +71,133 @@ struct bio_slab { · 12 unmodified lines
58	char name[8];	71	char name[8];
59	};	72	};
60	static DEFINE_MUTEX(bio_slab_lock);	73	static DEFINE_MUTEX(bio_slab_lock);
61	static struct bio_slab *bio_slabs;	74	static DEFINE_XARRAY(bio_slabs);
62	static unsigned int bio_slab_nr, bio_slab_max;
63		75
64	static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)	76	static struct bio_slab *create_bio_slab(unsigned int size)
65	{	77	{
66	unsigned int sz = sizeof(struct bio) + extra_size;	78	struct bio_slab bslab = kzalloc(sizeof(bslab), GFP_KERNEL);
67	struct kmem_cache *slab = NULL;
68	struct bio_slab bslab, new_bio_slabs;
69	unsigned int new_bio_slab_max;
70	unsigned int i, entry = -1;
71		79
72	mutex_lock(&bio_slab_lock);	80	if (!bslab)
		81	return NULL;
73		82
74	i = 0;	83	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size);
75	while (i < bio_slab_nr) {	84	bslab->slab = kmem_cache_create(bslab->name, size,
76	bslab = &bio_slabs[i];	85	ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN, NULL);
		86	if (!bslab->slab)
		87	goto fail_alloc_slab;
77		88
78	if (!bslab->slab && entry == -1)	89	bslab->slab_ref = 1;
79	entry = i;	90	bslab->slab_size = size;
80	else if (bslab->slab_size == sz) {
81	slab = bslab->slab;
82	bslab->slab_ref++;
83	break;
84	}
85	i++;
86	}
87		91
88	if (slab)	92	if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL)))
89	goto out_unlock;	93	return bslab;
90		94
91	if (bio_slab_nr == bio_slab_max && entry == -1) {	95	kmem_cache_destroy(bslab->slab);
92	new_bio_slab_max = bio_slab_max << 1;
93	new_bio_slabs = krealloc(bio_slabs,
94	new_bio_slab_max * sizeof(struct bio_slab),
95	GFP_KERNEL);
96	if (!new_bio_slabs)
97	goto out_unlock;
98	bio_slab_max = new_bio_slab_max;
99	bio_slabs = new_bio_slabs;
100	}
101	if (entry == -1)
102	entry = bio_slab_nr++;
103		96
104	bslab = &bio_slabs[entry];	97	fail_alloc_slab:
		98	kfree(bslab);
		99	return NULL;
		100	}
		101
		102	static inline unsigned int bs_bio_slab_size(struct bio_set *bs)
		103	{
		104	return bs->front_pad + sizeof(struct bio) + bs->back_pad;
		105	}
105		106
106	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);	107	static struct kmem_cache bio_find_or_create_slab(struct bio_set bs)
107	slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,	108	{
108	SLAB_HWCACHE_ALIGN, NULL);	109	unsigned int size = bs_bio_slab_size(bs);
109	if (!slab)	110	struct bio_slab *bslab;
110	goto out_unlock;
111		111
112	bslab->slab = slab;	112	mutex_lock(&bio_slab_lock);
113	bslab->slab_ref = 1;	113	bslab = xa_load(&bio_slabs, size);
114	bslab->slab_size = sz;	114	if (bslab)
115	out_unlock:	115	bslab->slab_ref++;
		116	else
		117	bslab = create_bio_slab(size);
116	mutex_unlock(&bio_slab_lock);	118	mutex_unlock(&bio_slab_lock);
117	return slab;	119
		120	if (bslab)
		121	return bslab->slab;
		122	return NULL;
118	}	123	}
119		124
120	static void bio_put_slab(struct bio_set *bs)	125	static void bio_put_slab(struct bio_set *bs)
121	{	126	{
122	struct bio_slab *bslab = NULL;	127	struct bio_slab *bslab = NULL;
123	unsigned int i;	128	unsigned int slab_size = bs_bio_slab_size(bs);
124		129
125	mutex_lock(&bio_slab_lock);	130	mutex_lock(&bio_slab_lock);
126		131
127	for (i = 0; i < bio_slab_nr; i++) {	132	bslab = xa_load(&bio_slabs, slab_size);
128	if (bs->bio_slab == bio_slabs[i].slab) {
129	bslab = &bio_slabs[i];
130	break;
131	}
132	}
133
134	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))	133	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
135	goto out;	134	goto out;
136		135
		136	WARN_ON_ONCE(bslab->slab != bs->bio_slab);
		137
137	WARN_ON(!bslab->slab_ref);	138	WARN_ON(!bslab->slab_ref);
138		139
139	if (--bslab->slab_ref)	140	if (--bslab->slab_ref)
140	goto out;	141	goto out;
141		142
		143	xa_erase(&bio_slabs, slab_size);
		144
142	kmem_cache_destroy(bslab->slab);	145	kmem_cache_destroy(bslab->slab);
143	bslab->slab = NULL;	146	kfree(bslab);
144		147
145	out:	148	out:
146	mutex_unlock(&bio_slab_lock);	149	mutex_unlock(&bio_slab_lock);
147	}	150	}
148		151
149	unsigned int bvec_nr_vecs(unsigned short idx)	152	void bvec_free(mempool_t pool, struct bio_vec bv, unsigned short nr_vecs)
150	{	153	{
151	return bvec_slabs[--idx].nr_vecs;	154	BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES);
152	}
153
154	void bvec_free(mempool_t pool, struct bio_vec bv, unsigned int idx)
155	{
156	if (!idx)
157	return;
158	idx--;
159		155
160	BIO_BUG_ON(idx >= BVEC_POOL_NR);	156	if (nr_vecs == BIO_MAX_PAGES)
161
162	if (idx == BVEC_POOL_MAX) {
163	mempool_free(bv, pool);	157	mempool_free(bv, pool);
164	} else {	158	else if (nr_vecs > BIO_INLINE_VECS)
165	struct biovec_slab *bvs = bvec_slabs + idx;	159	kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
		160	}
166		161
167	kmem_cache_free(bvs->slab, bv);	162	/*
168	}	163	* Make the first allocation restricted and don't dump info on allocation
		164	* failures, since we'll fall back to the mempool in case of failure.
		165	*/
		166	static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
		167	{
		168	return (gfp & ~(__GFP_DIRECT_RECLAIM \| __GFP_IO)) \|
		169	__GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_NOWARN;
169	}	170	}
170		171
171	struct bio_vec bvec_alloc(gfp_t gfp_mask, int nr, unsigned long idx,	172	struct bio_vec bvec_alloc(mempool_t pool, unsigned short *nr_vecs,
172	mempool_t *pool)	173	gfp_t gfp_mask)
173	{	174	{
174	struct bio_vec *bvl;	175	struct biovec_slab bvs = biovec_slab(nr_vecs);
175		176
176	/*	177	if (WARN_ON_ONCE(!bvs))
177	* see comment near bvec_array define!
178	*/
179	switch (nr) {
180	case 1:
181	*idx = 0;
182	break;
183	case 2 ... 4:
184	*idx = 1;
185	break;
186	case 5 ... 16:
187	*idx = 2;
188	break;
189	case 17 ... 64:
190	*idx = 3;
191	break;
192	case 65 ... 128:
193	*idx = 4;
194	break;
195	case 129 ... BIO_MAX_PAGES:
196	*idx = 5;
197	break;
198	default:
199	return NULL;	178	return NULL;
200	}
201		179
202	/*	180	/*
203	* idx now points to the pool we want to allocate from. only the	181	* Upgrade the nr_vecs request to take full advantage of the allocation.
204	* 1-vec entry pool is mempool backed.	182	* We also rely on this in the bvec_free path.
205	*/	183	*/
206	if (*idx == BVEC_POOL_MAX) {	184	*nr_vecs = bvs->nr_vecs;
207	fallback:
208	bvl = mempool_alloc(pool, gfp_mask);
209	} else {
210	struct biovec_slab bvs = bvec_slabs + idx;
211	gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM \| __GFP_IO);
212		185
213	/*	186	/*
214	* Make this allocation restricted and don't dump info on	187	* Try a slab allocation first for all smaller allocations. If that
215	* allocation failures, since we'll fallback to the mempool	188	* fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
216	* in case of failure.	189	* The mempool is sized to handle up to BIO_MAX_PAGES entries.
217	*/	190	*/
218	__gfp_mask \|= __GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_NOWARN;	191	if (*nr_vecs < BIO_MAX_PAGES) {
		192	struct bio_vec *bvl;
219		193
220	/*	194	bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
221	* Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM	195	if (likely(bvl) \|\| !(gfp_mask & __GFP_DIRECT_RECLAIM))
222	* is set, retry with the 1-entry mempool	196	return bvl;
223	*/	197	*nr_vecs = BIO_MAX_PAGES;
224	bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
225	if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
226	*idx = BVEC_POOL_MAX;
227	goto fallback;
228	}
229	}	198	}
230		199
231	(*idx)++;	200	return mempool_alloc(pool, gfp_mask);
232	return bvl;
233	}	201	}
234		202
235	void bio_uninit(struct bio *bio)	203	void bio_uninit(struct bio *bio)
@@ -255,7 +223,7 @@ static void bio_free(struct bio *bio) · 19 unmodified lines
255	bio_uninit(bio);	223	bio_uninit(bio);
256		224
257	if (bs) {	225	if (bs) {
258	bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));	226	bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
259		227
260	/*	228	/*
261	* If we have front padding, adjust the bio pointer before freeing	229	* If we have front padding, adjust the bio pointer before freeing
@@ -299,12 +267,8 @@ EXPORT_SYMBOL(bio_init); · 37 unmodified lines
299	*/	267	*/
300	void bio_reset(struct bio *bio)	268	void bio_reset(struct bio *bio)
301	{	269	{
302	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
303
304	bio_uninit(bio);	270	bio_uninit(bio);
305
306	memset(bio, 0, BIO_RESET_BYTES);	271	memset(bio, 0, BIO_RESET_BYTES);
307	bio->bi_flags = flags;
308	atomic_set(&bio->__bi_remaining, 1);	272	atomic_set(&bio->__bi_remaining, 1);
309	}	273	}
310	EXPORT_SYMBOL(bio_reset);	274	EXPORT_SYMBOL(bio_reset);
@@ -405,122 +369,97 @@ static void punt_bios_to_rescuer(struct bio_set *bs) · 94 unmodified lines
405	* @nr_iovecs: number of iovecs to pre-allocate	369	* @nr_iovecs: number of iovecs to pre-allocate
406	* @bs: the bio_set to allocate from.	370	* @bs: the bio_set to allocate from.
407	*	371	*
408	* Description:	372	* Allocate a bio from the mempools in @bs.
409	* If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
410	* backed by the @bs's mempool.
411	*	373	*
412	* When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will	374	* If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to
413	* always be able to allocate a bio. This is due to the mempool guarantees.	375	* allocate a bio. This is due to the mempool guarantees. To make this work,
414	* To make this work, callers must never allocate more than 1 bio at a time	376	* callers must never allocate more than 1 bio at a time from the general pool.
415	* from this pool. Callers that need to allocate more than 1 bio must always	377	* Callers that need to allocate more than 1 bio must always submit the
416	* submit the previously allocated bio for IO before attempting to allocate	378	* previously allocated bio for IO before attempting to allocate a new one.
417	* a new one. Failure to do so can cause deadlocks under memory pressure.	379	* Failure to do so can cause deadlocks under memory pressure.
418	*	380	*
419	* Note that when running under submit_bio_noacct() (i.e. any block	381	* Note that when running under submit_bio_noacct() (i.e. any block driver),
420	* driver), bios are not submitted until after you return - see the code in	382	* bios are not submitted until after you return - see the code in
421	* submit_bio_noacct() that converts recursion into iteration, to prevent	383	* submit_bio_noacct() that converts recursion into iteration, to prevent
422	* stack overflows.	384	* stack overflows.
423	*	385	*
424	* This would normally mean allocating multiple bios under	386	* This would normally mean allocating multiple bios under submit_bio_noacct()
425	* submit_bio_noacct() would be susceptible to deadlocks, but we have	387	* would be susceptible to deadlocks, but we have
426	* deadlock avoidance code that resubmits any blocked bios from a rescuer	388	* deadlock avoidance code that resubmits any blocked bios from a rescuer
427	* thread.	389	* thread.
428	*	390	*
429	* However, we do not guarantee forward progress for allocations from other	391	* However, we do not guarantee forward progress for allocations from other
430	* mempools. Doing multiple allocations from the same mempool under	392	* mempools. Doing multiple allocations from the same mempool under
431	* submit_bio_noacct() should be avoided - instead, use bio_set's front_pad	393	* submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
432	* for per bio allocations.	394	* for per bio allocations.
433	*	395	*
434	* RETURNS:	396	* Returns: Pointer to new bio on success, NULL on failure.
435	* Pointer to new bio on success, NULL on failure.
436	*/	397	*/
437	struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,	398	struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs,
438	struct bio_set *bs)	399	struct bio_set *bs)
439	{	400	{
440	gfp_t saved_gfp = gfp_mask;	401	gfp_t saved_gfp = gfp_mask;
441	unsigned front_pad;
442	unsigned inline_vecs;
443	struct bio_vec *bvl = NULL;
444	struct bio *bio;	402	struct bio *bio;
445	void *p;	403	void *p;
446		404
447	if (!bs) {	405	/* should not use nobvec bioset for nr_iovecs > 0 */
448	if (nr_iovecs > UIO_MAXIOV)	406	if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_iovecs > 0))
449	return NULL;	407	return NULL;
450
451	p = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask);
452	front_pad = 0;
453	inline_vecs = nr_iovecs;
454	} else {
455	/* should not use nobvec bioset for nr_iovecs > 0 */
456	if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
457	nr_iovecs > 0))
458	return NULL;
459	/*
460	* submit_bio_noacct() converts recursion to iteration; this
461	* means if we're running beneath it, any bios we allocate and
462	* submit will not be submitted (and thus freed) until after we
463	* return.
464	*
465	* This exposes us to a potential deadlock if we allocate
466	* multiple bios from the same bio_set() while running
467	* underneath submit_bio_noacct(). If we were to allocate
468	* multiple bios (say a stacking block driver that was splitting
469	* bios), we would deadlock if we exhausted the mempool's
470	* reserve.
471	*
472	* We solve this, and guarantee forward progress, with a rescuer
473	* workqueue per bio_set. If we go to allocate and there are
474	* bios on current->bio_list, we first try the allocation
475	* without __GFP_DIRECT_RECLAIM; if that fails, we punt those
476	* bios we would be blocking to the rescuer workqueue before
477	* we retry with the original gfp_flags.
478	*/
479
480	if (current->bio_list &&
481	(!bio_list_empty(&current->bio_list[0]) \|\|
482	!bio_list_empty(&current->bio_list[1])) &&
483	bs->rescue_workqueue)
484	gfp_mask &= ~__GFP_DIRECT_RECLAIM;
485		408
		409	/*
		410	* submit_bio_noacct() converts recursion to iteration; this means if
		411	* we're running beneath it, any bios we allocate and submit will not be
		412	* submitted (and thus freed) until after we return.
		413	*
		414	* This exposes us to a potential deadlock if we allocate multiple bios
		415	* from the same bio_set() while running underneath submit_bio_noacct().
		416	* If we were to allocate multiple bios (say a stacking block driver
		417	* that was splitting bios), we would deadlock if we exhausted the
		418	* mempool's reserve.
		419	*
		420	* We solve this, and guarantee forward progress, with a rescuer
		421	* workqueue per bio_set. If we go to allocate and there are bios on
		422	* current->bio_list, we first try the allocation without
		423	* __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be
		424	* blocking to the rescuer workqueue before we retry with the original
		425	* gfp_flags.
		426	*/
		427	if (current->bio_list &&
		428	(!bio_list_empty(&current->bio_list[0]) \|\|
		429	!bio_list_empty(&current->bio_list[1])) &&
		430	bs->rescue_workqueue)
		431	gfp_mask &= ~__GFP_DIRECT_RECLAIM;
		432
		433	p = mempool_alloc(&bs->bio_pool, gfp_mask);
		434	if (!p && gfp_mask != saved_gfp) {
		435	punt_bios_to_rescuer(bs);
		436	gfp_mask = saved_gfp;
486	p = mempool_alloc(&bs->bio_pool, gfp_mask);	437	p = mempool_alloc(&bs->bio_pool, gfp_mask);
487	if (!p && gfp_mask != saved_gfp) {
488	punt_bios_to_rescuer(bs);
489	gfp_mask = saved_gfp;
490	p = mempool_alloc(&bs->bio_pool, gfp_mask);
491	}
492
493	front_pad = bs->front_pad;
494	inline_vecs = BIO_INLINE_VECS;
495	}	438	}
496
497	if (unlikely(!p))	439	if (unlikely(!p))
498	return NULL;	440	return NULL;
499		441
500	bio = p + front_pad;	442	bio = p + bs->front_pad;
501	bio_init(bio, NULL, 0);	443	if (nr_iovecs > BIO_INLINE_VECS) {
		444	struct bio_vec *bvl = NULL;
502		445
503	if (nr_iovecs > inline_vecs) {	446	bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
504	unsigned long idx = 0;
505
506	bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
507	if (!bvl && gfp_mask != saved_gfp) {	447	if (!bvl && gfp_mask != saved_gfp) {
508	punt_bios_to_rescuer(bs);	448	punt_bios_to_rescuer(bs);
509	gfp_mask = saved_gfp;	449	gfp_mask = saved_gfp;
510	bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);	450	bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
511	}	451	}
512
513	if (unlikely(!bvl))	452	if (unlikely(!bvl))
514	goto err_free;	453	goto err_free;
515		454
516	bio->bi_flags \|= idx << BVEC_POOL_OFFSET;	455	bio_init(bio, bvl, nr_iovecs);
517	} else if (nr_iovecs) {	456	} else if (nr_iovecs) {
518	bvl = bio->bi_inline_vecs;	457	bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);
		458	} else {
		459	bio_init(bio, NULL, 0);
519	}	460	}
520		461
521	bio->bi_pool = bs;	462	bio->bi_pool = bs;
522	bio->bi_max_vecs = nr_iovecs;
523	bio->bi_io_vec = bvl;
524	return bio;	463	return bio;
525		464
526	err_free:	465	err_free:
@@ -529,6 +468,31 @@ err_free: · 2 unmodified lines
529	}	468	}
530	EXPORT_SYMBOL(bio_alloc_bioset);	469	EXPORT_SYMBOL(bio_alloc_bioset);
531		470
		471	/**
		472	* bio_kmalloc - kmalloc a bio for I/O
		473	* @gfp_mask: the GFP_* mask given to the slab allocator
		474	* @nr_iovecs: number of iovecs to pre-allocate
		475	*
		476	* Use kmalloc to allocate and initialize a bio.
		477	*
		478	* Returns: Pointer to new bio on success, NULL on failure.
		479	*/
		480	struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs)
		481	{
		482	struct bio *bio;
		483
		484	if (nr_iovecs > UIO_MAXIOV)
		485	return NULL;
		486
		487	bio = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask);
		488	if (unlikely(!bio))
		489	return NULL;
		490	bio_init(bio, nr_iovecs ? bio->bi_inline_vecs : NULL, nr_iovecs);
		491	bio->bi_pool = NULL;
		492	return bio;
		493	}
		494	EXPORT_SYMBOL(bio_kmalloc);
		495
532	void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)	496	void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
533	{	497	{
534	unsigned long flags;	498	unsigned long flags;
@@ -607,16 +571,7 @@ void bio_truncate(struct bio *bio, unsigned new_size) · 72 unmodified lines
607	*/	571	*/
608	void guard_bio_eod(struct bio *bio)	572	void guard_bio_eod(struct bio *bio)
609	{	573	{
610	sector_t maxsector;	574	sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
611	struct block_device *part;
612
613	rcu_read_lock();
614	part = __disk_get_part(bio->bi_disk, bio->bi_partno);
615	if (part)
616	maxsector = bdev_nr_sectors(part);
617	else
618	maxsector = get_capacity(bio->bi_disk);
619	rcu_read_unlock();
620		575
621	if (!maxsector)	576	if (!maxsector)
622	return;	577	return;
@@ -673,17 +628,18 @@ EXPORT_SYMBOL(bio_put); · 50 unmodified lines
673	*/	628	*/
674	void __bio_clone_fast(struct bio bio, struct bio bio_src)	629	void __bio_clone_fast(struct bio bio, struct bio bio_src)
675	{	630	{
676	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));	631	WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs);
677		632
678	/*	633	/*
679	* most users will be overriding ->bi_disk with a new target,	634	* most users will be overriding ->bi_bdev with a new target,
680	* so we don't set nor calculate new physical/hw segment counts here	635	* so we don't set nor calculate new physical/hw segment counts here
681	*/	636	*/
682	bio->bi_disk = bio_src->bi_disk;	637	bio->bi_bdev = bio_src->bi_bdev;
683	bio->bi_partno = bio_src->bi_partno;
684	bio_set_flag(bio, BIO_CLONED);	638	bio_set_flag(bio, BIO_CLONED);
685	if (bio_flagged(bio_src, BIO_THROTTLED))	639	if (bio_flagged(bio_src, BIO_THROTTLED))
686	bio_set_flag(bio, BIO_THROTTLED);	640	bio_set_flag(bio, BIO_THROTTLED);
		641	if (bio_flagged(bio_src, BIO_REMAPPED))
		642	bio_set_flag(bio, BIO_REMAPPED);
687	bio->bi_opf = bio_src->bi_opf;	643	bio->bi_opf = bio_src->bi_opf;
688	bio->bi_ioprio = bio_src->bi_ioprio;	644	bio->bi_ioprio = bio_src->bi_ioprio;
689	bio->bi_write_hint = bio_src->bi_write_hint;	645	bio->bi_write_hint = bio_src->bi_write_hint;
@@ -730,7 +686,7 @@ EXPORT_SYMBOL(bio_clone_fast); · 40 unmodified lines
730		686
731	const char bio_devname(struct bio bio, char *buf)	687	const char bio_devname(struct bio bio, char *buf)
732	{	688	{
733	return disk_name(bio->bi_disk, bio->bi_partno, buf);	689	return bdevname(bio->bi_bdev, buf);
734	}	690	}
735	EXPORT_SYMBOL(bio_devname);	691	EXPORT_SYMBOL(bio_devname);
736		692
@@ -870,7 +826,7 @@ EXPORT_SYMBOL(bio_add_pc_page); · 133 unmodified lines
870	int bio_add_zone_append_page(struct bio bio, struct page page,	826	int bio_add_zone_append_page(struct bio bio, struct page page,
871	unsigned int len, unsigned int offset)	827	unsigned int len, unsigned int offset)
872	{	828	{
873	struct request_queue *q = bio->bi_disk->queue;	829	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
874	bool same_page = false;	830	bool same_page = false;
875		831
876	if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND))	832	if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND))
@@ -993,21 +949,18 @@ void bio_release_pages(struct bio *bio, bool mark_dirty) · 116 unmodified lines
993	}	949	}
994	EXPORT_SYMBOL_GPL(bio_release_pages);	950	EXPORT_SYMBOL_GPL(bio_release_pages);
995		951
996	static int __bio_iov_bvec_add_pages(struct bio bio, struct iov_iter iter)	952	static int bio_iov_bvec_set(struct bio bio, struct iov_iter iter)
997	{	953	{
998	const struct bio_vec *bv = iter->bvec;	954	WARN_ON_ONCE(bio->bi_max_vecs);
999	unsigned int len;	955
1000	size_t size;	956	bio->bi_vcnt = iter->nr_segs;
1001		957	bio->bi_io_vec = (struct bio_vec *)iter->bvec;
1002	if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len))	958	bio->bi_iter.bi_bvec_done = iter->iov_offset;
1003	return -EINVAL;	959	bio->bi_iter.bi_size = iter->count;
1004		960	bio_set_flag(bio, BIO_NO_PAGE_REF);
1005	len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count);	961	bio_set_flag(bio, BIO_CLONED);
1006	size = bio_add_page(bio, bv->bv_page, len,	962
1007	bv->bv_offset + iter->iov_offset);	963	iov_iter_advance(iter, iter->count);
1008	if (unlikely(size != len))
1009	return -EINVAL;
1010	iov_iter_advance(iter, size);
1011	return 0;	964	return 0;
1012	}	965	}
1013		966
@@ -1070,7 +1023,7 @@ static int __bio_iov_append_get_pages(struct bio bio, struct iov_iter iter) · 56 unmodified lines
1070	{	1023	{
1071	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;	1024	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
1072	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;	1025	unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
1073	struct request_queue *q = bio->bi_disk->queue;	1026	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
1074	unsigned int max_append_sectors = queue_max_zone_append_sectors(q);	1027	unsigned int max_append_sectors = queue_max_zone_append_sectors(q);
1075	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;	1028	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
1076	struct page pages = (struct page )bv;	1029	struct page pages = (struct page )bv;
@@ -1121,41 +1074,40 @@ static int __bio_iov_append_get_pages(struct bio bio, struct iov_iter iter) · 44 unmodified lines
1121	* This takes either an iterator pointing to user memory, or one pointing to	1074	* This takes either an iterator pointing to user memory, or one pointing to
1122	* kernel pages (BVEC iterator). If we're adding user pages, we pin them and	1075	* kernel pages (BVEC iterator). If we're adding user pages, we pin them and
1123	* map them into the kernel. On IO completion, the caller should put those	1076	* map them into the kernel. On IO completion, the caller should put those
1124	* pages. If we're adding kernel pages, and the caller told us it's safe to	1077	* pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided
1125	* do so, we just have to add the pages to the bio directly. We don't grab an	1078	* bvecs rather than copying them. Hence anyone issuing kiocb based IO needs
1126	* extra reference to those pages (the user should already have that), and we	1079	* to ensure the bvecs and pages stay referenced until the submitted I/O is
1127	* don't put the page on IO completion. The caller needs to check if the bio is	1080	* completed by a call to ->ki_complete() or returns with an error other than
1128	* flagged BIO_NO_PAGE_REF on IO completion. If it isn't, then pages should be	1081	* -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF
1129	* released.	1082	* on IO completion. If it isn't, then pages should be released.
1130	*	1083	*
1131	* The function tries, but does not guarantee, to pin as many pages as	1084	* The function tries, but does not guarantee, to pin as many pages as
1132	* fit into the bio, or are requested in @iter, whatever is smaller. If	1085	* fit into the bio, or are requested in @iter, whatever is smaller. If
1133	* MM encounters an error pinning the requested pages, it stops. Error	1086	* MM encounters an error pinning the requested pages, it stops. Error
1134	* is returned only if 0 pages could be pinned.	1087	* is returned only if 0 pages could be pinned.
		1088	*
		1089	* It's intended for direct IO, so doesn't do PSI tracking, the caller is
		1090	* responsible for setting BIO_WORKINGSET if necessary.
1135	*/	1091	*/
1136	int bio_iov_iter_get_pages(struct bio bio, struct iov_iter iter)	1092	int bio_iov_iter_get_pages(struct bio bio, struct iov_iter iter)
1137	{	1093	{
1138	const bool is_bvec = iov_iter_is_bvec(iter);	1094	int ret = 0;
1139	int ret;
1140		1095
1141	if (WARN_ON_ONCE(bio->bi_vcnt))	1096	if (iov_iter_is_bvec(iter)) {
1142	return -EINVAL;	1097	if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND))
		1098	return -EINVAL;
		1099	return bio_iov_bvec_set(bio, iter);
		1100	}
1143		1101
1144	do {	1102	do {
1145	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {	1103	if (bio_op(bio) == REQ_OP_ZONE_APPEND)
1146	if (WARN_ON_ONCE(is_bvec))
1147	return -EINVAL;
1148	ret = __bio_iov_append_get_pages(bio, iter);	1104	ret = __bio_iov_append_get_pages(bio, iter);
1149	} else {	1105	else
1150	if (is_bvec)	1106	ret = __bio_iov_iter_get_pages(bio, iter);
1151	ret = __bio_iov_bvec_add_pages(bio, iter);
1152	else
1153	ret = __bio_iov_iter_get_pages(bio, iter);
1154	}
1155	} while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));	1107	} while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
1156		1108
1157	if (is_bvec)	1109	/* don't account direct I/O as memory stall */
1158	bio_set_flag(bio, BIO_NO_PAGE_REF);	1110	bio_clear_flag(bio, BIO_WORKINGSET);
1159	return bio->bi_vcnt ? 0 : ret;	1111	return bio->bi_vcnt ? 0 : ret;
1160	}	1112	}
1161	EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);	1113	EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
@@ -1178,7 +1130,8 @@ static void submit_bio_wait_endio(struct bio *bio) · 16 unmodified lines
1178	*/	1130	*/
1179	int submit_bio_wait(struct bio *bio)	1131	int submit_bio_wait(struct bio *bio)
1180	{	1132	{
1181	DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map);	1133	DECLARE_COMPLETION_ONSTACK_MAP(done,
		1134	bio->bi_bdev->bd_disk->lockdep_map);
1182	unsigned long hang_check;	1135	unsigned long hang_check;
1183		1136
1184	bio->bi_private = &done;	1137	bio->bi_private = &done;
@@ -1455,8 +1408,8 @@ again: · 270 unmodified lines
1455	if (!bio_integrity_endio(bio))	1408	if (!bio_integrity_endio(bio))
1456	return;	1409	return;
1457		1410
1458	if (bio->bi_disk)	1411	if (bio->bi_bdev)
1459	rq_qos_done_bio(bio->bi_disk->queue, bio);	1412	rq_qos_done_bio(bio->bi_bdev->bd_disk->queue, bio);
1460		1413
1461	/*	1414	/*
1462	* Need to have a real endio function for chained bios, otherwise	1415	* Need to have a real endio function for chained bios, otherwise
@@ -1471,8 +1424,8 @@ again: · 8 unmodified lines
1471	goto again;	1424	goto again;
1472	}	1425	}
1473		1426
1474	if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) {	1427	if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1475	trace_block_bio_complete(bio->bi_disk->queue, bio);	1428	trace_block_bio_complete(bio->bi_bdev->bd_disk->queue, bio);
1476	bio_clear_flag(bio, BIO_TRACE_COMPLETION);	1429	bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1477	}	1430	}
1478		1431
@@ -1559,7 +1512,7 @@ EXPORT_SYMBOL_GPL(bio_trim); · 80 unmodified lines
1559	*/	1512	*/
1560	int biovec_init_pool(mempool_t *pool, int pool_entries)	1513	int biovec_init_pool(mempool_t *pool, int pool_entries)
1561	{	1514	{
1562	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;	1515	struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
1563		1516
1564	return mempool_init_slab_pool(pool, pool_entries, bp->slab);	1517	return mempool_init_slab_pool(pool, pool_entries, bp->slab);
1565	}	1518	}
@@ -1612,15 +1565,17 @@ int bioset_init(struct bio_set *bs, · 46 unmodified lines
1612	unsigned int front_pad,	1565	unsigned int front_pad,
1613	int flags)	1566	int flags)
1614	{	1567	{
1615	unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1616
1617	bs->front_pad = front_pad;	1568	bs->front_pad = front_pad;
		1569	if (flags & BIOSET_NEED_BVECS)
		1570	bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
		1571	else
		1572	bs->back_pad = 0;
1618		1573
1619	spin_lock_init(&bs->rescue_lock);	1574	spin_lock_init(&bs->rescue_lock);
1620	bio_list_init(&bs->rescue_list);	1575	bio_list_init(&bs->rescue_list);
1621	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);	1576	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1622		1577
1623	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);	1578	bs->bio_slab = bio_find_or_create_slab(bs);
1624	if (!bs->bio_slab)	1579	if (!bs->bio_slab)
1625	return -ENOMEM;	1580	return -ENOMEM;
1626		1581
@@ -1663,39 +1618,19 @@ int bioset_init_from_src(struct bio_set bs, struct bio_set src) · 36 unmodified lines
1663	}	1618	}
1664	EXPORT_SYMBOL(bioset_init_from_src);	1619	EXPORT_SYMBOL(bioset_init_from_src);
1665		1620
1666	static void __init biovec_init_slabs(void)	1621	static int __init init_bio(void)
1667	{	1622	{
1668	int i;	1623	int i;
1669		1624
1670	for (i = 0; i < BVEC_POOL_NR; i++) {	1625	bio_integrity_init();
1671	int size;
1672	struct biovec_slab *bvs = bvec_slabs + i;
1673		1626
1674	if (bvs->nr_vecs <= BIO_INLINE_VECS) {	1627	for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
1675	bvs->slab = NULL;	1628	struct biovec_slab *bvs = bvec_slabs + i;
1676	continue;
1677	}
1678		1629
1679	size = bvs->nr_vecs * sizeof(struct bio_vec);	1630	bvs->slab = kmem_cache_create(bvs->name,
1680	bvs->slab = kmem_cache_create(bvs->name, size, 0,	1631	bvs->nr_vecs * sizeof(struct bio_vec), 0,
1681	SLAB_HWCACHE_ALIGN\|SLAB_PANIC, NULL);	1632	SLAB_HWCACHE_ALIGN \| SLAB_PANIC, NULL);
1682	}	1633	}
1683	}
1684
1685	static int __init init_bio(void)
1686	{
1687	bio_slab_max = 2;
1688	bio_slab_nr = 0;
1689	bio_slabs = kcalloc(bio_slab_max, sizeof(struct bio_slab),
1690	GFP_KERNEL);
1691
1692	BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);
1693
1694	if (!bio_slabs)
1695	panic("bio: can't allocate bios\n");
1696
1697	bio_integrity_init();
1698	biovec_init_slabs();
1699		1634
1700	if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))	1635	if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
1701	panic("bio: can't allocate bios\n");	1636	panic("bio: can't allocate bios\n");

block/blk-cgroup.c+12−10

 #include <linux/psi.h>
 #include "blk.h"
-#define MAX_KEY_LEN 100
 /*
  * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
  * blkcg_pol_register_mutex nests outside of it and synchronizes entire
 	if (unlikely(current->flags & PF_KTHREAD))
 		return;
-	if (!blk_get_queue(q))
+	if (current->throttle_queue != q) {
-		return;
+		if (!blk_get_queue(q))
+			return;
+		if (current->throttle_queue)
+			blk_put_queue(current->throttle_queue);
+		current->throttle_queue = q;
+	}
-	if (current->throttle_queue)
-		blk_put_queue(current->throttle_queue);
-	current->throttle_queue = q;
 	if (use_memdelay)
 		current->use_memdelay = use_memdelay;
 	set_notify_resume(current);
 	struct blkcg_gq *blkg, *ret_blkg = NULL;
 	rcu_read_lock();
-	blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_disk->queue);
+	blkg = blkg_lookup_create(css_to_blkcg(css),
+				  bio->bi_bdev->bd_disk->queue);
 	while (blkg) {
 		if (blkg_tryget(blkg)) {
 			ret_blkg = blkg;
 	if (css && css->parent) {
 		bio->bi_blkg = blkg_tryget_closest(bio, css);
 	} else {
-		blkg_get(bio->bi_disk->queue->root_blkg);
+		blkg_get(bio->bi_bdev->bd_disk->queue->root_blkg);
-		bio->bi_blkg = bio->bi_disk->queue->root_blkg;
+		bio->bi_blkg = bio->bi_bdev->bd_disk->queue->root_blkg;
 	}
 }
 EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);

block/blk-core.c+46−53

@@ -476,7 +476,7 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) · 475 unmodified lines
476		476
477	static inline int bio_queue_enter(struct bio *bio)	477	static inline int bio_queue_enter(struct bio *bio)
478	{	478	{
479	struct request_queue *q = bio->bi_disk->queue;	479	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
480	bool nowait = bio->bi_opf & REQ_NOWAIT;	480	bool nowait = bio->bi_opf & REQ_NOWAIT;
481	int ret;	481	int ret;
482		482
@@ -531,7 +531,7 @@ struct request_queue *blk_alloc_queue(int node_id) · 48 unmodified lines
531	if (q->id < 0)	531	if (q->id < 0)
532	goto fail_q;	532	goto fail_q;
533		533
534	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);	534	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
535	if (ret)	535	if (ret)
536	goto fail_id;	536	goto fail_id;
537		537
@@ -692,11 +692,9 @@ static inline bool should_fail_request(struct block_device *part, · 154 unmodified lines
692		692
693	#endif /* CONFIG_FAIL_MAKE_REQUEST */	693	#endif /* CONFIG_FAIL_MAKE_REQUEST */
694		694
695	static inline bool bio_check_ro(struct bio bio, struct block_device part)	695	static inline bool bio_check_ro(struct bio *bio)
696	{	696	{
697	const int op = bio_op(bio);	697	if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) {
698
699	if (part->bd_read_only && op_is_write(op)) {
700	char b[BDEVNAME_SIZE];	698	char b[BDEVNAME_SIZE];
701		699
702	if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))	700	if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
@@ -704,7 +702,7 @@ static inline bool bio_check_ro(struct bio bio, struct block_device part) · 1 unmodified line
704		702
705	WARN_ONCE(1,	703	WARN_ONCE(1,
706	"Trying to write to read-only block-device %s (partno %d)\n",	704	"Trying to write to read-only block-device %s (partno %d)\n",
707	bio_devname(bio, b), part->bd_partno);	705	bio_devname(bio, b), bio->bi_bdev->bd_partno);
708	/* Older lvm-tools actually trigger this */	706	/* Older lvm-tools actually trigger this */
709	return false;	707	return false;
710	}	708	}
@@ -714,7 +712,7 @@ static inline bool bio_check_ro(struct bio bio, struct block_device part) · 3 unmodified lines
714		712
715	static noinline int should_fail_bio(struct bio *bio)	713	static noinline int should_fail_bio(struct bio *bio)
716	{	714	{
717	if (should_fail_request(bio->bi_disk->part0, bio->bi_iter.bi_size))	715	if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size))
718	return -EIO;	716	return -EIO;
719	return 0;	717	return 0;
720	}	718	}
@@ -725,8 +723,9 @@ ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO); · 4 unmodified lines
725	* This may well happen - the kernel calls bread() without checking the size of	723	* This may well happen - the kernel calls bread() without checking the size of
726	* the device, e.g., when mounting a file system.	724	* the device, e.g., when mounting a file system.
727	*/	725	*/
728	static inline int bio_check_eod(struct bio *bio, sector_t maxsector)	726	static inline int bio_check_eod(struct bio *bio)
729	{	727	{
		728	sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
730	unsigned int nr_sectors = bio_sectors(bio);	729	unsigned int nr_sectors = bio_sectors(bio);
731		730
732	if (nr_sectors && maxsector &&	731	if (nr_sectors && maxsector &&
@@ -741,33 +740,20 @@ static inline int bio_check_eod(struct bio *bio, sector_t maxsector) · 8 unmodified lines
741	/*	740	/*
742	* Remap block n of partition p to block n+start(p) of the disk.	741	* Remap block n of partition p to block n+start(p) of the disk.
743	*/	742	*/
744	static inline int blk_partition_remap(struct bio *bio)	743	static int blk_partition_remap(struct bio *bio)
745	{	744	{
746	struct block_device *p;	745	struct block_device *p = bio->bi_bdev;
747	int ret = -EIO;
748		746
749	rcu_read_lock();
750	p = __disk_get_part(bio->bi_disk, bio->bi_partno);
751	if (unlikely(!p))
752	goto out;
753	if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))	747	if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
754	goto out;	748	return -EIO;
755	if (unlikely(bio_check_ro(bio, p)))
756	goto out;
757
758	if (bio_sectors(bio)) {	749	if (bio_sectors(bio)) {
759	if (bio_check_eod(bio, bdev_nr_sectors(p)))
760	goto out;
761	bio->bi_iter.bi_sector += p->bd_start_sect;	750	bio->bi_iter.bi_sector += p->bd_start_sect;
762	trace_block_bio_remap(bio, p->bd_dev,	751	trace_block_bio_remap(bio, p->bd_dev,
763	bio->bi_iter.bi_sector -	752	bio->bi_iter.bi_sector -
764	p->bd_start_sect);	753	p->bd_start_sect);
765	}	754	}
766	bio->bi_partno = 0;	755	bio_set_flag(bio, BIO_REMAPPED);
767	ret = 0;	756	return 0;
768	out:
769	rcu_read_unlock();
770	return ret;
771	}	757	}
772		758
773	/*	759	/*
@@ -807,7 +793,8 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, · 33 unmodified lines
807		793
808	static noinline_for_stack bool submit_bio_checks(struct bio *bio)	794	static noinline_for_stack bool submit_bio_checks(struct bio *bio)
809	{	795	{
810	struct request_queue *q = bio->bi_disk->queue;	796	struct block_device *bdev = bio->bi_bdev;
		797	struct request_queue *q = bdev->bd_disk->queue;
811	blk_status_t status = BLK_STS_IOERR;	798	blk_status_t status = BLK_STS_IOERR;
812	struct blk_plug *plug;	799	struct blk_plug *plug;
813		800
@@ -826,14 +813,12 @@ static noinline_for_stack bool submit_bio_checks(struct bio *bio) · 12 unmodified lines
826		813
827	if (should_fail_bio(bio))	814	if (should_fail_bio(bio))
828	goto end_io;	815	goto end_io;
829		816	if (unlikely(bio_check_ro(bio)))
830	if (bio->bi_partno) {	817	goto end_io;
831	if (unlikely(blk_partition_remap(bio)))	818	if (!bio_flagged(bio, BIO_REMAPPED)) {
832	goto end_io;	819	if (unlikely(bio_check_eod(bio)))
833	} else {
834	if (unlikely(bio_check_ro(bio, bio->bi_disk->part0)))
835	goto end_io;	820	goto end_io;
836	if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))	821	if (bdev->bd_partno && unlikely(blk_partition_remap(bio)))
837	goto end_io;	822	goto end_io;
838	}	823	}
839		824
@@ -926,7 +911,7 @@ end_io: · 86 unmodified lines
926		911
927	static blk_qc_t __submit_bio(struct bio *bio)	912	static blk_qc_t __submit_bio(struct bio *bio)
928	{	913	{
929	struct gendisk *disk = bio->bi_disk;	914	struct gendisk *disk = bio->bi_bdev->bd_disk;
930	blk_qc_t ret = BLK_QC_T_NONE;	915	blk_qc_t ret = BLK_QC_T_NONE;
931		916
932	if (blk_crypto_bio_prep(&bio)) {	917	if (blk_crypto_bio_prep(&bio)) {
@@ -968,7 +953,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) · 35 unmodified lines
968	current->bio_list = bio_list_on_stack;	953	current->bio_list = bio_list_on_stack;
969		954
970	do {	955	do {
971	struct request_queue *q = bio->bi_disk->queue;	956	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
972	struct bio_list lower, same;	957	struct bio_list lower, same;
973		958
974	if (unlikely(bio_queue_enter(bio) != 0))	959	if (unlikely(bio_queue_enter(bio) != 0))
@@ -989,7 +974,7 @@ static blk_qc_t __submit_bio_noacct(struct bio *bio) · 14 unmodified lines
989	bio_list_init(&lower);	974	bio_list_init(&lower);
990	bio_list_init(&same);	975	bio_list_init(&same);
991	while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)	976	while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
992	if (q == bio->bi_disk->queue)	977	if (q == bio->bi_bdev->bd_disk->queue)
993	bio_list_add(&same, bio);	978	bio_list_add(&same, bio);
994	else	979	else
995	bio_list_add(&lower, bio);	980	bio_list_add(&lower, bio);
@@ -1014,7 +999,7 @@ static blk_qc_t __submit_bio_noacct_mq(struct bio *bio) · 18 unmodified lines
1014	current->bio_list = bio_list;	999	current->bio_list = bio_list;
1015		1000
1016	do {	1001	do {
1017	struct gendisk *disk = bio->bi_disk;	1002	struct gendisk *disk = bio->bi_bdev->bd_disk;
1018		1003
1019	if (unlikely(bio_queue_enter(bio) != 0))	1004	if (unlikely(bio_queue_enter(bio) != 0))
1020	continue;	1005	continue;
@@ -1057,7 +1042,7 @@ blk_qc_t submit_bio_noacct(struct bio *bio) · 36 unmodified lines
1057	return BLK_QC_T_NONE;	1042	return BLK_QC_T_NONE;
1058	}	1043	}
1059		1044
1060	if (!bio->bi_disk->fops->submit_bio)	1045	if (!bio->bi_bdev->bd_disk->fops->submit_bio)
1061	return __submit_bio_noacct_mq(bio);	1046	return __submit_bio_noacct_mq(bio);
1062	return __submit_bio_noacct(bio);	1047	return __submit_bio_noacct(bio);
1063	}	1048	}
@@ -1069,7 +1054,7 @@ EXPORT_SYMBOL(submit_bio_noacct); · 5 unmodified lines
1069	*	1054	*
1070	* submit_bio() is used to submit I/O requests to block devices. It is passed a	1055	* submit_bio() is used to submit I/O requests to block devices. It is passed a
1071	* fully set up &struct bio that describes the I/O that needs to be done. The	1056	* fully set up &struct bio that describes the I/O that needs to be done. The
1072	* bio will be send to the device described by the bi_disk and bi_partno fields.	1057	* bio will be send to the device described by the bi_bdev field.
1073	*	1058	*
1074	* The success/failure status of the request, along with notification of	1059	* The success/failure status of the request, along with notification of
1075	* completion, is delivered asynchronously through the ->bi_end_io() callback	1060	* completion, is delivered asynchronously through the ->bi_end_io() callback
@@ -1089,7 +1074,8 @@ blk_qc_t submit_bio(struct bio *bio) · 13 unmodified lines
1089	unsigned int count;	1074	unsigned int count;
1090		1075
1091	if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))	1076	if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
1092	count = queue_logical_block_size(bio->bi_disk->queue) >> 9;	1077	count = queue_logical_block_size(
		1078	bio->bi_bdev->bd_disk->queue) >> 9;
1093	else	1079	else
1094	count = bio_sectors(bio);	1080	count = bio_sectors(bio);
1095		1081
@@ -1313,7 +1299,11 @@ void blk_account_io_start(struct request *rq) · 217 unmodified lines
1313	if (!blk_do_io_stat(rq))	1299	if (!blk_do_io_stat(rq))
1314	return;	1300	return;
1315		1301
1316	rq->part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));	1302	/* passthrough requests can hold bios that do not have ->bi_bdev set */
		1303	if (rq->bio && rq->bio->bi_bdev)
		1304	rq->part = rq->bio->bi_bdev;
		1305	else
		1306	rq->part = rq->rq_disk->part0;
1317		1307
1318	part_stat_lock();	1308	part_stat_lock();
1319	update_io_ticks(rq->part, jiffies, false);	1309	update_io_ticks(rq->part, jiffies, false);
@@ -1336,14 +1326,17 @@ static unsigned long __part_start_io_acct(struct block_device *part, · 16 unmodified lines
1336	return now;	1326	return now;
1337	}	1327	}
1338		1328
1339	unsigned long part_start_io_acct(struct gendisk disk, struct block_device *part,	1329	/**
1340	struct bio *bio)	1330	* bio_start_io_acct - start I/O accounting for bio based drivers
		1331	* @bio: bio to start account for
		1332	*
		1333	* Returns the start time that should be passed back to bio_end_io_acct().
		1334	*/
		1335	unsigned long bio_start_io_acct(struct bio *bio)
1341	{	1336	{
1342	*part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector);	1337	return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio), bio_op(bio));
1343
1344	return __part_start_io_acct(*part, bio_sectors(bio), bio_op(bio));
1345	}	1338	}
1346	EXPORT_SYMBOL_GPL(part_start_io_acct);	1339	EXPORT_SYMBOL_GPL(bio_start_io_acct);
1347		1340
1348	unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,	1341	unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1349	unsigned int op)	1342	unsigned int op)
@@ -1366,12 +1359,12 @@ static void __part_end_io_acct(struct block_device *part, unsigned int op, · 16 unmodified lines
1366	part_stat_unlock();	1359	part_stat_unlock();
1367	}	1360	}
1368		1361
1369	void part_end_io_acct(struct block_device part, struct bio bio,	1362	void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1370	unsigned long start_time)	1363	struct block_device *orig_bdev)
1371	{	1364	{
1372	__part_end_io_acct(part, bio_op(bio), start_time);	1365	__part_end_io_acct(orig_bdev, bio_op(bio), start_time);
1373	}	1366	}
1374	EXPORT_SYMBOL_GPL(part_end_io_acct);	1367	EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped);
1375		1368
1376	void disk_end_io_acct(struct gendisk *disk, unsigned int op,	1369	void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1377	unsigned long start_time)	1370	unsigned long start_time)

block/blk-crypto-fallback.c+4−2

 	struct bio_vec bv;
 	struct bio *bio;
-	bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL);
+	bio = bio_kmalloc(GFP_NOIO, bio_segments(bio_src));
 	if (!bio)
 		return NULL;
-	bio->bi_disk		= bio_src->bi_disk;
+	bio->bi_bdev		= bio_src->bi_bdev;
+	if (bio_flagged(bio_src, BIO_REMAPPED))
+		bio_set_flag(bio, BIO_REMAPPED);
 	bio->bi_opf		= bio_src->bi_opf;
 	bio->bi_ioprio		= bio_src->bi_ioprio;
 	bio->bi_write_hint	= bio_src->bi_write_hint;

block/blk-crypto.c+1−1

@@ -280,7 +280,7 @@ bool __blk_crypto_bio_prep(struct bio **bio_ptr) · 279 unmodified lines
280	* Success if device supports the encryption context, or if we succeeded	280	* Success if device supports the encryption context, or if we succeeded
281	* in falling back to the crypto API.	281	* in falling back to the crypto API.
282	*/	282	*/
283	if (blk_ksm_crypto_cfg_supported(bio->bi_disk->queue->ksm,	283	if (blk_ksm_crypto_cfg_supported(bio->bi_bdev->bd_disk->queue->ksm,
284	&bc_key->crypto_cfg))	284	&bc_key->crypto_cfg))
285	return true;	285	return true;
286		286

block/blk-exec.c+5−9

 }
 /**
- * blk_execute_rq_nowait - insert a request into queue for execution
+ * blk_execute_rq_nowait - insert a request to I/O scheduler for execution
- * @q:		queue to insert the request in
  * @bd_disk:	matching gendisk
  * @rq:		request to insert
  * @at_head:    insert request at head or tail of queue
  * Note:
  *    This function will invoke @done directly if the queue is dead.
  */
-void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
+void blk_execute_rq_nowait(struct gendisk *bd_disk, struct request *rq,
-			   struct request *rq, int at_head,
+			   int at_head, rq_end_io_fn *done)
-			   rq_end_io_fn *done)
 {
 	WARN_ON(irqs_disabled());
 	WARN_ON(!blk_rq_is_passthrough(rq));
 /**
  * blk_execute_rq - insert a request into queue for execution
- * @q:		queue to insert the request in
  * @bd_disk:	matching gendisk
  * @rq:		request to insert
  * @at_head:    insert request at head or tail of queue
  *    Insert a fully prepared request at the back of the I/O scheduler queue
  *    for execution and wait for completion.
  */
-void blk_execute_rq(struct request_queue *q, struct gendisk *bd_disk,
+void blk_execute_rq(struct gendisk *bd_disk, struct request *rq, int at_head)
-		   struct request *rq, int at_head)
 {
 	DECLARE_COMPLETION_ONSTACK(wait);
 	unsigned long hang_check;
 	rq->end_io_data = &wait;
-	blk_execute_rq_nowait(q, bd_disk, rq, at_head, blk_end_sync_rq);
+	blk_execute_rq_nowait(bd_disk, rq, at_head, blk_end_sync_rq);
 	/* Prevent hang_check timer from firing at us during very long I/O */
 	hang_check = sysctl_hung_task_timeout_secs;

block/blk-flush.c+6−11

@@ -432,23 +432,18 @@ void blk_insert_flush(struct request *rq) · 431 unmodified lines
432	/**	432	/**
433	* blkdev_issue_flush - queue a flush	433	* blkdev_issue_flush - queue a flush
434	* @bdev: blockdev to issue flush for	434	* @bdev: blockdev to issue flush for
435	* @gfp_mask: memory allocation flags (for bio_alloc)
436	*	435	*
437	* Description:	436	* Description:
438	* Issue a flush for the block device in question.	437	* Issue a flush for the block device in question.
439	*/	438	*/
440	int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)	439	int blkdev_issue_flush(struct block_device *bdev)
441	{	440	{
442	struct bio *bio;	441	struct bio bio;
443	int ret = 0;
444		442
445	bio = bio_alloc(gfp_mask, 0);	443	bio_init(&bio, NULL, 0);
446	bio_set_dev(bio, bdev);	444	bio_set_dev(&bio, bdev);
447	bio->bi_opf = REQ_OP_WRITE \| REQ_PREFLUSH;	445	bio.bi_opf = REQ_OP_WRITE \| REQ_PREFLUSH;
448		446	return submit_bio_wait(&bio);
449	ret = submit_bio_wait(bio);
450	bio_put(bio);
451	return ret;
452	}	447	}
453	EXPORT_SYMBOL(blkdev_issue_flush);	448	EXPORT_SYMBOL(blkdev_issue_flush);
454		449

block/blk-merge.c+8−9

  * Split a bio into two bios, chain the two bios, submit the second half and
  * store a pointer to the first half in *@bio. If the second bio is still too
  * big it will be split by a recursive call to this function. Since this
- * function may allocate a new bio from @bio->bi_disk->queue->bio_split, it is
+ * function may allocate a new bio from q->bio_split, it is the responsibility
- * the responsibility of the caller to ensure that
+ * of the caller to ensure that q->bio_split is only released after processing
- * @bio->bi_disk->queue->bio_split is only released after processing of the
+ * of the split bio has finished.
- * split bio has finished.
  */
 void __blk_queue_split(struct bio **bio, unsigned int *nr_segs)
 {
-	struct request_queue *q = (*bio)->bi_disk->queue;
+	struct request_queue *q = (*bio)->bi_bdev->bd_disk->queue;
 	struct bio *split = NULL;
 	switch (bio_op(*bio)) {
  *
  * Split a bio into two bios, chains the two bios, submit the second half and
  * store a pointer to the first half in *@bio. Since this function may allocate
- * a new bio from @bio->bi_disk->queue->bio_split, it is the responsibility of
+ * a new bio from q->bio_split, it is the responsibility of the caller to ensure
- * the caller to ensure that @bio->bi_disk->queue->bio_split is only released
+ * that q->bio_split is only released after processing of the split bio has
- * after processing of the split bio has finished.
+ * finished.
  */
 void blk_queue_split(struct bio **bio)
 {
 		return false;
 	/* must be same device */
-	if (rq->rq_disk != bio->bi_disk)
+	if (rq->rq_disk != bio->bi_bdev->bd_disk)
 		return false;
 	/* only merge integrity protected bio into ditto rq */

block/blk-mq.c+61−8

@@ -1646,6 +1646,42 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) · 1645 unmodified lines
1646	}	1646	}
1647	EXPORT_SYMBOL(blk_mq_run_hw_queue);	1647	EXPORT_SYMBOL(blk_mq_run_hw_queue);
1648		1648
		1649	/*
		1650	* Is the request queue handled by an IO scheduler that does not respect
		1651	* hardware queues when dispatching?
		1652	*/
		1653	static bool blk_mq_has_sqsched(struct request_queue *q)
		1654	{
		1655	struct elevator_queue *e = q->elevator;
		1656
		1657	if (e && e->type->ops.dispatch_request &&
		1658	!(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
		1659	return true;
		1660	return false;
		1661	}
		1662
		1663	/*
		1664	* Return prefered queue to dispatch from (if any) for non-mq aware IO
		1665	* scheduler.
		1666	*/
		1667	static struct blk_mq_hw_ctx blk_mq_get_sq_hctx(struct request_queue q)
		1668	{
		1669	struct blk_mq_hw_ctx *hctx;
		1670
		1671	/*
		1672	* If the IO scheduler does not respect hardware queues when
		1673	* dispatching, we just don't bother with multiple HW queues and
		1674	* dispatch from hctx for the current CPU since running multiple queues
		1675	* just causes lock contention inside the scheduler and pointless cache
		1676	* bouncing.
		1677	*/
		1678	hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT,
		1679	raw_smp_processor_id());
		1680	if (!blk_mq_hctx_stopped(hctx))
		1681	return hctx;
		1682	return NULL;
		1683	}
		1684
1649	/**	1685	/**
1650	* blk_mq_run_hw_queues - Run all hardware queues in a request queue.	1686	* blk_mq_run_hw_queues - Run all hardware queues in a request queue.
1651	* @q: Pointer to the request queue to run.	1687	* @q: Pointer to the request queue to run.
@@ -1653,14 +1689,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queue); · 1 unmodified line
1653	*/	1689	*/
1654	void blk_mq_run_hw_queues(struct request_queue *q, bool async)	1690	void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1655	{	1691	{
1656	struct blk_mq_hw_ctx *hctx;	1692	struct blk_mq_hw_ctx hctx, sq_hctx;
1657	int i;	1693	int i;
1658		1694
		1695	sq_hctx = NULL;
		1696	if (blk_mq_has_sqsched(q))
		1697	sq_hctx = blk_mq_get_sq_hctx(q);
1659	queue_for_each_hw_ctx(q, hctx, i) {	1698	queue_for_each_hw_ctx(q, hctx, i) {
1660	if (blk_mq_hctx_stopped(hctx))	1699	if (blk_mq_hctx_stopped(hctx))
1661	continue;	1700	continue;
1662		1701	/*
1663	blk_mq_run_hw_queue(hctx, async);	1702	* Dispatch from this hctx either if there's no hctx preferred
		1703	* by IO scheduler or if it has requests that bypass the
		1704	* scheduler.
		1705	*/
		1706	if (!sq_hctx \|\| sq_hctx == hctx \|\|
		1707	!list_empty_careful(&hctx->dispatch))
		1708	blk_mq_run_hw_queue(hctx, async);
1664	}	1709	}
1665	}	1710	}
1666	EXPORT_SYMBOL(blk_mq_run_hw_queues);	1711	EXPORT_SYMBOL(blk_mq_run_hw_queues);
@@ -1672,14 +1717,23 @@ EXPORT_SYMBOL(blk_mq_run_hw_queues); · 5 unmodified lines
1672	*/	1717	*/
1673	void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)	1718	void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
1674	{	1719	{
1675	struct blk_mq_hw_ctx *hctx;	1720	struct blk_mq_hw_ctx hctx, sq_hctx;
1676	int i;	1721	int i;
1677		1722
		1723	sq_hctx = NULL;
		1724	if (blk_mq_has_sqsched(q))
		1725	sq_hctx = blk_mq_get_sq_hctx(q);
1678	queue_for_each_hw_ctx(q, hctx, i) {	1726	queue_for_each_hw_ctx(q, hctx, i) {
1679	if (blk_mq_hctx_stopped(hctx))	1727	if (blk_mq_hctx_stopped(hctx))
1680	continue;	1728	continue;
1681		1729	/*
1682	blk_mq_delay_run_hw_queue(hctx, msecs);	1730	* Dispatch from this hctx either if there's no hctx preferred
		1731	* by IO scheduler or if it has requests that bypass the
		1732	* scheduler.
		1733	*/
		1734	if (!sq_hctx \|\| sq_hctx == hctx \|\|
		1735	!list_empty_careful(&hctx->dispatch))
		1736	blk_mq_delay_run_hw_queue(hctx, msecs);
1683	}	1737	}
1684	}	1738	}
1685	EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);	1739	EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
@@ -2128,7 +2182,7 @@ static void blk_add_rq_to_plug(struct blk_plug plug, struct request rq) · 442 unmodified lines
2128	*/	2182	*/
2129	blk_qc_t blk_mq_submit_bio(struct bio *bio)	2183	blk_qc_t blk_mq_submit_bio(struct bio *bio)
2130	{	2184	{
2131	struct request_queue *q = bio->bi_disk->queue;	2185	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
2132	const int is_sync = op_is_sync(bio->bi_opf);	2186	const int is_sync = op_is_sync(bio->bi_opf);
2133	const int is_flush_fua = op_is_flush(bio->bi_opf);	2187	const int is_flush_fua = op_is_flush(bio->bi_opf);
2134	struct blk_mq_alloc_data data = {	2188	struct blk_mq_alloc_data data = {
@@ -2653,7 +2707,6 @@ blk_mq_alloc_hctx(struct request_queue q, struct blk_mq_tag_set set, · 518 unmodified lines
2653	goto free_hctx;	2707	goto free_hctx;
2654		2708
2655	atomic_set(&hctx->nr_active, 0);	2709	atomic_set(&hctx->nr_active, 0);
2656	atomic_set(&hctx->elevator_queued, 0);
2657	if (node == NUMA_NO_NODE)	2710	if (node == NUMA_NO_NODE)
2658	node = set->numa_node;	2711	node = set->numa_node;
2659	hctx->numa_node = node;	2712	hctx->numa_node = node;

block/blk-settings.c+39−2

 	lim->io_opt = 0;
 	lim->misaligned = 0;
 	lim->zoned = BLK_ZONED_NONE;
+	lim->zone_write_granularity = 0;
 }
 EXPORT_SYMBOL(blk_set_default_limits);
 }
 EXPORT_SYMBOL(blk_queue_physical_block_size);
+/**
+ * blk_queue_zone_write_granularity - set zone write granularity for the queue
+ * @q:  the request queue for the zoned device
+ * @size:  the zone write granularity size, in bytes
+ *
+ * Description:
+ *   This should be set to the lowest possible size allowing to write in
+ *   sequential zones of a zoned block device.
+ */
+void blk_queue_zone_write_granularity(struct request_queue *q,
+				      unsigned int size)
+{
+	if (WARN_ON_ONCE(!blk_queue_is_zoned(q)))
+		return;
+	q->limits.zone_write_granularity = size;
+	if (q->limits.zone_write_granularity < q->limits.logical_block_size)
+		q->limits.zone_write_granularity = q->limits.logical_block_size;
+}
+EXPORT_SYMBOL_GPL(blk_queue_zone_write_granularity);
 /**
  * blk_queue_alignment_offset - set physical block alignment offset
  * @q:	the request queue for the device
 			t->discard_granularity;
 	}
+	t->zone_write_granularity = max(t->zone_write_granularity,
+					b->zone_write_granularity);
 	t->zoned = max(t->zoned, b->zoned);
 	return ret;
 }
  */
 void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model)
 {
+	struct request_queue *q = disk->queue;
 	switch (model) {
 	case BLK_ZONED_HM:
 		/*
 		 * we do nothing special as far as the block layer is concerned.
 		 */
 		if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) ||
-		    disk_has_partitions(disk))
+		    !xa_empty(&disk->part_tbl))
 			model = BLK_ZONED_NONE;
 		break;
 	case BLK_ZONED_NONE:
 		break;
 	}
-	disk->queue->limits.zoned = model;
+	q->limits.zoned = model;
+	if (model != BLK_ZONED_NONE) {
+		/*
+		 * Set the zone write granularity to the device logical block
+		 * size by default. The driver can change this value if needed.
+		 */
+		blk_queue_zone_write_granularity(q,
+						queue_logical_block_size(q));
+	} else {
+		blk_queue_clear_zone_settings(q);
+	}
 }
 EXPORT_SYMBOL_GPL(blk_queue_set_zoned);

block/blk-sysfs.c+8−0

 		(unsigned long long)q->limits.max_write_zeroes_sectors << 9);
 }
+static ssize_t queue_zone_write_granularity_show(struct request_queue *q,
+						 char *page)
+{
+	return queue_var_show(queue_zone_write_granularity(q), page);
+}
 static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page)
 {
 	unsigned long long max_sectors = q->limits.max_zone_append_sectors;
 QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
 QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes");
 QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
+QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
 QUEUE_RO_ENTRY(queue_zoned, "zoned");
 QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
 	&queue_write_same_max_entry.attr,
 	&queue_write_zeroes_max_entry.attr,
 	&queue_zone_append_max_entry.attr,
+	&queue_zone_write_granularity_entry.attr,
 	&queue_nonrot_entry.attr,
 	&queue_zoned_entry.attr,
 	&queue_nr_zones_entry.attr,

block/blk-throttle.c+1−1

@@ -2178,7 +2178,7 @@ static inline void throtl_update_latency_buckets(struct throtl_data *td) · 2177 unmodified lines
2178		2178
2179	bool blk_throtl_bio(struct bio *bio)	2179	bool blk_throtl_bio(struct bio *bio)
2180	{	2180	{
2181	struct request_queue *q = bio->bi_disk->queue;	2181	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
2182	struct blkcg_gq *blkg = bio->bi_blkg;	2182	struct blkcg_gq *blkg = bio->bi_blkg;
2183	struct throtl_qnode *qn = NULL;	2183	struct throtl_qnode *qn = NULL;
2184	struct throtl_grp *tg = blkg_to_tg(blkg);	2184	struct throtl_grp *tg = blkg_to_tg(blkg);

block/blk-wbt.c+2−2

@@ -518,7 +518,7 @@ static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct, · 517 unmodified lines
518	rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);	518	rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);
519	}	519	}
520		520
521	static inline bool wbt_should_throttle(struct rq_wb rwb, struct bio bio)	521	static inline bool wbt_should_throttle(struct bio *bio)
522	{	522	{
523	switch (bio_op(bio)) {	523	switch (bio_op(bio)) {
524	case REQ_OP_WRITE:	524	case REQ_OP_WRITE:
@@ -545,7 +545,7 @@ static enum wbt_flags bio_to_wbt_flags(struct rq_wb rwb, struct bio bio) · 20 unmodified lines
545		545
546	if (bio_op(bio) == REQ_OP_READ) {	546	if (bio_op(bio) == REQ_OP_READ) {
547	flags = WBT_READ;	547	flags = WBT_READ;
548	} else if (wbt_should_throttle(rwb, bio)) {	548	} else if (wbt_should_throttle(bio)) {
549	if (current_is_kswapd())	549	if (current_is_kswapd())
550	flags \|= WBT_KSWAPD;	550	flags \|= WBT_KSWAPD;
551	if (bio_op(bio) == REQ_OP_DISCARD)	551	if (bio_op(bio) == REQ_OP_DISCARD)

block/blk-zoned.c+17−0

@@ -549,3 +549,20 @@ int blk_revalidate_disk_zones(struct gendisk *disk, · 548 unmodified lines
549	return ret;	549	return ret;
550	}	550	}
551	EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones);	551	EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones);
		552
		553	void blk_queue_clear_zone_settings(struct request_queue *q)
		554	{
		555	blk_mq_freeze_queue(q);
		556
		557	blk_queue_free_zone_bitmaps(q);
		558	blk_queue_flag_clear(QUEUE_FLAG_ZONE_RESETALL, q);
		559	q->required_elevator_features &= ~ELEVATOR_F_ZBD_SEQ_WRITE;
		560	q->nr_zones = 0;
		561	q->max_open_zones = 0;
		562	q->max_active_zones = 0;
		563	q->limits.chunk_sectors = 0;
		564	q->limits.zone_write_granularity = 0;
		565	q->limits.max_zone_append_sectors = 0;
		566
		567	blk_mq_unfreeze_queue(q);
		568	}

block/blk.h+7−5

 void blk_freeze_queue(struct request_queue *q);
+#define BIO_INLINE_VECS 4
+struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
+		gfp_t gfp_mask);
+void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);
 static inline bool biovec_phys_mergeable(struct request_queue *q,
 		struct bio_vec *vec1, struct bio_vec *vec2)
 {
 	__elevator_exit(q, e);
 }
-struct block_device *__disk_get_part(struct gendisk *disk, int partno);
 ssize_t part_size_show(struct device *dev, struct device_attribute *attr,
 		char *buf);
 ssize_t part_stat_show(struct device *dev, struct device_attribute *attr,
 #ifdef CONFIG_BLK_DEV_ZONED
 void blk_queue_free_zone_bitmaps(struct request_queue *q);
+void blk_queue_clear_zone_settings(struct request_queue *q);
 #else
 static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {}
+static inline void blk_queue_clear_zone_settings(struct request_queue *q) {}
 #endif
-struct block_device *disk_map_sector_rcu(struct gendisk *disk, sector_t sector);
 int blk_alloc_devt(struct block_device *part, dev_t *devt);
 void blk_free_devt(dev_t devt);
 char *disk_name(struct gendisk *hd, int partno, char *buf);
 int bdev_del_partition(struct block_device *bdev, int partno);
 int bdev_resize_partition(struct block_device *bdev, int partno,
 		sector_t start, sector_t length);
-int disk_expand_part_tbl(struct gendisk *disk, int target);
 int bio_add_hw_page(struct request_queue *q, struct bio *bio,
 		struct page *page, unsigned int len, unsigned int offset,

block/bounce.c+3−1

 	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
 	if (!bio)
 		return NULL;
-	bio->bi_disk		= bio_src->bi_disk;
+	bio->bi_bdev		= bio_src->bi_bdev;
+	if (bio_flagged(bio_src, BIO_REMAPPED))
+		bio_set_flag(bio, BIO_REMAPPED);
 	bio->bi_opf		= bio_src->bi_opf;
 	bio->bi_ioprio		= bio_src->bi_ioprio;
 	bio->bi_write_hint	= bio_src->bi_write_hint;

block/bsg.c+4−2

 		return PTR_ERR(rq);
 	ret = q->bsg_dev.ops->fill_hdr(rq, &hdr, mode);
-	if (ret)
+	if (ret) {
+		blk_put_request(rq);
 		return ret;
+	}
 	rq->timeout = msecs_to_jiffies(hdr.timeout);
 	if (!rq->timeout)
 	bio = rq->bio;
-	blk_execute_rq(q, NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL));
+	blk_execute_rq(NULL, rq, !(hdr.flags & BSG_FLAG_Q_AT_TAIL));
 	ret = rq->q->bsg_dev.ops->complete_rq(rq, &hdr);
 	blk_rq_unmap_user(bio);

block/genhd.c+49−257

@@ -162,15 +162,6 @@ static void part_in_flight_rw(struct block_device *part, · 161 unmodified lines
162	inflight[1] = 0;	162	inflight[1] = 0;
163	}	163	}
164		164
165	struct block_device __disk_get_part(struct gendisk disk, int partno)
166	{
167	struct disk_part_tbl *ptbl = rcu_dereference(disk->part_tbl);
168
169	if (unlikely(partno < 0 \|\| partno >= ptbl->len))
170	return NULL;
171	return rcu_dereference(ptbl->part[partno]);
172	}
173
174	/**	165	/**
175	* disk_part_iter_init - initialize partition iterator	166	* disk_part_iter_init - initialize partition iterator
176	* @piter: iterator to initialize	167	* @piter: iterator to initialize
@@ -185,26 +176,14 @@ struct block_device __disk_get_part(struct gendisk disk, int partno) · 8 unmodified lines
185	void disk_part_iter_init(struct disk_part_iter piter, struct gendisk disk,	176	void disk_part_iter_init(struct disk_part_iter piter, struct gendisk disk,
186	unsigned int flags)	177	unsigned int flags)
187	{	178	{
188	struct disk_part_tbl *ptbl;
189
190	rcu_read_lock();
191	ptbl = rcu_dereference(disk->part_tbl);
192
193	piter->disk = disk;	179	piter->disk = disk;
194	piter->part = NULL;	180	piter->part = NULL;
195		181	if (flags & (DISK_PITER_INCL_PART0 \| DISK_PITER_INCL_EMPTY_PART0))
196	if (flags & DISK_PITER_REVERSE)
197	piter->idx = ptbl->len - 1;
198	else if (flags & (DISK_PITER_INCL_PART0 \| DISK_PITER_INCL_EMPTY_PART0))
199	piter->idx = 0;	182	piter->idx = 0;
200	else	183	else
201	piter->idx = 1;	184	piter->idx = 1;
202
203	piter->flags = flags;	185	piter->flags = flags;
204
205	rcu_read_unlock();
206	}	186	}
207	EXPORT_SYMBOL_GPL(disk_part_iter_init);
208		187
209	/**	188	/**
210	* disk_part_iter_next - proceed iterator to the next partition and return it	189	* disk_part_iter_next - proceed iterator to the next partition and return it
@@ -217,57 +196,30 @@ EXPORT_SYMBOL_GPL(disk_part_iter_init); · 6 unmodified lines
217	*/	196	*/
218	struct block_device disk_part_iter_next(struct disk_part_iter piter)	197	struct block_device disk_part_iter_next(struct disk_part_iter piter)
219	{	198	{
220	struct disk_part_tbl *ptbl;	199	struct block_device *part;
221	int inc, end;	200	unsigned long idx;
222		201
223	/* put the last partition */	202	/* put the last partition */
224	disk_part_iter_exit(piter);	203	disk_part_iter_exit(piter);
225		204
226	/* get part_tbl */
227	rcu_read_lock();	205	rcu_read_lock();
228	ptbl = rcu_dereference(piter->disk->part_tbl);	206	xa_for_each_start(&piter->disk->part_tbl, idx, part, piter->idx) {
229
230	/* determine iteration parameters */
231	if (piter->flags & DISK_PITER_REVERSE) {
232	inc = -1;
233	if (piter->flags & (DISK_PITER_INCL_PART0 \|
234	DISK_PITER_INCL_EMPTY_PART0))
235	end = -1;
236	else
237	end = 0;
238	} else {
239	inc = 1;
240	end = ptbl->len;
241	}
242
243	/* iterate to the next partition */
244	for (; piter->idx != end; piter->idx += inc) {
245	struct block_device *part;
246
247	part = rcu_dereference(ptbl->part[piter->idx]);
248	if (!part)
249	continue;
250	piter->part = bdgrab(part);
251	if (!piter->part)
252	continue;
253	if (!bdev_nr_sectors(part) &&	207	if (!bdev_nr_sectors(part) &&
254	!(piter->flags & DISK_PITER_INCL_EMPTY) &&	208	!(piter->flags & DISK_PITER_INCL_EMPTY) &&
255	!(piter->flags & DISK_PITER_INCL_EMPTY_PART0 &&	209	!(piter->flags & DISK_PITER_INCL_EMPTY_PART0 &&
256	piter->idx == 0)) {	210	piter->idx == 0))
257	bdput(piter->part);
258	piter->part = NULL;
259	continue;	211	continue;
260	}
261		212
262	piter->idx += inc;	213	piter->part = bdgrab(part);
		214	if (!piter->part)
		215	continue;
		216	piter->idx = idx + 1;
263	break;	217	break;
264	}	218	}
265
266	rcu_read_unlock();	219	rcu_read_unlock();
267		220
268	return piter->part;	221	return piter->part;
269	}	222	}
270	EXPORT_SYMBOL_GPL(disk_part_iter_next);
271		223
272	/**	224	/**
273	* disk_part_iter_exit - finish up partition iteration	225	* disk_part_iter_exit - finish up partition iteration
@@ -284,91 +236,6 @@ void disk_part_iter_exit(struct disk_part_iter *piter) · 10 unmodified lines
284	bdput(piter->part);	236	bdput(piter->part);
285	piter->part = NULL;	237	piter->part = NULL;
286	}	238	}
287	EXPORT_SYMBOL_GPL(disk_part_iter_exit);
288
289	static inline int sector_in_part(struct block_device *part, sector_t sector)
290	{
291	return part->bd_start_sect <= sector &&
292	sector < part->bd_start_sect + bdev_nr_sectors(part);
293	}
294
295	/**
296	* disk_map_sector_rcu - map sector to partition
297	* @disk: gendisk of interest
298	* @sector: sector to map
299	*
300	* Find out which partition @sector maps to on @disk. This is
301	* primarily used for stats accounting.
302	*
303	* CONTEXT:
304	* RCU read locked.
305	*
306	* RETURNS:
307	* Found partition on success, part0 is returned if no partition matches
308	* or the matched partition is being deleted.
309	*/
310	struct block_device disk_map_sector_rcu(struct gendisk disk, sector_t sector)
311	{
312	struct disk_part_tbl *ptbl;
313	struct block_device *part;
314	int i;
315
316	rcu_read_lock();
317	ptbl = rcu_dereference(disk->part_tbl);
318
319	part = rcu_dereference(ptbl->last_lookup);
320	if (part && sector_in_part(part, sector))
321	goto out_unlock;
322
323	for (i = 1; i < ptbl->len; i++) {
324	part = rcu_dereference(ptbl->part[i]);
325	if (part && sector_in_part(part, sector)) {
326	rcu_assign_pointer(ptbl->last_lookup, part);
327	goto out_unlock;
328	}
329	}
330
331	part = disk->part0;
332	out_unlock:
333	rcu_read_unlock();
334	return part;
335	}
336
337	/**
338	* disk_has_partitions
339	* @disk: gendisk of interest
340	*
341	* Walk through the partition table and check if valid partition exists.
342	*
343	* CONTEXT:
344	* Don't care.
345	*
346	* RETURNS:
347	* True if the gendisk has at least one valid non-zero size partition.
348	* Otherwise false.
349	*/
350	bool disk_has_partitions(struct gendisk *disk)
351	{
352	struct disk_part_tbl *ptbl;
353	int i;
354	bool ret = false;
355
356	rcu_read_lock();
357	ptbl = rcu_dereference(disk->part_tbl);
358
359	/* Iterate partitions skipping the whole device at index 0 */
360	for (i = 1; i < ptbl->len; i++) {
361	if (rcu_dereference(ptbl->part[i])) {
362	ret = true;
363	break;
364	}
365	}
366
367	rcu_read_unlock();
368
369	return ret;
370	}
371	EXPORT_SYMBOL_GPL(disk_has_partitions);
372		239
373	/*	240	/*
374	* Can be deleted altogether. Later.	241	* Can be deleted altogether. Later.
@@ -604,6 +471,18 @@ static char bdevt_str(dev_t devt, char buf) · 229 unmodified lines
604	return buf;	471	return buf;
605	}	472	}
606		473
		474	void disk_uevent(struct gendisk *disk, enum kobject_action action)
		475	{
		476	struct disk_part_iter piter;
		477	struct block_device *part;
		478
		479	disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
		480	while ((part = disk_part_iter_next(&piter)))
		481	kobject_uevent(bdev_kobj(part), action);
		482	disk_part_iter_exit(&piter);
		483	}
		484	EXPORT_SYMBOL_GPL(disk_uevent);
		485
607	static void disk_scan_partitions(struct gendisk *disk)	486	static void disk_scan_partitions(struct gendisk *disk)
608	{	487	{
609	struct block_device *bdev;	488	struct block_device *bdev;
@@ -621,8 +500,6 @@ static void register_disk(struct device parent, struct gendisk disk, · 11 unmodified lines
621	const struct attribute_group **groups)	500	const struct attribute_group **groups)
622	{	501	{
623	struct device *ddev = disk_to_dev(disk);	502	struct device *ddev = disk_to_dev(disk);
624	struct disk_part_iter piter;
625	struct block_device *part;
626	int err;	503	int err;
627		504
628	ddev->parent = parent;	505	ddev->parent = parent;
@@ -665,15 +542,9 @@ static void register_disk(struct device parent, struct gendisk disk, · 36 unmodified lines
665		542
666	disk_scan_partitions(disk);	543	disk_scan_partitions(disk);
667		544
668	/* announce disk after possible partitions are created */	545	/* announce the disk and partitions after all partitions are created */
669	dev_set_uevent_suppress(ddev, 0);	546	dev_set_uevent_suppress(ddev, 0);
670	kobject_uevent(&ddev->kobj, KOBJ_ADD);	547	disk_uevent(disk, KOBJ_ADD);
671
672	/* announce possible partitions */
673	disk_part_iter_init(&piter, disk, 0);
674	while ((part = disk_part_iter_next(&piter)))
675	kobject_uevent(bdev_kobj(part), KOBJ_ADD);
676	disk_part_iter_exit(&piter);
677		548
678	if (disk->queue->backing_dev_info->dev) {	549	if (disk->queue->backing_dev_info->dev) {
679	err = sysfs_create_link(&ddev->kobj,	550	err = sysfs_create_link(&ddev->kobj,
@@ -829,8 +700,7 @@ void del_gendisk(struct gendisk *disk) · 149 unmodified lines
829	down_write(&bdev_lookup_sem);	700	down_write(&bdev_lookup_sem);
830		701
831	/* invalidate stuff */	702	/* invalidate stuff */
832	disk_part_iter_init(&piter, disk,	703	disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY);
833	DISK_PITER_INCL_EMPTY \| DISK_PITER_REVERSE);
834	while ((part = disk_part_iter_next(&piter))) {	704	while ((part = disk_part_iter_next(&piter))) {
835	invalidate_partition(part);	705	invalidate_partition(part);
836	delete_partition(part);	706	delete_partition(part);
@@ -929,7 +799,7 @@ struct block_device bdget_disk(struct gendisk disk, int partno) · 92 unmodified lines
929	struct block_device *bdev = NULL;	799	struct block_device *bdev = NULL;
930		800
931	rcu_read_lock();	801	rcu_read_lock();
932	bdev = __disk_get_part(disk, partno);	802	bdev = xa_load(&disk->part_tbl, partno);
933	if (bdev && !bdgrab(bdev))	803	if (bdev && !bdgrab(bdev))
934	bdev = NULL;	804	bdev = NULL;
935	rcu_read_unlock();	805	rcu_read_unlock();
@@ -1319,83 +1189,6 @@ static const struct attribute_group *disk_attr_groups[] = { · 383 unmodified lines
1319	NULL	1189	NULL
1320	};	1190	};
1321		1191
1322	/**
1323	* disk_replace_part_tbl - replace disk->part_tbl in RCU-safe way
1324	* @disk: disk to replace part_tbl for
1325	* @new_ptbl: new part_tbl to install
1326	*
1327	* Replace disk->part_tbl with @new_ptbl in RCU-safe way. The
1328	* original ptbl is freed using RCU callback.
1329	*
1330	* LOCKING:
1331	* Matching bd_mutex locked or the caller is the only user of @disk.
1332	*/
1333	static void disk_replace_part_tbl(struct gendisk *disk,
1334	struct disk_part_tbl *new_ptbl)
1335	{
1336	struct disk_part_tbl *old_ptbl =
1337	rcu_dereference_protected(disk->part_tbl, 1);
1338
1339	rcu_assign_pointer(disk->part_tbl, new_ptbl);
1340
1341	if (old_ptbl) {
1342	rcu_assign_pointer(old_ptbl->last_lookup, NULL);
1343	kfree_rcu(old_ptbl, rcu_head);
1344	}
1345	}
1346
1347	/**
1348	* disk_expand_part_tbl - expand disk->part_tbl
1349	* @disk: disk to expand part_tbl for
1350	* @partno: expand such that this partno can fit in
1351	*
1352	* Expand disk->part_tbl such that @partno can fit in. disk->part_tbl
1353	* uses RCU to allow unlocked dereferencing for stats and other stuff.
1354	*
1355	* LOCKING:
1356	* Matching bd_mutex locked or the caller is the only user of @disk.
1357	* Might sleep.
1358	*
1359	* RETURNS:
1360	* 0 on success, -errno on failure.
1361	*/
1362	int disk_expand_part_tbl(struct gendisk *disk, int partno)
1363	{
1364	struct disk_part_tbl *old_ptbl =
1365	rcu_dereference_protected(disk->part_tbl, 1);
1366	struct disk_part_tbl *new_ptbl;
1367	int len = old_ptbl ? old_ptbl->len : 0;
1368	int i, target;
1369
1370	/*
1371	* check for int overflow, since we can get here from blkpg_ioctl()
1372	* with a user passed 'partno'.
1373	*/
1374	target = partno + 1;
1375	if (target < 0)
1376	return -EINVAL;
1377
1378	/* disk_max_parts() is zero during initialization, ignore if so */
1379	if (disk_max_parts(disk) && target > disk_max_parts(disk))
1380	return -EINVAL;
1381
1382	if (target <= len)
1383	return 0;
1384
1385	new_ptbl = kzalloc_node(struct_size(new_ptbl, part, target), GFP_KERNEL,
1386	disk->node_id);
1387	if (!new_ptbl)
1388	return -ENOMEM;
1389
1390	new_ptbl->len = target;
1391
1392	for (i = 0; i < len; i++)
1393	rcu_assign_pointer(new_ptbl->part[i], old_ptbl->part[i]);
1394
1395	disk_replace_part_tbl(disk, new_ptbl);
1396	return 0;
1397	}
1398
1399	/**	1192	/**
1400	* disk_release - releases all allocated resources of the gendisk	1193	* disk_release - releases all allocated resources of the gendisk
1401	* @dev: the device representing this disk	1194	* @dev: the device representing this disk
@@ -1419,7 +1212,7 @@ static void disk_release(struct device *dev) · 17 unmodified lines
1419	blk_free_devt(dev->devt);	1212	blk_free_devt(dev->devt);
1420	disk_release_events(disk);	1213	disk_release_events(disk);
1421	kfree(disk->random);	1214	kfree(disk->random);
1422	disk_replace_part_tbl(disk, NULL);	1215	xa_destroy(&disk->part_tbl);
1423	bdput(disk->part0);	1216	bdput(disk->part0);
1424	if (disk->queue)	1217	if (disk->queue)
1425	blk_put_queue(disk->queue);	1218	blk_put_queue(disk->queue);
@@ -1572,7 +1365,6 @@ dev_t blk_lookup_devt(const char *name, int partno) · 146 unmodified lines
1572	struct gendisk *__alloc_disk_node(int minors, int node_id)	1365	struct gendisk *__alloc_disk_node(int minors, int node_id)
1573	{	1366	{
1574	struct gendisk *disk;	1367	struct gendisk *disk;
1575	struct disk_part_tbl *ptbl;
1576		1368
1577	if (minors > DISK_MAX_PARTS) {	1369	if (minors > DISK_MAX_PARTS) {
1578	printk(KERN_ERR	1370	printk(KERN_ERR
@@ -1590,11 +1382,9 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) · 11 unmodified lines
1590	goto out_free_disk;	1382	goto out_free_disk;
1591		1383
1592	disk->node_id = node_id;	1384	disk->node_id = node_id;
1593	if (disk_expand_part_tbl(disk, 0))	1385	xa_init(&disk->part_tbl);
1594	goto out_bdput;	1386	if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL))
1595		1387	goto out_destroy_part_tbl;
1596	ptbl = rcu_dereference_protected(disk->part_tbl, 1);
1597	rcu_assign_pointer(ptbl->part[0], disk->part0);
1598		1388
1599	disk->minors = minors;	1389	disk->minors = minors;
1600	rand_initialize_disk(disk);	1390	rand_initialize_disk(disk);
@@ -1603,7 +1393,8 @@ struct gendisk *__alloc_disk_node(int minors, int node_id) · 2 unmodified lines
1603	device_initialize(disk_to_dev(disk));	1393	device_initialize(disk_to_dev(disk));
1604	return disk;	1394	return disk;
1605		1395
1606	out_bdput:	1396	out_destroy_part_tbl:
		1397	xa_destroy(&disk->part_tbl);
1607	bdput(disk->part0);	1398	bdput(disk->part0);
1608	out_free_disk:	1399	out_free_disk:
1609	kfree(disk);	1400	kfree(disk);
@@ -1638,31 +1429,32 @@ static void set_disk_ro_uevent(struct gendisk *gd, int ro) · 28 unmodified lines
1638	kobject_uevent_env(&disk_to_dev(gd)->kobj, KOBJ_CHANGE, envp);	1429	kobject_uevent_env(&disk_to_dev(gd)->kobj, KOBJ_CHANGE, envp);
1639	}	1430	}
1640		1431
1641	void set_disk_ro(struct gendisk *disk, int flag)	1432	/**
		1433	* set_disk_ro - set a gendisk read-only
		1434	* @disk: gendisk to operate on
		1435	* @read_only: %true to set the disk read-only, %false set the disk read/write
		1436	*
		1437	* This function is used to indicate whether a given disk device should have its
		1438	* read-only flag set. set_disk_ro() is typically used by device drivers to
		1439	* indicate whether the underlying physical device is write-protected.
		1440	*/
		1441	void set_disk_ro(struct gendisk *disk, bool read_only)
1642	{	1442	{
1643	struct disk_part_iter piter;	1443	if (read_only) {
1644	struct block_device *part;	1444	if (test_and_set_bit(GD_READ_ONLY, &disk->state))
1645		1445	return;
1646	if (disk->part0->bd_read_only != flag) {	1446	} else {
1647	set_disk_ro_uevent(disk, flag);	1447	if (!test_and_clear_bit(GD_READ_ONLY, &disk->state))
1648	disk->part0->bd_read_only = flag;	1448	return;
1649	}	1449	}
1650		1450	set_disk_ro_uevent(disk, read_only);
1651	disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY);
1652	while ((part = disk_part_iter_next(&piter)))
1653	part->bd_read_only = flag;
1654	disk_part_iter_exit(&piter);
1655	}	1451	}
1656
1657	EXPORT_SYMBOL(set_disk_ro);	1452	EXPORT_SYMBOL(set_disk_ro);
1658		1453
1659	int bdev_read_only(struct block_device *bdev)	1454	int bdev_read_only(struct block_device *bdev)
1660	{	1455	{
1661	if (!bdev)	1456	return bdev->bd_read_only \|\| get_disk_ro(bdev->bd_disk);
1662	return 0;
1663	return bdev->bd_read_only;
1664	}	1457	}
1665
1666	EXPORT_SYMBOL(bdev_read_only);	1458	EXPORT_SYMBOL(bdev_read_only);
1667		1459
1668	/*	1460	/*

block/kyber-iosched.c+1−0

@@ -1029,6 +1029,7 @@ static struct elevator_type kyber_sched = { · 1028 unmodified lines
1029	#endif	1029	#endif
1030	.elevator_attrs = kyber_sched_attrs,	1030	.elevator_attrs = kyber_sched_attrs,
1031	.elevator_name = "kyber",	1031	.elevator_name = "kyber",
		1032	.elevator_features = ELEVATOR_F_MQ_AWARE,
1032	.elevator_owner = THIS_MODULE,	1033	.elevator_owner = THIS_MODULE,
1033	};	1034	};
1034		1035

block/mq-deadline.c+0−6

 	spin_lock(&dd->lock);
 	rq = __dd_dispatch_request(dd);
 	spin_unlock(&dd->lock);
-	if (rq)
-		atomic_dec(&rq->mq_hctx->elevator_queued);
 	return rq;
 }
 		rq = list_first_entry(list, struct request, queuelist);
 		list_del_init(&rq->queuelist);
 		dd_insert_request(hctx, rq, at_head);
-		atomic_inc(&hctx->elevator_queued);
 	}
 	spin_unlock(&dd->lock);
 }
 {
 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;
-	if (!atomic_read(&hctx->elevator_queued))
-		return false;
 	return !list_empty_careful(&dd->dispatch) ||
 		!list_empty_careful(&dd->fifo_list[0]) ||
 		!list_empty_careful(&dd->fifo_list[1]);

block/partitions/core.c+8−28

 static ssize_t part_ro_show(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", dev_to_bdev(dev)->bd_read_only);
+	return sprintf(buf, "%d\n", bdev_read_only(dev_to_bdev(dev)));
 }
 static ssize_t part_alignment_offset_show(struct device *dev,
  */
 void delete_partition(struct block_device *part)
 {
-	struct gendisk *disk = part->bd_disk;
+	xa_erase(&part->bd_disk->part_tbl, part->bd_partno);
-	struct disk_part_tbl *ptbl =
-		rcu_dereference_protected(disk->part_tbl, 1);
-	rcu_assign_pointer(ptbl->part[part->bd_partno], NULL);
-	rcu_assign_pointer(ptbl->last_lookup, NULL);
 	kobject_put(part->bd_holder_dir);
 	device_del(&part->bd_device);
 	struct device *ddev = disk_to_dev(disk);
 	struct device *pdev;
 	struct block_device *bdev;
-	struct disk_part_tbl *ptbl;
 	const char *dname;
 	int err;
 	case BLK_ZONED_HA:
 		pr_info("%s: disabling host aware zoned block device support due to partitions\n",
 			disk->disk_name);
-		disk->queue->limits.zoned = BLK_ZONED_NONE;
+		blk_queue_set_zoned(disk, BLK_ZONED_NONE);
 		break;
 	case BLK_ZONED_NONE:
 		break;
 	}
-	err = disk_expand_part_tbl(disk, partno);
+	if (xa_load(&disk->part_tbl, partno))
-	if (err)
-		return ERR_PTR(err);
-	ptbl = rcu_dereference_protected(disk->part_tbl, 1);
-	if (ptbl->part[partno])
 		return ERR_PTR(-EBUSY);
 	bdev = bdev_alloc(disk, partno);
 	bdev->bd_start_sect = start;
 	bdev_set_nr_sectors(bdev, len);
-	bdev->bd_read_only = get_disk_ro(disk);
 	if (info) {
 		err = -ENOMEM;
 	}
 	/* everything is up and running, commence */
+	err = xa_insert(&disk->part_tbl, partno, bdev, GFP_KERNEL);
+	if (err)
+		goto out_del;
 	bdev_add(bdev, devt);
-	rcu_assign_pointer(ptbl->part[partno], bdev);
 	/* suppress uevent if the disk suppresses it */
 	if (!dev_get_uevent_suppress(ddev))
 int blk_add_partitions(struct gendisk *disk, struct block_device *bdev)
 {
 	struct parsed_partitions *state;
-	int ret = -EAGAIN, p, highest;
+	int ret = -EAGAIN, p;
 	if (!disk_part_scan_enabled(disk))
 		return 0;
 	/* tell userspace that the media / partition table may have changed */
 	kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
-	/*
-	 * Detect the highest partition number and preallocate disk->part_tbl.
-	 * This is an optimization and not strictly necessary.
-	 */
-	for (p = 1, highest = 0; p < state->limit; p++)
-		if (state->parts[p].size)
-			highest = p;
-	disk_expand_part_tbl(disk, highest);
 	for (p = 1; p < state->limit; p++)
 		if (!blk_add_partition(disk, bdev, state, p))
 			goto out_free_state;

block/scsi_ioctl.c+3−3

@@ -357,7 +357,7 @@ static int sg_io(struct request_queue q, struct gendisk bd_disk, · 356 unmodified lines
357	* (if he doesn't check that is his problem).	357	* (if he doesn't check that is his problem).
358	* N.B. a non-zero SCSI status is _not_ necessarily an error.	358	* N.B. a non-zero SCSI status is _not_ necessarily an error.
359	*/	359	*/
360	blk_execute_rq(q, bd_disk, rq, at_head);	360	blk_execute_rq(bd_disk, rq, at_head);
361		361
362	hdr->duration = jiffies_to_msecs(jiffies - start_time);	362	hdr->duration = jiffies_to_msecs(jiffies - start_time);
363		363
@@ -493,7 +493,7 @@ int sg_scsi_ioctl(struct request_queue q, struct gendisk disk, fmode_t mode, · 129 unmodified lines
493	goto error;	493	goto error;
494	}	494	}
495		495
496	blk_execute_rq(q, disk, rq, 0);	496	blk_execute_rq(disk, rq, 0);
497		497
498	err = req->result & 0xff; /* only 8 bit SCSI status */	498	err = req->result & 0xff; /* only 8 bit SCSI status */
499	if (err) {	499	if (err) {
@@ -532,7 +532,7 @@ static int __blk_send_generic(struct request_queue q, struct gendisk bd_disk, · 32 unmodified lines
532	scsi_req(rq)->cmd[0] = cmd;	532	scsi_req(rq)->cmd[0] = cmd;
533	scsi_req(rq)->cmd[4] = data;	533	scsi_req(rq)->cmd[4] = data;
534	scsi_req(rq)->cmd_len = 6;	534	scsi_req(rq)->cmd_len = 6;
535	blk_execute_rq(q, bd_disk, rq, 0);	535	blk_execute_rq(bd_disk, rq, 0);
536	err = scsi_req(rq)->result ? -EIO : 0;	536	err = scsi_req(rq)->result ? -EIO : 0;
537	blk_put_request(rq);	537	blk_put_request(rq);
538		538

drivers/block/brd.c+2−6

 static blk_qc_t brd_submit_bio(struct bio *bio)
 {
-	struct brd_device *brd = bio->bi_disk->private_data;
+	struct brd_device *brd = bio->bi_bdev->bd_disk->private_data;
+	sector_t sector = bio->bi_iter.bi_sector;
 	struct bio_vec bvec;
-	sector_t sector;
 	struct bvec_iter iter;
-	sector = bio->bi_iter.bi_sector;
-	if (bio_end_sector(bio) > get_capacity(bio->bi_disk))
-		goto io_error;
 	bio_for_each_segment(bvec, bio, iter) {
 		unsigned int len = bvec.bv_len;
 		int err;

drivers/block/drbd/drbd_actlog.c+1−1

@@ -138,7 +138,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, · 137 unmodified lines
138	op_flags \|= REQ_FUA \| REQ_PREFLUSH;	138	op_flags \|= REQ_FUA \| REQ_PREFLUSH;
139	op_flags \|= REQ_SYNC;	139	op_flags \|= REQ_SYNC;
140		140
141	bio = bio_alloc_drbd(GFP_NOIO);	141	bio = bio_alloc_bioset(GFP_NOIO, 1, &drbd_md_io_bio_set);
142	bio_set_dev(bio, bdev->md_bdev);	142	bio_set_dev(bio, bdev->md_bdev);
143	bio->bi_iter.bi_sector = sector;	143	bio->bi_iter.bi_sector = sector;
144	err = -EIO;	144	err = -EIO;

drivers/block/drbd/drbd_bitmap.c+1−1

@@ -976,7 +976,7 @@ static void drbd_bm_endio(struct bio *bio) · 975 unmodified lines
976		976
977	static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_hold(local)	977	static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_hold(local)
978	{	978	{
979	struct bio *bio = bio_alloc_drbd(GFP_NOIO);	979	struct bio *bio = bio_alloc_bioset(GFP_NOIO, 1, &drbd_md_io_bio_set);
980	struct drbd_device *device = ctx->device;	980	struct drbd_device *device = ctx->device;
981	struct drbd_bitmap *b = device->bitmap;	981	struct drbd_bitmap *b = device->bitmap;
982	struct page *page;	982	struct page *page;

drivers/block/drbd/drbd_int.h+2−4

 /* We also need to make sure we get a bio
  * when we need it for housekeeping purposes */
 extern struct bio_set drbd_md_io_bio_set;
-/* to allocate from that set */
-extern struct bio *bio_alloc_drbd(gfp_t gfp_mask);
 /* And a bio_set for cloning */
 extern struct bio_set drbd_io_bio_set;
 					     int fault_type, struct bio *bio)
 {
 	__release(local);
-	if (!bio->bi_disk) {
+	if (!bio->bi_bdev) {
-		drbd_err(device, "drbd_submit_bio_noacct: bio->bi_disk == NULL\n");
+		drbd_err(device, "drbd_submit_bio_noacct: bio->bi_bdev == NULL\n");
 		bio->bi_status = BLK_STS_IOERR;
 		bio_endio(bio);
 		return;

drivers/block/drbd/drbd_main.c+0−13

@@ -138,19 +138,6 @@ static const struct block_device_operations drbd_ops = { · 137 unmodified lines
138	.release = drbd_release,	138	.release = drbd_release,
139	};	139	};
140		140
141	struct bio *bio_alloc_drbd(gfp_t gfp_mask)
142	{
143	struct bio *bio;
144
145	if (!bioset_initialized(&drbd_md_io_bio_set))
146	return bio_alloc(gfp_mask, 1);
147
148	bio = bio_alloc_bioset(gfp_mask, 1, &drbd_md_io_bio_set);
149	if (!bio)
150	return NULL;
151	return bio;
152	}
153
154	#ifdef __CHECKER__	141	#ifdef __CHECKER__
155	/* When checking with sparse, and this is an inline function, sparse will	142	/* When checking with sparse, and this is an inline function, sparse will
156	give tons of false positives. When this is a real functions sparse works.	143	give tons of false positives. When this is a real functions sparse works.

drivers/block/drbd/drbd_req.c+5−2

 		return NULL;
 	memset(req, 0, sizeof(*req));
-	drbd_req_make_private_bio(req, bio_src);
+	req->private_bio = bio_clone_fast(bio_src, GFP_NOIO, &drbd_io_bio_set);
+	req->private_bio->bi_private = req;
+	req->private_bio->bi_end_io = drbd_request_endio;
 	req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0)
 		      | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0)
 		      | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_ZEROES : 0)
 blk_qc_t drbd_submit_bio(struct bio *bio)
 {
-	struct drbd_device *device = bio->bi_disk->private_data;
+	struct drbd_device *device = bio->bi_bdev->bd_disk->private_data;
 	unsigned long start_jif;
 	blk_queue_split(&bio);

drivers/block/drbd/drbd_req.h+0−12

@@ -256,18 +256,6 @@ enum drbd_req_state_bits { · 255 unmodified lines
256	#define MR_WRITE 1	256	#define MR_WRITE 1
257	#define MR_READ 2	257	#define MR_READ 2
258		258
259	static inline void drbd_req_make_private_bio(struct drbd_request req, struct bio bio_src)
260	{
261	struct bio *bio;
262	bio = bio_clone_fast(bio_src, GFP_NOIO, &drbd_io_bio_set);
263
264	req->private_bio = bio;
265
266	bio->bi_private = req;
267	bio->bi_end_io = drbd_request_endio;
268	bio->bi_next = NULL;
269	}
270
271	/* Short lived temporary struct on the stack.	259	/* Short lived temporary struct on the stack.
272	* We could squirrel the error to be returned into	260	* We could squirrel the error to be returned into
273	* bio->bi_iter.bi_size, or similar. But that would be too ugly. */	261	* bio->bi_iter.bi_size, or similar. But that would be too ugly. */

drivers/block/drbd/drbd_worker.c+4−1

 	if (bio_data_dir(req->master_bio) == WRITE && req->rq_state & RQ_IN_ACT_LOG)
 		drbd_al_begin_io(device, &req->i);
-	drbd_req_make_private_bio(req, req->master_bio);
+	req->private_bio = bio_clone_fast(req->master_bio, GFP_NOIO,
+					  &drbd_io_bio_set);
 	bio_set_dev(req->private_bio, device->ldev->backing_bdev);
+	req->private_bio->bi_private = req;
+	req->private_bio->bi_end_io = drbd_request_endio;
 	submit_bio_noacct(req->private_bio);
 	return 0;

drivers/block/mtip32xx/mtip32xx.c+1−1

@@ -1015,7 +1015,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, · 1014 unmodified lines
1015	rq->timeout = timeout;	1015	rq->timeout = timeout;
1016		1016
1017	/* insert request and run queue */	1017	/* insert request and run queue */
1018	blk_execute_rq(rq->q, NULL, rq, true);	1018	blk_execute_rq(NULL, rq, true);
1019		1019
1020	if (int_cmd->status) {	1020	if (int_cmd->status) {
1021	dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",	1021	dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",

drivers/block/null_blk/main.c+1−1

@@ -1420,7 +1420,7 @@ static blk_qc_t null_submit_bio(struct bio *bio) · 1419 unmodified lines
1420	{	1420	{
1421	sector_t sector = bio->bi_iter.bi_sector;	1421	sector_t sector = bio->bi_iter.bi_sector;
1422	sector_t nr_sectors = bio_sectors(bio);	1422	sector_t nr_sectors = bio_sectors(bio);
1423	struct nullb *nullb = bio->bi_disk->private_data;	1423	struct nullb *nullb = bio->bi_bdev->bd_disk->private_data;
1424	struct nullb_queue *nq = nullb_to_queue(nullb);	1424	struct nullb_queue *nq = nullb_to_queue(nullb);
1425	struct nullb_cmd *cmd;	1425	struct nullb_cmd *cmd;
1426		1426

drivers/block/null_blk/zoned.c+4−4

 		sector += dev->zone_size_sects;
 	}
-	q->limits.zoned = BLK_ZONED_HM;
-	blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
-	blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE);
 	return 0;
 }
 	struct nullb_device *dev = nullb->dev;
 	struct request_queue *q = nullb->q;
+	blk_queue_set_zoned(nullb->disk, BLK_ZONED_HM);
+	blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
+	blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE);
 	if (queue_is_mq(q)) {
 		int ret = blk_revalidate_disk_zones(nullb->disk, NULL);

drivers/block/paride/pd.c+1−1

@@ -781,7 +781,7 @@ static int pd_special_command(struct pd_unit *disk, · 780 unmodified lines
781	req = blk_mq_rq_to_pdu(rq);	781	req = blk_mq_rq_to_pdu(rq);
782		782
783	req->func = func;	783	req->func = func;
784	blk_execute_rq(disk->gd->queue, disk->gd, rq, 0);	784	blk_execute_rq(disk->gd, rq, 0);
785	blk_put_request(rq);	785	blk_put_request(rq);
786	return 0;	786	return 0;
787	}	787	}

drivers/block/pktcdvd.c+3−3

@@ -722,7 +722,7 @@ static int pkt_generic_packet(struct pktcdvd_device pd, struct packet_command · 721 unmodified lines
722	if (cgc->quiet)	722	if (cgc->quiet)
723	rq->rq_flags \|= RQF_QUIET;	723	rq->rq_flags \|= RQF_QUIET;
724		724
725	blk_execute_rq(rq->q, pd->bdev->bd_disk, rq, 0);	725	blk_execute_rq(pd->bdev->bd_disk, rq, 0);
726	if (scsi_req(rq)->result)	726	if (scsi_req(rq)->result)
727	ret = -EIO;	727	ret = -EIO;
728	out:	728	out:
@@ -2374,7 +2374,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) · 1645 unmodified lines
2374		2374
2375	blk_queue_split(&bio);	2375	blk_queue_split(&bio);
2376		2376
2377	pd = bio->bi_disk->queue->queuedata;	2377	pd = bio->bi_bdev->bd_disk->queue->queuedata;
2378	if (!pd) {	2378	if (!pd) {
2379	pr_err("%s incorrect request queue\n", bio_devname(bio, b));	2379	pr_err("%s incorrect request queue\n", bio_devname(bio, b));
2380	goto end_io;	2380	goto end_io;
@@ -2418,7 +2418,7 @@ static blk_qc_t pkt_submit_bio(struct bio *bio) · 37 unmodified lines
2418	split = bio;	2418	split = bio;
2419	}	2419	}
2420		2420
2421	pkt_make_request_write(bio->bi_disk->queue, split);	2421	pkt_make_request_write(bio->bi_bdev->bd_disk->queue, split);
2422	} while (split != bio);	2422	} while (split != bio);
2423		2423
2424	return BLK_QC_T_NONE;	2424	return BLK_QC_T_NONE;

drivers/block/ps3vram.c+1−1

@@ -581,7 +581,7 @@ out: · 580 unmodified lines
581		581
582	static blk_qc_t ps3vram_submit_bio(struct bio *bio)	582	static blk_qc_t ps3vram_submit_bio(struct bio *bio)
583	{	583	{
584	struct ps3_system_bus_device *dev = bio->bi_disk->private_data;	584	struct ps3_system_bus_device *dev = bio->bi_bdev->bd_disk->private_data;
585	struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev);	585	struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev);
586	int busy;	586	int busy;
587		587

drivers/block/rbd.c+0−19

@@ -692,29 +692,10 @@ static void rbd_release(struct gendisk *disk, fmode_t mode) · 691 unmodified lines
692	put_device(&rbd_dev->dev);	692	put_device(&rbd_dev->dev);
693	}	693	}
694		694
695	static int rbd_set_read_only(struct block_device *bdev, bool ro)
696	{
697	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
698
699	/*
700	* Both images mapped read-only and snapshots can't be marked
701	* read-write.
702	*/
703	if (!ro) {
704	if (rbd_is_ro(rbd_dev))
705	return -EROFS;
706
707	rbd_assert(!rbd_is_snap(rbd_dev));
708	}
709
710	return 0;
711	}
712
713	static const struct block_device_operations rbd_bd_ops = {	695	static const struct block_device_operations rbd_bd_ops = {
714	.owner = THIS_MODULE,	696	.owner = THIS_MODULE,
715	.open = rbd_open,	697	.open = rbd_open,
716	.release = rbd_release,	698	.release = rbd_release,
717	.set_read_only = rbd_set_read_only,
718	};	699	};
719		700
720	/*	701	/*

drivers/block/rsxx/dev.c+1−1

@@ -122,7 +122,7 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card, · 121 unmodified lines
122		122
123	static blk_qc_t rsxx_submit_bio(struct bio *bio)	123	static blk_qc_t rsxx_submit_bio(struct bio *bio)
124	{	124	{
125	struct rsxx_cardinfo *card = bio->bi_disk->private_data;	125	struct rsxx_cardinfo *card = bio->bi_bdev->bd_disk->private_data;
126	struct rsxx_bio_meta *bio_meta;	126	struct rsxx_bio_meta *bio_meta;
127	blk_status_t st = BLK_STS_IOERR;	127	blk_status_t st = BLK_STS_IOERR;
128		128

drivers/block/sx8.c+2−2

@@ -539,7 +539,7 @@ static int carm_array_info (struct carm_host *host, unsigned int array_idx) · 538 unmodified lines
539	spin_unlock_irq(&host->lock);	539	spin_unlock_irq(&host->lock);
540		540
541	DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);	541	DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);
542	blk_execute_rq_nowait(host->oob_q, NULL, rq, true, NULL);	542	blk_execute_rq_nowait(NULL, rq, true, NULL);
543		543
544	return 0;	544	return 0;
545		545
@@ -578,7 +578,7 @@ static int carm_send_special (struct carm_host *host, carm_sspc_t func) · 32 unmodified lines
578	crq->msg_bucket = (u32) rc;	578	crq->msg_bucket = (u32) rc;
579		579
580	DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);	580	DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);
581	blk_execute_rq_nowait(host->oob_q, NULL, rq, true, NULL);	581	blk_execute_rq_nowait(NULL, rq, true, NULL);
582		582
583	return 0;	583	return 0;
584	}	584	}

drivers/block/umem.c+1−1

@@ -521,7 +521,7 @@ static int mm_check_plugged(struct cardinfo *card) · 520 unmodified lines
521		521
522	static blk_qc_t mm_submit_bio(struct bio *bio)	522	static blk_qc_t mm_submit_bio(struct bio *bio)
523	{	523	{
524	struct cardinfo *card = bio->bi_disk->private_data;	524	struct cardinfo *card = bio->bi_bdev->bd_disk->private_data;
525		525
526	pr_debug("mm_make_request %llu %u\n",	526	pr_debug("mm_make_request %llu %u\n",
527	(unsigned long long)bio->bi_iter.bi_sector,	527	(unsigned long long)bio->bi_iter.bi_sector,

drivers/block/virtio_blk.c+1−1

@@ -320,7 +320,7 @@ static int virtblk_get_id(struct gendisk disk, char id_str) · 319 unmodified lines
320	if (err)	320	if (err)
321	goto out;	321	goto out;
322		322
323	blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);	323	blk_execute_rq(vblk->disk, req, false);
324	err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req)));	324	err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req)));
325	out:	325	out:
326	blk_put_request(req);	326	blk_put_request(req);

drivers/block/zram/zram_drv.c+1−1

@@ -1596,7 +1596,7 @@ static void __zram_make_request(struct zram zram, struct bio bio) · 1595 unmodified lines
1596	*/	1596	*/
1597	static blk_qc_t zram_submit_bio(struct bio *bio)	1597	static blk_qc_t zram_submit_bio(struct bio *bio)
1598	{	1598	{
1599	struct zram *zram = bio->bi_disk->private_data;	1599	struct zram *zram = bio->bi_bdev->bd_disk->private_data;
1600		1600
1601	if (!valid_io_request(zram, bio->bi_iter.bi_sector,	1601	if (!valid_io_request(zram, bio->bi_iter.bi_sector,
1602	bio->bi_iter.bi_size)) {	1602	bio->bi_iter.bi_size)) {

drivers/cdrom/cdrom.c+1−1

@@ -2214,7 +2214,7 @@ static int cdrom_read_cdda_bpc(struct cdrom_device_info cdi, __u8 __user ubuf, · 2213 unmodified lines
2214	rq->timeout = 60 * HZ;	2214	rq->timeout = 60 * HZ;
2215	bio = rq->bio;	2215	bio = rq->bio;
2216		2216
2217	blk_execute_rq(q, cdi->disk, rq, 0);	2217	blk_execute_rq(cdi->disk, rq, 0);
2218	if (scsi_req(rq)->result) {	2218	if (scsi_req(rq)->result) {
2219	struct scsi_sense_hdr sshdr;	2219	struct scsi_sense_hdr sshdr;
2220		2220

drivers/ide/ide-atapi.c+1−1

@@ -107,7 +107,7 @@ int ide_queue_pc_tail(ide_drive_t drive, struct gendisk disk, · 106 unmodified lines
107	memcpy(scsi_req(rq)->cmd, pc->c, 12);	107	memcpy(scsi_req(rq)->cmd, pc->c, 12);
108	if (drive->media == ide_tape)	108	if (drive->media == ide_tape)
109	scsi_req(rq)->cmd[13] = REQ_IDETAPE_PC1;	109	scsi_req(rq)->cmd[13] = REQ_IDETAPE_PC1;
110	blk_execute_rq(drive->queue, disk, rq, 0);	110	blk_execute_rq(disk, rq, 0);
111	error = scsi_req(rq)->result ? -EIO : 0;	111	error = scsi_req(rq)->result ? -EIO : 0;
112	put_req:	112	put_req:
113	blk_put_request(rq);	113	blk_put_request(rq);

drivers/ide/ide-cd.c+1−1

@@ -467,7 +467,7 @@ int ide_cd_queue_pc(ide_drive_t drive, const unsigned char cmd, · 466 unmodified lines
467	}	467	}
468	}	468	}
469		469
470	blk_execute_rq(drive->queue, info->disk, rq, 0);	470	blk_execute_rq(info->disk, rq, 0);
471	error = scsi_req(rq)->result ? -EIO : 0;	471	error = scsi_req(rq)->result ? -EIO : 0;
472		472
473	if (buffer)	473	if (buffer)

drivers/ide/ide-cd_ioctl.c+1−1

@@ -299,7 +299,7 @@ int ide_cdrom_reset(struct cdrom_device_info *cdi) · 298 unmodified lines
299	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);	299	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
300	ide_req(rq)->type = ATA_PRIV_MISC;	300	ide_req(rq)->type = ATA_PRIV_MISC;
301	rq->rq_flags = RQF_QUIET;	301	rq->rq_flags = RQF_QUIET;
302	blk_execute_rq(drive->queue, cd->disk, rq, 0);	302	blk_execute_rq(cd->disk, rq, 0);
303	ret = scsi_req(rq)->result ? -EIO : 0;	303	ret = scsi_req(rq)->result ? -EIO : 0;
304	blk_put_request(rq);	304	blk_put_request(rq);
305	/*	305	/*

drivers/ide/ide-devsets.c+1−1

@@ -173,7 +173,7 @@ int ide_devset_execute(ide_drive_t drive, const struct ide_devset setting, · 172 unmodified lines
173	(int )&scsi_req(rq)->cmd[1] = arg;	173	(int )&scsi_req(rq)->cmd[1] = arg;
174	ide_req(rq)->special = setting->set;	174	ide_req(rq)->special = setting->set;
175		175
176	blk_execute_rq(q, NULL, rq, 0);	176	blk_execute_rq(NULL, rq, 0);
177	ret = scsi_req(rq)->result;	177	ret = scsi_req(rq)->result;
178	blk_put_request(rq);	178	blk_put_request(rq);
179		179

drivers/ide/ide-disk.c+1−1

@@ -482,7 +482,7 @@ static int set_multcount(ide_drive_t *drive, int arg) · 481 unmodified lines
482		482
483	drive->mult_req = arg;	483	drive->mult_req = arg;
484	drive->special_flags \|= IDE_SFLAG_SET_MULTMODE;	484	drive->special_flags \|= IDE_SFLAG_SET_MULTMODE;
485	blk_execute_rq(drive->queue, NULL, rq, 0);	485	blk_execute_rq(NULL, rq, 0);
486	blk_put_request(rq);	486	blk_put_request(rq);
487		487
488	return (drive->mult_count == arg) ? 0 : -EIO;	488	return (drive->mult_count == arg) ? 0 : -EIO;

drivers/ide/ide-ioctls.c+2−2

@@ -137,7 +137,7 @@ static int ide_cmd_ioctl(ide_drive_t drive, void __user argp) · 136 unmodified lines
137		137
138	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);	138	rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
139	ide_req(rq)->type = ATA_PRIV_TASKFILE;	139	ide_req(rq)->type = ATA_PRIV_TASKFILE;
140	blk_execute_rq(drive->queue, NULL, rq, 0);	140	blk_execute_rq(NULL, rq, 0);
141	err = scsi_req(rq)->result ? -EIO : 0;	141	err = scsi_req(rq)->result ? -EIO : 0;
142	blk_put_request(rq);	142	blk_put_request(rq);
143		143
@@ -235,7 +235,7 @@ static int generic_drive_reset(ide_drive_t *drive) · 91 unmodified lines
235	ide_req(rq)->type = ATA_PRIV_MISC;	235	ide_req(rq)->type = ATA_PRIV_MISC;
236	scsi_req(rq)->cmd_len = 1;	236	scsi_req(rq)->cmd_len = 1;
237	scsi_req(rq)->cmd[0] = REQ_DRIVE_RESET;	237	scsi_req(rq)->cmd[0] = REQ_DRIVE_RESET;
238	blk_execute_rq(drive->queue, NULL, rq, 1);	238	blk_execute_rq(NULL, rq, 1);
239	ret = scsi_req(rq)->result;	239	ret = scsi_req(rq)->result;
240	blk_put_request(rq);	240	blk_put_request(rq);
241	return ret;	241	return ret;

drivers/ide/ide-park.c+1−1

@@ -37,7 +37,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) · 36 unmodified lines
37	scsi_req(rq)->cmd_len = 1;	37	scsi_req(rq)->cmd_len = 1;
38	ide_req(rq)->type = ATA_PRIV_MISC;	38	ide_req(rq)->type = ATA_PRIV_MISC;
39	ide_req(rq)->special = &timeout;	39	ide_req(rq)->special = &timeout;
40	blk_execute_rq(q, NULL, rq, 1);	40	blk_execute_rq(NULL, rq, 1);
41	rc = scsi_req(rq)->result ? -EIO : 0;	41	rc = scsi_req(rq)->result ? -EIO : 0;
42	blk_put_request(rq);	42	blk_put_request(rq);
43	if (rc)	43	if (rc)

drivers/ide/ide-pm.c+2−2

@@ -27,7 +27,7 @@ int generic_ide_suspend(struct device *dev, pm_message_t mesg) · 26 unmodified lines
27	mesg.event = PM_EVENT_FREEZE;	27	mesg.event = PM_EVENT_FREEZE;
28	rqpm.pm_state = mesg.event;	28	rqpm.pm_state = mesg.event;
29		29
30	blk_execute_rq(drive->queue, NULL, rq, 0);	30	blk_execute_rq(NULL, rq, 0);
31	ret = scsi_req(rq)->result ? -EIO : 0;	31	ret = scsi_req(rq)->result ? -EIO : 0;
32	blk_put_request(rq);	32	blk_put_request(rq);
33		33
@@ -50,7 +50,7 @@ static int ide_pm_execute_rq(struct request *rq) · 16 unmodified lines
50	blk_mq_end_request(rq, BLK_STS_OK);	50	blk_mq_end_request(rq, BLK_STS_OK);
51	return -ENXIO;	51	return -ENXIO;
52	}	52	}
53	blk_execute_rq(q, NULL, rq, true);	53	blk_execute_rq(NULL, rq, true);
54		54
55	return scsi_req(rq)->result ? -EIO : 0;	55	return scsi_req(rq)->result ? -EIO : 0;
56	}	56	}

drivers/ide/ide-tape.c+1−1

@@ -868,7 +868,7 @@ static int idetape_queue_rw_tail(ide_drive_t *drive, int cmd, int size) · 867 unmodified lines
868	goto out_put;	868	goto out_put;
869	}	869	}
870		870
871	blk_execute_rq(drive->queue, tape->disk, rq, 0);	871	blk_execute_rq(tape->disk, rq, 0);
872		872
873	/* calculate the number of transferred bytes and update buffer state */	873	/* calculate the number of transferred bytes and update buffer state */
874	size -= scsi_req(rq)->resid_len;	874	size -= scsi_req(rq)->resid_len;

drivers/ide/ide-taskfile.c+1−1

@@ -443,7 +443,7 @@ int ide_raw_taskfile(ide_drive_t drive, struct ide_cmd cmd, u8 *buf, · 442 unmodified lines
443	ide_req(rq)->special = cmd;	443	ide_req(rq)->special = cmd;
444	cmd->rq = rq;	444	cmd->rq = rq;
445		445
446	blk_execute_rq(drive->queue, NULL, rq, 0);	446	blk_execute_rq(NULL, rq, 0);
447	error = scsi_req(rq)->result ? -EIO : 0;	447	error = scsi_req(rq)->result ? -EIO : 0;
448	put_req:	448	put_req:
449	blk_put_request(rq);	449	blk_put_request(rq);

drivers/lightnvm/pblk-init.c+1−1

@@ -49,7 +49,7 @@ struct bio_set pblk_bio_set; · 48 unmodified lines
49		49
50	static blk_qc_t pblk_submit_bio(struct bio *bio)	50	static blk_qc_t pblk_submit_bio(struct bio *bio)
51	{	51	{
52	struct pblk *pblk = bio->bi_disk->queue->queuedata;	52	struct pblk *pblk = bio->bi_bdev->bd_disk->queue->queuedata;
53		53
54	if (bio_op(bio) == REQ_OP_DISCARD) {	54	if (bio_op(bio) == REQ_OP_DISCARD) {
55	pblk_discard(pblk, bio);	55	pblk_discard(pblk, bio);

drivers/md/bcache/debug.c+1−1

@@ -114,7 +114,7 @@ void bch_data_verify(struct cached_dev dc, struct bio bio) · 113 unmodified lines
114	check = bio_kmalloc(GFP_NOIO, bio_segments(bio));	114	check = bio_kmalloc(GFP_NOIO, bio_segments(bio));
115	if (!check)	115	if (!check)
116	return;	116	return;
117	check->bi_disk = bio->bi_disk;	117	bio_set_dev(check, bio->bi_bdev);
118	check->bi_opf = REQ_OP_READ;	118	check->bi_opf = REQ_OP_READ;
119	check->bi_iter.bi_sector = bio->bi_iter.bi_sector;	119	check->bi_iter.bi_sector = bio->bi_iter.bi_sector;
120	check->bi_iter.bi_size = bio->bi_iter.bi_size;	120	check->bi_iter.bi_size = bio->bi_iter.bi_size;

drivers/md/bcache/request.c+24−15

@@ -475,7 +475,7 @@ struct search { · 474 unmodified lines
475	unsigned int read_dirty_data:1;	475	unsigned int read_dirty_data:1;
476	unsigned int cache_missed:1;	476	unsigned int cache_missed:1;
477		477
478	struct block_device *part;	478	struct block_device *orig_bdev;
479	unsigned long start_time;	479	unsigned long start_time;
480		480
481	struct btree_op op;	481	struct btree_op op;
@@ -670,8 +670,8 @@ static void bio_complete(struct search *s) · 188 unmodified lines
670	{	670	{
671	if (s->orig_bio) {	671	if (s->orig_bio) {
672	/* Count on bcache device */	672	/* Count on bcache device */
673	part_end_io_acct(s->part, s->orig_bio, s->start_time);	673	bio_end_io_acct_remapped(s->orig_bio, s->start_time,
674		674	s->orig_bdev);
675	trace_bcache_request_end(s->d, s->orig_bio);	675	trace_bcache_request_end(s->d, s->orig_bio);
676	s->orig_bio->bi_status = s->iop.status;	676	s->orig_bio->bi_status = s->iop.status;
677	bio_endio(s->orig_bio);	677	bio_endio(s->orig_bio);
@@ -714,7 +714,8 @@ static void search_free(struct closure *cl) · 36 unmodified lines
714	}	714	}
715		715
716	static inline struct search search_alloc(struct bio bio,	716	static inline struct search search_alloc(struct bio bio,
717	struct bcache_device *d)	717	struct bcache_device d, struct block_device orig_bdev,
		718	unsigned long start_time)
718	{	719	{
719	struct search *s;	720	struct search *s;
720		721
@@ -732,7 +733,8 @@ static inline struct search search_alloc(struct bio bio, · 11 unmodified lines
732	s->write = op_is_write(bio_op(bio));	733	s->write = op_is_write(bio_op(bio));
733	s->read_dirty_data = 0;	734	s->read_dirty_data = 0;
734	/* Count on the bcache device */	735	/* Count on the bcache device */
735	s->start_time = part_start_io_acct(d->disk, &s->part, bio);	736	s->orig_bdev = orig_bdev;
		737	s->start_time = start_time;
736	s->iop.c = d->c;	738	s->iop.c = d->c;
737	s->iop.bio = NULL;	739	s->iop.bio = NULL;
738	s->iop.inode = d->id;	740	s->iop.inode = d->id;
@@ -894,7 +896,8 @@ static int cached_dev_cache_miss(struct btree b, struct search s, · 155 unmodified lines
894	!(bio->bi_opf & (REQ_META\|REQ_PRIO)) &&	896	!(bio->bi_opf & (REQ_META\|REQ_PRIO)) &&
895	s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA)	897	s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA)
896	reada = min_t(sector_t, dc->readahead >> 9,	898	reada = min_t(sector_t, dc->readahead >> 9,
897	get_capacity(bio->bi_disk) - bio_end_sector(bio));	899	get_capacity(bio->bi_bdev->bd_disk) -
		900	bio_end_sector(bio));
898		901
899	s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);	902	s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);
900		903
@@ -1073,7 +1076,7 @@ struct detached_dev_io_private { · 172 unmodified lines
1073	unsigned long start_time;	1076	unsigned long start_time;
1074	bio_end_io_t *bi_end_io;	1077	bio_end_io_t *bi_end_io;
1075	void *bi_private;	1078	void *bi_private;
1076	struct block_device *part;	1079	struct block_device *orig_bdev;
1077	};	1080	};
1078		1081
1079	static void detached_dev_end_io(struct bio *bio)	1082	static void detached_dev_end_io(struct bio *bio)
@@ -1085,7 +1088,7 @@ static void detached_dev_end_io(struct bio *bio) · 5 unmodified lines
1085	bio->bi_private = ddip->bi_private;	1088	bio->bi_private = ddip->bi_private;
1086		1089
1087	/* Count on the bcache device */	1090	/* Count on the bcache device */
1088	part_end_io_acct(ddip->part, bio, ddip->start_time);	1091	bio_end_io_acct_remapped(bio, ddip->start_time, ddip->orig_bdev);
1089		1092
1090	if (bio->bi_status) {	1093	if (bio->bi_status) {
1091	struct cached_dev *dc = container_of(ddip->d,	1094	struct cached_dev *dc = container_of(ddip->d,
@@ -1098,7 +1101,8 @@ static void detached_dev_end_io(struct bio *bio) · 6 unmodified lines
1098	bio->bi_end_io(bio);	1101	bio->bi_end_io(bio);
1099	}	1102	}
1100		1103
1101	static void detached_dev_do_request(struct bcache_device d, struct bio bio)	1104	static void detached_dev_do_request(struct bcache_device d, struct bio bio,
		1105	struct block_device *orig_bdev, unsigned long start_time)
1102	{	1106	{
1103	struct detached_dev_io_private *ddip;	1107	struct detached_dev_io_private *ddip;
1104	struct cached_dev *dc = container_of(d, struct cached_dev, disk);	1108	struct cached_dev *dc = container_of(d, struct cached_dev, disk);
@@ -1111,7 +1115,8 @@ static void detached_dev_do_request(struct bcache_device d, struct bio bio) · 6 unmodified lines
1111	ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);	1115	ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);
1112	ddip->d = d;	1116	ddip->d = d;
1113	/* Count on the bcache device */	1117	/* Count on the bcache device */
1114	ddip->start_time = part_start_io_acct(d->disk, &ddip->part, bio);	1118	ddip->orig_bdev = orig_bdev;
		1119	ddip->start_time = start_time;
1115	ddip->bi_end_io = bio->bi_end_io;	1120	ddip->bi_end_io = bio->bi_end_io;
1116	ddip->bi_private = bio->bi_private;	1121	ddip->bi_private = bio->bi_private;
1117	bio->bi_end_io = detached_dev_end_io;	1122	bio->bi_end_io = detached_dev_end_io;
@@ -1167,8 +1172,10 @@ static void quit_max_writeback_rate(struct cache_set *c, · 49 unmodified lines
1167	blk_qc_t cached_dev_submit_bio(struct bio *bio)	1172	blk_qc_t cached_dev_submit_bio(struct bio *bio)
1168	{	1173	{
1169	struct search *s;	1174	struct search *s;
1170	struct bcache_device *d = bio->bi_disk->private_data;	1175	struct block_device *orig_bdev = bio->bi_bdev;
		1176	struct bcache_device *d = orig_bdev->bd_disk->private_data;
1171	struct cached_dev *dc = container_of(d, struct cached_dev, disk);	1177	struct cached_dev *dc = container_of(d, struct cached_dev, disk);
		1178	unsigned long start_time;
1172	int rw = bio_data_dir(bio);	1179	int rw = bio_data_dir(bio);
1173		1180
1174	if (unlikely((d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags)) \|\|	1181	if (unlikely((d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags)) \|\|
@@ -1193,11 +1200,13 @@ blk_qc_t cached_dev_submit_bio(struct bio *bio) · 18 unmodified lines
1193	}	1200	}
1194	}	1201	}
1195		1202
		1203	start_time = bio_start_io_acct(bio);
		1204
1196	bio_set_dev(bio, dc->bdev);	1205	bio_set_dev(bio, dc->bdev);
1197	bio->bi_iter.bi_sector += dc->sb.data_offset;	1206	bio->bi_iter.bi_sector += dc->sb.data_offset;
1198		1207
1199	if (cached_dev_get(dc)) {	1208	if (cached_dev_get(dc)) {
1200	s = search_alloc(bio, d);	1209	s = search_alloc(bio, d, orig_bdev, start_time);
1201	trace_bcache_request_start(s->d, bio);	1210	trace_bcache_request_start(s->d, bio);
1202		1211
1203	if (!bio->bi_iter.bi_size) {	1212	if (!bio->bi_iter.bi_size) {
@@ -1218,7 +1227,7 @@ blk_qc_t cached_dev_submit_bio(struct bio *bio) · 14 unmodified lines
1218	}	1227	}
1219	} else	1228	} else
1220	/* I/O request sent to backing device */	1229	/* I/O request sent to backing device */
1221	detached_dev_do_request(d, bio);	1230	detached_dev_do_request(d, bio, orig_bdev, start_time);
1222		1231
1223	return BLK_QC_T_NONE;	1232	return BLK_QC_T_NONE;
1224	}	1233	}
@@ -1274,7 +1283,7 @@ blk_qc_t flash_dev_submit_bio(struct bio *bio) · 49 unmodified lines
1274	{	1283	{
1275	struct search *s;	1284	struct search *s;
1276	struct closure *cl;	1285	struct closure *cl;
1277	struct bcache_device *d = bio->bi_disk->private_data;	1286	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
1278		1287
1279	if (unlikely(d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags))) {	1288	if (unlikely(d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags))) {
1280	bio->bi_status = BLK_STS_IOERR;	1289	bio->bi_status = BLK_STS_IOERR;
@@ -1282,7 +1291,7 @@ blk_qc_t flash_dev_submit_bio(struct bio *bio) · 1 unmodified line
1282	return BLK_QC_T_NONE;	1291	return BLK_QC_T_NONE;
1283	}	1292	}
1284		1293
1285	s = search_alloc(bio, d);	1294	s = search_alloc(bio, d, bio->bi_bdev, bio_start_io_acct(bio));
1286	cl = &s->cl;	1295	cl = &s->cl;
1287	bio = &s->bio.bio;	1296	bio = &s->bio.bio;
1288		1297

drivers/md/bcache/super.c+1−1

@@ -1939,7 +1939,7 @@ struct cache_set bch_cache_set_alloc(struct cache_sb sb) · 1938 unmodified lines
1939	goto err;	1939	goto err;
1940		1940
1941	if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio),	1941	if (bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio),
1942	BIOSET_NEED_BVECS\|BIOSET_NEED_RESCUER))	1942	BIOSET_NEED_RESCUER))
1943	goto err;	1943	goto err;
1944		1944
1945	c->uuids = alloc_meta_bucket_pages(GFP_KERNEL, sb);	1945	c->uuids = alloc_meta_bucket_pages(GFP_KERNEL, sb);

drivers/md/dm-bio-record.h+3−6

  */
 struct dm_bio_details {
-	struct gendisk *bi_disk;
+	struct block_device *bi_bdev;
-	u8 bi_partno;
 	int __bi_remaining;
 	unsigned long bi_flags;
 	struct bvec_iter bi_iter;
 static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio)
 {
-	bd->bi_disk = bio->bi_disk;
+	bd->bi_bdev = bio->bi_bdev;
-	bd->bi_partno = bio->bi_partno;
 	bd->bi_flags = bio->bi_flags;
 	bd->bi_iter = bio->bi_iter;
 	bd->__bi_remaining = atomic_read(&bio->__bi_remaining);
 static inline void dm_bio_restore(struct dm_bio_details *bd, struct bio *bio)
 {
-	bio->bi_disk = bd->bi_disk;
+	bio->bi_bdev = bd->bi_bdev;
-	bio->bi_partno = bd->bi_partno;
 	bio->bi_flags = bd->bi_flags;
 	bio->bi_iter = bd->bi_iter;
 	atomic_set(&bio->__bi_remaining, bd->__bi_remaining);

drivers/md/dm-cache-metadata.c+1−1

@@ -449,7 +449,7 @@ static int __check_incompat_features(struct cache_disk_superblock *disk_super, · 448 unmodified lines
449	/*	449	/*
450	* Check for read-only metadata to skip the following RDWR checks.	450	* Check for read-only metadata to skip the following RDWR checks.
451	*/	451	*/
452	if (get_disk_ro(cmd->bdev->bd_disk))	452	if (bdev_read_only(cmd->bdev))
453	return 0;	453	return 0;
454		454
455	features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;	455	features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;

drivers/md/dm-clone-target.c+1−13

 	struct dm_clone_metadata *cmd;
-	/*
-	 * bio used to flush the destination device, before committing the
-	 * metadata.
-	 */
-	struct bio flush_bio;
 	/* Region hydration hash table */
 	struct hash_table_bucket *ht;
 		goto out;
 	}
-	bio_reset(&clone->flush_bio);
+	r = blkdev_issue_flush(clone->dest_dev->bdev);
-	bio_set_dev(&clone->flush_bio, clone->dest_dev->bdev);
-	clone->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
-	r = submit_bio_wait(&clone->flush_bio);
 	if (unlikely(r)) {
 		__metadata_operation_failed(clone, "flush destination device", r);
 		goto out;
 	bio_list_init(&clone->deferred_flush_completions);
 	clone->hydration_offset = 0;
 	atomic_set(&clone->hydrations_in_flight, 0);
-	bio_init(&clone->flush_bio, NULL, 0);
 	clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
 	if (!clone->wq) {
 	struct clone *clone = ti->private;
 	mutex_destroy(&clone->commit_lock);
-	bio_uninit(&clone->flush_bio);
 	for (i = 0; i < clone->nr_ctr_args; i++)
 		kfree(clone->ctr_args[i]);

drivers/md/dm-raid1.c+5−5

@@ -145,7 +145,7 @@ static void dispatch_bios(void context, struct bio_list bio_list) · 144 unmodified lines
145		145
146	struct dm_raid1_bio_record {	146	struct dm_raid1_bio_record {
147	struct mirror *m;	147	struct mirror *m;
148	/* if details->bi_disk == NULL, details were not saved */	148	/* if details->bi_bdev == NULL, details were not saved */
149	struct dm_bio_details details;	149	struct dm_bio_details details;
150	region_t write_region;	150	region_t write_region;
151	};	151	};
@@ -1190,7 +1190,7 @@ static int mirror_map(struct dm_target ti, struct bio bio) · 1038 unmodified lines
1190	struct dm_raid1_bio_record *bio_record =	1190	struct dm_raid1_bio_record *bio_record =
1191	dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record));	1191	dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record));
1192		1192
1193	bio_record->details.bi_disk = NULL;	1193	bio_record->details.bi_bdev = NULL;
1194		1194
1195	if (rw == WRITE) {	1195	if (rw == WRITE) {
1196	/* Save region for mirror_end_io() handler */	1196	/* Save region for mirror_end_io() handler */
@@ -1257,7 +1257,7 @@ static int mirror_end_io(struct dm_target ti, struct bio bio, · 60 unmodified lines
1257	goto out;	1257	goto out;
1258		1258
1259	if (unlikely(*error)) {	1259	if (unlikely(*error)) {
1260	if (!bio_record->details.bi_disk) {	1260	if (!bio_record->details.bi_bdev) {
1261	/*	1261	/*
1262	* There wasn't enough memory to record necessary	1262	* There wasn't enough memory to record necessary
1263	* information for a retry or there was no other	1263	* information for a retry or there was no other
@@ -1282,7 +1282,7 @@ static int mirror_end_io(struct dm_target ti, struct bio bio, · 18 unmodified lines
1282	bd = &bio_record->details;	1282	bd = &bio_record->details;
1283		1283
1284	dm_bio_restore(bd, bio);	1284	dm_bio_restore(bd, bio);
1285	bio_record->details.bi_disk = NULL;	1285	bio_record->details.bi_bdev = NULL;
1286	bio->bi_status = 0;	1286	bio->bi_status = 0;
1287		1287
1288	queue_bio(ms, bio, rw);	1288	queue_bio(ms, bio, rw);
@@ -1292,7 +1292,7 @@ static int mirror_end_io(struct dm_target ti, struct bio bio, · 3 unmodified lines
1292	}	1292	}
1293		1293
1294	out:	1294	out:
1295	bio_record->details.bi_disk = NULL;	1295	bio_record->details.bi_bdev = NULL;
1296		1296
1297	return DM_ENDIO_DONE;	1297	return DM_ENDIO_DONE;
1298	}	1298	}

drivers/md/dm-thin-metadata.c+1−1

@@ -636,7 +636,7 @@ static int __check_incompat_features(struct thin_disk_superblock *disk_super, · 635 unmodified lines
636	/*	636	/*
637	* Check for read-only metadata to skip the following RDWR checks.	637	* Check for read-only metadata to skip the following RDWR checks.
638	*/	638	*/
639	if (get_disk_ro(pmd->bdev->bd_disk))	639	if (bdev_read_only(pmd->bdev))
640	return 0;	640	return 0;
641		641
642	features = le32_to_cpu(disk_super->compat_ro_flags) & ~THIN_FEATURE_COMPAT_RO_SUPP;	642	features = le32_to_cpu(disk_super->compat_ro_flags) & ~THIN_FEATURE_COMPAT_RO_SUPP;

drivers/md/dm-zoned-metadata.c+3−3

@@ -819,7 +819,7 @@ static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set) · 818 unmodified lines
819	ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,	819	ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,
820	mblk->page);	820	mblk->page);
821	if (ret == 0)	821	if (ret == 0)
822	ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);	822	ret = blkdev_issue_flush(dev->bdev);
823		823
824	return ret;	824	return ret;
825	}	825	}
@@ -862,7 +862,7 @@ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd, · 36 unmodified lines
862		862
863	/* Flush drive cache (this will also sync data) */	863	/* Flush drive cache (this will also sync data) */
864	if (ret == 0)	864	if (ret == 0)
865	ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);	865	ret = blkdev_issue_flush(dev->bdev);
866		866
867	return ret;	867	return ret;
868	}	868	}
@@ -933,7 +933,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd) · 64 unmodified lines
933		933
934	/* If there are no dirty metadata blocks, just flush the device cache */	934	/* If there are no dirty metadata blocks, just flush the device cache */
935	if (list_empty(&write_list)) {	935	if (list_empty(&write_list)) {
936	ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);	936	ret = blkdev_issue_flush(dev->bdev);
937	goto err;	937	goto err;
938	}	938	}
939		939

drivers/md/dm.c+7−7

 	struct mapped_device *md = tio->io->md;
 	dm_endio_fn endio = tio->ti->type->end_io;
 	struct bio *orig_bio = io->orig_bio;
+	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
 	if (unlikely(error == BLK_STS_TARGET)) {
 		if (bio_op(bio) == REQ_OP_DISCARD &&
-		    !bio->bi_disk->queue->limits.max_discard_sectors)
+		    !q->limits.max_discard_sectors)
 			disable_discard(md);
 		else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
-			 !bio->bi_disk->queue->limits.max_write_same_sectors)
+			 !q->limits.max_write_same_sectors)
 			disable_write_same(md);
 		else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
-			 !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
+			 !q->limits.max_write_zeroes_sectors)
 			disable_write_zeroes(md);
 	}
 	 */
 	if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
 		sector_t written_sector = bio->bi_iter.bi_sector;
-		struct request_queue *q = orig_bio->bi_disk->queue;
+		struct request_queue *q = orig_bio->bi_bdev->bd_disk->queue;
 		u64 mask = (u64)blk_queue_zone_sectors(q) - 1;
 		orig_bio->bi_iter.bi_sector += written_sector & mask;
 	 */
 	bio_init(&flush_bio, NULL, 0);
 	flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
-	flush_bio.bi_disk = ci->io->md->disk;
+	bio_set_dev(&flush_bio, ci->io->md->disk->part0);
-	bio_associate_blkg(&flush_bio);
 	ci->bio = &flush_bio;
 	ci->sector_count = 0;
 static blk_qc_t dm_submit_bio(struct bio *bio)
 {
-	struct mapped_device *md = bio->bi_disk->private_data;
+	struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
 	blk_qc_t ret = BLK_QC_T_NONE;
 	int srcu_idx;
 	struct dm_table *map;

drivers/md/md-linear.c+1−1

@@ -252,7 +252,7 @@ static bool linear_make_request(struct mddev mddev, struct bio bio) · 251 unmodified lines
252	start_sector + data_offset;	252	start_sector + data_offset;
253		253
254	if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&	254	if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
255	!blk_queue_discard(bio->bi_disk->queue))) {	255	!blk_queue_discard(bio->bi_bdev->bd_disk->queue))) {
256	/* Just ignore it */	256	/* Just ignore it */
257	bio_endio(bio);	257	bio_endio(bio);
258	} else {	258	} else {

drivers/md/md.c+29−44

  */
 static bool create_on_open = true;
-struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs,
-			    struct mddev *mddev)
-{
-	if (!mddev || !bioset_initialized(&mddev->bio_set))
-		return bio_alloc(gfp_mask, nr_iovecs);
-	return bio_alloc_bioset(gfp_mask, nr_iovecs, &mddev->bio_set);
-}
-EXPORT_SYMBOL_GPL(bio_alloc_mddev);
-static struct bio *md_bio_alloc_sync(struct mddev *mddev)
-{
-	if (!mddev || !bioset_initialized(&mddev->sync_set))
-		return bio_alloc(GFP_NOIO, 1);
-	return bio_alloc_bioset(GFP_NOIO, 1, &mddev->sync_set);
-}
 /*
  * We have a system wide 'event count' that is incremented
  * on any 'interesting' event, and readers of /proc/mdstat
 	struct mddev *mddev;
 	bio_end_io_t *orig_bi_end_io;
 	void *orig_bi_private;
+	struct block_device *orig_bi_bdev;
 	unsigned long start_time;
-	struct block_device *part;
 };
 static void md_end_io(struct bio *bio)
 	struct md_io *md_io = bio->bi_private;
 	struct mddev *mddev = md_io->mddev;
-	part_end_io_acct(md_io->part, bio, md_io->start_time);
+	bio_end_io_acct_remapped(bio, md_io->start_time, md_io->orig_bi_bdev);
 	bio->bi_end_io = md_io->orig_bi_end_io;
 	bio->bi_private = md_io->orig_bi_private;
 static blk_qc_t md_submit_bio(struct bio *bio)
 {
 	const int rw = bio_data_dir(bio);
-	struct mddev *mddev = bio->bi_disk->private_data;
+	struct mddev *mddev = bio->bi_bdev->bd_disk->private_data;
 	if (mddev == NULL || mddev->pers == NULL) {
 		bio_io_error(bio);
 		md_io->mddev = mddev;
 		md_io->orig_bi_end_io = bio->bi_end_io;
 		md_io->orig_bi_private = bio->bi_private;
+		md_io->orig_bi_bdev = bio->bi_bdev;
 		bio->bi_end_io = md_end_io;
 		bio->bi_private = md_io;
-		md_io->start_time = part_start_io_acct(mddev->gendisk,
+		md_io->start_time = bio_start_io_acct(bio);
-						       &md_io->part, bio);
 	}
 	/* bio could be mergeable after passing to underlayer */
 			atomic_inc(&rdev->nr_pending);
 			atomic_inc(&rdev->nr_pending);
 			rcu_read_unlock();
-			bi = bio_alloc_mddev(GFP_NOIO, 0, mddev);
+			bi = bio_alloc_bioset(GFP_NOIO, 0, &mddev->bio_set);
 			bi->bi_end_io = md_end_flush;
 			bi->bi_private = rdev;
 			bio_set_dev(bi, rdev->bdev);
 	if (test_bit(Faulty, &rdev->flags))
 		return;
-	bio = md_bio_alloc_sync(mddev);
+	bio = bio_alloc_bioset(GFP_NOIO, 1, &mddev->sync_set);
 	atomic_inc(&rdev->nr_pending);
 int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
 		 struct page *page, int op, int op_flags, bool metadata_op)
 {
-	struct bio *bio = md_bio_alloc_sync(rdev->mddev);
+	struct bio bio;
-	int ret;
+	struct bio_vec bvec;
+	bio_init(&bio, &bvec, 1);
 	if (metadata_op && rdev->meta_bdev)
-		bio_set_dev(bio, rdev->meta_bdev);
+		bio_set_dev(&bio, rdev->meta_bdev);
 	else
-		bio_set_dev(bio, rdev->bdev);
+		bio_set_dev(&bio, rdev->bdev);
-	bio_set_op_attrs(bio, op, op_flags);
+	bio.bi_opf = op | op_flags;
 	if (metadata_op)
-		bio->bi_iter.bi_sector = sector + rdev->sb_start;
+		bio.bi_iter.bi_sector = sector + rdev->sb_start;
 	else if (rdev->mddev->reshape_position != MaxSector &&
 		 (rdev->mddev->reshape_backwards ==
 		  (sector >= rdev->mddev->reshape_position)))
-		bio->bi_iter.bi_sector = sector + rdev->new_data_offset;
+		bio.bi_iter.bi_sector = sector + rdev->new_data_offset;
 	else
-		bio->bi_iter.bi_sector = sector + rdev->data_offset;
+		bio.bi_iter.bi_sector = sector + rdev->data_offset;
-	bio_add_page(bio, page, size, 0);
+	bio_add_page(&bio, page, size, 0);
-	submit_bio_wait(bio);
+	submit_bio_wait(&bio);
-	ret = !bio->bi_status;
+	return !bio.bi_status;
-	bio_put(bio);
-	return ret;
 }
 EXPORT_SYMBOL_GPL(sync_page_io);
 }
 EXPORT_SYMBOL(md_integrity_add_rdev);
+static bool rdev_read_only(struct md_rdev *rdev)
+{
+	return bdev_read_only(rdev->bdev) ||
+		(rdev->meta_bdev && bdev_read_only(rdev->meta_bdev));
+}
 static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev)
 {
 	char b[BDEVNAME_SIZE];
 	if (find_rdev(mddev, rdev->bdev->bd_dev))
 		return -EEXIST;
-	if ((bdev_read_only(rdev->bdev) || bdev_read_only(rdev->meta_bdev)) &&
+	if (rdev_read_only(rdev) && mddev->pers)
-	    mddev->pers)
 		return -EROFS;
 	/* make sure rdev->sectors exceeds mddev->dev_sectors */
 			continue;
 		sync_blockdev(rdev->bdev);
 		invalidate_bdev(rdev->bdev);
-		if (mddev->ro != 1 &&
+		if (mddev->ro != 1 && rdev_read_only(rdev)) {
-		    (bdev_read_only(rdev->bdev) ||
-		     bdev_read_only(rdev->meta_bdev))) {
 			mddev->ro = 1;
 			if (mddev->gendisk)
 				set_disk_ro(mddev->gendisk, 1);
 		if (test_bit(Journal, &rdev->flags) &&
 		    !test_bit(Faulty, &rdev->flags))
 			has_journal = true;
-		if (bdev_read_only(rdev->bdev))
+		if (rdev_read_only(rdev))
 			has_readonly = true;
 	}
 	rcu_read_unlock();

drivers/md/md.h+3−5

 static inline void md_sync_acct_bio(struct bio *bio, unsigned long nr_sectors)
 {
-	atomic_add(nr_sectors, &bio->bi_disk->sync_io);
+	md_sync_acct(bio->bi_bdev, nr_sectors);
 }
 struct md_personality
 extern void md_handle_request(struct mddev *mddev, struct bio *bio);
 extern void mddev_suspend(struct mddev *mddev);
 extern void mddev_resume(struct mddev *mddev);
-extern struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs,
-				   struct mddev *mddev);
 extern void md_reload_sb(struct mddev *mddev, int raid_disk);
 extern void md_update_sb(struct mddev *mddev, int force);
 static inline void mddev_check_writesame(struct mddev *mddev, struct bio *bio)
 {
 	if (bio_op(bio) == REQ_OP_WRITE_SAME &&
-	    !bio->bi_disk->queue->limits.max_write_same_sectors)
+	    !bio->bi_bdev->bd_disk->queue->limits.max_write_same_sectors)
 		mddev->queue->limits.max_write_same_sectors = 0;
 }
 static inline void mddev_check_write_zeroes(struct mddev *mddev, struct bio *bio)
 {
 	if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
-	    !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
+	    !bio->bi_bdev->bd_disk->queue->limits.max_write_zeroes_sectors)
 		mddev->queue->limits.max_write_zeroes_sectors = 0;
 }

drivers/md/raid1.c+4−4

@@ -794,13 +794,13 @@ static void flush_bio_list(struct r1conf conf, struct bio bio) · 793 unmodified lines
794		794
795	while (bio) { /* submit pending writes */	795	while (bio) { /* submit pending writes */
796	struct bio *next = bio->bi_next;	796	struct bio *next = bio->bi_next;
797	struct md_rdev rdev = (void )bio->bi_disk;	797	struct md_rdev rdev = (void )bio->bi_bdev;
798	bio->bi_next = NULL;	798	bio->bi_next = NULL;
799	bio_set_dev(bio, rdev->bdev);	799	bio_set_dev(bio, rdev->bdev);
800	if (test_bit(Faulty, &rdev->flags)) {	800	if (test_bit(Faulty, &rdev->flags)) {
801	bio_io_error(bio);	801	bio_io_error(bio);
802	} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&	802	} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
803	!blk_queue_discard(bio->bi_disk->queue)))	803	!blk_queue_discard(bio->bi_bdev->bd_disk->queue)))
804	/* Just ignore it */	804	/* Just ignore it */
805	bio_endio(bio);	805	bio_endio(bio);
806	else	806	else
@@ -1104,7 +1104,7 @@ static void alloc_behind_master_bio(struct r1bio *r1_bio, · 297 unmodified lines
1104	int i = 0;	1104	int i = 0;
1105	struct bio *behind_bio = NULL;	1105	struct bio *behind_bio = NULL;
1106		1106
1107	behind_bio = bio_alloc_mddev(GFP_NOIO, vcnt, r1_bio->mddev);	1107	behind_bio = bio_alloc_bioset(GFP_NOIO, vcnt, &r1_bio->mddev->bio_set);
1108	if (!behind_bio)	1108	if (!behind_bio)
1109	return;	1109	return;
1110		1110
@@ -1520,7 +1520,7 @@ static void raid1_write_request(struct mddev mddev, struct bio bio, · 409 unmodified lines
1520	trace_block_bio_remap(mbio, disk_devt(mddev->gendisk),	1520	trace_block_bio_remap(mbio, disk_devt(mddev->gendisk),
1521	r1_bio->sector);	1521	r1_bio->sector);
1522	/* flush_pending_writes() needs access to the rdev so...*/	1522	/* flush_pending_writes() needs access to the rdev so...*/
1523	mbio->bi_disk = (void *)conf->mirrors[i].rdev;	1523	mbio->bi_bdev = (void *)conf->mirrors[i].rdev;
1524		1524
1525	cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));	1525	cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));
1526	if (cb)	1526	if (cb)

drivers/md/raid10.c+7−11

 		while (bio) { /* submit pending writes */
 			struct bio *next = bio->bi_next;
-			struct md_rdev *rdev = (void*)bio->bi_disk;
+			struct md_rdev *rdev = (void*)bio->bi_bdev;
 			bio->bi_next = NULL;
 			bio_set_dev(bio, rdev->bdev);
 			if (test_bit(Faulty, &rdev->flags)) {
 				bio_io_error(bio);
 			} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
-					    !blk_queue_discard(bio->bi_disk->queue)))
+					    !blk_queue_discard(bio->bi_bdev->bd_disk->queue)))
 				/* Just ignore it */
 				bio_endio(bio);
 			else
 	while (bio) { /* submit pending writes */
 		struct bio *next = bio->bi_next;
-		struct md_rdev *rdev = (void*)bio->bi_disk;
+		struct md_rdev *rdev = (void*)bio->bi_bdev;
 		bio->bi_next = NULL;
 		bio_set_dev(bio, rdev->bdev);
 		if (test_bit(Faulty, &rdev->flags)) {
 			bio_io_error(bio);
 		} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
-				    !blk_queue_discard(bio->bi_disk->queue)))
+				    !blk_queue_discard(bio->bi_bdev->bd_disk->queue)))
 			/* Just ignore it */
 			bio_endio(bio);
 		else
 		trace_block_bio_remap(mbio, disk_devt(conf->mddev->gendisk),
 				      r10_bio->sector);
 	/* flush_pending_writes() needs access to the rdev so...*/
-	mbio->bi_disk = (void *)rdev;
+	mbio->bi_bdev = (void *)rdev;
 	atomic_inc(&r10_bio->remaining);
 	/* Again, very different code for resync and recovery.
 	 * Both must result in an r10bio with a list of bios that
-	 * have bi_end_io, bi_sector, bi_disk set,
+	 * have bi_end_io, bi_sector, bi_bdev set,
 	 * and bi_private set to the r10bio.
 	 * For recovery, we may actually create several r10bios
 	 * with 2 bios in each, that correspond to the bios in the main one.
 		return sectors_done;
 	}
-	read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);
+	read_bio = bio_alloc_bioset(GFP_KERNEL, RESYNC_PAGES, &mddev->bio_set);
 	bio_set_dev(read_bio, rdev->bdev);
 	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
 	read_bio->bi_private = r10_bio;
 	read_bio->bi_end_io = end_reshape_read;
 	bio_set_op_attrs(read_bio, REQ_OP_READ, 0);
-	read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
-	read_bio->bi_status = 0;
-	read_bio->bi_vcnt = 0;
-	read_bio->bi_iter.bi_size = 0;
 	r10_bio->master_bio = read_bio;
 	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;

drivers/md/raid5-ppl.c+1−1

@@ -1037,7 +1037,7 @@ static int ppl_recover(struct ppl_log log, struct ppl_header pplhdr, · 1036 unmodified lines
1037	}	1037	}
1038		1038
1039	/* flush the disk cache after recovery if necessary */	1039	/* flush the disk cache after recovery if necessary */
1040	ret = blkdev_issue_flush(rdev->bdev, GFP_KERNEL);	1040	ret = blkdev_issue_flush(rdev->bdev);
1041	out:	1041	out:
1042	__free_page(page);	1042	__free_page(page);
1043	return ret;	1043	return ret;

drivers/md/raid5.c+46−64

@@ -5310,7 +5310,7 @@ static int in_chunk_boundary(struct mddev mddev, struct bio bio) · 5309 unmodified lines
5310	unsigned int chunk_sectors;	5310	unsigned int chunk_sectors;
5311	unsigned int bio_sectors = bio_sectors(bio);	5311	unsigned int bio_sectors = bio_sectors(bio);
5312		5312
5313	WARN_ON_ONCE(bio->bi_partno);	5313	WARN_ON_ONCE(bio->bi_bdev->bd_partno);
5314		5314
5315	chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);	5315	chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);
5316	return chunk_sectors >=	5316	return chunk_sectors >=
@@ -5393,90 +5393,72 @@ static void raid5_align_endio(struct bio *bi) · 76 unmodified lines
5393	static int raid5_read_one_chunk(struct mddev mddev, struct bio raid_bio)	5393	static int raid5_read_one_chunk(struct mddev mddev, struct bio raid_bio)
5394	{	5394	{
5395	struct r5conf *conf = mddev->private;	5395	struct r5conf *conf = mddev->private;
5396	int dd_idx;	5396	struct bio *align_bio;
5397	struct bio* align_bi;
5398	struct md_rdev *rdev;	5397	struct md_rdev *rdev;
5399	sector_t end_sector;	5398	sector_t sector, end_sector, first_bad;
		5399	int bad_sectors, dd_idx;
5400		5400
5401	if (!in_chunk_boundary(mddev, raid_bio)) {	5401	if (!in_chunk_boundary(mddev, raid_bio)) {
5402	pr_debug("%s: non aligned\n", __func__);	5402	pr_debug("%s: non aligned\n", __func__);
5403	return 0;	5403	return 0;
5404	}	5404	}
5405	/*
5406	* use bio_clone_fast to make a copy of the bio
5407	*/
5408	align_bi = bio_clone_fast(raid_bio, GFP_NOIO, &mddev->bio_set);
5409	if (!align_bi)
5410	return 0;
5411	/*
5412	* set bi_end_io to a new function, and set bi_private to the
5413	* original bio.
5414	*/
5415	align_bi->bi_end_io = raid5_align_endio;
5416	align_bi->bi_private = raid_bio;
5417	/*
5418	* compute position
5419	*/
5420	align_bi->bi_iter.bi_sector =
5421	raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
5422	0, &dd_idx, NULL);
5423		5405
5424	end_sector = bio_end_sector(align_bi);	5406	sector = raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 0,
		5407	&dd_idx, NULL);
		5408	end_sector = bio_end_sector(raid_bio);
		5409
5425	rcu_read_lock();	5410	rcu_read_lock();
		5411	if (r5c_big_stripe_cached(conf, sector))
		5412	goto out_rcu_unlock;
		5413
5426	rdev = rcu_dereference(conf->disks[dd_idx].replacement);	5414	rdev = rcu_dereference(conf->disks[dd_idx].replacement);
5427	if (!rdev \|\| test_bit(Faulty, &rdev->flags) \|\|	5415	if (!rdev \|\| test_bit(Faulty, &rdev->flags) \|\|
5428	rdev->recovery_offset < end_sector) {	5416	rdev->recovery_offset < end_sector) {
5429	rdev = rcu_dereference(conf->disks[dd_idx].rdev);	5417	rdev = rcu_dereference(conf->disks[dd_idx].rdev);
5430	if (rdev &&	5418	if (!rdev)
5431	(test_bit(Faulty, &rdev->flags) \|\|	5419	goto out_rcu_unlock;
		5420	if (test_bit(Faulty, &rdev->flags) \|\|
5432	!(test_bit(In_sync, &rdev->flags) \|\|	5421	!(test_bit(In_sync, &rdev->flags) \|\|
5433	rdev->recovery_offset >= end_sector)))	5422	rdev->recovery_offset >= end_sector))
5434	rdev = NULL;	5423	goto out_rcu_unlock;
5435	}	5424	}
5436		5425
5437	if (r5c_big_stripe_cached(conf, align_bi->bi_iter.bi_sector)) {	5426	atomic_inc(&rdev->nr_pending);
5438	rcu_read_unlock();	5427	rcu_read_unlock();
5439	bio_put(align_bi);	5428
		5429	align_bio = bio_clone_fast(raid_bio, GFP_NOIO, &mddev->bio_set);
		5430	bio_set_dev(align_bio, rdev->bdev);
		5431	align_bio->bi_end_io = raid5_align_endio;
		5432	align_bio->bi_private = raid_bio;
		5433	align_bio->bi_iter.bi_sector = sector;
		5434
		5435	raid_bio->bi_next = (void *)rdev;
		5436
		5437	if (is_badblock(rdev, sector, bio_sectors(align_bio), &first_bad,
		5438	&bad_sectors)) {
		5439	bio_put(align_bio);
		5440	rdev_dec_pending(rdev, mddev);
5440	return 0;	5441	return 0;
5441	}	5442	}
5442		5443
5443	if (rdev) {	5444	/* No reshape active, so we can trust rdev->data_offset */
5444	sector_t first_bad;	5445	align_bio->bi_iter.bi_sector += rdev->data_offset;
5445	int bad_sectors;
5446
5447	atomic_inc(&rdev->nr_pending);
5448	rcu_read_unlock();
5449	raid_bio->bi_next = (void*)rdev;
5450	bio_set_dev(align_bi, rdev->bdev);
5451
5452	if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
5453	bio_sectors(align_bi),
5454	&first_bad, &bad_sectors)) {
5455	bio_put(align_bi);
5456	rdev_dec_pending(rdev, mddev);
5457	return 0;
5458	}
5459		5446
5460	/* No reshape active, so we can trust rdev->data_offset */	5447	spin_lock_irq(&conf->device_lock);
5461	align_bi->bi_iter.bi_sector += rdev->data_offset;	5448	wait_event_lock_irq(conf->wait_for_quiescent, conf->quiesce == 0,
		5449	conf->device_lock);
		5450	atomic_inc(&conf->active_aligned_reads);
		5451	spin_unlock_irq(&conf->device_lock);
5462		5452
5463	spin_lock_irq(&conf->device_lock);	5453	if (mddev->gendisk)
5464	wait_event_lock_irq(conf->wait_for_quiescent,	5454	trace_block_bio_remap(align_bio, disk_devt(mddev->gendisk),
5465	conf->quiesce == 0,	5455	raid_bio->bi_iter.bi_sector);
5466	conf->device_lock);	5456	submit_bio_noacct(align_bio);
5467	atomic_inc(&conf->active_aligned_reads);	5457	return 1;
5468	spin_unlock_irq(&conf->device_lock);
5469		5458
5470	if (mddev->gendisk)	5459	out_rcu_unlock:
5471	trace_block_bio_remap(align_bi, disk_devt(mddev->gendisk),	5460	rcu_read_unlock();
5472	raid_bio->bi_iter.bi_sector);	5461	return 0;
5473	submit_bio_noacct(align_bi);
5474	return 1;
5475	} else {
5476	rcu_read_unlock();
5477	bio_put(align_bi);
5478	return 0;
5479	}
5480	}	5462	}
5481		5463
5482	static struct bio chunk_aligned_read(struct mddev mddev, struct bio *raid_bio)	5464	static struct bio chunk_aligned_read(struct mddev mddev, struct bio *raid_bio)

drivers/mmc/core/block.c+5−5

@@ -253,7 +253,7 @@ static ssize_t power_ro_lock_store(struct device *dev, · 252 unmodified lines
253	goto out_put;	253	goto out_put;
254	}	254	}
255	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP;	255	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP;
256	blk_execute_rq(mq->queue, NULL, req, 0);	256	blk_execute_rq(NULL, req, 0);
257	ret = req_to_mmc_queue_req(req)->drv_op_result;	257	ret = req_to_mmc_queue_req(req)->drv_op_result;
258	blk_put_request(req);	258	blk_put_request(req);
259		259
@@ -629,7 +629,7 @@ static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md, · 369 unmodified lines
629	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;	629	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
630	req_to_mmc_queue_req(req)->drv_op_data = idatas;	630	req_to_mmc_queue_req(req)->drv_op_data = idatas;
631	req_to_mmc_queue_req(req)->ioc_count = 1;	631	req_to_mmc_queue_req(req)->ioc_count = 1;
632	blk_execute_rq(mq->queue, NULL, req, 0);	632	blk_execute_rq(NULL, req, 0);
633	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;	633	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
634	err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);	634	err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);
635	blk_put_request(req);	635	blk_put_request(req);
@@ -698,7 +698,7 @@ static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md, · 62 unmodified lines
698	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;	698	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
699	req_to_mmc_queue_req(req)->drv_op_data = idata;	699	req_to_mmc_queue_req(req)->drv_op_data = idata;
700	req_to_mmc_queue_req(req)->ioc_count = num_of_cmds;	700	req_to_mmc_queue_req(req)->ioc_count = num_of_cmds;
701	blk_execute_rq(mq->queue, NULL, req, 0);	701	blk_execute_rq(NULL, req, 0);
702	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;	702	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
703		703
704	/* copy to user if data and response */	704	/* copy to user if data and response */
@@ -2722,7 +2722,7 @@ static int mmc_dbg_card_status_get(void data, u64 val) · 2017 unmodified lines
2722	if (IS_ERR(req))	2722	if (IS_ERR(req))
2723	return PTR_ERR(req);	2723	return PTR_ERR(req);
2724	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;	2724	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
2725	blk_execute_rq(mq->queue, NULL, req, 0);	2725	blk_execute_rq(NULL, req, 0);
2726	ret = req_to_mmc_queue_req(req)->drv_op_result;	2726	ret = req_to_mmc_queue_req(req)->drv_op_result;
2727	if (ret >= 0) {	2727	if (ret >= 0) {
2728	*val = ret;	2728	*val = ret;
@@ -2761,7 +2761,7 @@ static int mmc_ext_csd_open(struct inode inode, struct file filp) · 32 unmodified lines
2761	}	2761	}
2762	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;	2762	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
2763	req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;	2763	req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;
2764	blk_execute_rq(mq->queue, NULL, req, 0);	2764	blk_execute_rq(NULL, req, 0);
2765	err = req_to_mmc_queue_req(req)->drv_op_result;	2765	err = req_to_mmc_queue_req(req)->drv_op_result;
2766	blk_put_request(req);	2766	blk_put_request(req);
2767	if (err) {	2767	if (err) {

drivers/nvdimm/blk.c+2−2

@@ -165,7 +165,7 @@ static int nsblk_do_bvec(struct nd_namespace_blk *nsblk, · 164 unmodified lines
165	static blk_qc_t nd_blk_submit_bio(struct bio *bio)	165	static blk_qc_t nd_blk_submit_bio(struct bio *bio)
166	{	166	{
167	struct bio_integrity_payload *bip;	167	struct bio_integrity_payload *bip;
168	struct nd_namespace_blk *nsblk = bio->bi_disk->private_data;	168	struct nd_namespace_blk *nsblk = bio->bi_bdev->bd_disk->private_data;
169	struct bvec_iter iter;	169	struct bvec_iter iter;
170	unsigned long start;	170	unsigned long start;
171	struct bio_vec bvec;	171	struct bio_vec bvec;
@@ -177,7 +177,7 @@ static blk_qc_t nd_blk_submit_bio(struct bio *bio) · 5 unmodified lines
177		177
178	bip = bio_integrity(bio);	178	bip = bio_integrity(bio);
179	rw = bio_data_dir(bio);	179	rw = bio_data_dir(bio);
180	do_acct = blk_queue_io_stat(bio->bi_disk->queue);	180	do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
181	if (do_acct)	181	if (do_acct)
182	start = bio_start_io_acct(bio);	182	start = bio_start_io_acct(bio);
183	bio_for_each_segment(bvec, bio, iter) {	183	bio_for_each_segment(bvec, bio, iter) {

drivers/nvdimm/btt.c+2−2

@@ -1442,7 +1442,7 @@ static int btt_do_bvec(struct btt btt, struct bio_integrity_payload bip, · 1441 unmodified lines
1442	static blk_qc_t btt_submit_bio(struct bio *bio)	1442	static blk_qc_t btt_submit_bio(struct bio *bio)
1443	{	1443	{
1444	struct bio_integrity_payload *bip = bio_integrity(bio);	1444	struct bio_integrity_payload *bip = bio_integrity(bio);
1445	struct btt *btt = bio->bi_disk->private_data;	1445	struct btt *btt = bio->bi_bdev->bd_disk->private_data;
1446	struct bvec_iter iter;	1446	struct bvec_iter iter;
1447	unsigned long start;	1447	unsigned long start;
1448	struct bio_vec bvec;	1448	struct bio_vec bvec;
@@ -1452,7 +1452,7 @@ static blk_qc_t btt_submit_bio(struct bio *bio) · 3 unmodified lines
1452	if (!bio_integrity_prep(bio))	1452	if (!bio_integrity_prep(bio))
1453	return BLK_QC_T_NONE;	1453	return BLK_QC_T_NONE;
1454		1454
1455	do_acct = blk_queue_io_stat(bio->bi_disk->queue);	1455	do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
1456	if (do_acct)	1456	if (do_acct)
1457	start = bio_start_io_acct(bio);	1457	start = bio_start_io_acct(bio);
1458	bio_for_each_segment(bvec, bio, iter) {	1458	bio_for_each_segment(bvec, bio, iter) {

drivers/nvdimm/pmem.c+2−2

@@ -196,13 +196,13 @@ static blk_qc_t pmem_submit_bio(struct bio *bio) · 195 unmodified lines
196	unsigned long start;	196	unsigned long start;
197	struct bio_vec bvec;	197	struct bio_vec bvec;
198	struct bvec_iter iter;	198	struct bvec_iter iter;
199	struct pmem_device *pmem = bio->bi_disk->private_data;	199	struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data;
200	struct nd_region *nd_region = to_region(pmem);	200	struct nd_region *nd_region = to_region(pmem);
201		201
202	if (bio->bi_opf & REQ_PREFLUSH)	202	if (bio->bi_opf & REQ_PREFLUSH)
203	ret = nvdimm_flush(nd_region, bio);	203	ret = nvdimm_flush(nd_region, bio);
204		204
205	do_acct = blk_queue_io_stat(bio->bi_disk->queue);	205	do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue);
206	if (do_acct)	206	if (do_acct)
207	start = bio_start_io_acct(bio);	207	start = bio_start_io_acct(bio);
208	bio_for_each_segment(bvec, bio, iter) {	208	bio_for_each_segment(bvec, bio, iter) {

drivers/nvme/host/core.c+16−15

 	rq->cmd_flags |= REQ_HIPRI;
 	rq->end_io_data = &wait;
-	blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
+	blk_execute_rq_nowait(bd_disk, rq, at_head, nvme_end_sync_rq);
 	while (!completion_done(&wait)) {
 		blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
 	if (poll)
 		nvme_execute_rq_polled(req->q, NULL, req, at_head);
 	else
-		blk_execute_rq(req->q, NULL, req, at_head);
+		blk_execute_rq(NULL, req, at_head);
 	if (result)
 		*result = nvme_req(req)->result;
 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 	u32 effects;
 	effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
-	blk_execute_rq(rq->q, disk, rq, 0);
+	blk_execute_rq(disk, rq, 0);
 	nvme_passthru_end(ctrl, effects);
 }
 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
 {
 	bool write = nvme_is_write(cmd);
 	struct nvme_ns *ns = q->queuedata;
-	struct gendisk *disk = ns ? ns->disk : NULL;
+	struct block_device *bdev = ns ? ns->disk->part0 : NULL;
 	struct request *req;
 	struct bio *bio = NULL;
 	void *meta = NULL;
 		if (ret)
 			goto out;
 		bio = req->bio;
-		bio->bi_disk = disk;
+		if (bdev)
-		if (disk && meta_buffer && meta_len) {
+			bio_set_dev(bio, bdev);
+		if (bdev && meta_buffer && meta_len) {
 			meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
 					meta_seed, write);
 			if (IS_ERR(meta)) {
 	rq->timeout = ctrl->kato * HZ;
 	rq->end_io_data = ctrl;
-	blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
+	blk_execute_rq_nowait(NULL, rq, 0, nvme_keep_alive_end_io);
 	return 0;
 }
 	nvme_config_discard(disk, ns);
 	nvme_config_write_zeroes(disk, ns);
-	if ((id->nsattr & NVME_NS_ATTR_RO) ||
+	set_disk_ro(disk, (id->nsattr & NVME_NS_ATTR_RO) ||
-	    test_bit(NVME_NS_FORCE_RO, &ns->flags))
+		test_bit(NVME_NS_FORCE_RO, &ns->flags));
-		set_disk_ro(disk, true);
 }
 static inline bool nvme_first_scan(struct gendisk *disk)
 	ns->lba_shift = id->lbaf[lbaf].ds;
 	nvme_set_queue_limits(ns->ctrl, ns->queue);
+	ret = nvme_configure_metadata(ns, id);
+	if (ret)
+		goto out_unfreeze;
+	nvme_set_chunk_sectors(ns, id);
+	nvme_update_disk_info(ns->disk, ns, id);
 	if (ns->head->ids.csi == NVME_CSI_ZNS) {
 		ret = nvme_update_zone_info(ns, lbaf);
 		if (ret)
 			goto out_unfreeze;
 	}
-	ret = nvme_configure_metadata(ns, id);
-	if (ret)
-		goto out_unfreeze;
-	nvme_set_chunk_sectors(ns, id);
-	nvme_update_disk_info(ns->disk, ns, id);
 	blk_mq_unfreeze_queue(ns->disk->queue);
 	if (blk_queue_is_zoned(ns->queue)) {

drivers/nvme/host/lightnvm.c+3−4

 	rq->end_io_data = rqd;
-	blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io);
+	blk_execute_rq_nowait(NULL, rq, 0, nvme_nvm_end_io);
 	return 0;
 {
 	bool write = nvme_is_write((struct nvme_command *)vcmd);
 	struct nvm_dev *dev = ns->ndev;
-	struct gendisk *disk = ns->disk;
 	struct request *rq;
 	struct bio *bio = NULL;
 	__le64 *ppa_list = NULL;
 			vcmd->ph_rw.metadata = cpu_to_le64(metadata_dma);
 		}
-		bio->bi_disk = disk;
+		bio_set_dev(bio, ns->disk->part0);
 	}
-	blk_execute_rq(q, NULL, rq, 0);
+	blk_execute_rq(NULL, rq, 0);
 	if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
 		ret = -EINTR;

drivers/nvme/host/multipath.c+3−3

@@ -296,7 +296,7 @@ static bool nvme_available_path(struct nvme_ns_head *head) · 295 unmodified lines
296		296
297	blk_qc_t nvme_ns_head_submit_bio(struct bio *bio)	297	blk_qc_t nvme_ns_head_submit_bio(struct bio *bio)
298	{	298	{
299	struct nvme_ns_head *head = bio->bi_disk->private_data;	299	struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
300	struct device *dev = disk_to_dev(head->disk);	300	struct device *dev = disk_to_dev(head->disk);
301	struct nvme_ns *ns;	301	struct nvme_ns *ns;
302	blk_qc_t ret = BLK_QC_T_NONE;	302	blk_qc_t ret = BLK_QC_T_NONE;
@@ -312,7 +312,7 @@ blk_qc_t nvme_ns_head_submit_bio(struct bio *bio) · 9 unmodified lines
312	srcu_idx = srcu_read_lock(&head->srcu);	312	srcu_idx = srcu_read_lock(&head->srcu);
313	ns = nvme_find_path(head);	313	ns = nvme_find_path(head);
314	if (likely(ns)) {	314	if (likely(ns)) {
315	bio->bi_disk = ns->disk;	315	bio_set_dev(bio, ns->disk->part0);
316	bio->bi_opf \|= REQ_NVME_MPATH;	316	bio->bi_opf \|= REQ_NVME_MPATH;
317	trace_block_bio_remap(bio, disk_devt(ns->head->disk),	317	trace_block_bio_remap(bio, disk_devt(ns->head->disk),
318	bio->bi_iter.bi_sector);	318	bio->bi_iter.bi_sector);
@@ -352,7 +352,7 @@ static void nvme_requeue_work(struct work_struct *work) · 33 unmodified lines
352	* Reset disk to the mpath node and resubmit to select a new	352	* Reset disk to the mpath node and resubmit to select a new
353	* path.	353	* path.
354	*/	354	*/
355	bio->bi_disk = head->disk;	355	bio_set_dev(bio, head->disk->part0);
356	submit_bio_noacct(bio);	356	submit_bio_noacct(bio);
357	}	357	}
358	}	358	}

drivers/nvme/host/pci.c+2−2

@@ -1357,7 +1357,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) · 1356 unmodified lines
1357	}	1357	}
1358		1358
1359	abort_req->end_io_data = NULL;	1359	abort_req->end_io_data = NULL;
1360	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);	1360	blk_execute_rq_nowait(NULL, abort_req, 0, abort_endio);
1361		1361
1362	/*	1362	/*
1363	* The aborted req will be completed on receiving the abort req.	1363	* The aborted req will be completed on receiving the abort req.
@@ -2281,7 +2281,7 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) · 917 unmodified lines
2281	req->end_io_data = nvmeq;	2281	req->end_io_data = nvmeq;
2282		2282
2283	init_completion(&nvmeq->delete_done);	2283	init_completion(&nvmeq->delete_done);
2284	blk_execute_rq_nowait(q, NULL, req, false,	2284	blk_execute_rq_nowait(NULL, req, false,
2285	opcode == nvme_admin_delete_cq ?	2285	opcode == nvme_admin_delete_cq ?
2286	nvme_del_cq_end : nvme_del_queue_end);	2286	nvme_del_cq_end : nvme_del_queue_end);
2287	return 0;	2287	return 0;

drivers/nvme/host/rdma.c+1−1

@@ -1468,7 +1468,7 @@ static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue, · 1467 unmodified lines
1468	if (unlikely(nr))	1468	if (unlikely(nr))
1469	goto mr_put;	1469	goto mr_put;
1470		1470
1471	nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_disk), c,	1471	nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_bdev->bd_disk), c,
1472	req->mr->sig_attrs, ns->pi_type);	1472	req->mr->sig_attrs, ns->pi_type);
1473	nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask);	1473	nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask);
1474		1474

drivers/nvme/host/zns.c+3−8

 int nvme_revalidate_zones(struct nvme_ns *ns)
 {
-	struct request_queue *q = ns->queue;
+	return blk_revalidate_disk_zones(ns->disk, NULL);
-	int ret;
-	ret = blk_revalidate_disk_zones(ns->disk, NULL);
-	if (!ret)
-		blk_queue_max_zone_append_sectors(q, ns->ctrl->max_zone_append);
-	return ret;
 }
 static int nvme_set_max_append(struct nvme_ctrl *ctrl)
 		goto free_data;
 	}
-	q->limits.zoned = BLK_ZONED_HM;
+	blk_queue_set_zoned(ns->disk, BLK_ZONED_HM);
 	blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q);
 	blk_queue_max_open_zones(q, le32_to_cpu(id->mor) + 1);
 	blk_queue_max_active_zones(q, le32_to_cpu(id->mar) + 1);
+	blk_queue_max_zone_append_sectors(q, ns->ctrl->max_zone_append);
 free_data:
 	kfree(id);
 	return status;

drivers/nvme/target/io-cmd-bdev.c+1−1

@@ -333,7 +333,7 @@ static void nvmet_bdev_execute_flush(struct nvmet_req *req) · 332 unmodified lines
333		333
334	u16 nvmet_bdev_flush(struct nvmet_req *req)	334	u16 nvmet_bdev_flush(struct nvmet_req *req)
335	{	335	{
336	if (blkdev_issue_flush(req->ns->bdev, GFP_KERNEL))	336	if (blkdev_issue_flush(req->ns->bdev))
337	return NVME_SC_INTERNAL \| NVME_SC_DNR;	337	return NVME_SC_INTERNAL \| NVME_SC_DNR;
338	return 0;	338	return 0;
339	}	339	}

drivers/nvme/target/passthru.c+1−1

@@ -275,7 +275,7 @@ static void nvmet_passthru_execute_cmd(struct nvmet_req *req) · 274 unmodified lines
275	schedule_work(&req->p.work);	275	schedule_work(&req->p.work);
276	} else {	276	} else {
277	rq->end_io_data = req;	277	rq->end_io_data = req;
278	blk_execute_rq_nowait(rq->q, ns ? ns->disk : NULL, rq, 0,	278	blk_execute_rq_nowait(ns ? ns->disk : NULL, rq, 0,
279	nvmet_passthru_req_done);	279	nvmet_passthru_req_done);
280	}	280	}
281		281

drivers/s390/block/dasd.c+5−21

 static int
 dasd_state_ready_to_online(struct dasd_device * device)
 {
-	struct gendisk *disk;
-	struct disk_part_iter piter;
-	struct block_device *part;
 	device->state = DASD_STATE_ONLINE;
 	if (device->block) {
 		dasd_schedule_block_bh(device->block);
 		if ((device->features & DASD_FEATURE_USERAW)) {
-			disk = device->block->gdp;
+			kobject_uevent(&disk_to_dev(device->block->gdp)->kobj,
-			kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
+					KOBJ_CHANGE);
 			return 0;
 		}
-		disk = device->block->bdev->bd_disk;
+		disk_uevent(device->block->bdev->bd_disk, KOBJ_CHANGE);
-		disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
-		while ((part = disk_part_iter_next(&piter)))
-			kobject_uevent(bdev_kobj(part), KOBJ_CHANGE);
-		disk_part_iter_exit(&piter);
 	}
 	return 0;
 }
 static int dasd_state_online_to_ready(struct dasd_device *device)
 {
 	int rc;
-	struct gendisk *disk;
-	struct disk_part_iter piter;
-	struct block_device *part;
 	if (device->discipline->online_to_ready) {
 		rc = device->discipline->online_to_ready(device);
 	}
 	device->state = DASD_STATE_READY;
-	if (device->block && !(device->features & DASD_FEATURE_USERAW)) {
+	if (device->block && !(device->features & DASD_FEATURE_USERAW))
-		disk = device->block->bdev->bd_disk;
+		disk_uevent(device->block->bdev->bd_disk, KOBJ_CHANGE);
-		disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0);
-		while ((part = disk_part_iter_next(&piter)))
-			kobject_uevent(bdev_kobj(part), KOBJ_CHANGE);
-		disk_part_iter_exit(&piter);
-	}
 	return 0;
 }

drivers/s390/block/dcssblk.c+1−5

 	blk_queue_split(&bio);
 	bytes_done = 0;
-	dev_info = bio->bi_disk->private_data;
+	dev_info = bio->bi_bdev->bd_disk->private_data;
 	if (dev_info == NULL)
 		goto fail;
 	if ((bio->bi_iter.bi_sector & 7) != 0 ||
 	    (bio->bi_iter.bi_size & 4095) != 0)
 		/* Request is not page-aligned. */
 		goto fail;
-	if (bio_end_sector(bio) > get_capacity(bio->bi_disk)) {
-		/* Request beyond end of DCSS segment. */
-		goto fail;
-	}
 	/* verify data transfer direction */
 	if (dev_info->is_shared) {
 		switch (dev_info->segment_type) {

drivers/s390/block/xpram.c+1−1

@@ -184,7 +184,7 @@ static unsigned long xpram_highest_page_index(void) · 183 unmodified lines
184	*/	184	*/
185	static blk_qc_t xpram_submit_bio(struct bio *bio)	185	static blk_qc_t xpram_submit_bio(struct bio *bio)
186	{	186	{
187	xpram_device_t *xdev = bio->bi_disk->private_data;	187	xpram_device_t *xdev = bio->bi_bdev->bd_disk->private_data;
188	struct bio_vec bvec;	188	struct bio_vec bvec;
189	struct bvec_iter iter;	189	struct bvec_iter iter;
190	unsigned int index;	190	unsigned int index;

drivers/scsi/scsi_error.c+1−1

@@ -2007,7 +2007,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev) · 2006 unmodified lines
2007	req->timeout = 10 * HZ;	2007	req->timeout = 10 * HZ;
2008	rq->retries = 5;	2008	rq->retries = 5;
2009		2009
2010	blk_execute_rq_nowait(req->q, NULL, req, 1, eh_lock_door_done);	2010	blk_execute_rq_nowait(NULL, req, 1, eh_lock_door_done);
2011	}	2011	}
2012		2012
2013	/**	2013	/**

drivers/scsi/scsi_lib.c+1−1

@@ -269,7 +269,7 @@ int __scsi_execute(struct scsi_device sdev, const unsigned char cmd, · 268 unmodified lines
269	/*	269	/*
270	* head injection required here otherwise quiesce won't work	270	* head injection required here otherwise quiesce won't work
271	*/	271	*/
272	blk_execute_rq(req->q, NULL, req, 1);	272	blk_execute_rq(NULL, req, 1);
273		273
274	/*	274	/*
275	* Some devices (USB mass-storage in particular) may transfer	275	* Some devices (USB mass-storage in particular) may transfer

drivers/scsi/sd_zbc.c+39−4

@@ -665,12 +665,28 @@ static int sd_zbc_init_disk(struct scsi_disk *sdkp) · 664 unmodified lines
665	return 0;	665	return 0;
666	}	666	}
667		667
668	void sd_zbc_release_disk(struct scsi_disk *sdkp)	668	static void sd_zbc_clear_zone_info(struct scsi_disk *sdkp)
669	{	669	{
		670	/* Serialize against revalidate zones */
		671	mutex_lock(&sdkp->rev_mutex);
		672
670	kvfree(sdkp->zones_wp_offset);	673	kvfree(sdkp->zones_wp_offset);
671	sdkp->zones_wp_offset = NULL;	674	sdkp->zones_wp_offset = NULL;
672	kfree(sdkp->zone_wp_update_buf);	675	kfree(sdkp->zone_wp_update_buf);
673	sdkp->zone_wp_update_buf = NULL;	676	sdkp->zone_wp_update_buf = NULL;
		677
		678	sdkp->nr_zones = 0;
		679	sdkp->rev_nr_zones = 0;
		680	sdkp->zone_blocks = 0;
		681	sdkp->rev_zone_blocks = 0;
		682
		683	mutex_unlock(&sdkp->rev_mutex);
		684	}
		685
		686	void sd_zbc_release_disk(struct scsi_disk *sdkp)
		687	{
		688	if (sd_is_zoned(sdkp))
		689	sd_zbc_clear_zone_info(sdkp);
674	}	690	}
675		691
676	static void sd_zbc_revalidate_zones_cb(struct gendisk *disk)	692	static void sd_zbc_revalidate_zones_cb(struct gendisk *disk)
@@ -769,6 +785,21 @@ int sd_zbc_read_zones(struct scsi_disk sdkp, unsigned char buf) · 92 unmodified lines
769	*/	785	*/
770	return 0;	786	return 0;
771		787
		788	/* READ16/WRITE16 is mandatory for ZBC disks */
		789	sdkp->device->use_16_for_rw = 1;
		790	sdkp->device->use_10_for_rw = 0;
		791
		792	if (!blk_queue_is_zoned(q)) {
		793	/*
		794	* This can happen for a host aware disk with partitions.
		795	* The block device zone information was already cleared
		796	* by blk_queue_set_zoned(). Only clear the scsi disk zone
		797	* information and exit early.
		798	*/
		799	sd_zbc_clear_zone_info(sdkp);
		800	return 0;
		801	}
		802
772	/* Check zoned block device characteristics (unconstrained reads) */	803	/* Check zoned block device characteristics (unconstrained reads) */
773	ret = sd_zbc_check_zoned_characteristics(sdkp, buf);	804	ret = sd_zbc_check_zoned_characteristics(sdkp, buf);
774	if (ret)	805	if (ret)
@@ -789,9 +820,13 @@ int sd_zbc_read_zones(struct scsi_disk sdkp, unsigned char buf) · 14 unmodified lines
789	blk_queue_max_active_zones(q, 0);	820	blk_queue_max_active_zones(q, 0);
790	nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks);	821	nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks);
791		822
792	/* READ16/WRITE16 is mandatory for ZBC disks */	823	/*
793	sdkp->device->use_16_for_rw = 1;	824	* Per ZBC and ZAC specifications, writes in sequential write required
794	sdkp->device->use_10_for_rw = 0;	825	* zones of host-managed devices must be aligned to the device physical
		826	* block size.
		827	*/
		828	if (blk_queue_zoned_model(q) == BLK_ZONED_HM)
		829	blk_queue_zone_write_granularity(q, sdkp->physical_block_size);
795		830
796	sdkp->rev_nr_zones = nr_zones;	831	sdkp->rev_nr_zones = nr_zones;
797	sdkp->rev_zone_blocks = zone_blocks;	832	sdkp->rev_zone_blocks = zone_blocks;

drivers/scsi/sg.c+1−2

@@ -829,8 +829,7 @@ sg_common_write(Sg_fd * sfp, Sg_request * srp, · 828 unmodified lines
829		829
830	srp->rq->timeout = timeout;	830	srp->rq->timeout = timeout;
831	kref_get(&sfp->f_ref); /* sg_rq_end_io() does kref_put(). */	831	kref_get(&sfp->f_ref); /* sg_rq_end_io() does kref_put(). */
832	blk_execute_rq_nowait(sdp->device->request_queue, sdp->disk,	832	blk_execute_rq_nowait(sdp->disk, srp->rq, at_head, sg_rq_end_io);
833	srp->rq, at_head, sg_rq_end_io);
834	return 0;	833	return 0;
835	}	834	}
836		835

drivers/scsi/st.c+1−1

@@ -585,7 +585,7 @@ static int st_scsi_execute(struct st_request SRpnt, const unsigned char cmd, · 584 unmodified lines
585	rq->retries = retries;	585	rq->retries = retries;
586	req->end_io_data = SRpnt;	586	req->end_io_data = SRpnt;
587		587
588	blk_execute_rq_nowait(req->q, NULL, req, 1, st_scsi_execute_end);	588	blk_execute_rq_nowait(NULL, req, 1, st_scsi_execute_end);
589	return 0;	589	return 0;
590	}	590	}
591		591

drivers/target/target_core_file.c+6−14

 	unsigned long	len;
 	struct se_cmd	*cmd;
 	struct kiocb	iocb;
+	struct bio_vec	bvecs[];
 };
 static void cmd_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
 	struct target_core_file_cmd *aio_cmd;
 	struct iov_iter iter = {};
 	struct scatterlist *sg;
-	struct bio_vec *bvec;
 	ssize_t len = 0;
 	int ret = 0, i;
-	aio_cmd = kmalloc(sizeof(struct target_core_file_cmd), GFP_KERNEL);
+	aio_cmd = kmalloc(struct_size(aio_cmd, bvecs, sgl_nents), GFP_KERNEL);
 	if (!aio_cmd)
 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
-	bvec = kcalloc(sgl_nents, sizeof(struct bio_vec), GFP_KERNEL);
-	if (!bvec) {
-		kfree(aio_cmd);
-		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
-	}
 	for_each_sg(sgl, sg, sgl_nents, i) {
-		bvec[i].bv_page = sg_page(sg);
+		aio_cmd->bvecs[i].bv_page = sg_page(sg);
-		bvec[i].bv_len = sg->length;
+		aio_cmd->bvecs[i].bv_len = sg->length;
-		bvec[i].bv_offset = sg->offset;
+		aio_cmd->bvecs[i].bv_offset = sg->offset;
 		len += sg->length;
 	}
-	iov_iter_bvec(&iter, is_write, bvec, sgl_nents, len);
+	iov_iter_bvec(&iter, is_write, aio_cmd->bvecs, sgl_nents, len);
 	aio_cmd->cmd = cmd;
 	aio_cmd->len = len;
 	else
 		ret = call_read_iter(file, &aio_cmd->iocb, &iter);
-	kfree(bvec);
 	if (ret != -EIOCBQUEUED)
 		cmd_rw_aio_complete(&aio_cmd->iocb, ret, 0);

drivers/target/target_core_pscsi.c+1−2

@@ -1000,8 +1000,7 @@ pscsi_execute_cmd(struct se_cmd *cmd) · 999 unmodified lines
1000	req->timeout = PS_TIMEOUT_OTHER;	1000	req->timeout = PS_TIMEOUT_OTHER;
1001	scsi_req(req)->retries = PS_RETRY;	1001	scsi_req(req)->retries = PS_RETRY;
1002		1002
1003	blk_execute_rq_nowait(pdv->pdv_sd->request_queue, NULL, req,	1003	blk_execute_rq_nowait(NULL, req, (cmd->sam_task_attr == TCM_HEAD_TAG),
1004	(cmd->sam_task_attr == TCM_HEAD_TAG),
1005	pscsi_req_done);	1004	pscsi_req_done);
1006		1005
1007	return 0;	1006	return 0;

fs/block_dev.c+9−11

 		bd_abort_claiming(bdev, truncate_bdev_range);
 	return 0;
 }
-EXPORT_SYMBOL(truncate_bdev_range);
 static void set_init_blocksize(struct block_device *bdev)
 {
 		dio->size += bio->bi_iter.bi_size;
 		pos += bio->bi_iter.bi_size;
-		nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
+		nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES);
 		if (!nr_pages) {
 			bool polled = false;
 {
 	int nr_pages;
-	nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1);
+	if (!iov_iter_count(iter))
-	if (!nr_pages)
 		return 0;
+	nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_PAGES + 1);
 	if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)
 		return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
 	 * i_mutex and doing so causes performance issues with concurrent
 	 * O_SYNC writers to a block device.
 	 */
-	error = blkdev_issue_flush(bdev, GFP_KERNEL);
+	error = blkdev_issue_flush(bdev);
 	if (error == -EOPNOTSUPP)
 		error = 0;
 		return error;
 	/*
-	 * Invalidate again; if someone wandered in and dirtied a page,
+	 * Invalidate the page cache again; if someone wandered in and dirtied
-	 * the caller will be given -EBUSY.  The third argument is
+	 * a page, we just discard it - userspace has no way of knowing whether
-	 * inclusive, so the rounding here is safe.
+	 * the write happened before or after discard completing...
 	 */
-	return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
+	return truncate_bdev_range(bdev, file->f_mode, start, end);
-					     start >> PAGE_SHIFT,
-					     end >> PAGE_SHIFT);
 }
 const struct file_operations def_blk_fops = {

fs/btrfs/check-integrity.c+5−5

fs/btrfs/raid56.c+2−5

 		 * devices or if they are not contiguous
 		 */
 		if (last_end == disk_start && !last->bi_status &&
-		    last->bi_disk == stripe->dev->bdev->bd_disk &&
+		    last->bi_bdev == stripe->dev->bdev) {
-		    last->bi_partno == stripe->dev->bdev->bd_partno) {
 			ret = bio_add_page(last, page, PAGE_SIZE, 0);
 			if (ret == PAGE_SIZE)
 				return 0;
 	for (i = 0; i < rbio->bbio->num_stripes; i++) {
 		stripe = &rbio->bbio->stripes[i];
 		if (in_range(physical, stripe->physical, rbio->stripe_len) &&
-		    stripe->dev->bdev &&
+		    stripe->dev->bdev && bio->bi_bdev == stripe->dev->bdev) {
-		    bio->bi_disk == stripe->dev->bdev->bd_disk &&
-		    bio->bi_partno == stripe->dev->bdev->bd_partno) {
 			return i;
 		}
 	}

fs/btrfs/scrub.c+1−1

@@ -1728,7 +1728,7 @@ static void scrub_wr_submit(struct scrub_ctx *sctx) · 1727 unmodified lines
1728		1728
1729	sbio = sctx->wr_curr_bio;	1729	sbio = sctx->wr_curr_bio;
1730	sctx->wr_curr_bio = NULL;	1730	sctx->wr_curr_bio = NULL;
1731	WARN_ON(!sbio->bio->bi_disk);	1731	WARN_ON(!sbio->bio->bi_bdev);
1732	scrub_pending_bio_inc(sctx);	1732	scrub_pending_bio_inc(sctx);
1733	/* process all writes in a single worker thread. Then the block layer	1733	/* process all writes in a single worker thread. Then the block layer
1734	* orders the requests before sending them to the driver which	1734	* orders the requests before sending them to the driver which

fs/btrfs/volumes.c+1−1

@@ -421,7 +421,7 @@ static struct btrfs_device __alloc_device(struct btrfs_fs_info fs_info) · 420 unmodified lines
421	* Preallocate a bio that's always going to be used for flushing device	421	* Preallocate a bio that's always going to be used for flushing device
422	* barriers and matches the device lifespan	422	* barriers and matches the device lifespan
423	*/	423	*/
424	dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL);	424	dev->flush_bio = bio_kmalloc(GFP_KERNEL, 0);
425	if (!dev->flush_bio) {	425	if (!dev->flush_bio) {
426	kfree(dev);	426	kfree(dev);
427	return ERR_PTR(-ENOMEM);	427	return ERR_PTR(-ENOMEM);

fs/btrfs/zoned.c+2−2

@@ -1275,8 +1275,8 @@ void btrfs_record_physical_zoned(struct inode *inode, u64 file_offset, · 1274 unmodified lines
1275	return;	1275	return;
1276		1276
1277	ordered->physical = physical;	1277	ordered->physical = physical;
1278	ordered->disk = bio->bi_disk;	1278	ordered->disk = bio->bi_bdev->bd_disk;
1279	ordered->partno = bio->bi_partno;	1279	ordered->partno = bio->bi_bdev->bd_partno;
1280		1280
1281	btrfs_put_ordered_extent(ordered);	1281	btrfs_put_ordered_extent(ordered);
1282	}	1282	}

fs/direct-io.c+3−1

 	unsigned long flags;
 	bio->bi_private = dio;
+	/* don't account direct I/O as memory stall */
+	bio_clear_flag(bio, BIO_WORKINGSET);
 	spin_lock_irqsave(&dio->bio_lock, flags);
 	dio->refcount++;
 	if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty)
 		bio_set_pages_dirty(bio);
-	dio->bio_disk = bio->bi_disk;
+	dio->bio_disk = bio->bi_bdev->bd_disk;
 	if (sdio->submit_io) {
 		sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio);

fs/exfat/file.c+1−1

@@ -361,7 +361,7 @@ int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) · 360 unmodified lines
361	if (err)	361	if (err)
362	return err;	362	return err;
363		363
364	return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);	364	return blkdev_issue_flush(inode->i_sb->s_bdev);
365	}	365	}
366		366
367	const struct file_operations exfat_file_operations = {	367	const struct file_operations exfat_file_operations = {

fs/ext4/fast_commit.c+2−2

@@ -1076,7 +1076,7 @@ static int ext4_fc_perform_commit(journal_t *journal) · 1075 unmodified lines
1076	* flush before we start writing fast commit blocks.	1076	* flush before we start writing fast commit blocks.
1077	*/	1077	*/
1078	if (journal->j_fs_dev != journal->j_dev)	1078	if (journal->j_fs_dev != journal->j_dev)
1079	blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);	1079	blkdev_issue_flush(journal->j_fs_dev);
1080		1080
1081	blk_start_plug(&plug);	1081	blk_start_plug(&plug);
1082	if (sbi->s_fc_bytes == 0) {	1082	if (sbi->s_fc_bytes == 0) {
@@ -1535,7 +1535,7 @@ static int ext4_fc_replay_inode(struct super_block sb, struct ext4_fc_tl tl) · 452 unmodified lines
1535	out:	1535	out:
1536	iput(inode);	1536	iput(inode);
1537	if (!ret)	1537	if (!ret)
1538	blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);	1538	blkdev_issue_flush(sb->s_bdev);
1539		1539
1540	return 0;	1540	return 0;
1541	}	1541	}

fs/ext4/fsync.c+1−1

@@ -174,7 +174,7 @@ int ext4_sync_file(struct file *file, loff_t start, loff_t end, int datasync) · 173 unmodified lines
174	ret = ext4_fsync_journal(inode, datasync, &needs_barrier);	174	ret = ext4_fsync_journal(inode, datasync, &needs_barrier);
175		175
176	if (needs_barrier) {	176	if (needs_barrier) {
177	err = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);	177	err = blkdev_issue_flush(inode->i_sb->s_bdev);
178	if (!ret)	178	if (!ret)
179	ret = err;	179	ret = err;
180	}	180	}

fs/ext4/ialloc.c+1−1

@@ -1583,7 +1583,7 @@ int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, · 1582 unmodified lines
1583	if (ret < 0)	1583	if (ret < 0)
1584	goto err_out;	1584	goto err_out;
1585	if (barrier)	1585	if (barrier)
1586	blkdev_issue_flush(sb->s_bdev, GFP_NOFS);	1586	blkdev_issue_flush(sb->s_bdev);
1587		1587
1588	skip_zeroout:	1588	skip_zeroout:
1589	ext4_lock_group(sb, group);	1589	ext4_lock_group(sb, group);

fs/ext4/super.c+1−1

@@ -5709,7 +5709,7 @@ static int ext4_sync_fs(struct super_block *sb, int wait) · 5708 unmodified lines
5709	needs_barrier = true;	5709	needs_barrier = true;
5710	if (needs_barrier) {	5710	if (needs_barrier) {
5711	int err;	5711	int err;
5712	err = blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);	5712	err = blkdev_issue_flush(sb->s_bdev);
5713	if (!ret)	5713	if (!ret)
5714	ret = err;	5714	ret = err;
5715	}	5715	}

fs/f2fs/data.c+5−35

 	bioset_exit(&f2fs_bioset);
 }
-static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask,
-						unsigned int nr_iovecs)
-{
-	return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset);
-}
-struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio)
-{
-	if (noio) {
-		/* No failure on bio allocation */
-		return __f2fs_bio_alloc(GFP_NOIO, npages);
-	}
-	if (time_to_inject(sbi, FAULT_ALLOC_BIO)) {
-		f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO);
-		return NULL;
-	}
-	return __f2fs_bio_alloc(GFP_KERNEL, npages);
-}
 static bool __is_cp_guaranteed(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
 	return 0;
 }
-/*
- * Return true, if pre_bio's bdev is same as its target device.
- */
-static bool __same_bdev(struct f2fs_sb_info *sbi,
-				block_t blk_addr, struct bio *bio)
-{
-	struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
-	return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
-}
 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
 {
 	struct f2fs_sb_info *sbi = fio->sbi;
 	struct bio *bio;
-	bio = f2fs_bio_alloc(sbi, npages, true);
+	bio = bio_alloc_bioset(GFP_NOIO, npages, &f2fs_bioset);
 	f2fs_target_device(sbi, fio->new_blkaddr, bio);
 	if (is_read_io(fio->op)) {
 		return false;
 	if (last_blkaddr + 1 != cur_blkaddr)
 		return false;
-	return __same_bdev(sbi, cur_blkaddr, bio);
+	return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL);
 }
 static bool io_type_is_mergeable(struct f2fs_bio_info *io,
 	struct bio_post_read_ctx *ctx;
 	unsigned int post_read_steps = 0;
-	bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES),
+	bio = bio_alloc_bioset(for_write ? GFP_NOIO : GFP_KERNEL,
-								for_write);
+			       min_t(int, nr_pages, BIO_MAX_PAGES),
+			       &f2fs_bioset);
 	if (!bio)
 		return ERR_PTR(-ENOMEM);

fs/f2fs/f2fs.h+0−2

 	FAULT_KVMALLOC,
 	FAULT_PAGE_ALLOC,
 	FAULT_PAGE_GET,
-	FAULT_ALLOC_BIO,
 	FAULT_ALLOC_NID,
 	FAULT_ORPHAN,
 	FAULT_BLOCK,
  */
 int __init f2fs_init_bioset(void);
 void f2fs_destroy_bioset(void);
-struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio);
 int f2fs_init_bio_entry_cache(void);
 void f2fs_destroy_bio_entry_cache(void);
 void f2fs_submit_bio(struct f2fs_sb_info *sbi,

fs/f2fs/segment.c+1−11

@@ -563,17 +563,7 @@ do_sync: · 562 unmodified lines
563	static int __submit_flush_wait(struct f2fs_sb_info *sbi,	563	static int __submit_flush_wait(struct f2fs_sb_info *sbi,
564	struct block_device *bdev)	564	struct block_device *bdev)
565	{	565	{
566	struct bio *bio;	566	int ret = blkdev_issue_flush(bdev);
567	int ret;
568
569	bio = f2fs_bio_alloc(sbi, 0, false);
570	if (!bio)
571	return -ENOMEM;
572
573	bio->bi_opf = REQ_OP_WRITE \| REQ_SYNC \| REQ_PREFLUSH;
574	bio_set_dev(bio, bdev);
575	ret = submit_bio_wait(bio);
576	bio_put(bio);
577		567
578	trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),	568	trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
579	test_opt(sbi, FLUSH_MERGE), ret);	569	test_opt(sbi, FLUSH_MERGE), ret);

fs/f2fs/super.c+0−1

@@ -46,7 +46,6 @@ const char *f2fs_fault_name[FAULT_MAX] = { · 45 unmodified lines
46	[FAULT_KVMALLOC] = "kvmalloc",	46	[FAULT_KVMALLOC] = "kvmalloc",
47	[FAULT_PAGE_ALLOC] = "page alloc",	47	[FAULT_PAGE_ALLOC] = "page alloc",
48	[FAULT_PAGE_GET] = "page get",	48	[FAULT_PAGE_GET] = "page get",
49	[FAULT_ALLOC_BIO] = "alloc bio",
50	[FAULT_ALLOC_NID] = "alloc nid",	49	[FAULT_ALLOC_NID] = "alloc nid",
51	[FAULT_ORPHAN] = "orphan",	50	[FAULT_ORPHAN] = "orphan",
52	[FAULT_BLOCK] = "no more block",	51	[FAULT_BLOCK] = "no more block",

fs/fat/file.c+1−1

@@ -195,7 +195,7 @@ int fat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) · 194 unmodified lines
195	if (err)	195	if (err)
196	return err;	196	return err;
197		197
198	return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);	198	return blkdev_issue_flush(inode->i_sb->s_bdev);
199	}	199	}
200		200
201		201

fs/hfsplus/inode.c+1−1

@@ -340,7 +340,7 @@ int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, · 339 unmodified lines
340	}	340	}
341		341
342	if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags))	342	if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags))
343	blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);	343	blkdev_issue_flush(inode->i_sb->s_bdev);
344		344
345	inode_unlock(inode);	345	inode_unlock(inode);
346		346

fs/hfsplus/super.c+1−1

@@ -239,7 +239,7 @@ out: · 238 unmodified lines
239	mutex_unlock(&sbi->vh_mutex);	239	mutex_unlock(&sbi->vh_mutex);
240		240
241	if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags))	241	if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags))
242	blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);	242	blkdev_issue_flush(sb->s_bdev);
243		243
244	return error;	244	return error;
245	}	245	}

fs/iomap/direct-io.c+4−5

 	orig_count = iov_iter_count(dio->submit.iter);
 	iov_iter_truncate(dio->submit.iter, length);
-	nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES);
+	if (!iov_iter_count(dio->submit.iter))
-	if (nr_pages <= 0) {
-		ret = nr_pages;
 		goto out;
-	}
 	if (need_zeroout) {
 		/* zero out from the start of the block to the write offset */
 	 */
 	bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua);
+	nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_PAGES);
 	do {
 		size_t n;
 		if (dio->error) {
 		dio->size += n;
 		copied += n;
-		nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES);
+		nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,
+						 BIO_MAX_PAGES);
 		iomap_dio_submit_bio(dio, iomap, bio, pos);
 		pos += n;
 	} while (nr_pages);

fs/jbd2/checkpoint.c+1−1

@@ -416,7 +416,7 @@ int jbd2_cleanup_journal_tail(journal_t *journal) · 415 unmodified lines
416	* jbd2_cleanup_journal_tail() doesn't get called all that often.	416	* jbd2_cleanup_journal_tail() doesn't get called all that often.
417	*/	417	*/
418	if (journal->j_flags & JBD2_BARRIER)	418	if (journal->j_flags & JBD2_BARRIER)
419	blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);	419	blkdev_issue_flush(journal->j_fs_dev);
420		420
421	return __jbd2_update_log_tail(journal, first_tid, blocknr);	421	return __jbd2_update_log_tail(journal, first_tid, blocknr);
422	}	422	}

fs/jbd2/commit.c+2−2

@@ -825,7 +825,7 @@ start_journal_io: · 824 unmodified lines
825	if (commit_transaction->t_need_data_flush &&	825	if (commit_transaction->t_need_data_flush &&
826	(journal->j_fs_dev != journal->j_dev) &&	826	(journal->j_fs_dev != journal->j_dev) &&
827	(journal->j_flags & JBD2_BARRIER))	827	(journal->j_flags & JBD2_BARRIER))
828	blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);	828	blkdev_issue_flush(journal->j_fs_dev);
829		829
830	/* Done it all: now write the commit record asynchronously. */	830	/* Done it all: now write the commit record asynchronously. */
831	if (jbd2_has_feature_async_commit(journal)) {	831	if (jbd2_has_feature_async_commit(journal)) {
@@ -932,7 +932,7 @@ start_journal_io: · 100 unmodified lines
932	stats.run.rs_blocks_logged++;	932	stats.run.rs_blocks_logged++;
933	if (jbd2_has_feature_async_commit(journal) &&	933	if (jbd2_has_feature_async_commit(journal) &&
934	journal->j_flags & JBD2_BARRIER) {	934	journal->j_flags & JBD2_BARRIER) {
935	blkdev_issue_flush(journal->j_dev, GFP_NOFS);	935	blkdev_issue_flush(journal->j_dev);
936	}	936	}
937		937
938	if (err)	938	if (err)

fs/jbd2/recovery.c+1−1

@@ -326,7 +326,7 @@ int jbd2_journal_recover(journal_t *journal) · 325 unmodified lines
326	err = err2;	326	err = err2;
327	/* Make sure all replayed data is on permanent storage */	327	/* Make sure all replayed data is on permanent storage */
328	if (journal->j_flags & JBD2_BARRIER) {	328	if (journal->j_flags & JBD2_BARRIER) {
329	err2 = blkdev_issue_flush(journal->j_fs_dev, GFP_KERNEL);	329	err2 = blkdev_issue_flush(journal->j_fs_dev);
330	if (!err)	330	if (!err)
331	err = err2;	331	err = err2;
332	}	332	}

fs/libfs.c+1−1

@@ -1117,7 +1117,7 @@ int generic_file_fsync(struct file *file, loff_t start, loff_t end, · 1116 unmodified lines
1117	err = __generic_file_fsync(file, start, end, datasync);	1117	err = __generic_file_fsync(file, start, end, datasync);
1118	if (err)	1118	if (err)
1119	return err;	1119	return err;
1120	return blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);	1120	return blkdev_issue_flush(inode->i_sb->s_bdev);
1121	}	1121	}
1122	EXPORT_SYMBOL(generic_file_fsync);	1122	EXPORT_SYMBOL(generic_file_fsync);
1123		1123

fs/nfs/blocklayout/blocklayout.c+0−5

 	npg = min(npg, BIO_MAX_PAGES);
 	bio = bio_alloc(GFP_NOIO, npg);
-	if (!bio && (current->flags & PF_MEMALLOC)) {
-		while (!bio && (npg /= 2))
-			bio = bio_alloc(GFP_NOIO, npg);
-	}
 	if (bio) {
 		bio->bi_iter.bi_sector = disk_sector;
 		bio_set_dev(bio, bdev);

fs/nfsd/blocklayout.c+1−1

@@ -254,7 +254,7 @@ again: · 253 unmodified lines
254	req->cmd[4] = bufflen & 0xff;	254	req->cmd[4] = bufflen & 0xff;
255	req->cmd_len = COMMAND_SIZE(INQUIRY);	255	req->cmd_len = COMMAND_SIZE(INQUIRY);
256		256
257	blk_execute_rq(rq->q, NULL, rq, 1);	257	blk_execute_rq(NULL, rq, 1);
258	if (req->result) {	258	if (req->result) {
259	pr_err("pNFS: INQUIRY 0x83 failed with: %x\n",	259	pr_err("pNFS: INQUIRY 0x83 failed with: %x\n",
260	req->result);	260	req->result);

fs/nilfs2/segbuf.c+0−4

 	struct bio *bio;
 	bio = bio_alloc(GFP_NOIO, nr_vecs);
-	if (bio == NULL) {
-		while (!bio && (nr_vecs >>= 1))
-			bio = bio_alloc(GFP_NOIO, nr_vecs);
-	}
 	if (likely(bio)) {
 		bio_set_dev(bio, nilfs->ns_bdev);
 		bio->bi_iter.bi_sector =

fs/nilfs2/the_nilfs.h+1−1

@@ -375,7 +375,7 @@ static inline int nilfs_flush_device(struct the_nilfs *nilfs) · 374 unmodified lines
375	*/	375	*/
376	smp_wmb();	376	smp_wmb();
377		377
378	err = blkdev_issue_flush(nilfs->ns_bdev, GFP_KERNEL);	378	err = blkdev_issue_flush(nilfs->ns_bdev);
379	if (err != -EIO)	379	if (err != -EIO)
380	err = 0;	380	err = 0;
381	return err;	381	return err;

fs/ocfs2/file.c+1−1

@@ -194,7 +194,7 @@ static int ocfs2_sync_file(struct file *file, loff_t start, loff_t end, · 193 unmodified lines
194	needs_barrier = true;	194	needs_barrier = true;
195	err = jbd2_complete_transaction(journal, commit_tid);	195	err = jbd2_complete_transaction(journal, commit_tid);
196	if (needs_barrier) {	196	if (needs_barrier) {
197	ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);	197	ret = blkdev_issue_flush(inode->i_sb->s_bdev);
198	if (!err)	198	if (!err)
199	err = ret;	199	err = ret;
200	}	200	}

fs/reiserfs/file.c+1−1

@@ -159,7 +159,7 @@ static int reiserfs_sync_file(struct file *filp, loff_t start, loff_t end, · 158 unmodified lines
159	barrier_done = reiserfs_commit_for_inode(inode);	159	barrier_done = reiserfs_commit_for_inode(inode);
160	reiserfs_write_unlock(inode->i_sb);	160	reiserfs_write_unlock(inode->i_sb);
161	if (barrier_done != 1 && reiserfs_barrier_flush(inode->i_sb))	161	if (barrier_done != 1 && reiserfs_barrier_flush(inode->i_sb))
162	blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);	162	blkdev_issue_flush(inode->i_sb->s_bdev);
163	inode_unlock(inode);	163	inode_unlock(inode);
164	if (barrier_done < 0)	164	if (barrier_done < 0)
165	return barrier_done;	165	return barrier_done;

fs/splice.c+6−3

 		/* build the vector */
 		left = sd.total_len;
-		for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) {
+		for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
 			struct pipe_buffer *buf = &pipe->bufs[tail & mask];
 			size_t this_len = buf->len;
-			if (this_len > left)
+			/* zero-length bvecs are not supported, skip them */
-				this_len = left;
+			if (!this_len)
+				continue;
+			this_len = min(this_len, left);
 			ret = pipe_buf_confirm(pipe, buf);
 			if (unlikely(ret)) {
 			array[n].bv_len = this_len;
 			array[n].bv_offset = buf->offset;
 			left -= this_len;
+			n++;
 		}
 		iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left);

fs/super.c+2−1

@@ -865,7 +865,8 @@ int reconfigure_super(struct fs_context *fc) · 864 unmodified lines
865		865
866	if (fc->sb_flags_mask & SB_RDONLY) {	866	if (fc->sb_flags_mask & SB_RDONLY) {
867	#ifdef CONFIG_BLOCK	867	#ifdef CONFIG_BLOCK
868	if (!(fc->sb_flags & SB_RDONLY) && bdev_read_only(sb->s_bdev))	868	if (!(fc->sb_flags & SB_RDONLY) && sb->s_bdev &&
		869	bdev_read_only(sb->s_bdev))
869	return -EACCES;	870	return -EACCES;
870	#endif	871	#endif
871		872

fs/xfs/xfs_super.c+1−1

@@ -343,7 +343,7 @@ void · 342 unmodified lines
343	xfs_blkdev_issue_flush(	343	xfs_blkdev_issue_flush(
344	xfs_buftarg_t *buftarg)	344	xfs_buftarg_t *buftarg)
345	{	345	{
346	blkdev_issue_flush(buftarg->bt_bdev, GFP_NOFS);	346	blkdev_issue_flush(buftarg->bt_bdev);
347	}	347	}
348		348
349	STATIC void	349	STATIC void

fs/zonefs/super.c+6−7

 	if (ZONEFS_I(inode)->i_ztype == ZONEFS_ZTYPE_CNV)
 		ret = file_write_and_wait_range(file, start, end);
 	if (!ret)
-		ret = blkdev_issue_flush(inode->i_sb->s_bdev, GFP_KERNEL);
+		ret = blkdev_issue_flush(inode->i_sb->s_bdev);
 	if (ret)
 		zonefs_io_error(inode, true);
 	if (!nr_pages)
 		return 0;
-	bio = bio_alloc_bioset(GFP_NOFS, nr_pages, &fs_bio_set);
+	bio = bio_alloc(GFP_NOFS, nr_pages);
 	if (!bio)
 		return -ENOMEM;
 	sb->s_time_gran	= 1;
 	/*
-	 * The block size is set to the device physical sector size to ensure
+	 * The block size is set to the device zone write granularity to ensure
-	 * that write operations on 512e devices (512B logical block and 4KB
+	 * that write operations are always aligned according to the device
-	 * physical block) are always aligned to the device physical blocks,
+	 * interface constraints.
-	 * as mandated by the ZBC/ZAC specifications.
 	 */
-	sb_set_blocksize(sb, bdev_physical_block_size(sb->s_bdev));
+	sb_set_blocksize(sb, bdev_zone_write_granularity(sb->s_bdev));
 	sbi->s_zone_sectors_shift = ilog2(bdev_zone_sectors(sb->s_bdev));
 	sbi->s_uid = GLOBAL_ROOT_UID;
 	sbi->s_gid = GLOBAL_ROOT_GID;

include/linux/bio.h+28−27

 #include <linux/ioprio.h>
 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
 #include <linux/blk_types.h>
+#include <linux/uio.h>
 #define BIO_DEBUG
 	struct bvec_iter	bip_iter;
-	unsigned short		bip_slab;	/* slab the bip came from */
 	unsigned short		bip_vcnt;	/* # of integrity bio_vecs */
 	unsigned short		bip_max_vcnt;	/* integrity bio_vec slots */
 	unsigned short		bip_flags;	/* control flags */
 extern int biovec_init_pool(mempool_t *pool, int pool_entries);
 extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
-extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
+struct bio *bio_alloc_bioset(gfp_t gfp, unsigned short nr_iovecs,
+		struct bio_set *bs);
+struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs);
 extern void bio_put(struct bio *);
 extern void __bio_clone_fast(struct bio *, struct bio *);
 extern struct bio_set fs_bio_set;
-static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
+static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned short nr_iovecs)
 {
 	return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
 }
-static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
-{
-	return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
-}
 extern blk_qc_t submit_bio(struct bio *);
 extern void bio_endio(struct bio *);
 	bio_endio(bio);
 }
+/*
+ * Calculate number of bvec segments that should be allocated to fit data
+ * pointed by @iter. If @iter is backed by bvec it's going to be reused
+ * instead of allocating a new one.
+ */
+static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
+{
+	if (iov_iter_is_bvec(iter))
+		return 0;
+	return iov_iter_npages(iter, max_segs);
+}
 struct request_queue;
 extern int submit_bio_wait(struct bio *bio);
 	zero_fill_bio_iter(bio, bio->bi_iter);
 }
-extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
-extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
-extern unsigned int bvec_nr_vecs(unsigned short idx);
 extern const char *bio_devname(struct bio *bio, char *buffer);
-#define bio_set_dev(bio, bdev) 			\
+#define bio_set_dev(bio, bdev) 				\
-do {						\
+do {							\
-	if ((bio)->bi_disk != (bdev)->bd_disk)	\
+	bio_clear_flag(bio, BIO_REMAPPED);		\
-		bio_clear_flag(bio, BIO_THROTTLED);\
+	if ((bio)->bi_bdev != (bdev))			\
-	(bio)->bi_disk = (bdev)->bd_disk;	\
+		bio_clear_flag(bio, BIO_THROTTLED);	\
-	(bio)->bi_partno = (bdev)->bd_partno;	\
+	(bio)->bi_bdev = (bdev);			\
-	bio_associate_blkg(bio);		\
+	bio_associate_blkg(bio);			\
 } while (0)
 #define bio_copy_dev(dst, src)			\
 do {						\
-	(dst)->bi_disk = (src)->bi_disk;	\
+	bio_clear_flag(dst, BIO_REMAPPED);		\
-	(dst)->bi_partno = (src)->bi_partno;	\
+	(dst)->bi_bdev = (src)->bi_bdev;	\
 	bio_clone_blkg_association(dst, src);	\
 } while (0)
 #define bio_dev(bio) \
-	disk_devt((bio)->bi_disk)
+	disk_devt((bio)->bi_bdev->bd_disk)
 #ifdef CONFIG_BLK_CGROUP
 void bio_associate_blkg(struct bio *bio);
 	mempool_t bvec_integrity_pool;
 #endif
+	unsigned int back_pad;
 	/*
 	 * Deadlock avoidance for stacking block drivers: see comments in
 	 * bio_alloc_bioset() for details
 	struct workqueue_struct	*rescue_workqueue;
 };
-struct biovec_slab {
-	int nr_vecs;
-	char *name;
-	struct kmem_cache *slab;
-};
 static inline bool bioset_initialized(struct bio_set *bs)
 {
 	return bs->bio_slab != NULL;

include/linux/blk-mq.h+2−6

 	 * shared across request queues.
 	 */
 	atomic_t		nr_active;
-	/**
-	 * @elevator_queued: Number of queued requests on hctx.
-	 */
-	atomic_t                elevator_queued;
 	/** @cpuhp_online: List to store request if CPU is going to die */
 	struct hlist_node	cpuhp_online;
 	rq->bio = rq->biotail = bio;
 	rq->ioprio = bio_prio(bio);
-	if (bio->bi_disk)
+	if (bio->bi_bdev)
-		rq->rq_disk = bio->bi_disk;
+		rq->rq_disk = bio->bi_bdev->bd_disk;
 }
 blk_qc_t blk_mq_submit_bio(struct bio *bio);

include/linux/blk_types.h+3−30

@@ -222,16 +222,15 @@ static inline void bio_issue_init(struct bio_issue *issue, · 221 unmodified lines
222	*/	222	*/
223	struct bio {	223	struct bio {
224	struct bio bi_next; / request queue link */	224	struct bio bi_next; / request queue link */
225	struct gendisk *bi_disk;	225	struct block_device *bi_bdev;
226	unsigned int bi_opf; /* bottom bits req flags,	226	unsigned int bi_opf; /* bottom bits req flags,
227	* top bits REQ_OP. Use	227	* top bits REQ_OP. Use
228	* accessors.	228	* accessors.
229	*/	229	*/
230	unsigned short bi_flags; /* status, etc and bvec pool number */	230	unsigned short bi_flags; /* BIO_* below */
231	unsigned short bi_ioprio;	231	unsigned short bi_ioprio;
232	unsigned short bi_write_hint;	232	unsigned short bi_write_hint;
233	blk_status_t bi_status;	233	blk_status_t bi_status;
234	u8 bi_partno;
235	atomic_t __bi_remaining;	234	atomic_t __bi_remaining;
236		235
237	struct bvec_iter bi_iter;	236	struct bvec_iter bi_iter;
@@ -304,36 +303,10 @@ enum { · 66 unmodified lines
304	* of this bio. */	303	* of this bio. */
305	BIO_CGROUP_ACCT, /* has been accounted to a cgroup */	304	BIO_CGROUP_ACCT, /* has been accounted to a cgroup */
306	BIO_TRACKED, /* set if bio goes through the rq_qos path */	305	BIO_TRACKED, /* set if bio goes through the rq_qos path */
		306	BIO_REMAPPED,
307	BIO_FLAG_LAST	307	BIO_FLAG_LAST
308	};	308	};
309		309
310	/* See BVEC_POOL_OFFSET below before adding new flags */
311
312	/*
313	* We support 6 different bvec pools, the last one is magic in that it
314	* is backed by a mempool.
315	*/
316	#define BVEC_POOL_NR 6
317	#define BVEC_POOL_MAX (BVEC_POOL_NR - 1)
318
319	/*
320	* Top 3 bits of bio flags indicate the pool the bvecs came from. We add
321	* 1 to the actual index so that 0 indicates that there are no bvecs to be
322	* freed.
323	*/
324	#define BVEC_POOL_BITS (3)
325	#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
326	#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
327	#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
328	# error "BVEC_POOL_BITS is too small"
329	#endif
330
331	/*
332	* Flags starting here get preserved by bio_reset() - this includes
333	* only BVEC_POOL_IDX()
334	*/
335	#define BIO_RESET_BITS BVEC_POOL_OFFSET
336
337	typedef __u32 __bitwise blk_mq_req_flags_t;	310	typedef __u32 __bitwise blk_mq_req_flags_t;
338		311
339	/*	312	/*

include/linux/blkdev.h+25−28

 	unsigned int		max_zone_append_sectors;
 	unsigned int		discard_granularity;
 	unsigned int		discard_alignment;
+	unsigned int		zone_write_granularity;
 	unsigned short		max_segments;
 	unsigned short		max_integrity_segments;
 extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
 			       struct rq_map_data *, const struct iov_iter *,
 			       gfp_t);
-extern void blk_execute_rq(struct request_queue *, struct gendisk *,
+extern void blk_execute_rq(struct gendisk *, struct request *, int);
-			  struct request *, int);
+extern void blk_execute_rq_nowait(struct gendisk *,
-extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
 				  struct request *, int, rq_end_io_fn *);
 /* Helper to convert REQ_OP_XXX to its string format XXX */
 extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
 		unsigned int max_zone_append_sectors);
 extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
+void blk_queue_zone_write_granularity(struct request_queue *q,
+				      unsigned int size);
 extern void blk_queue_alignment_offset(struct request_queue *q,
 				       unsigned int alignment);
 void blk_queue_update_readahead(struct request_queue *q);
 		 !list_empty(&plug->cb_list));
 }
-int blkdev_issue_flush(struct block_device *, gfp_t);
+int blkdev_issue_flush(struct block_device *bdev);
 long nr_blockdev_pages(void);
 #else /* CONFIG_BLOCK */
 struct blk_plug {
 	return false;
 }
-static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)
+static inline int blkdev_issue_flush(struct block_device *bdev)
 {
 	return 0;
 }
 	return queue_io_opt(bdev_get_queue(bdev));
 }
+static inline unsigned int
+queue_zone_write_granularity(const struct request_queue *q)
+{
+	return q->limits.zone_write_granularity;
+}
+static inline unsigned int
+bdev_zone_write_granularity(struct block_device *bdev)
+{
+	return queue_zone_write_granularity(bdev_get_queue(bdev));
+}
 static inline int queue_alignment_offset(const struct request_queue *q)
 {
 	if (q->limits.misaligned)
 void disk_end_io_acct(struct gendisk *disk, unsigned int op,
 		unsigned long start_time);
-unsigned long part_start_io_acct(struct gendisk *disk,
+unsigned long bio_start_io_acct(struct bio *bio);
-		struct block_device **part, struct bio *bio);
+void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
-void part_end_io_acct(struct block_device *part, struct bio *bio,
+		struct block_device *orig_bdev);
-		      unsigned long start_time);
-/**
- * bio_start_io_acct - start I/O accounting for bio based drivers
- * @bio:	bio to start account for
- *
- * Returns the start time that should be passed back to bio_end_io_acct().
- */
-static inline unsigned long bio_start_io_acct(struct bio *bio)
-{
-	return disk_start_io_acct(bio->bi_disk, bio_sectors(bio), bio_op(bio));
-}
 /**
  * bio_end_io_acct - end I/O accounting for bio based drivers
  */
 static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
 {
-	return disk_end_io_acct(bio->bi_disk, bio_op(bio), start_time);
+	return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
 }
 int bdev_read_only(struct block_device *bdev);
 struct block_device *I_BDEV(struct inode *inode);
 struct block_device *bdgrab(struct block_device *bdev);
 void bdput(struct block_device *);
+int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
+		loff_t lend);
 #ifdef CONFIG_BLOCK
 void invalidate_bdev(struct block_device *bdev);
-int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
-			loff_t lend);
 int sync_blockdev(struct block_device *bdev);
 #else
 static inline void invalidate_bdev(struct block_device *bdev)
 {
 }
-static inline int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
-				      loff_t lstart, loff_t lend)
-{
-	return 0;
-}
 static inline int sync_blockdev(struct block_device *bdev)
 {
 	return 0;

include/linux/elevator.h+2−0

@@ -172,6 +172,8 @@ extern struct request elv_rb_find(struct rb_root , sector_t); · 171 unmodified lines
172		172
173	/* Supports zoned block devices sequential write constraint */	173	/* Supports zoned block devices sequential write constraint */
174	#define ELEVATOR_F_ZBD_SEQ_WRITE (1U << 0)	174	#define ELEVATOR_F_ZBD_SEQ_WRITE (1U << 0)
		175	/* Supports scheduling on multiple hardware queues */
		176	#define ELEVATOR_F_MQ_AWARE (1U << 1)
175		177
176	#endif /* CONFIG_BLOCK */	178	#endif /* CONFIG_BLOCK */
177	#endif	179	#endif

include/linux/genhd.h+9−18

 #include <linux/string.h>
 #include <linux/fs.h>
 #include <linux/workqueue.h>
+#include <linux/xarray.h>
 #define PARTITION_META_INFO_VOLNAMELTH	64
 /*
 	DISK_EVENT_FLAG_UEVENT			= 1 << 1,
 };
-struct disk_part_tbl {
-	struct rcu_head rcu_head;
-	int len;
-	struct block_device __rcu *last_lookup;
-	struct block_device __rcu *part[];
-};
 struct disk_events;
 struct badblocks;
 	unsigned short events;		/* supported events */
 	unsigned short event_flags;	/* flags related to event processing */
-	/* Array of pointers to partitions indexed by partno.
+	struct xarray part_tbl;
-	 * Protected with matching bdev lock but stat and other
-	 * non-critical accesses use RCU.  Always access through
-	 * helpers.
-	 */
-	struct disk_part_tbl __rcu *part_tbl;
 	struct block_device *part0;
 	const struct block_device_operations *fops;
 	int flags;
 	unsigned long state;
 #define GD_NEED_PART_SCAN		0
+#define GD_READ_ONLY			1
 	struct kobject *slave_dir;
 	struct timer_rand_state *random;
 	return MKDEV(disk->major, disk->first_minor);
 }
+void disk_uevent(struct gendisk *disk, enum kobject_action action);
 /*
  * Smarter partition iterator without context limits.
  */
-#define DISK_PITER_REVERSE	(1 << 0) /* iterate in the reverse direction */
 #define DISK_PITER_INCL_EMPTY	(1 << 1) /* include 0-sized parts */
 #define DISK_PITER_INCL_PART0	(1 << 2) /* include partition 0 */
 #define DISK_PITER_INCL_EMPTY_PART0 (1 << 3) /* include empty partition 0 */
 struct disk_part_iter {
 	struct gendisk		*disk;
 	struct block_device	*part;
-	int			idx;
+	unsigned long		idx;
 	unsigned int		flags;
 };
 				 struct gendisk *disk, unsigned int flags);
 struct block_device *disk_part_iter_next(struct disk_part_iter *piter);
 extern void disk_part_iter_exit(struct disk_part_iter *piter);
-extern bool disk_has_partitions(struct gendisk *disk);
 /* block/genhd.c */
 extern void device_add_disk(struct device *parent, struct gendisk *disk,
 extern void del_gendisk(struct gendisk *gp);
 extern struct block_device *bdget_disk(struct gendisk *disk, int partno);
-extern void set_disk_ro(struct gendisk *disk, int flag);
+void set_disk_ro(struct gendisk *disk, bool read_only);
 static inline int get_disk_ro(struct gendisk *disk)
 {
-	return disk->part0->bd_read_only;
+	return disk->part0->bd_read_only ||
+		test_bit(GD_READ_ONLY, &disk->state);
 }
 extern void disk_block_events(struct gendisk *disk);

include/linux/swap.h+0−1

@@ -468,7 +468,6 @@ extern int free_swap_and_cache(swp_entry_t); · 467 unmodified lines
468	int swap_type_of(dev_t device, sector_t offset);	468	int swap_type_of(dev_t device, sector_t offset);
469	int find_first_swap(dev_t *device);	469	int find_first_swap(dev_t *device);
470	extern unsigned int count_swap_pages(int, int);	470	extern unsigned int count_swap_pages(int, int);
471	extern sector_t map_swap_page(struct page , struct block_device *);
472	extern sector_t swapdev_block(int, pgoff_t);	471	extern sector_t swapdev_block(int, pgoff_t);
473	extern int page_swapcount(struct page *);	472	extern int page_swapcount(struct page *);
474	extern int __swap_count(swp_entry_t entry);	473	extern int __swap_count(swp_entry_t entry);

kernel/trace/blktrace.c+9−7

 static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio)
 {
-	blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BOUNCE, 0);
+	blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0);
 }
 static void blk_add_trace_bio_complete(void *ignore,
 static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio)
 {
-	blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BACKMERGE, 0);
+	blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE,
+			0);
 }
 static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio)
 {
-	blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_FRONTMERGE, 0);
+	blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE,
+			0);
 }
 static void blk_add_trace_bio_queue(void *ignore, struct bio *bio)
 {
-	blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_QUEUE, 0);
+	blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0);
 }
 static void blk_add_trace_getrq(void *ignore, struct bio *bio)
 {
-	blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_GETRQ, 0);
+	blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0);
 }
 static void blk_add_trace_plug(void *ignore, struct request_queue *q)
 static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu)
 {
-	struct request_queue *q = bio->bi_disk->queue;
+	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
 	struct blk_trace *bt;
 	rcu_read_lock();
 static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev,
 				    sector_t from)
 {
-	struct request_queue *q = bio->bi_disk->queue;
+	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
 	struct blk_trace *bt;
 	struct blk_io_trace_remap r;

lib/iov_iter.c+19−2

 	__start.bi_bvec_done = skip;			\
 	__start.bi_idx = 0;				\
 	for_each_bvec(__v, i->bvec, __bi, __start) {	\
-		if (!__v.bv_len)			\
-			continue;			\
 		(void)(STEP);				\
 	}						\
 }
 	pipe_truncate(i);
 }
+static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
+{
+	struct bvec_iter bi;
+	bi.bi_size = i->count;
+	bi.bi_bvec_done = i->iov_offset;
+	bi.bi_idx = 0;
+	bvec_iter_advance(i->bvec, &bi, size);
+	i->bvec += bi.bi_idx;
+	i->nr_segs -= bi.bi_idx;
+	i->count = bi.bi_size;
+	i->iov_offset = bi.bi_bvec_done;
+}
 void iov_iter_advance(struct iov_iter *i, size_t size)
 {
 	if (unlikely(iov_iter_is_pipe(i))) {
 		i->count -= size;
 		return;
 	}
+	if (iov_iter_is_bvec(i)) {
+		iov_iter_bvec_advance(i, size);
+		return;
+	}
 	iterate_and_advance(i, size, v, 0, 0, 0)
 }
 EXPORT_SYMBOL(iov_iter_advance);

mm/page_io.c+14−33

@@ -26,25 +26,6 @@ · 25 unmodified lines
26	#include <linux/uio.h>	26	#include <linux/uio.h>
27	#include <linux/sched/task.h>	27	#include <linux/sched/task.h>
28		28
29	static struct bio *get_swap_bio(gfp_t gfp_flags,
30	struct page *page, bio_end_io_t end_io)
31	{
32	struct bio *bio;
33
34	bio = bio_alloc(gfp_flags, 1);
35	if (bio) {
36	struct block_device *bdev;
37
38	bio->bi_iter.bi_sector = map_swap_page(page, &bdev);
39	bio_set_dev(bio, bdev);
40	bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
41	bio->bi_end_io = end_io;
42
43	bio_add_page(bio, page, thp_size(page), 0);
44	}
45	return bio;
46	}
47
48	void end_swap_bio_write(struct bio *bio)	29	void end_swap_bio_write(struct bio *bio)
49	{	30	{
50	struct page *page = bio_first_page_all(bio);	31	struct page *page = bio_first_page_all(bio);
@@ -361,13 +342,13 @@ int __swap_writepage(struct page page, struct writeback_control wbc, · 310 unmodified lines
361	return 0;	342	return 0;
362	}	343	}
363		344
364	bio = get_swap_bio(GFP_NOIO, page, end_write_func);	345	bio = bio_alloc(GFP_NOIO, 1);
365	if (bio == NULL) {	346	bio_set_dev(bio, sis->bdev);
366	set_page_dirty(page);	347	bio->bi_iter.bi_sector = swap_page_sector(page);
367	unlock_page(page);
368	return -ENOMEM;
369	}
370	bio->bi_opf = REQ_OP_WRITE \| REQ_SWAP \| wbc_to_write_flags(wbc);	348	bio->bi_opf = REQ_OP_WRITE \| REQ_SWAP \| wbc_to_write_flags(wbc);
		349	bio->bi_end_io = end_write_func;
		350	bio_add_page(bio, page, thp_size(page), 0);
		351
371	bio_associate_blkg_from_page(bio, page);	352	bio_associate_blkg_from_page(bio, page);
372	count_swpout_vm_event(page);	353	count_swpout_vm_event(page);
373	set_page_writeback(page);	354	set_page_writeback(page);
@@ -427,18 +408,18 @@ int swap_readpage(struct page *page, bool synchronous) · 53 unmodified lines
427	}	408	}
428		409
429	ret = 0;	410	ret = 0;
430	bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);	411	bio = bio_alloc(GFP_KERNEL, 1);
431	if (bio == NULL) {	412	bio_set_dev(bio, sis->bdev);
432	unlock_page(page);	413	bio->bi_opf = REQ_OP_READ;
433	ret = -ENOMEM;	414	bio->bi_iter.bi_sector = swap_page_sector(page);
434	goto out;	415	bio->bi_end_io = end_swap_bio_read;
435	}	416	bio_add_page(bio, page, thp_size(page), 0);
436	disk = bio->bi_disk;	417
		418	disk = bio->bi_bdev->bd_disk;
437	/*	419	/*
438	* Keep this task valid during swap readpage because the oom killer may	420	* Keep this task valid during swap readpage because the oom killer may
439	* attempt to access it in the page fault retry time check.	421	* attempt to access it in the page fault retry time check.
440	*/	422	*/
441	bio_set_op_attrs(bio, REQ_OP_READ, 0);
442	if (synchronous) {	423	if (synchronous) {
443	bio->bi_opf \|= REQ_HIPRI;	424	bio->bi_opf \|= REQ_HIPRI;
444	get_task_struct(current);	425	get_task_struct(current);

mm/swapfile.c+3−33

@@ -47,7 +47,6 @@ · 46 unmodified lines
47	static bool swap_count_continued(struct swap_info_struct *, pgoff_t,	47	static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
48	unsigned char);	48	unsigned char);
49	static void free_swap_count_continuations(struct swap_info_struct *);	49	static void free_swap_count_continuations(struct swap_info_struct *);
50	static sector_t map_swap_entry(swp_entry_t, struct block_device**);
51		50
52	DEFINE_SPINLOCK(swap_lock);	51	DEFINE_SPINLOCK(swap_lock);
53	static unsigned int nr_swapfiles;	52	static unsigned int nr_swapfiles;
@@ -1850,12 +1849,13 @@ int find_first_swap(dev_t *device) · 1796 unmodified lines
1850	*/	1849	*/
1851	sector_t swapdev_block(int type, pgoff_t offset)	1850	sector_t swapdev_block(int type, pgoff_t offset)
1852	{	1851	{
1853	struct block_device *bdev;
1854	struct swap_info_struct *si = swap_type_to_swap_info(type);	1852	struct swap_info_struct *si = swap_type_to_swap_info(type);
		1853	struct swap_extent *se;
1855		1854
1856	if (!si \|\| !(si->flags & SWP_WRITEOK))	1855	if (!si \|\| !(si->flags & SWP_WRITEOK))
1857	return 0;	1856	return 0;
1858	return map_swap_entry(swp_entry(type, offset), &bdev);	1857	se = offset_to_swap_extent(si, offset);
		1858	return se->start_block + (offset - se->start_page);
1859	}	1859	}
1860		1860
1861	/*	1861	/*
@@ -2281,36 +2281,6 @@ static void drain_mmlist(void) · 419 unmodified lines
2281	spin_unlock(&mmlist_lock);	2281	spin_unlock(&mmlist_lock);
2282	}	2282	}
2283		2283
2284	/*
2285	* Use this swapdev's extent info to locate the (PAGE_SIZE) block which
2286	* corresponds to page offset for the specified swap entry.
2287	* Note that the type of this function is sector_t, but it returns page offset
2288	* into the bdev, not sector offset.
2289	*/
2290	static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
2291	{
2292	struct swap_info_struct *sis;
2293	struct swap_extent *se;
2294	pgoff_t offset;
2295
2296	sis = swp_swap_info(entry);
2297	*bdev = sis->bdev;
2298
2299	offset = swp_offset(entry);
2300	se = offset_to_swap_extent(sis, offset);
2301	return se->start_block + (offset - se->start_page);
2302	}
2303
2304	/*
2305	* Returns the page offset into bdev for the specified page's swap entry.
2306	*/
2307	sector_t map_swap_page(struct page page, struct block_device *bdev)
2308	{
2309	swp_entry_t entry;
2310	entry.val = page_private(page);
2311	return map_swap_entry(entry, bdev);
2312	}
2313
2314	/*	2284	/*
2315	* Free all of a swapdev's extent information	2285	* Free all of a swapdev's extent information
2316	*/	2286	*/

Merge tag 'for-5.12/block-2021-02-17' of git://git.kernel.dk/linux-block

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -2674,7 +2674,7 @@ static void __btrfsic_submit_bio(struct bio *bio) · 2673 unmodified lines
2674	mutex_lock(&btrfsic_mutex);	2674	mutex_lock(&btrfsic_mutex);
2675	/* since btrfsic_submit_bio() is also called before	2675	/* since btrfsic_submit_bio() is also called before
2676	* btrfsic_mount(), this might return NULL */	2676	* btrfsic_mount(), this might return NULL */
2677	dev_state = btrfsic_dev_state_lookup(bio_dev(bio) + bio->bi_partno);	2677	dev_state = btrfsic_dev_state_lookup(bio->bi_bdev->bd_dev);
2678	if (NULL != dev_state &&	2678	if (NULL != dev_state &&
2679	(bio_op(bio) == REQ_OP_WRITE) && bio_has_data(bio)) {	2679	(bio_op(bio) == REQ_OP_WRITE) && bio_has_data(bio)) {
2680	unsigned int i = 0;	2680	unsigned int i = 0;
@@ -2690,9 +2690,9 @@ static void __btrfsic_submit_bio(struct bio *bio) · 9 unmodified lines
2690	bio_is_patched = 0;	2690	bio_is_patched = 0;
2691	if (dev_state->state->print_mask &	2691	if (dev_state->state->print_mask &
2692	BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)	2692	BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2693	pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_disk=%p)\n",	2693	pr_info("submit_bio(rw=%d,0x%x, bi_vcnt=%u, bi_sector=%llu (bytenr %llu), bi_bdev=%p)\n",
2694	bio_op(bio), bio->bi_opf, segs,	2694	bio_op(bio), bio->bi_opf, segs,
2695	bio->bi_iter.bi_sector, dev_bytenr, bio->bi_disk);	2695	bio->bi_iter.bi_sector, dev_bytenr, bio->bi_bdev);
2696		2696
2697	mapped_datav = kmalloc_array(segs,	2697	mapped_datav = kmalloc_array(segs,
2698	sizeof(*mapped_datav), GFP_NOFS);	2698	sizeof(*mapped_datav), GFP_NOFS);
@@ -2721,8 +2721,8 @@ static void __btrfsic_submit_bio(struct bio *bio) · 22 unmodified lines
2721	} else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) {	2721	} else if (NULL != dev_state && (bio->bi_opf & REQ_PREFLUSH)) {
2722	if (dev_state->state->print_mask &	2722	if (dev_state->state->print_mask &
2723	BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)	2723	BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
2724	pr_info("submit_bio(rw=%d,0x%x FLUSH, disk=%p)\n",	2724	pr_info("submit_bio(rw=%d,0x%x FLUSH, bdev=%p)\n",
2725	bio_op(bio), bio->bi_opf, bio->bi_disk);	2725	bio_op(bio), bio->bi_opf, bio->bi_bdev);
2726	if (!dev_state->dummy_block_for_bio_bh_flush.is_iodone) {	2726	if (!dev_state->dummy_block_for_bio_bh_flush.is_iodone) {
2727	if ((dev_state->state->print_mask &	2727	if ((dev_state->state->print_mask &
2728	(BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH \|	2728	(BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH \|