History

block: refactor the bounce buffering code

Get rid of all the PFN arithmetics and just use an enum for the two remaining options, and use PageHighMem for the actual bounce decision. Add a fast path to entirely avoid the call for the common case of a queue not using the legacy bouncing code.

Christoph Hellwig committed 5 years ago

Commit 9bb33f21 parent ce288e05 files · +38 −90Browse files

Unified · Split

5 files
  1. block/blk-core.c +2 −4
  2. block/blk-settings.c +8 −34
  3. block/blk.h +12 −4
  4. block/bounce.c +5 −30
  5. include/linux/blkdev.h +11 −18
block/blk-core.c+2−4
@@ -1161,10 +1161,8 @@ static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q, · 1160 unmodified lines
11611161 }
11621162
11631163 /*
1164− * queue's settings related to segment counting like q->bounce_pfn
1165− * may differ from that of other stacking queues.
1166− * Recalculate it to check the request correctly on this queue's
1167− * limitation.
1164+ * The queue settings related to segment counting may differ from the
1165+ * original queue.
11681166 */
11691167 rq->nr_phys_segments = blk_recalc_rq_segments(rq);
11701168 if (rq->nr_phys_segments > queue_max_segments(q)) {
block/blk-settings.c+8−34
@@ -7,7 +7,6 @@ · 6 unmodified lines
77 #include <linux/init.h>
88 #include <linux/bio.h>
99 #include <linux/blkdev.h>
10−#include <linux/memblock.h> /* for max_pfn/max_low_pfn */
1110 #include <linux/gcd.h>
1211 #include <linux/lcm.h>
1312 #include <linux/jiffies.h>
@@ -17,11 +16,6 @@ · 3 unmodified lines
1716 #include "blk.h"
1817 #include "blk-wbt.h"
1918
20−unsigned long blk_max_low_pfn;
21−EXPORT_SYMBOL(blk_max_low_pfn);
22−
23−unsigned long blk_max_pfn;
24−
2519 void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout)
2620 {
2721 q->rq_timeout = timeout;
@@ -55,7 +49,7 @@ void blk_set_default_limits(struct queue_limits *lim) · 27 unmodified lines
5549 lim->discard_alignment = 0;
5650 lim->discard_misaligned = 0;
5751 lim->logical_block_size = lim->physical_block_size = lim->io_min = 512;
58− lim->bounce_pfn = (unsigned long)(BLK_BOUNCE_ANY >> PAGE_SHIFT);
52+ lim->bounce = BLK_BOUNCE_NONE;
5953 lim->alignment_offset = 0;
6054 lim->io_opt = 0;
6155 lim->misaligned = 0;
@@ -92,28 +86,16 @@ EXPORT_SYMBOL(blk_set_stacking_limits); · 30 unmodified lines
9286 /**
9387 * blk_queue_bounce_limit - set bounce buffer limit for queue
9488 * @q: the request queue for the device
95− * @max_addr: the maximum address the device can handle
89+ * @bounce: bounce limit to enforce
9690 *
9791 * Description:
98− * Different hardware can have different requirements as to what pages
99− * it can do I/O directly to. A low level driver can call
100− * blk_queue_bounce_limit to have lower memory pages allocated as bounce
101− * buffers for doing I/O to pages residing above @max_addr.
92+ * Force bouncing for ISA DMA ranges or highmem.
93+ *
94+ * DEPRECATED, don't use in new code.
10295 **/
103−void blk_queue_bounce_limit(struct request_queue *q, u64 max_addr)
96+void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce bounce)
10497 {
105− unsigned long b_pfn = max_addr >> PAGE_SHIFT;
106−
107−#if BITS_PER_LONG == 64
108− /*
109− * Assume anything <= 4GB can be handled by IOMMU. Actually
110− * some IOMMUs can handle everything, but I don't know of a
111− * way to test this here.
112− */
113− q->limits.bounce_pfn = max(max_low_pfn, b_pfn);
114−#else
115− q->limits.bounce_pfn = b_pfn;
116−#endif
98+ q->limits.bounce = bounce;
11799 }
118100 EXPORT_SYMBOL(blk_queue_bounce_limit);
119101
@@ -536,7 +518,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, · 416 unmodified lines
536518 b->max_write_zeroes_sectors);
537519 t->max_zone_append_sectors = min(t->max_zone_append_sectors,
538520 b->max_zone_append_sectors);
539− t->bounce_pfn = min_not_zero(t->bounce_pfn, b->bounce_pfn);
521+ t->bounce = max(t->bounce, b->bounce);
540522
541523 t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask,
542524 b->seg_boundary_mask);
@@ -916,11 +898,3 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) · 373 unmodified lines
916898 }
917899 }
918900 EXPORT_SYMBOL_GPL(blk_queue_set_zoned);
919−
920−static int __init blk_settings_init(void)
921−{
922− blk_max_low_pfn = max_low_pfn - 1;
923− blk_max_pfn = max_pfn - 1;
924− return 0;
925−}
926−subsys_initcall(blk_settings_init);
block/blk.h+12−4
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #include <linux/blk-mq.h>
77 #include <linux/part_stat.h>
88 #include <linux/blk-crypto.h>
9+#include <linux/memblock.h> /* for max_pfn/max_low_pfn */
910 #include <xen/xen.h>
1011 #include "blk-crypto-internal.h"
1112 #include "blk-mq.h"
@@ -311,13 +312,20 @@ static inline void blk_throtl_bio_endio(struct bio *bio) { } · 299 unmodified lines
311312 static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
312313 #endif
313314
314−#ifdef CONFIG_BOUNCE
315−extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
316−#else
315+void __blk_queue_bounce(struct request_queue *q, struct bio **bio);
316+
317+static inline bool blk_queue_may_bounce(struct request_queue *q)
318+{
319+ return IS_ENABLED(CONFIG_BOUNCE) &&
320+ q->limits.bounce == BLK_BOUNCE_HIGH &&
321+ max_low_pfn >= max_pfn;
322+}
323+
317324 static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
318325 {
326+ if (unlikely(blk_queue_may_bounce(q) && bio_has_data(*bio)))
327+ __blk_queue_bounce(q, bio);
319328 }
320−#endif /* CONFIG_BOUNCE */
321329
322330 #ifdef CONFIG_BLK_CGROUP_IOLATENCY
323331 extern int blk_iolatency_init(struct request_queue *q);
block/bounce.c+5−30
@@ -18,7 +18,6 @@ · 17 unmodified lines
1818 #include <linux/init.h>
1919 #include <linux/hash.h>
2020 #include <linux/highmem.h>
21−#include <linux/memblock.h>
2221 #include <linux/printk.h>
2322 #include <asm/tlbflush.h>
2423
@@ -49,11 +48,11 @@ static void init_bounce_bioset(void) · 24 unmodified lines
4948 bounce_bs_setup = true;
5049 }
5150
52−#if defined(CONFIG_HIGHMEM)
5351 static __init int init_emergency_pool(void)
5452 {
5553 int ret;
56−#if defined(CONFIG_HIGHMEM) && !defined(CONFIG_MEMORY_HOTPLUG)
54+
55+#ifndef CONFIG_MEMORY_HOTPLUG
5756 if (max_pfn <= max_low_pfn)
5857 return 0;
5958 #endif
@@ -67,9 +66,7 @@ static __init int init_emergency_pool(void) · 7 unmodified lines
6766 }
6867
6968 __initcall(init_emergency_pool);
70−#endif
7169
72−#ifdef CONFIG_HIGHMEM
7370 /*
7471 * highmem version, map in to vec
7572 */
@@ -82,13 +79,6 @@ static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom) · 6 unmodified lines
8279 kunmap_atomic(vto);
8380 }
8481
85−#else /* CONFIG_HIGHMEM */
86−
87−#define bounce_copy_vec(to, vfrom) \
88− memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
89−
90−#endif /* CONFIG_HIGHMEM */
91−
9282 /*
9383 * Simple bounce buffer support for highmem pages. Depending on the
9484 * queue gfp mask set, *to may or may not be a highmem page. kmap it
@@ -236,8 +226,7 @@ err_put: · 141 unmodified lines
236226 return NULL;
237227 }
238228
239−
240−void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
229+void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
241230 {
242231 struct bio *bio;
243232 int rw = bio_data_dir(*bio_orig);
@@ -247,24 +236,10 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · 3 unmodified lines
247236 bool bounce = false;
248237 int sectors = 0;
249238
250− /*
251− * Data-less bio, nothing to bounce
252− */
253− if (!bio_has_data(*bio_orig))
254− return;
255−
256− /*
257− * Just check if the bounce pfn is equal to or bigger than the highest
258− * pfn in the system -- in that case, don't waste time iterating over
259− * bio segments
260− */
261− if (q->limits.bounce_pfn >= blk_max_pfn)
262− return;
263−
264239 bio_for_each_segment(from, *bio_orig, iter) {
265240 if (i++ < BIO_MAX_VECS)
266241 sectors += from.bv_len >> 9;
267− if (page_to_pfn(from.bv_page) > q->limits.bounce_pfn)
242+ if (PageHighMem(from.bv_page))
268243 bounce = true;
269244 }
270245 if (!bounce)
@@ -287,7 +262,7 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · 16 unmodified lines
287262 for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {
288263 struct page *page = to->bv_page;
289264
290− if (page_to_pfn(page) <= q->limits.bounce_pfn)
265+ if (!PageHighMem(page))
291266 continue;
292267
293268 to->bv_page = mempool_alloc(&page_pool, GFP_NOIO);
include/linux/blkdev.h+11−18
@@ -313,8 +313,17 @@ enum blk_zoned_model { · 312 unmodified lines
313313 BLK_ZONED_HM, /* Host-managed zoned block device */
314314 };
315315
316+/*
317+ * BLK_BOUNCE_NONE: never bounce (default)
318+ * BLK_BOUNCE_HIGH: bounce all highmem pages
319+ */
320+enum blk_bounce {
321+ BLK_BOUNCE_NONE,
322+ BLK_BOUNCE_HIGH,
323+};
324+
316325 struct queue_limits {
317− unsigned long bounce_pfn;
326+ enum blk_bounce bounce;
318327 unsigned long seg_boundary_mask;
319328 unsigned long virt_boundary_mask;
320329
@@ -835,22 +844,6 @@ static inline unsigned int blk_queue_depth(struct request_queue *q) · 514 unmodified lines
835844 return q->nr_requests;
836845 }
837846
838−extern unsigned long blk_max_low_pfn, blk_max_pfn;
839−
840−/*
841− * standard bounce addresses:
842− *
843− * BLK_BOUNCE_HIGH : bounce all highmem pages
844− * BLK_BOUNCE_ANY : don't bounce anything
845− */
846−
847−#if BITS_PER_LONG == 32
848−#define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
849−#else
850−#define BLK_BOUNCE_HIGH -1ULL
851−#endif
852−#define BLK_BOUNCE_ANY (-1ULL)
853−
854847 /*
855848 * default timeout for SG_IO if none specified
856849 */
@@ -1134,7 +1127,7 @@ extern void blk_abort_request(struct request *); · 277 unmodified lines
11341127 * Access functions for manipulating queue properties
11351128 */
11361129 extern void blk_cleanup_queue(struct request_queue *);
1137−extern void blk_queue_bounce_limit(struct request_queue *, u64);
1130+void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce limit);
11381131 extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
11391132 extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
11401133 extern void blk_queue_max_segments(struct request_queue *, unsigned short);