History

block: refactor the bounce buffering code

Get rid of all the PFN arithmetics and just use an enum for the two remaining options, and use PageHighMem for the actual bounce decision. Add a fast path to entirely avoid the call for the common case of a queue not using the legacy bouncing code.

Christoph Hellwig committed 5 years ago

Commit 9bb33f21 parent ce288e05 files · +38 −90Browse files

Unified · Split

5 files
  1. block/blk-core.c +2 −4
  2. block/blk-settings.c +8 −34
  3. block/blk.h +12 −4
  4. block/bounce.c +5 −30
  5. include/linux/blkdev.h +11 −18
block/blk-core.c+2−4
@@ -1161,10 +1161,8 @@ static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q, · 1160 unmodified lines
1161 }1161 }
1162 1162
1163 /*1163 /*
1164 * queue's settings related to segment counting like q->bounce_pfn1164 * The queue settings related to segment counting may differ from the
1165 * may differ from that of other stacking queues.1165 * original queue.
1166 * Recalculate it to check the request correctly on this queue's
1167 * limitation.
1168 */1166 */
1169 rq->nr_phys_segments = blk_recalc_rq_segments(rq);1167 rq->nr_phys_segments = blk_recalc_rq_segments(rq);
1170 if (rq->nr_phys_segments > queue_max_segments(q)) {1168 if (rq->nr_phys_segments > queue_max_segments(q)) {
block/blk-settings.c+8−34
@@ -7,7 +7,6 @@ · 6 unmodified lines
7#include <linux/init.h>7#include <linux/init.h>
8#include <linux/bio.h>8#include <linux/bio.h>
9#include <linux/blkdev.h>9#include <linux/blkdev.h>
10#include <linux/memblock.h> /* for max_pfn/max_low_pfn */
11#include <linux/gcd.h>10#include <linux/gcd.h>
12#include <linux/lcm.h>11#include <linux/lcm.h>
13#include <linux/jiffies.h>12#include <linux/jiffies.h>
@@ -17,11 +16,6 @@ · 3 unmodified lines
17#include "blk.h"16#include "blk.h"
18#include "blk-wbt.h"17#include "blk-wbt.h"
19 18
20unsigned long blk_max_low_pfn;
21EXPORT_SYMBOL(blk_max_low_pfn);
22
23unsigned long blk_max_pfn;
24
25void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout)19void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout)
26{20{
27 q->rq_timeout = timeout;21 q->rq_timeout = timeout;
@@ -55,7 +49,7 @@ void blk_set_default_limits(struct queue_limits *lim) · 27 unmodified lines
55 lim->discard_alignment = 0;49 lim->discard_alignment = 0;
56 lim->discard_misaligned = 0;50 lim->discard_misaligned = 0;
57 lim->logical_block_size = lim->physical_block_size = lim->io_min = 512;51 lim->logical_block_size = lim->physical_block_size = lim->io_min = 512;
58 lim->bounce_pfn = (unsigned long)(BLK_BOUNCE_ANY >> PAGE_SHIFT);52 lim->bounce = BLK_BOUNCE_NONE;
59 lim->alignment_offset = 0;53 lim->alignment_offset = 0;
60 lim->io_opt = 0;54 lim->io_opt = 0;
61 lim->misaligned = 0;55 lim->misaligned = 0;
@@ -92,28 +86,16 @@ EXPORT_SYMBOL(blk_set_stacking_limits); · 30 unmodified lines
92/**86/**
93 * blk_queue_bounce_limit - set bounce buffer limit for queue87 * blk_queue_bounce_limit - set bounce buffer limit for queue
94 * @q: the request queue for the device88 * @q: the request queue for the device
95 * @max_addr: the maximum address the device can handle89 * @bounce: bounce limit to enforce
96 *90 *
97 * Description:91 * Description:
98 * Different hardware can have different requirements as to what pages92 * Force bouncing for ISA DMA ranges or highmem.
99 * it can do I/O directly to. A low level driver can call93 *
100 * blk_queue_bounce_limit to have lower memory pages allocated as bounce94 * DEPRECATED, don't use in new code.
101 * buffers for doing I/O to pages residing above @max_addr.
102 **/95 **/
103void blk_queue_bounce_limit(struct request_queue *q, u64 max_addr)96void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce bounce)
104{97{
105 unsigned long b_pfn = max_addr >> PAGE_SHIFT;98 q->limits.bounce = bounce;
106
107#if BITS_PER_LONG == 64
108 /*
109 * Assume anything <= 4GB can be handled by IOMMU. Actually
110 * some IOMMUs can handle everything, but I don't know of a
111 * way to test this here.
112 */
113 q->limits.bounce_pfn = max(max_low_pfn, b_pfn);
114#else
115 q->limits.bounce_pfn = b_pfn;
116#endif
117}99}
118EXPORT_SYMBOL(blk_queue_bounce_limit);100EXPORT_SYMBOL(blk_queue_bounce_limit);
119 101
@@ -536,7 +518,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, · 416 unmodified lines
536 b->max_write_zeroes_sectors);518 b->max_write_zeroes_sectors);
537 t->max_zone_append_sectors = min(t->max_zone_append_sectors,519 t->max_zone_append_sectors = min(t->max_zone_append_sectors,
538 b->max_zone_append_sectors);520 b->max_zone_append_sectors);
539 t->bounce_pfn = min_not_zero(t->bounce_pfn, b->bounce_pfn);521 t->bounce = max(t->bounce, b->bounce);
540 522
541 t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask,523 t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask,
542 b->seg_boundary_mask);524 b->seg_boundary_mask);
@@ -916,11 +898,3 @@ void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model) · 373 unmodified lines
916 }898 }
917}899}
918EXPORT_SYMBOL_GPL(blk_queue_set_zoned);900EXPORT_SYMBOL_GPL(blk_queue_set_zoned);
919
920static int __init blk_settings_init(void)
921{
922 blk_max_low_pfn = max_low_pfn - 1;
923 blk_max_pfn = max_pfn - 1;
924 return 0;
925}
926subsys_initcall(blk_settings_init);
block/blk.h+12−4
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#include <linux/blk-mq.h>6#include <linux/blk-mq.h>
7#include <linux/part_stat.h>7#include <linux/part_stat.h>
8#include <linux/blk-crypto.h>8#include <linux/blk-crypto.h>
9#include <linux/memblock.h> /* for max_pfn/max_low_pfn */
9#include <xen/xen.h>10#include <xen/xen.h>
10#include "blk-crypto-internal.h"11#include "blk-crypto-internal.h"
11#include "blk-mq.h"12#include "blk-mq.h"
@@ -311,13 +312,20 @@ static inline void blk_throtl_bio_endio(struct bio *bio) { } · 299 unmodified lines
311static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }312static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
312#endif313#endif
313 314
314#ifdef CONFIG_BOUNCE315void __blk_queue_bounce(struct request_queue *q, struct bio **bio);
315extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);316
316#else317static inline bool blk_queue_may_bounce(struct request_queue *q)
318{
319 return IS_ENABLED(CONFIG_BOUNCE) &&
320 q->limits.bounce == BLK_BOUNCE_HIGH &&
321 max_low_pfn >= max_pfn;
322}
323
317static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)324static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
318{325{
326 if (unlikely(blk_queue_may_bounce(q) && bio_has_data(*bio)))
327 __blk_queue_bounce(q, bio);
319}328}
320#endif /* CONFIG_BOUNCE */
321 329
322#ifdef CONFIG_BLK_CGROUP_IOLATENCY330#ifdef CONFIG_BLK_CGROUP_IOLATENCY
323extern int blk_iolatency_init(struct request_queue *q);331extern int blk_iolatency_init(struct request_queue *q);
block/bounce.c+5−30
@@ -18,7 +18,6 @@ · 17 unmodified lines
18#include <linux/init.h>18#include <linux/init.h>
19#include <linux/hash.h>19#include <linux/hash.h>
20#include <linux/highmem.h>20#include <linux/highmem.h>
21#include <linux/memblock.h>
22#include <linux/printk.h>21#include <linux/printk.h>
23#include <asm/tlbflush.h>22#include <asm/tlbflush.h>
24 23
@@ -49,11 +48,11 @@ static void init_bounce_bioset(void) · 24 unmodified lines
49 bounce_bs_setup = true;48 bounce_bs_setup = true;
50}49}
51 50
52#if defined(CONFIG_HIGHMEM)
53static __init int init_emergency_pool(void)51static __init int init_emergency_pool(void)
54{52{
55 int ret;53 int ret;
56#if defined(CONFIG_HIGHMEM) && !defined(CONFIG_MEMORY_HOTPLUG)54
55#ifndef CONFIG_MEMORY_HOTPLUG
57 if (max_pfn <= max_low_pfn)56 if (max_pfn <= max_low_pfn)
58 return 0;57 return 0;
59#endif58#endif
@@ -67,9 +66,7 @@ static __init int init_emergency_pool(void) · 7 unmodified lines
67}66}
68 67
69__initcall(init_emergency_pool);68__initcall(init_emergency_pool);
70#endif
71 69
72#ifdef CONFIG_HIGHMEM
73/*70/*
74 * highmem version, map in to vec71 * highmem version, map in to vec
75 */72 */
@@ -82,13 +79,6 @@ static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom) · 6 unmodified lines
82 kunmap_atomic(vto);79 kunmap_atomic(vto);
83}80}
84 81
85#else /* CONFIG_HIGHMEM */
86
87#define bounce_copy_vec(to, vfrom) \
88 memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
89
90#endif /* CONFIG_HIGHMEM */
91
92/*82/*
93 * Simple bounce buffer support for highmem pages. Depending on the83 * Simple bounce buffer support for highmem pages. Depending on the
94 * queue gfp mask set, *to may or may not be a highmem page. kmap it84 * queue gfp mask set, *to may or may not be a highmem page. kmap it
@@ -236,8 +226,7 @@ err_put: · 141 unmodified lines
236 return NULL;226 return NULL;
237}227}
238 228
239 229void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
240void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
241{230{
242 struct bio *bio;231 struct bio *bio;
243 int rw = bio_data_dir(*bio_orig);232 int rw = bio_data_dir(*bio_orig);
@@ -247,24 +236,10 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · 3 unmodified lines
247 bool bounce = false;236 bool bounce = false;
248 int sectors = 0;237 int sectors = 0;
249 238
250 /*
251 * Data-less bio, nothing to bounce
252 */
253 if (!bio_has_data(*bio_orig))
254 return;
255
256 /*
257 * Just check if the bounce pfn is equal to or bigger than the highest
258 * pfn in the system -- in that case, don't waste time iterating over
259 * bio segments
260 */
261 if (q->limits.bounce_pfn >= blk_max_pfn)
262 return;
263
264 bio_for_each_segment(from, *bio_orig, iter) {239 bio_for_each_segment(from, *bio_orig, iter) {
265 if (i++ < BIO_MAX_VECS)240 if (i++ < BIO_MAX_VECS)
266 sectors += from.bv_len >> 9;241 sectors += from.bv_len >> 9;
267 if (page_to_pfn(from.bv_page) > q->limits.bounce_pfn)242 if (PageHighMem(from.bv_page))
268 bounce = true;243 bounce = true;
269 }244 }
270 if (!bounce)245 if (!bounce)
@@ -287,7 +262,7 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) · 16 unmodified lines
287 for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {262 for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {
288 struct page *page = to->bv_page;263 struct page *page = to->bv_page;
289 264
290 if (page_to_pfn(page) <= q->limits.bounce_pfn)265 if (!PageHighMem(page))
291 continue;266 continue;
292 267
293 to->bv_page = mempool_alloc(&page_pool, GFP_NOIO);268 to->bv_page = mempool_alloc(&page_pool, GFP_NOIO);
include/linux/blkdev.h+11−18
@@ -313,8 +313,17 @@ enum blk_zoned_model { · 312 unmodified lines
313 BLK_ZONED_HM, /* Host-managed zoned block device */313 BLK_ZONED_HM, /* Host-managed zoned block device */
314};314};
315 315
316/*
317 * BLK_BOUNCE_NONE: never bounce (default)
318 * BLK_BOUNCE_HIGH: bounce all highmem pages
319 */
320enum blk_bounce {
321 BLK_BOUNCE_NONE,
322 BLK_BOUNCE_HIGH,
323};
324
316struct queue_limits {325struct queue_limits {
317 unsigned long bounce_pfn;326 enum blk_bounce bounce;
318 unsigned long seg_boundary_mask;327 unsigned long seg_boundary_mask;
319 unsigned long virt_boundary_mask;328 unsigned long virt_boundary_mask;
320 329
@@ -835,22 +844,6 @@ static inline unsigned int blk_queue_depth(struct request_queue *q) · 514 unmodified lines
835 return q->nr_requests;844 return q->nr_requests;
836}845}
837 846
838extern unsigned long blk_max_low_pfn, blk_max_pfn;
839
840/*
841 * standard bounce addresses:
842 *
843 * BLK_BOUNCE_HIGH : bounce all highmem pages
844 * BLK_BOUNCE_ANY : don't bounce anything
845 */
846
847#if BITS_PER_LONG == 32
848#define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
849#else
850#define BLK_BOUNCE_HIGH -1ULL
851#endif
852#define BLK_BOUNCE_ANY (-1ULL)
853
854/*847/*
855 * default timeout for SG_IO if none specified848 * default timeout for SG_IO if none specified
856 */849 */
@@ -1134,7 +1127,7 @@ extern void blk_abort_request(struct request *); · 277 unmodified lines
1134 * Access functions for manipulating queue properties1127 * Access functions for manipulating queue properties
1135 */1128 */
1136extern void blk_cleanup_queue(struct request_queue *);1129extern void blk_cleanup_queue(struct request_queue *);
1137extern void blk_queue_bounce_limit(struct request_queue *, u64);1130void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce limit);
1138extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);1131extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1139extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);1132extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
1140extern void blk_queue_max_segments(struct request_queue *, unsigned short);1133extern void blk_queue_max_segments(struct request_queue *, unsigned short);