Merge block/IO core bits from Jens Axboe: "This is a bit bigger on the core side than usual, but that is purely because we decided to hold off on parts of Tejun's submission on 3.4 to give it a bit more time to simmer. As a consequence, it's seen a long cycle in for-next. It contains: - Bug fix from Dan, wrong locking type. - Relax splice gifting restriction from Eric. - A ton of updates from Tejun, primarily for blkcg. This improves the code a lot, making the API nicer and cleaner, and also includes fixes for how we handle and tie policies and re-activate on switches. The changes also include generic bug fixes. - A simple fix from Vivek, along with a fix for doing proper delayed allocation of the blkcg stats." Fix up annoying conflict just due to different merge resolution in Documentation/feature-removal-schedule.txt * 'for-3.5/core' of git://git.kernel.dk/linux-block: (92 commits) blkcg: tg_stats_alloc_lock is an irq lock vmsplice: relax alignement requirements for SPLICE_F_GIFT blkcg: use radix tree to index blkgs from blkcg blkcg: fix blkcg->css ref leak in __blkg_lookup_create() block: fix elvpriv allocation failure handling block: collapse blk_alloc_request() into get_request() blkcg: collapse blkcg_policy_ops into blkcg_policy blkcg: embed struct blkg_policy_data in policy specific data blkcg: mass rename of blkcg API blkcg: style cleanups for blk-cgroup.h blkcg: remove blkio_group->path[] blkcg: blkg_rwstat_read() was missing inline blkcg: shoot down blkgs if all policies are deactivated blkcg: drop stuff unused after per-queue policy activation update blkcg: implement per-queue policy activation blkcg: add request_queue->root_blkg blkcg: make request_queue bypassing on allocation blkcg: make sure blkg_lookup() returns %NULL if @q is bypassing blkcg: make blkg_conf_prep() take @pol and return with queue lock held blkcg: remove static policy ID enums ...
Showing the changes to include (6 of 24 files). Show all
Unified · Split
| @@ -269,6 +269,14 @@ extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set · 268 unmodified lines | ||
| 269 | 269 | extern void bvec_free_bs(struct bio_set *, struct bio_vec *, unsigned int); |
| 270 | 270 | extern unsigned int bvec_nr_vecs(unsigned short idx); |
| 271 | 271 | |
| 272 | +#ifdef CONFIG_BLK_CGROUP | |
| 273 | +int bio_associate_current(struct bio *bio); | |
| 274 | +void bio_disassociate_task(struct bio *bio); | |
| 275 | +#else /* CONFIG_BLK_CGROUP */ | |
| 276 | +static inline int bio_associate_current(struct bio *bio) { return -ENOENT; } | |
| 277 | +static inline void bio_disassociate_task(struct bio *bio) { } | |
| 278 | +#endif /* CONFIG_BLK_CGROUP */ | |
| 279 | + | |
| 272 | 280 | /* |
| 273 | 281 | * bio_set is used to allow other portions of the IO system to |
| 274 | 282 | * allocate their own private memory pools for bio and iovec structures. |
| @@ -14,6 +14,8 @@ struct bio; · 13 unmodified lines | ||
| 14 | 14 | struct bio_integrity_payload; |
| 15 | 15 | struct page; |
| 16 | 16 | struct block_device; |
| 17 | +struct io_context; | |
| 18 | +struct cgroup_subsys_state; | |
| 17 | 19 | typedef void (bio_end_io_t) (struct bio *, int); |
| 18 | 20 | typedef void (bio_destructor_t) (struct bio *); |
| 19 | 21 | |
| @@ -66,6 +68,14 @@ struct bio { · 46 unmodified lines | ||
| 66 | 68 | bio_end_io_t *bi_end_io; |
| 67 | 69 | |
| 68 | 70 | void *bi_private; |
| 71 | +#ifdef CONFIG_BLK_CGROUP | |
| 72 | + /* | |
| 73 | + * Optional ioc and css associated with this bio. Put on bio | |
| 74 | + * release. Read comment on top of bio_associate_current(). | |
| 75 | + */ | |
| 76 | + struct io_context *bi_ioc; | |
| 77 | + struct cgroup_subsys_state *bi_css; | |
| 78 | +#endif | |
| 69 | 79 | #if defined(CONFIG_BLK_DEV_INTEGRITY) |
| 70 | 80 | struct bio_integrity_payload *bi_integrity; /* data integrity */ |
| 71 | 81 | #endif |
| @@ -32,10 +32,17 @@ struct blk_trace; · 31 unmodified lines | ||
| 32 | 32 | struct request; |
| 33 | 33 | struct sg_io_hdr; |
| 34 | 34 | struct bsg_job; |
| 35 | +struct blkcg_gq; | |
| 35 | 36 | |
| 36 | 37 | #define BLKDEV_MIN_RQ 4 |
| 37 | 38 | #define BLKDEV_MAX_RQ 128 /* Default maximum */ |
| 38 | 39 | |
| 40 | +/* | |
| 41 | + * Maximum number of blkcg policies allowed to be registered concurrently. | |
| 42 | + * Defined here to simplify include dependency. | |
| 43 | + */ | |
| 44 | +#define BLKCG_MAX_POLS 2 | |
| 45 | + | |
| 39 | 46 | struct request; |
| 40 | 47 | typedef void (rq_end_io_fn)(struct request *, int); |
| 41 | 48 | |
| @@ -363,6 +370,11 @@ struct request_queue { · 321 unmodified lines | ||
| 363 | 370 | struct list_head timeout_list; |
| 364 | 371 | |
| 365 | 372 | struct list_head icq_list; |
| 373 | +#ifdef CONFIG_BLK_CGROUP | |
| 374 | + DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS); | |
| 375 | + struct blkcg_gq *root_blkg; | |
| 376 | + struct list_head blkg_list; | |
| 377 | +#endif | |
| 366 | 378 | |
| 367 | 379 | struct queue_limits limits; |
| 368 | 380 | |
| @@ -390,12 +402,17 @@ struct request_queue { · 21 unmodified lines | ||
| 390 | 402 | |
| 391 | 403 | struct mutex sysfs_lock; |
| 392 | 404 | |
| 405 | + int bypass_depth; | |
| 406 | + | |
| 393 | 407 | #if defined(CONFIG_BLK_DEV_BSG) |
| 394 | 408 | bsg_job_fn *bsg_job_fn; |
| 395 | 409 | int bsg_job_size; |
| 396 | 410 | struct bsg_class_device bsg_dev; |
| 397 | 411 | #endif |
| 398 | 412 | |
| 413 | +#ifdef CONFIG_BLK_CGROUP | |
| 414 | + struct list_head all_q_node; | |
| 415 | +#endif | |
| 399 | 416 | #ifdef CONFIG_BLK_DEV_THROTTLING |
| 400 | 417 | /* Throttle data */ |
| 401 | 418 | struct throtl_data *td; |
| @@ -407,7 +424,7 @@ struct request_queue { · 5 unmodified lines | ||
| 407 | 424 | #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */ |
| 408 | 425 | #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ |
| 409 | 426 | #define QUEUE_FLAG_DEAD 5 /* queue being torn down */ |
| 410 | −#define QUEUE_FLAG_ELVSWITCH 6 /* don't use elevator, just do FIFO */ | |
| 427 | +#define QUEUE_FLAG_BYPASS 6 /* act as dumb FIFO queue */ | |
| 411 | 428 | #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */ |
| 412 | 429 | #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */ |
| 413 | 430 | #define QUEUE_FLAG_SAME_COMP 9 /* complete on same CPU-group */ |
| @@ -491,6 +508,7 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) · 77 unmodified lines | ||
| 491 | 508 | #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) |
| 492 | 509 | #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) |
| 493 | 510 | #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags) |
| 511 | +#define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags) | |
| 494 | 512 | #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) |
| 495 | 513 | #define blk_queue_noxmerges(q) \ |
| 496 | 514 | test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) |
| @@ -28,12 +28,13 @@ typedef int (elevator_may_queue_fn) (struct request_queue *, int); · 27 unmodified lines | ||
| 28 | 28 | |
| 29 | 29 | typedef void (elevator_init_icq_fn) (struct io_cq *); |
| 30 | 30 | typedef void (elevator_exit_icq_fn) (struct io_cq *); |
| 31 | −typedef int (elevator_set_req_fn) (struct request_queue *, struct request *, gfp_t); | |
| 31 | +typedef int (elevator_set_req_fn) (struct request_queue *, struct request *, | |
| 32 | + struct bio *, gfp_t); | |
| 32 | 33 | typedef void (elevator_put_req_fn) (struct request *); |
| 33 | 34 | typedef void (elevator_activate_req_fn) (struct request_queue *, struct request *); |
| 34 | 35 | typedef void (elevator_deactivate_req_fn) (struct request_queue *, struct request *); |
| 35 | 36 | |
| 36 | −typedef void *(elevator_init_fn) (struct request_queue *); | |
| 37 | +typedef int (elevator_init_fn) (struct request_queue *); | |
| 37 | 38 | typedef void (elevator_exit_fn) (struct elevator_queue *); |
| 38 | 39 | |
| 39 | 40 | struct elevator_ops |
| @@ -129,7 +130,8 @@ extern void elv_unregister_queue(struct request_queue *q); · 89 unmodified lines | ||
| 129 | 130 | extern int elv_may_queue(struct request_queue *, int); |
| 130 | 131 | extern void elv_abort_queue(struct request_queue *); |
| 131 | 132 | extern void elv_completed_request(struct request_queue *, struct request *); |
| 132 | −extern int elv_set_request(struct request_queue *, struct request *, gfp_t); | |
| 133 | +extern int elv_set_request(struct request_queue *q, struct request *rq, | |
| 134 | + struct bio *bio, gfp_t gfp_mask); | |
| 133 | 135 | extern void elv_put_request(struct request_queue *, struct request *); |
| 134 | 136 | extern void elv_drain_elevator(struct request_queue *); |
| 135 | 137 | |
| @@ -6,11 +6,7 @@ · 5 unmodified lines | ||
| 6 | 6 | #include <linux/workqueue.h> |
| 7 | 7 | |
| 8 | 8 | enum { |
| 9 | − ICQ_IOPRIO_CHANGED = 1 << 0, | |
| 10 | − ICQ_CGROUP_CHANGED = 1 << 1, | |
| 11 | 9 | ICQ_EXITED = 1 << 2, |
| 12 | − | |
| 13 | − ICQ_CHANGED_MASK = ICQ_IOPRIO_CHANGED | ICQ_CGROUP_CHANGED, | |
| 14 | 10 | }; |
| 15 | 11 | |
| 16 | 12 | /* |
| @@ -100,6 +96,7 @@ struct io_cq { · 83 unmodified lines | ||
| 100 | 96 | */ |
| 101 | 97 | struct io_context { |
| 102 | 98 | atomic_long_t refcount; |
| 99 | + atomic_t active_ref; | |
| 103 | 100 | atomic_t nr_tasks; |
| 104 | 101 | |
| 105 | 102 | /* all the fields below are protected by this lock */ |
| @@ -120,29 +117,37 @@ struct io_context { · 14 unmodified lines | ||
| 120 | 117 | struct work_struct release_work; |
| 121 | 118 | }; |
| 122 | 119 | |
| 123 | −static inline struct io_context *ioc_task_link(struct io_context *ioc) | |
| 120 | +/** | |
| 121 | + * get_io_context_active - get active reference on ioc | |
| 122 | + * @ioc: ioc of interest | |
| 123 | + * | |
| 124 | + * Only iocs with active reference can issue new IOs. This function | |
| 125 | + * acquires an active reference on @ioc. The caller must already have an | |
| 126 | + * active reference on @ioc. | |
| 127 | + */ | |
| 128 | +static inline void get_io_context_active(struct io_context *ioc) | |
| 124 | 129 | { |
| 125 | − /* | |
| 126 | − * if ref count is zero, don't allow sharing (ioc is going away, it's | |
| 127 | − * a race). | |
| 128 | − */ | |
| 129 | − if (ioc && atomic_long_inc_not_zero(&ioc->refcount)) { | |
| 130 | − atomic_inc(&ioc->nr_tasks); | |
| 131 | − return ioc; | |
| 132 | − } | |
| 130 | + WARN_ON_ONCE(atomic_long_read(&ioc->refcount) <= 0); | |
| 131 | + WARN_ON_ONCE(atomic_read(&ioc->active_ref) <= 0); | |
| 132 | + atomic_long_inc(&ioc->refcount); | |
| 133 | + atomic_inc(&ioc->active_ref); | |
| 134 | +} | |
| 135 | + | |
| 136 | +static inline void ioc_task_link(struct io_context *ioc) | |
| 137 | +{ | |
| 138 | + get_io_context_active(ioc); | |
| 133 | 139 | |
| 134 | − return NULL; | |
| 140 | + WARN_ON_ONCE(atomic_read(&ioc->nr_tasks) <= 0); | |
| 141 | + atomic_inc(&ioc->nr_tasks); | |
| 135 | 142 | } |
| 136 | 143 | |
| 137 | 144 | struct task_struct; |
| 138 | 145 | #ifdef CONFIG_BLOCK |
| 139 | 146 | void put_io_context(struct io_context *ioc); |
| 147 | +void put_io_context_active(struct io_context *ioc); | |
| 140 | 148 | void exit_io_context(struct task_struct *task); |
| 141 | 149 | struct io_context *get_task_io_context(struct task_struct *task, |
| 142 | 150 | gfp_t gfp_flags, int node); |
| 143 | −void ioc_ioprio_changed(struct io_context *ioc, int ioprio); | |
| 144 | −void ioc_cgroup_changed(struct io_context *ioc); | |
| 145 | −unsigned int icq_get_changed(struct io_cq *icq); | |
| 146 | 151 | #else |
| 147 | 152 | struct io_context; |
| 148 | 153 | static inline void put_io_context(struct io_context *ioc) { } |
| @@ -42,26 +42,14 @@ enum { · 41 unmodified lines | ||
| 42 | 42 | }; |
| 43 | 43 | |
| 44 | 44 | /* |
| 45 | − * if process has set io priority explicitly, use that. if not, convert | |
| 46 | − * the cpu scheduler nice value to an io priority | |
| 45 | + * Fallback BE priority | |
| 47 | 46 | */ |
| 48 | 47 | #define IOPRIO_NORM (4) |
| 49 | −static inline int task_ioprio(struct io_context *ioc) | |
| 50 | −{ | |
| 51 | − if (ioprio_valid(ioc->ioprio)) | |
| 52 | − return IOPRIO_PRIO_DATA(ioc->ioprio); | |
| 53 | − | |
| 54 | − return IOPRIO_NORM; | |
| 55 | −} | |
| 56 | − | |
| 57 | −static inline int task_ioprio_class(struct io_context *ioc) | |
| 58 | −{ | |
| 59 | − if (ioprio_valid(ioc->ioprio)) | |
| 60 | − return IOPRIO_PRIO_CLASS(ioc->ioprio); | |
| 61 | − | |
| 62 | − return IOPRIO_CLASS_BE; | |
| 63 | −} | |
| 64 | 48 | |
| 49 | +/* | |
| 50 | + * if process has set io priority explicitly, use that. if not, convert | |
| 51 | + * the cpu scheduler nice value to an io priority | |
| 52 | + */ | |
| 65 | 53 | static inline int task_nice_ioprio(struct task_struct *task) |
| 66 | 54 | { |
| 67 | 55 | return (task_nice(task) + 20) / 5; |