History

Merge branch 'for-3.5/core' of git://git.kernel.dk/linux-block

Merge block/IO core bits from Jens Axboe: "This is a bit bigger on the core side than usual, but that is purely because we decided to hold off on parts of Tejun's submission on 3.4 to give it a bit more time to simmer. As a consequence, it's seen a long cycle in for-next. It contains: - Bug fix from Dan, wrong locking type. - Relax splice gifting restriction from Eric. - A ton of updates from Tejun, primarily for blkcg. This improves the code a lot, making the API nicer and cleaner, and also includes fixes for how we handle and tie policies and re-activate on switches. The changes also include generic bug fixes. - A simple fix from Vivek, along with a fix for doing proper delayed allocation of the blkcg stats." Fix up annoying conflict just due to different merge resolution in Documentation/feature-removal-schedule.txt * 'for-3.5/core' of git://git.kernel.dk/linux-block: (92 commits) blkcg: tg_stats_alloc_lock is an irq lock vmsplice: relax alignement requirements for SPLICE_F_GIFT blkcg: use radix tree to index blkgs from blkcg blkcg: fix blkcg->css ref leak in __blkg_lookup_create() block: fix elvpriv allocation failure handling block: collapse blk_alloc_request() into get_request() blkcg: collapse blkcg_policy_ops into blkcg_policy blkcg: embed struct blkg_policy_data in policy specific data blkcg: mass rename of blkcg API blkcg: style cleanups for blk-cgroup.h blkcg: remove blkio_group->path[] blkcg: blkg_rwstat_read() was missing inline blkcg: shoot down blkgs if all policies are deactivated blkcg: drop stuff unused after per-queue policy activation update blkcg: implement per-queue policy activation blkcg: add request_queue->root_blkg blkcg: make request_queue bypassing on allocation blkcg: make sure blkg_lookup() returns %NULL if @q is bypassing blkcg: make blkg_conf_prep() take @pol and return with queue lock held blkcg: remove static policy ID enums ...

Linus Torvalds committed 14 years ago

Commit 0d167512 parents 2f83766 ff26eaa24 files · +2446 −2952Browse files

Unified · Split

24 files
  1. block/Kconfig.iosched +0 −4
  2. block/blk-cgroup.c +669 −1431
  3. block/blk-cgroup.h +334 −313
  4. block/blk-core.c +188 −93
  5. block/blk-ioc.c +35 −91
  6. block/blk-sysfs.c +3 −3
  7. block/blk-throttle.c +325 −372
  8. block/blk.h +13 −19
  9. block/cfq-iosched.c +685 −387
  10. block/cfq.h +0 −115
  11. block/deadline-iosched.c +5 −3
  12. block/elevator.c +48 −73
  13. block/noop-iosched.c +5 −3
  14. fs/bio.c +61 −0
  15. fs/ioprio.c +1 −1
  16. fs/splice.c +2 −2
  17. include/linux/bio.h +8 −0
  18. include/linux/blk_types.h +10 −0
  19. include/linux/blkdev.h +19 −1
  20. include/linux/elevator.h +5 −3
  21. include/linux/iocontext.h +22 −17
  22. include/linux/ioprio.h +5 −17
  23. init/Kconfig +1 −1
  24. kernel/fork.c +2 −3
block/Kconfig.iosched+0−4
@@ -23,8 +23,6 @@ config IOSCHED_DEADLINE · 22 unmodified lines
2323
2424 config IOSCHED_CFQ
2525 tristate "CFQ I/O scheduler"
26− # If BLK_CGROUP is a module, CFQ has to be built as module.
27− depends on (BLK_CGROUP=m && m) || !BLK_CGROUP || BLK_CGROUP=y
2826 default y
2927 ---help---
3028 The CFQ I/O scheduler tries to distribute bandwidth equally
@@ -34,8 +32,6 @@ config IOSCHED_CFQ · 3 unmodified lines
3432
3533 This is the default I/O scheduler.
3634
37− Note: If BLK_CGROUP=m, then CFQ can be built only as module.
38−
3935 config CFQ_GROUP_IOSCHED
4036 bool "CFQ Group Scheduling support"
4137 depends on IOSCHED_CFQ && BLK_CGROUP
block/blk-cgroup.c+669−1431
@@ -11,1570 +11,612 @@ · 10 unmodified lines
1111 * Nauman Rafique <nauman@google.com>
1212 */
1313 #include <linux/ioprio.h>
14−#include <linux/seq_file.h>
1514 #include <linux/kdev_t.h>
1615 #include <linux/module.h>
1716 #include <linux/err.h>
1817 #include <linux/blkdev.h>
1918 #include <linux/slab.h>
20−#include "blk-cgroup.h"
2119 #include <linux/genhd.h>
20+#include <linux/delay.h>
21+#include <linux/atomic.h>
22+#include "blk-cgroup.h"
23+#include "blk.h"
2224
2325 #define MAX_KEY_LEN 100
2426
25−static DEFINE_SPINLOCK(blkio_list_lock);
26−static LIST_HEAD(blkio_list);
27+static DEFINE_MUTEX(blkcg_pol_mutex);
2728
28−struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
29−EXPORT_SYMBOL_GPL(blkio_root_cgroup);
29+struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT };
30+EXPORT_SYMBOL_GPL(blkcg_root);
3031
31−/* for encoding cft->private value on file */
32−#define BLKIOFILE_PRIVATE(x, val) (((x) << 16) | (val))
33−/* What policy owns the file, proportional or throttle */
34−#define BLKIOFILE_POLICY(val) (((val) >> 16) & 0xffff)
35−#define BLKIOFILE_ATTR(val) ((val) & 0xffff)
32+static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
3633
37−static inline void blkio_policy_insert_node(struct blkio_cgroup *blkcg,
38− struct blkio_policy_node *pn)
34+struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)
3935 {
40− list_add(&pn->node, &blkcg->policy_list);
36+ return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
37+ struct blkcg, css);
4138 }
39+EXPORT_SYMBOL_GPL(cgroup_to_blkcg);
4240
43−static inline bool cftype_blkg_same_policy(struct cftype *cft,
44− struct blkio_group *blkg)
41+static struct blkcg *task_blkcg(struct task_struct *tsk)
4542 {
46− enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
47−
48− if (blkg->plid == plid)
49− return 1;
50−
51− return 0;
43+ return container_of(task_subsys_state(tsk, blkio_subsys_id),
44+ struct blkcg, css);
5245 }
5346
54−/* Determines if policy node matches cgroup file being accessed */
55−static inline bool pn_matches_cftype(struct cftype *cft,
56− struct blkio_policy_node *pn)
47+struct blkcg *bio_blkcg(struct bio *bio)
5748 {
58− enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
59− int fileid = BLKIOFILE_ATTR(cft->private);
60−
61− return (plid == pn->plid && fileid == pn->fileid);
49+ if (bio && bio->bi_css)
50+ return container_of(bio->bi_css, struct blkcg, css);
51+ return task_blkcg(current);
6252 }
53+EXPORT_SYMBOL_GPL(bio_blkcg);
6354
64−/* Must be called with blkcg->lock held */
65−static inline void blkio_policy_delete_node(struct blkio_policy_node *pn)
55+static bool blkcg_policy_enabled(struct request_queue *q,
56+ const struct blkcg_policy *pol)
6657 {
67− list_del(&pn->node);
58+ return pol && test_bit(pol->plid, q->blkcg_pols);
6859 }
6960
70−/* Must be called with blkcg->lock held */
71−static struct blkio_policy_node *
72−blkio_policy_search_node(const struct blkio_cgroup *blkcg, dev_t dev,
73− enum blkio_policy_id plid, int fileid)
61+/**
62+ * blkg_free - free a blkg
63+ * @blkg: blkg to free
64+ *
65+ * Free @blkg which may be partially allocated.
66+ */
67+static void blkg_free(struct blkcg_gq *blkg)
7468 {
75− struct blkio_policy_node *pn;
76−
77− list_for_each_entry(pn, &blkcg->policy_list, node) {
78− if (pn->dev == dev && pn->plid == plid && pn->fileid == fileid)
79− return pn;
80− }
69+ int i;
8170
82− return NULL;
83−}
71+ if (!blkg)
72+ return;
8473
85−struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
86−{
87− return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
88− struct blkio_cgroup, css);
89−}
90−EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
74+ for (i = 0; i < BLKCG_MAX_POLS; i++) {
75+ struct blkcg_policy *pol = blkcg_policy[i];
76+ struct blkg_policy_data *pd = blkg->pd[i];
9177
92−struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
93−{
94− return container_of(task_subsys_state(tsk, blkio_subsys_id),
95− struct blkio_cgroup, css);
96−}
97−EXPORT_SYMBOL_GPL(task_blkio_cgroup);
78+ if (!pd)
79+ continue;
9880
99−static inline void
100−blkio_update_group_weight(struct blkio_group *blkg, unsigned int weight)
101−{
102− struct blkio_policy_type *blkiop;
81+ if (pol && pol->pd_exit_fn)
82+ pol->pd_exit_fn(blkg);
10383
104− list_for_each_entry(blkiop, &blkio_list, list) {
105− /* If this policy does not own the blkg, do not send updates */
106− if (blkiop->plid != blkg->plid)
107− continue;
108− if (blkiop->ops.blkio_update_group_weight_fn)
109− blkiop->ops.blkio_update_group_weight_fn(blkg->key,
110− blkg, weight);
84+ kfree(pd);
11185 }
86+
87+ kfree(blkg);
11288 }
11389
114−static inline void blkio_update_group_bps(struct blkio_group *blkg, u64 bps,
115− int fileid)
90+/**
91+ * blkg_alloc - allocate a blkg
92+ * @blkcg: block cgroup the new blkg is associated with
93+ * @q: request_queue the new blkg is associated with
94+ *
95+ * Allocate a new blkg assocating @blkcg and @q.
96+ */
97+static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q)
11698 {
117− struct blkio_policy_type *blkiop;
118−
119− list_for_each_entry(blkiop, &blkio_list, list) {
120−
121− /* If this policy does not own the blkg, do not send updates */
122− if (blkiop->plid != blkg->plid)
123− continue;
124−
125− if (fileid == BLKIO_THROTL_read_bps_device
126− && blkiop->ops.blkio_update_group_read_bps_fn)
127− blkiop->ops.blkio_update_group_read_bps_fn(blkg->key,
128− blkg, bps);
99+ struct blkcg_gq *blkg;
100+ int i;
129101
130− if (fileid == BLKIO_THROTL_write_bps_device
131− && blkiop->ops.blkio_update_group_write_bps_fn)
132− blkiop->ops.blkio_update_group_write_bps_fn(blkg->key,
133− blkg, bps);
134− }
135−}
102+ /* alloc and init base part */
103+ blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node);
104+ if (!blkg)
105+ return NULL;
136106
137−static inline void blkio_update_group_iops(struct blkio_group *blkg,
138− unsigned int iops, int fileid)
139−{
140− struct blkio_policy_type *blkiop;
107+ blkg->q = q;
108+ INIT_LIST_HEAD(&blkg->q_node);
109+ blkg->blkcg = blkcg;
110+ blkg->refcnt = 1;
141111
142− list_for_each_entry(blkiop, &blkio_list, list) {
112+ for (i = 0; i < BLKCG_MAX_POLS; i++) {
113+ struct blkcg_policy *pol = blkcg_policy[i];
114+ struct blkg_policy_data *pd;
143115
144− /* If this policy does not own the blkg, do not send updates */
145− if (blkiop->plid != blkg->plid)
116+ if (!blkcg_policy_enabled(q, pol))
146117 continue;
147118
148− if (fileid == BLKIO_THROTL_read_iops_device
149− && blkiop->ops.blkio_update_group_read_iops_fn)
150− blkiop->ops.blkio_update_group_read_iops_fn(blkg->key,
151− blkg, iops);
119+ /* alloc per-policy data and attach it to blkg */
120+ pd = kzalloc_node(pol->pd_size, GFP_ATOMIC, q->node);
121+ if (!pd) {
122+ blkg_free(blkg);
123+ return NULL;
124+ }
152125
153− if (fileid == BLKIO_THROTL_write_iops_device
154− && blkiop->ops.blkio_update_group_write_iops_fn)
155− blkiop->ops.blkio_update_group_write_iops_fn(blkg->key,
156− blkg,iops);
126+ blkg->pd[i] = pd;
127+ pd->blkg = blkg;
157128 }
158−}
159129
160−/*
161− * Add to the appropriate stat variable depending on the request type.
162− * This should be called with the blkg->stats_lock held.
163− */
164−static void blkio_add_stat(uint64_t *stat, uint64_t add, bool direction,
165− bool sync)
166−{
167− if (direction)
168− stat[BLKIO_STAT_WRITE] += add;
169− else
170− stat[BLKIO_STAT_READ] += add;
171− if (sync)
172− stat[BLKIO_STAT_SYNC] += add;
173− else
174− stat[BLKIO_STAT_ASYNC] += add;
175−}
130+ /* invoke per-policy init */
131+ for (i = 0; i < BLKCG_MAX_POLS; i++) {
132+ struct blkcg_policy *pol = blkcg_policy[i];
176133
177−/*
178− * Decrements the appropriate stat variable if non-zero depending on the
179− * request type. Panics on value being zero.
180− * This should be called with the blkg->stats_lock held.
181− */
182−static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync)
183−{
184− if (direction) {
185− BUG_ON(stat[BLKIO_STAT_WRITE] == 0);
186− stat[BLKIO_STAT_WRITE]--;
187− } else {
188− BUG_ON(stat[BLKIO_STAT_READ] == 0);
189− stat[BLKIO_STAT_READ]--;
190− }
191− if (sync) {
192− BUG_ON(stat[BLKIO_STAT_SYNC] == 0);
193− stat[BLKIO_STAT_SYNC]--;
194− } else {
195− BUG_ON(stat[BLKIO_STAT_ASYNC] == 0);
196− stat[BLKIO_STAT_ASYNC]--;
134+ if (blkcg_policy_enabled(blkg->q, pol))
135+ pol->pd_init_fn(blkg);
197136 }
198−}
199137
200−#ifdef CONFIG_DEBUG_BLK_CGROUP
201−/* This should be called with the blkg->stats_lock held. */
202−static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
203− struct blkio_group *curr_blkg)
204−{
205− if (blkio_blkg_waiting(&blkg->stats))
206− return;
207− if (blkg == curr_blkg)
208− return;
209− blkg->stats.start_group_wait_time = sched_clock();
210− blkio_mark_blkg_waiting(&blkg->stats);
138+ return blkg;
211139 }
212140
213−/* This should be called with the blkg->stats_lock held. */
214−static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
141+static struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
142+ struct request_queue *q)
215143 {
216− unsigned long long now;
144+ struct blkcg_gq *blkg;
217145
218− if (!blkio_blkg_waiting(stats))
219− return;
146+ blkg = rcu_dereference(blkcg->blkg_hint);
147+ if (blkg && blkg->q == q)
148+ return blkg;
220149
221− now = sched_clock();
222− if (time_after64(now, stats->start_group_wait_time))
223− stats->group_wait_time += now - stats->start_group_wait_time;
224− blkio_clear_blkg_waiting(stats);
150+ /*
151+ * Hint didn't match. Look up from the radix tree. Note that we
152+ * may not be holding queue_lock and thus are not sure whether
153+ * @blkg from blkg_tree has already been removed or not, so we
154+ * can't update hint to the lookup result. Leave it to the caller.
155+ */
156+ blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
157+ if (blkg && blkg->q == q)
158+ return blkg;
159+
160+ return NULL;
225161 }
226162
227−/* This should be called with the blkg->stats_lock held. */
228−static void blkio_end_empty_time(struct blkio_group_stats *stats)
163+/**
164+ * blkg_lookup - lookup blkg for the specified blkcg - q pair
165+ * @blkcg: blkcg of interest
166+ * @q: request_queue of interest
167+ *
168+ * Lookup blkg for the @blkcg - @q pair. This function should be called
169+ * under RCU read lock and is guaranteed to return %NULL if @q is bypassing
170+ * - see blk_queue_bypass_start() for details.
171+ */
172+struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q)
229173 {
230− unsigned long long now;
231−
232− if (!blkio_blkg_empty(stats))
233− return;
174+ WARN_ON_ONCE(!rcu_read_lock_held());
234175
235− now = sched_clock();
236− if (time_after64(now, stats->start_empty_time))
237− stats->empty_time += now - stats->start_empty_time;
238− blkio_clear_blkg_empty(stats);
176+ if (unlikely(blk_queue_bypass(q)))
177+ return NULL;
178+ return __blkg_lookup(blkcg, q);
239179 }
180+EXPORT_SYMBOL_GPL(blkg_lookup);
240181
241−void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg)
182+static struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
183+ struct request_queue *q)
184+ __releases(q->queue_lock) __acquires(q->queue_lock)
242185 {
243− unsigned long flags;
186+ struct blkcg_gq *blkg;
187+ int ret;
244188
245− spin_lock_irqsave(&blkg->stats_lock, flags);
246− BUG_ON(blkio_blkg_idling(&blkg->stats));
247− blkg->stats.start_idle_time = sched_clock();
248− blkio_mark_blkg_idling(&blkg->stats);
249− spin_unlock_irqrestore(&blkg->stats_lock, flags);
250−}
251−EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);
189+ WARN_ON_ONCE(!rcu_read_lock_held());
190+ lockdep_assert_held(q->queue_lock);
252191
253−void blkiocg_update_idle_time_stats(struct blkio_group *blkg)
254−{
255− unsigned long flags;
256− unsigned long long now;
257− struct blkio_group_stats *stats;
258−
259− spin_lock_irqsave(&blkg->stats_lock, flags);
260− stats = &blkg->stats;
261− if (blkio_blkg_idling(stats)) {
262− now = sched_clock();
263− if (time_after64(now, stats->start_idle_time))
264− stats->idle_time += now - stats->start_idle_time;
265− blkio_clear_blkg_idling(stats);
192+ /* lookup and update hint on success, see __blkg_lookup() for details */
193+ blkg = __blkg_lookup(blkcg, q);
194+ if (blkg) {
195+ rcu_assign_pointer(blkcg->blkg_hint, blkg);
196+ return blkg;
266197 }
267− spin_unlock_irqrestore(&blkg->stats_lock, flags);
268−}
269−EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);
270198
271−void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg)
272−{
273− unsigned long flags;
274− struct blkio_group_stats *stats;
275−
276− spin_lock_irqsave(&blkg->stats_lock, flags);
277− stats = &blkg->stats;
278− stats->avg_queue_size_sum +=
279− stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] +
280− stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE];
281− stats->avg_queue_size_samples++;
282− blkio_update_group_wait_time(stats);
283− spin_unlock_irqrestore(&blkg->stats_lock, flags);
284−}
285−EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);
199+ /* blkg holds a reference to blkcg */
200+ if (!css_tryget(&blkcg->css))
201+ return ERR_PTR(-EINVAL);
286202
287−void blkiocg_set_start_empty_time(struct blkio_group *blkg)
288−{
289− unsigned long flags;
290− struct blkio_group_stats *stats;
203+ /* allocate */
204+ ret = -ENOMEM;
205+ blkg = blkg_alloc(blkcg, q);
206+ if (unlikely(!blkg))
207+ goto err_put;
291208
292− spin_lock_irqsave(&blkg->stats_lock, flags);
293− stats = &blkg->stats;
209+ /* insert */
210+ ret = radix_tree_preload(GFP_ATOMIC);
211+ if (ret)
212+ goto err_free;
294213
295− if (stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] ||
296− stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE]) {
297− spin_unlock_irqrestore(&blkg->stats_lock, flags);
298− return;
214+ spin_lock(&blkcg->lock);
215+ ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
216+ if (likely(!ret)) {
217+ hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
218+ list_add(&blkg->q_node, &q->blkg_list);
299219 }
220+ spin_unlock(&blkcg->lock);
300221
301− /*
302− * group is already marked empty. This can happen if cfqq got new
303− * request in parent group and moved to this group while being added
304− * to service tree. Just ignore the event and move on.
305− */
306− if(blkio_blkg_empty(stats)) {
307− spin_unlock_irqrestore(&blkg->stats_lock, flags);
308− return;
309− }
222+ radix_tree_preload_end();
310223
311− stats->start_empty_time = sched_clock();
312− blkio_mark_blkg_empty(stats);
313− spin_unlock_irqrestore(&blkg->stats_lock, flags);
224+ if (!ret)
225+ return blkg;
226+err_free:
227+ blkg_free(blkg);
228+err_put:
229+ css_put(&blkcg->css);
230+ return ERR_PTR(ret);
314231 }
315−EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);
316232
317−void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
318− unsigned long dequeue)
233+struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
234+ struct request_queue *q)
319235 {
320− blkg->stats.dequeue += dequeue;
321−}
322−EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
323−#else
324−static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
325− struct blkio_group *curr_blkg) {}
326−static inline void blkio_end_empty_time(struct blkio_group_stats *stats) {}
327−#endif
328−
329−void blkiocg_update_io_add_stats(struct blkio_group *blkg,
330− struct blkio_group *curr_blkg, bool direction,
331− bool sync)
332−{
333− unsigned long flags;
334−
335− spin_lock_irqsave(&blkg->stats_lock, flags);
336− blkio_add_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction,
337− sync);
338− blkio_end_empty_time(&blkg->stats);
339− blkio_set_start_group_wait_time(blkg, curr_blkg);
340− spin_unlock_irqrestore(&blkg->stats_lock, flags);
236+ /*
237+ * This could be the first entry point of blkcg implementation and
238+ * we shouldn't allow anything to go through for a bypassing queue.
239+ */
240+ if (unlikely(blk_queue_bypass(q)))
241+ return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
242+ return __blkg_lookup_create(blkcg, q);
341243 }
342−EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
244+EXPORT_SYMBOL_GPL(blkg_lookup_create);
343245
344−void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
345− bool direction, bool sync)
246+static void blkg_destroy(struct blkcg_gq *blkg)
346247 {
347− unsigned long flags;
248+ struct request_queue *q = blkg->q;
249+ struct blkcg *blkcg = blkg->blkcg;
348250
349− spin_lock_irqsave(&blkg->stats_lock, flags);
350− blkio_check_and_dec_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED],
351− direction, sync);
352− spin_unlock_irqrestore(&blkg->stats_lock, flags);
353−}
354−EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
251+ lockdep_assert_held(q->queue_lock);
252+ lockdep_assert_held(&blkcg->lock);
355253
356−void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time,
357− unsigned long unaccounted_time)
358−{
359− unsigned long flags;
360−
361− spin_lock_irqsave(&blkg->stats_lock, flags);
362− blkg->stats.time += time;
363−#ifdef CONFIG_DEBUG_BLK_CGROUP
364− blkg->stats.unaccounted_time += unaccounted_time;
365−#endif
366− spin_unlock_irqrestore(&blkg->stats_lock, flags);
367−}
368−EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
254+ /* Something wrong if we are trying to remove same group twice */
255+ WARN_ON_ONCE(list_empty(&blkg->q_node));
256+ WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
369257
370−/*
371− * should be called under rcu read lock or queue lock to make sure blkg pointer
372− * is valid.
373− */
374−void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
375− uint64_t bytes, bool direction, bool sync)
376−{
377− struct blkio_group_stats_cpu *stats_cpu;
378− unsigned long flags;
258+ radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
259+ list_del_init(&blkg->q_node);
260+ hlist_del_init_rcu(&blkg->blkcg_node);
379261
380262 /*
381− * Disabling interrupts to provide mutual exclusion between two
382− * writes on same cpu. It probably is not needed for 64bit. Not
383− * optimizing that case yet.
263+ * Both setting lookup hint to and clearing it from @blkg are done
264+ * under queue_lock. If it's not pointing to @blkg now, it never
265+ * will. Hint assignment itself can race safely.
384266 */
385− local_irq_save(flags);
386−
387− stats_cpu = this_cpu_ptr(blkg->stats_cpu);
388−
389− u64_stats_update_begin(&stats_cpu->syncp);
390− stats_cpu->sectors += bytes >> 9;
391− blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICED],
392− 1, direction, sync);
393− blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICE_BYTES],
394− bytes, direction, sync);
395− u64_stats_update_end(&stats_cpu->syncp);
396− local_irq_restore(flags);
397−}
398−EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
399−
400−void blkiocg_update_completion_stats(struct blkio_group *blkg,
401− uint64_t start_time, uint64_t io_start_time, bool direction, bool sync)
402−{
403− struct blkio_group_stats *stats;
404− unsigned long flags;
405− unsigned long long now = sched_clock();
406−
407− spin_lock_irqsave(&blkg->stats_lock, flags);
408− stats = &blkg->stats;
409− if (time_after64(now, io_start_time))
410− blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_TIME],
411− now - io_start_time, direction, sync);
412− if (time_after64(io_start_time, start_time))
413− blkio_add_stat(stats->stat_arr[BLKIO_STAT_WAIT_TIME],
414− io_start_time - start_time, direction, sync);
415− spin_unlock_irqrestore(&blkg->stats_lock, flags);
416−}
417−EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
418−
419−/* Merged stats are per cpu. */
420−void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction,
421− bool sync)
422−{
423− struct blkio_group_stats_cpu *stats_cpu;
424− unsigned long flags;
267+ if (rcu_dereference_raw(blkcg->blkg_hint) == blkg)
268+ rcu_assign_pointer(blkcg->blkg_hint, NULL);
425269
426270 /*
427− * Disabling interrupts to provide mutual exclusion between two
428− * writes on same cpu. It probably is not needed for 64bit. Not
429− * optimizing that case yet.
271+ * Put the reference taken at the time of creation so that when all
272+ * queues are gone, group can be destroyed.
430273 */
431− local_irq_save(flags);
432−
433− stats_cpu = this_cpu_ptr(blkg->stats_cpu);
434−
435− u64_stats_update_begin(&stats_cpu->syncp);
436− blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_MERGED], 1,
437− direction, sync);
438− u64_stats_update_end(&stats_cpu->syncp);
439− local_irq_restore(flags);
274+ blkg_put(blkg);
440275 }
441−EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
442276
443−/*
444− * This function allocates the per cpu stats for blkio_group. Should be called
445− * from sleepable context as alloc_per_cpu() requires that.
277+/**
278+ * blkg_destroy_all - destroy all blkgs associated with a request_queue
279+ * @q: request_queue of interest
280+ *
281+ * Destroy all blkgs associated with @q.
446282 */
447−int blkio_alloc_blkg_stats(struct blkio_group *blkg)
283+static void blkg_destroy_all(struct request_queue *q)
448284 {
449− /* Allocate memory for per cpu stats */
450− blkg->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
451− if (!blkg->stats_cpu)
452− return -ENOMEM;
453− return 0;
454−}
455−EXPORT_SYMBOL_GPL(blkio_alloc_blkg_stats);
285+ struct blkcg_gq *blkg, *n;
456286
457−void blkiocg_add_blkio_group(struct blkio_cgroup *blkcg,
458− struct blkio_group *blkg, void *key, dev_t dev,
459− enum blkio_policy_id plid)
460−{
461− unsigned long flags;
462−
463− spin_lock_irqsave(&blkcg->lock, flags);
464− spin_lock_init(&blkg->stats_lock);
465− rcu_assign_pointer(blkg->key, key);
466− blkg->blkcg_id = css_id(&blkcg->css);
467− hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
468− blkg->plid = plid;
469− spin_unlock_irqrestore(&blkcg->lock, flags);
470− /* Need to take css reference ? */
471− cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
472− blkg->dev = dev;
473−}
474−EXPORT_SYMBOL_GPL(blkiocg_add_blkio_group);
287+ lockdep_assert_held(q->queue_lock);
475288
476−static void __blkiocg_del_blkio_group(struct blkio_group *blkg)
477−{
478− hlist_del_init_rcu(&blkg->blkcg_node);
479− blkg->blkcg_id = 0;
480−}
289+ list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
290+ struct blkcg *blkcg = blkg->blkcg;
481291
482−/*
483− * returns 0 if blkio_group was still on cgroup list. Otherwise returns 1
484− * indicating that blk_group was unhashed by the time we got to it.
485− */
486−int blkiocg_del_blkio_group(struct blkio_group *blkg)
487−{
488− struct blkio_cgroup *blkcg;
489− unsigned long flags;
490− struct cgroup_subsys_state *css;
491− int ret = 1;
492−
493− rcu_read_lock();
494− css = css_lookup(&blkio_subsys, blkg->blkcg_id);
495− if (css) {
496− blkcg = container_of(css, struct blkio_cgroup, css);
497− spin_lock_irqsave(&blkcg->lock, flags);
498− if (!hlist_unhashed(&blkg->blkcg_node)) {
499− __blkiocg_del_blkio_group(blkg);
500− ret = 0;
501− }
502− spin_unlock_irqrestore(&blkcg->lock, flags);
292+ spin_lock(&blkcg->lock);
293+ blkg_destroy(blkg);
294+ spin_unlock(&blkcg->lock);
503295 }
504−
505− rcu_read_unlock();
506− return ret;
507296 }
508−EXPORT_SYMBOL_GPL(blkiocg_del_blkio_group);
509297
510−/* called under rcu_read_lock(). */
511−struct blkio_group *blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key)
298+static void blkg_rcu_free(struct rcu_head *rcu_head)
512299 {
513− struct blkio_group *blkg;
514− struct hlist_node *n;
515− void *__key;
516−
517− hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
518− __key = blkg->key;
519− if (__key == key)
520− return blkg;
521− }
522−
523− return NULL;
300+ blkg_free(container_of(rcu_head, struct blkcg_gq, rcu_head));
524301 }
525−EXPORT_SYMBOL_GPL(blkiocg_lookup_group);
526302
527−static void blkio_reset_stats_cpu(struct blkio_group *blkg)
303+void __blkg_release(struct blkcg_gq *blkg)
528304 {
529− struct blkio_group_stats_cpu *stats_cpu;
530− int i, j, k;
305+ /* release the extra blkcg reference this blkg has been holding */
306+ css_put(&blkg->blkcg->css);
307+
531308 /*
532− * Note: On 64 bit arch this should not be an issue. This has the
533− * possibility of returning some inconsistent value on 32bit arch
534− * as 64bit update on 32bit is non atomic. Taking care of this
535− * corner case makes code very complicated, like sending IPIs to
536− * cpus, taking care of stats of offline cpus etc.
309+ * A group is freed in rcu manner. But having an rcu lock does not
310+ * mean that one can access all the fields of blkg and assume these
311+ * are valid. For example, don't try to follow throtl_data and
312+ * request queue links.
537313 *
538− * reset stats is anyway more of a debug feature and this sounds a
539− * corner case. So I am not complicating the code yet until and
540− * unless this becomes a real issue.
314+ * Having a reference to blkg under an rcu allows acess to only
315+ * values local to groups like group stats and group rate limits
541316 */
542− for_each_possible_cpu(i) {
543− stats_cpu = per_cpu_ptr(blkg->stats_cpu, i);
544− stats_cpu->sectors = 0;
545− for(j = 0; j < BLKIO_STAT_CPU_NR; j++)
546− for (k = 0; k < BLKIO_STAT_TOTAL; k++)
547− stats_cpu->stat_arr_cpu[j][k] = 0;
548− }
317+ call_rcu(&blkg->rcu_head, blkg_rcu_free);
549318 }
319+EXPORT_SYMBOL_GPL(__blkg_release);
550320
551−static int
552−blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
321+static int blkcg_reset_stats(struct cgroup *cgroup, struct cftype *cftype,
322+ u64 val)
553323 {
554− struct blkio_cgroup *blkcg;
555− struct blkio_group *blkg;
556− struct blkio_group_stats *stats;
324+ struct blkcg *blkcg = cgroup_to_blkcg(cgroup);
325+ struct blkcg_gq *blkg;
557326 struct hlist_node *n;
558− uint64_t queued[BLKIO_STAT_TOTAL];
559327 int i;
560−#ifdef CONFIG_DEBUG_BLK_CGROUP
561− bool idling, waiting, empty;
562− unsigned long long now = sched_clock();
563−#endif
564328
565− blkcg = cgroup_to_blkio_cgroup(cgroup);
329+ mutex_lock(&blkcg_pol_mutex);
566330 spin_lock_irq(&blkcg->lock);
567− hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
568− spin_lock(&blkg->stats_lock);
569− stats = &blkg->stats;
570−#ifdef CONFIG_DEBUG_BLK_CGROUP
571− idling = blkio_blkg_idling(stats);
572− waiting = blkio_blkg_waiting(stats);
573− empty = blkio_blkg_empty(stats);
574−#endif
575− for (i = 0; i < BLKIO_STAT_TOTAL; i++)
576− queued[i] = stats->stat_arr[BLKIO_STAT_QUEUED][i];
577− memset(stats, 0, sizeof(struct blkio_group_stats));
578− for (i = 0; i < BLKIO_STAT_TOTAL; i++)
579− stats->stat_arr[BLKIO_STAT_QUEUED][i] = queued[i];
580−#ifdef CONFIG_DEBUG_BLK_CGROUP
581− if (idling) {
582− blkio_mark_blkg_idling(stats);
583− stats->start_idle_time = now;
584− }
585− if (waiting) {
586− blkio_mark_blkg_waiting(stats);
587− stats->start_group_wait_time = now;
588− }
589− if (empty) {
590− blkio_mark_blkg_empty(stats);
591− stats->start_empty_time = now;
592− }
593−#endif
594− spin_unlock(&blkg->stats_lock);
595−
596− /* Reset Per cpu stats which don't take blkg->stats_lock */
597− blkio_reset_stats_cpu(blkg);
598− }
599−
600− spin_unlock_irq(&blkcg->lock);
601− return 0;
602−}
603−
604−static void blkio_get_key_name(enum stat_sub_type type, dev_t dev, char *str,
605− int chars_left, bool diskname_only)
606−{
607− snprintf(str, chars_left, "%d:%d", MAJOR(dev), MINOR(dev));
608− chars_left -= strlen(str);
609− if (chars_left <= 0) {
610− printk(KERN_WARNING
611− "Possibly incorrect cgroup stat display format");
612− return;
613− }
614− if (diskname_only)
615− return;
616− switch (type) {
617− case BLKIO_STAT_READ:
618− strlcat(str, " Read", chars_left);
619− break;
620− case BLKIO_STAT_WRITE:
621− strlcat(str, " Write", chars_left);
622− break;
623− case BLKIO_STAT_SYNC:
624− strlcat(str, " Sync", chars_left);
625− break;
626− case BLKIO_STAT_ASYNC:
627− strlcat(str, " Async", chars_left);
628− break;
629− case BLKIO_STAT_TOTAL:
630− strlcat(str, " Total", chars_left);
631− break;
632− default:
633− strlcat(str, " Invalid", chars_left);
634− }
635−}
636−
637−static uint64_t blkio_fill_stat(char *str, int chars_left, uint64_t val,
638− struct cgroup_map_cb *cb, dev_t dev)
639−{
640− blkio_get_key_name(0, dev, str, chars_left, true);
641− cb->fill(cb, str, val);
642− return val;
643−}
644−
645−
646−static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg,
647− enum stat_type_cpu type, enum stat_sub_type sub_type)
648−{
649− int cpu;
650− struct blkio_group_stats_cpu *stats_cpu;
651− u64 val = 0, tval;
652−
653− for_each_possible_cpu(cpu) {
654− unsigned int start;
655− stats_cpu = per_cpu_ptr(blkg->stats_cpu, cpu);
656−
657− do {
658− start = u64_stats_fetch_begin(&stats_cpu->syncp);
659− if (type == BLKIO_STAT_CPU_SECTORS)
660− tval = stats_cpu->sectors;
661− else
662− tval = stats_cpu->stat_arr_cpu[type][sub_type];
663− } while(u64_stats_fetch_retry(&stats_cpu->syncp, start));
664−
665− val += tval;
666− }
667−
668− return val;
669−}
670−
671−static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg,
672− struct cgroup_map_cb *cb, dev_t dev, enum stat_type_cpu type)
673−{
674− uint64_t disk_total, val;
675− char key_str[MAX_KEY_LEN];
676− enum stat_sub_type sub_type;
677331
678− if (type == BLKIO_STAT_CPU_SECTORS) {
679− val = blkio_read_stat_cpu(blkg, type, 0);
680− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, val, cb, dev);
681− }
682−
683− for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
684− sub_type++) {
685− blkio_get_key_name(sub_type, dev, key_str, MAX_KEY_LEN, false);
686− val = blkio_read_stat_cpu(blkg, type, sub_type);
687− cb->fill(cb, key_str, val);
688− }
689−
690− disk_total = blkio_read_stat_cpu(blkg, type, BLKIO_STAT_READ) +
691− blkio_read_stat_cpu(blkg, type, BLKIO_STAT_WRITE);
692−
693− blkio_get_key_name(BLKIO_STAT_TOTAL, dev, key_str, MAX_KEY_LEN, false);
694− cb->fill(cb, key_str, disk_total);
695− return disk_total;
696−}
697−
698−/* This should be called with blkg->stats_lock held */
699−static uint64_t blkio_get_stat(struct blkio_group *blkg,
700− struct cgroup_map_cb *cb, dev_t dev, enum stat_type type)
701−{
702− uint64_t disk_total;
703− char key_str[MAX_KEY_LEN];
704− enum stat_sub_type sub_type;
705−
706− if (type == BLKIO_STAT_TIME)
707− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
708− blkg->stats.time, cb, dev);
709−#ifdef CONFIG_DEBUG_BLK_CGROUP
710− if (type == BLKIO_STAT_UNACCOUNTED_TIME)
711− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
712− blkg->stats.unaccounted_time, cb, dev);
713− if (type == BLKIO_STAT_AVG_QUEUE_SIZE) {
714− uint64_t sum = blkg->stats.avg_queue_size_sum;
715− uint64_t samples = blkg->stats.avg_queue_size_samples;
716− if (samples)
717− do_div(sum, samples);
718− else
719− sum = 0;
720− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, sum, cb, dev);
721− }
722− if (type == BLKIO_STAT_GROUP_WAIT_TIME)
723− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
724− blkg->stats.group_wait_time, cb, dev);
725− if (type == BLKIO_STAT_IDLE_TIME)
726− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
727− blkg->stats.idle_time, cb, dev);
728− if (type == BLKIO_STAT_EMPTY_TIME)
729− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
730− blkg->stats.empty_time, cb, dev);
731− if (type == BLKIO_STAT_DEQUEUE)
732− return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
733− blkg->stats.dequeue, cb, dev);
734−#endif
735−
736− for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
737− sub_type++) {
738− blkio_get_key_name(sub_type, dev, key_str, MAX_KEY_LEN, false);
739− cb->fill(cb, key_str, blkg->stats.stat_arr[type][sub_type]);
740− }
741− disk_total = blkg->stats.stat_arr[type][BLKIO_STAT_READ] +
742− blkg->stats.stat_arr[type][BLKIO_STAT_WRITE];
743− blkio_get_key_name(BLKIO_STAT_TOTAL, dev, key_str, MAX_KEY_LEN, false);
744− cb->fill(cb, key_str, disk_total);
745− return disk_total;
746−}
747−
748−static int blkio_policy_parse_and_set(char *buf,
749− struct blkio_policy_node *newpn, enum blkio_policy_id plid, int fileid)
750−{
751− struct gendisk *disk = NULL;
752− char *s[4], *p, *major_s = NULL, *minor_s = NULL;
753− unsigned long major, minor;
754− int i = 0, ret = -EINVAL;
755− int part;
756− dev_t dev;
757− u64 temp;
758−
759− memset(s, 0, sizeof(s));
760−
761− while ((p = strsep(&buf, " ")) != NULL) {
762− if (!*p)
763− continue;
764−
765− s[i++] = p;
766−
767− /* Prevent from inputing too many things */
768− if (i == 3)
769− break;
770− }
771−
772− if (i != 2)
773− goto out;
774−
775− p = strsep(&s[0], ":");
776− if (p != NULL)
777− major_s = p;
778− else
779− goto out;
780−
781− minor_s = s[0];
782− if (!minor_s)
783− goto out;
784−
785− if (strict_strtoul(major_s, 10, &major))
786− goto out;
787−
788− if (strict_strtoul(minor_s, 10, &minor))
789− goto out;
790−
791− dev = MKDEV(major, minor);
792−
793− if (strict_strtoull(s[1], 10, &temp))
794− goto out;
795−
796− /* For rule removal, do not check for device presence. */
797− if (temp) {
798− disk = get_gendisk(dev, &part);
799− if (!disk || part) {
800− ret = -ENODEV;
801− goto out;
802− }
803− }
804−
805− newpn->dev = dev;
806−
807− switch (plid) {
808− case BLKIO_POLICY_PROP:
809− if ((temp < BLKIO_WEIGHT_MIN && temp > 0) ||
810− temp > BLKIO_WEIGHT_MAX)
811− goto out;
812−
813− newpn->plid = plid;
814− newpn->fileid = fileid;
815− newpn->val.weight = temp;
816− break;
817− case BLKIO_POLICY_THROTL:
818− switch(fileid) {
819− case BLKIO_THROTL_read_bps_device:
820− case BLKIO_THROTL_write_bps_device:
821− newpn->plid = plid;
822− newpn->fileid = fileid;
823− newpn->val.bps = temp;
824− break;
825− case BLKIO_THROTL_read_iops_device:
826− case BLKIO_THROTL_write_iops_device:
827− if (temp > THROTL_IOPS_MAX)
828− goto out;
829−
830− newpn->plid = plid;
831− newpn->fileid = fileid;
832− newpn->val.iops = (unsigned int)temp;
833− break;
834− }
835− break;
836− default:
837− BUG();
838− }
839− ret = 0;
840−out:
841− put_disk(disk);
842− return ret;
843−}
844−
845−unsigned int blkcg_get_weight(struct blkio_cgroup *blkcg,
846− dev_t dev)
847−{
848− struct blkio_policy_node *pn;
849− unsigned long flags;
850− unsigned int weight;
851−
852− spin_lock_irqsave(&blkcg->lock, flags);
853−
854− pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_PROP,
855− BLKIO_PROP_weight_device);
856− if (pn)
857− weight = pn->val.weight;
858− else
859− weight = blkcg->weight;
860−
861− spin_unlock_irqrestore(&blkcg->lock, flags);
862−
863− return weight;
864−}
865−EXPORT_SYMBOL_GPL(blkcg_get_weight);
866−
867−uint64_t blkcg_get_read_bps(struct blkio_cgroup *blkcg, dev_t dev)
868−{
869− struct blkio_policy_node *pn;
870− unsigned long flags;
871− uint64_t bps = -1;
872−
873− spin_lock_irqsave(&blkcg->lock, flags);
874− pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
875− BLKIO_THROTL_read_bps_device);
876− if (pn)
877− bps = pn->val.bps;
878− spin_unlock_irqrestore(&blkcg->lock, flags);
879−
880− return bps;
881−}
882−
883−uint64_t blkcg_get_write_bps(struct blkio_cgroup *blkcg, dev_t dev)
884−{
885− struct blkio_policy_node *pn;
886− unsigned long flags;
887− uint64_t bps = -1;
888−
889− spin_lock_irqsave(&blkcg->lock, flags);
890− pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
891− BLKIO_THROTL_write_bps_device);
892− if (pn)
893− bps = pn->val.bps;
894− spin_unlock_irqrestore(&blkcg->lock, flags);
895−
896− return bps;
897−}
898−
899−unsigned int blkcg_get_read_iops(struct blkio_cgroup *blkcg, dev_t dev)
900−{
901− struct blkio_policy_node *pn;
902− unsigned long flags;
903− unsigned int iops = -1;
904−
905− spin_lock_irqsave(&blkcg->lock, flags);
906− pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
907− BLKIO_THROTL_read_iops_device);
908− if (pn)
909− iops = pn->val.iops;
910− spin_unlock_irqrestore(&blkcg->lock, flags);
911−
912− return iops;
913−}
914−
915−unsigned int blkcg_get_write_iops(struct blkio_cgroup *blkcg, dev_t dev)
916−{
917− struct blkio_policy_node *pn;
918− unsigned long flags;
919− unsigned int iops = -1;
920−
921− spin_lock_irqsave(&blkcg->lock, flags);
922− pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
923− BLKIO_THROTL_write_iops_device);
924− if (pn)
925− iops = pn->val.iops;
926− spin_unlock_irqrestore(&blkcg->lock, flags);
927−
928− return iops;
929−}
332+ /*
333+ * Note that stat reset is racy - it doesn't synchronize against
334+ * stat updates. This is a debug feature which shouldn't exist
335+ * anyway. If you get hit by a race, retry.
336+ */
337+ hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
338+ for (i = 0; i < BLKCG_MAX_POLS; i++) {
339+ struct blkcg_policy *pol = blkcg_policy[i];
930340
931−/* Checks whether user asked for deleting a policy rule */
932−static bool blkio_delete_rule_command(struct blkio_policy_node *pn)
933−{
934− switch(pn->plid) {
935− case BLKIO_POLICY_PROP:
936− if (pn->val.weight == 0)
937− return 1;
938− break;
939− case BLKIO_POLICY_THROTL:
940− switch(pn->fileid) {
941− case BLKIO_THROTL_read_bps_device:
942− case BLKIO_THROTL_write_bps_device:
943− if (pn->val.bps == 0)
944− return 1;
945− break;
946− case BLKIO_THROTL_read_iops_device:
947− case BLKIO_THROTL_write_iops_device:
948− if (pn->val.iops == 0)
949− return 1;
341+ if (blkcg_policy_enabled(blkg->q, pol) &&
342+ pol->pd_reset_stats_fn)
343+ pol->pd_reset_stats_fn(blkg);
950344 }
951− break;
952− default:
953− BUG();
954345 }
955346
347+ spin_unlock_irq(&blkcg->lock);
348+ mutex_unlock(&blkcg_pol_mutex);
956349 return 0;
957350 }
958351
959−static void blkio_update_policy_rule(struct blkio_policy_node *oldpn,
960− struct blkio_policy_node *newpn)
961−{
962− switch(oldpn->plid) {
963− case BLKIO_POLICY_PROP:
964− oldpn->val.weight = newpn->val.weight;
965− break;
966− case BLKIO_POLICY_THROTL:
967− switch(newpn->fileid) {
968− case BLKIO_THROTL_read_bps_device:
969− case BLKIO_THROTL_write_bps_device:
970− oldpn->val.bps = newpn->val.bps;
971− break;
972− case BLKIO_THROTL_read_iops_device:
973− case BLKIO_THROTL_write_iops_device:
974− oldpn->val.iops = newpn->val.iops;
975− }
976− break;
977− default:
978− BUG();
979− }
980−}
981−
982−/*
983− * Some rules/values in blkg have changed. Propagate those to respective
984− * policies.
985− */
986−static void blkio_update_blkg_policy(struct blkio_cgroup *blkcg,
987− struct blkio_group *blkg, struct blkio_policy_node *pn)
352+static const char *blkg_dev_name(struct blkcg_gq *blkg)
988353 {
989− unsigned int weight, iops;
990− u64 bps;
991−
992− switch(pn->plid) {
993− case BLKIO_POLICY_PROP:
994− weight = pn->val.weight ? pn->val.weight :
995− blkcg->weight;
996− blkio_update_group_weight(blkg, weight);
997− break;
998− case BLKIO_POLICY_THROTL:
999− switch(pn->fileid) {
1000− case BLKIO_THROTL_read_bps_device:
1001− case BLKIO_THROTL_write_bps_device:
1002− bps = pn->val.bps ? pn->val.bps : (-1);
1003− blkio_update_group_bps(blkg, bps, pn->fileid);
1004− break;
1005− case BLKIO_THROTL_read_iops_device:
1006− case BLKIO_THROTL_write_iops_device:
1007− iops = pn->val.iops ? pn->val.iops : (-1);
1008− blkio_update_group_iops(blkg, iops, pn->fileid);
1009− break;
1010− }
1011− break;
1012− default:
1013− BUG();
1014− }
354+ /* some drivers (floppy) instantiate a queue w/o disk registered */
355+ if (blkg->q->backing_dev_info.dev)
356+ return dev_name(blkg->q->backing_dev_info.dev);
357+ return NULL;
1015358 }
1016359
1017−/*
1018− * A policy node rule has been updated. Propagate this update to all the
1019− * block groups which might be affected by this update.
360+/**
361+ * blkcg_print_blkgs - helper for printing per-blkg data
362+ * @sf: seq_file to print to
363+ * @blkcg: blkcg of interest
364+ * @prfill: fill function to print out a blkg
365+ * @pol: policy in question
366+ * @data: data to be passed to @prfill
367+ * @show_total: to print out sum of prfill return values or not
368+ *
369+ * This function invokes @prfill on each blkg of @blkcg if pd for the
370+ * policy specified by @pol exists. @prfill is invoked with @sf, the
371+ * policy data and @data. If @show_total is %true, the sum of the return
372+ * values from @prfill is printed with "Total" label at the end.
373+ *
374+ * This is to be used to construct print functions for
375+ * cftype->read_seq_string method.
1020376 */
1021−static void blkio_update_policy_node_blkg(struct blkio_cgroup *blkcg,
1022− struct blkio_policy_node *pn)
377+void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
378+ u64 (*prfill)(struct seq_file *,
379+ struct blkg_policy_data *, int),
380+ const struct blkcg_policy *pol, int data,
381+ bool show_total)
1023382 {
1024− struct blkio_group *blkg;
383+ struct blkcg_gq *blkg;
1025384 struct hlist_node *n;
385+ u64 total = 0;
1026386
1027− spin_lock(&blkio_list_lock);
1028387 spin_lock_irq(&blkcg->lock);
1029−
1030− hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
1031− if (pn->dev != blkg->dev || pn->plid != blkg->plid)
1032− continue;
1033− blkio_update_blkg_policy(blkcg, blkg, pn);
1034− }
1035−
388+ hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
389+ if (blkcg_policy_enabled(blkg->q, pol))
390+ total += prfill(sf, blkg->pd[pol->plid], data);
1036391 spin_unlock_irq(&blkcg->lock);
1037− spin_unlock(&blkio_list_lock);
392+
393+ if (show_total)
394+ seq_printf(sf, "Total %llu\n", (unsigned long long)total);
1038395 }
396+EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
1039397
1040−static int blkiocg_file_write(struct cgroup *cgrp, struct cftype *cft,
1041− const char *buffer)
398+/**
399+ * __blkg_prfill_u64 - prfill helper for a single u64 value
400+ * @sf: seq_file to print to
401+ * @pd: policy private data of interest
402+ * @v: value to print
403+ *
404+ * Print @v to @sf for the device assocaited with @pd.
405+ */
406+u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
1042407 {
1043− int ret = 0;
1044− char *buf;
1045− struct blkio_policy_node *newpn, *pn;
1046− struct blkio_cgroup *blkcg;
1047− int keep_newpn = 0;
1048− enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1049− int fileid = BLKIOFILE_ATTR(cft->private);
1050−
1051− buf = kstrdup(buffer, GFP_KERNEL);
1052− if (!buf)
1053− return -ENOMEM;
1054−
1055− newpn = kzalloc(sizeof(*newpn), GFP_KERNEL);
1056− if (!newpn) {
1057− ret = -ENOMEM;
1058− goto free_buf;
1059− }
1060−
1061− ret = blkio_policy_parse_and_set(buf, newpn, plid, fileid);
1062− if (ret)
1063− goto free_newpn;
1064−
1065− blkcg = cgroup_to_blkio_cgroup(cgrp);
1066−
1067− spin_lock_irq(&blkcg->lock);
1068−
1069− pn = blkio_policy_search_node(blkcg, newpn->dev, plid, fileid);
1070− if (!pn) {
1071− if (!blkio_delete_rule_command(newpn)) {
1072− blkio_policy_insert_node(blkcg, newpn);
1073− keep_newpn = 1;
1074− }
1075− spin_unlock_irq(&blkcg->lock);
1076− goto update_io_group;
1077− }
1078−
1079− if (blkio_delete_rule_command(newpn)) {
1080− blkio_policy_delete_node(pn);
1081− kfree(pn);
1082− spin_unlock_irq(&blkcg->lock);
1083− goto update_io_group;
1084− }
1085− spin_unlock_irq(&blkcg->lock);
408+ const char *dname = blkg_dev_name(pd->blkg);
1086409
1087− blkio_update_policy_rule(pn, newpn);
410+ if (!dname)
411+ return 0;
1088412
1089−update_io_group:
1090− blkio_update_policy_node_blkg(blkcg, newpn);
1091−
1092−free_newpn:
1093− if (!keep_newpn)
1094− kfree(newpn);
1095−free_buf:
1096− kfree(buf);
1097− return ret;
413+ seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
414+ return v;
1098415 }
416+EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
1099417
1100−static void
1101−blkio_print_policy_node(struct seq_file *m, struct blkio_policy_node *pn)
1102−{
1103− switch(pn->plid) {
1104− case BLKIO_POLICY_PROP:
1105− if (pn->fileid == BLKIO_PROP_weight_device)
1106− seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
1107− MINOR(pn->dev), pn->val.weight);
1108− break;
1109− case BLKIO_POLICY_THROTL:
1110− switch(pn->fileid) {
1111− case BLKIO_THROTL_read_bps_device:
1112− case BLKIO_THROTL_write_bps_device:
1113− seq_printf(m, "%u:%u\t%llu\n", MAJOR(pn->dev),
1114− MINOR(pn->dev), pn->val.bps);
1115− break;
1116− case BLKIO_THROTL_read_iops_device:
1117− case BLKIO_THROTL_write_iops_device:
1118− seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
1119− MINOR(pn->dev), pn->val.iops);
1120− break;
1121− }
1122− break;
1123− default:
1124− BUG();
1125− }
1126−}
418+/**
419+ * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
420+ * @sf: seq_file to print to
421+ * @pd: policy private data of interest
422+ * @rwstat: rwstat to print
423+ *
424+ * Print @rwstat to @sf for the device assocaited with @pd.
425+ */
426+u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
427+ const struct blkg_rwstat *rwstat)
428+{
429+ static const char *rwstr[] = {
430+ [BLKG_RWSTAT_READ] = "Read",
431+ [BLKG_RWSTAT_WRITE] = "Write",
432+ [BLKG_RWSTAT_SYNC] = "Sync",
433+ [BLKG_RWSTAT_ASYNC] = "Async",
434+ };
435+ const char *dname = blkg_dev_name(pd->blkg);
436+ u64 v;
437+ int i;
1127438
1128−/* cgroup files which read their data from policy nodes end up here */
1129−static void blkio_read_policy_node_files(struct cftype *cft,
1130− struct blkio_cgroup *blkcg, struct seq_file *m)
1131−{
1132− struct blkio_policy_node *pn;
1133−
1134− if (!list_empty(&blkcg->policy_list)) {
1135− spin_lock_irq(&blkcg->lock);
1136− list_for_each_entry(pn, &blkcg->policy_list, node) {
1137− if (!pn_matches_cftype(cft, pn))
1138− continue;
1139− blkio_print_policy_node(m, pn);
1140− }
1141− spin_unlock_irq(&blkcg->lock);
1142− }
1143−}
439+ if (!dname)
440+ return 0;
1144441
1145−static int blkiocg_file_read(struct cgroup *cgrp, struct cftype *cft,
1146− struct seq_file *m)
1147−{
1148− struct blkio_cgroup *blkcg;
1149− enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1150− int name = BLKIOFILE_ATTR(cft->private);
1151−
1152− blkcg = cgroup_to_blkio_cgroup(cgrp);
1153−
1154− switch(plid) {
1155− case BLKIO_POLICY_PROP:
1156− switch(name) {
1157− case BLKIO_PROP_weight_device:
1158− blkio_read_policy_node_files(cft, blkcg, m);
1159− return 0;
1160− default:
1161− BUG();
1162− }
1163− break;
1164− case BLKIO_POLICY_THROTL:
1165− switch(name){
1166− case BLKIO_THROTL_read_bps_device:
1167− case BLKIO_THROTL_write_bps_device:
1168− case BLKIO_THROTL_read_iops_device:
1169− case BLKIO_THROTL_write_iops_device:
1170− blkio_read_policy_node_files(cft, blkcg, m);
1171− return 0;
1172− default:
1173− BUG();
1174− }
1175− break;
1176− default:
1177− BUG();
1178− }
442+ for (i = 0; i < BLKG_RWSTAT_NR; i++)
443+ seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
444+ (unsigned long long)rwstat->cnt[i]);
1179445
1180− return 0;
446+ v = rwstat->cnt[BLKG_RWSTAT_READ] + rwstat->cnt[BLKG_RWSTAT_WRITE];
447+ seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
448+ return v;
1181449 }
1182450
1183−static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg,
1184− struct cftype *cft, struct cgroup_map_cb *cb,
1185− enum stat_type type, bool show_total, bool pcpu)
451+/**
452+ * blkg_prfill_stat - prfill callback for blkg_stat
453+ * @sf: seq_file to print to
454+ * @pd: policy private data of interest
455+ * @off: offset to the blkg_stat in @pd
456+ *
457+ * prfill callback for printing a blkg_stat.
458+ */
459+u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
1186460 {
1187− struct blkio_group *blkg;
1188− struct hlist_node *n;
1189− uint64_t cgroup_total = 0;
1190−
1191− rcu_read_lock();
1192− hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
1193− if (blkg->dev) {
1194− if (!cftype_blkg_same_policy(cft, blkg))
1195− continue;
1196− if (pcpu)
1197− cgroup_total += blkio_get_stat_cpu(blkg, cb,
1198− blkg->dev, type);
1199− else {
1200− spin_lock_irq(&blkg->stats_lock);
1201− cgroup_total += blkio_get_stat(blkg, cb,
1202− blkg->dev, type);
1203− spin_unlock_irq(&blkg->stats_lock);
1204− }
1205− }
1206− }
1207− if (show_total)
1208− cb->fill(cb, "Total", cgroup_total);
1209− rcu_read_unlock();
1210− return 0;
461+ return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
1211462 }
463+EXPORT_SYMBOL_GPL(blkg_prfill_stat);
1212464
1213−/* All map kind of cgroup file get serviced by this function */
1214−static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft,
1215− struct cgroup_map_cb *cb)
465+/**
466+ * blkg_prfill_rwstat - prfill callback for blkg_rwstat
467+ * @sf: seq_file to print to
468+ * @pd: policy private data of interest
469+ * @off: offset to the blkg_rwstat in @pd
470+ *
471+ * prfill callback for printing a blkg_rwstat.
472+ */
473+u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
474+ int off)
1216475 {
1217− struct blkio_cgroup *blkcg;
1218− enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1219− int name = BLKIOFILE_ATTR(cft->private);
1220−
1221− blkcg = cgroup_to_blkio_cgroup(cgrp);
1222−
1223− switch(plid) {
1224− case BLKIO_POLICY_PROP:
1225− switch(name) {
1226− case BLKIO_PROP_time:
1227− return blkio_read_blkg_stats(blkcg, cft, cb,
1228− BLKIO_STAT_TIME, 0, 0);
1229− case BLKIO_PROP_sectors:
1230− return blkio_read_blkg_stats(blkcg, cft, cb,
1231− BLKIO_STAT_CPU_SECTORS, 0, 1);
1232− case BLKIO_PROP_io_service_bytes:
1233− return blkio_read_blkg_stats(blkcg, cft, cb,
1234− BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
1235− case BLKIO_PROP_io_serviced:
1236− return blkio_read_blkg_stats(blkcg, cft, cb,
1237− BLKIO_STAT_CPU_SERVICED, 1, 1);
1238− case BLKIO_PROP_io_service_time:
1239− return blkio_read_blkg_stats(blkcg, cft, cb,
1240− BLKIO_STAT_SERVICE_TIME, 1, 0);
1241− case BLKIO_PROP_io_wait_time:
1242− return blkio_read_blkg_stats(blkcg, cft, cb,
1243− BLKIO_STAT_WAIT_TIME, 1, 0);
1244− case BLKIO_PROP_io_merged:
1245− return blkio_read_blkg_stats(blkcg, cft, cb,
1246− BLKIO_STAT_CPU_MERGED, 1, 1);
1247− case BLKIO_PROP_io_queued:
1248− return blkio_read_blkg_stats(blkcg, cft, cb,
1249− BLKIO_STAT_QUEUED, 1, 0);
1250−#ifdef CONFIG_DEBUG_BLK_CGROUP
1251− case BLKIO_PROP_unaccounted_time:
1252− return blkio_read_blkg_stats(blkcg, cft, cb,
1253− BLKIO_STAT_UNACCOUNTED_TIME, 0, 0);
1254− case BLKIO_PROP_dequeue:
1255− return blkio_read_blkg_stats(blkcg, cft, cb,
1256− BLKIO_STAT_DEQUEUE, 0, 0);
1257− case BLKIO_PROP_avg_queue_size:
1258− return blkio_read_blkg_stats(blkcg, cft, cb,
1259− BLKIO_STAT_AVG_QUEUE_SIZE, 0, 0);
1260− case BLKIO_PROP_group_wait_time:
1261− return blkio_read_blkg_stats(blkcg, cft, cb,
1262− BLKIO_STAT_GROUP_WAIT_TIME, 0, 0);
1263− case BLKIO_PROP_idle_time:
1264− return blkio_read_blkg_stats(blkcg, cft, cb,
1265− BLKIO_STAT_IDLE_TIME, 0, 0);
1266− case BLKIO_PROP_empty_time:
1267− return blkio_read_blkg_stats(blkcg, cft, cb,
1268− BLKIO_STAT_EMPTY_TIME, 0, 0);
1269−#endif
1270− default:
1271− BUG();
1272− }
1273− break;
1274− case BLKIO_POLICY_THROTL:
1275− switch(name){
1276− case BLKIO_THROTL_io_service_bytes:
1277− return blkio_read_blkg_stats(blkcg, cft, cb,
1278− BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
1279− case BLKIO_THROTL_io_serviced:
1280− return blkio_read_blkg_stats(blkcg, cft, cb,
1281− BLKIO_STAT_CPU_SERVICED, 1, 1);
1282− default:
1283− BUG();
1284− }
1285− break;
1286− default:
1287− BUG();
1288− }
476+ struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
1289477
1290− return 0;
478+ return __blkg_prfill_rwstat(sf, pd, &rwstat);
1291479 }
480+EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
1292481
1293−static int blkio_weight_write(struct blkio_cgroup *blkcg, u64 val)
482+/**
483+ * blkg_conf_prep - parse and prepare for per-blkg config update
484+ * @blkcg: target block cgroup
485+ * @pol: target policy
486+ * @input: input string
487+ * @ctx: blkg_conf_ctx to be filled
488+ *
489+ * Parse per-blkg config update from @input and initialize @ctx with the
490+ * result. @ctx->blkg points to the blkg to be updated and @ctx->v the new
491+ * value. This function returns with RCU read lock and queue lock held and
492+ * must be paired with blkg_conf_finish().
493+ */
494+int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
495+ const char *input, struct blkg_conf_ctx *ctx)
496+ __acquires(rcu) __acquires(disk->queue->queue_lock)
1294497 {
1295− struct blkio_group *blkg;
1296− struct hlist_node *n;
1297− struct blkio_policy_node *pn;
498+ struct gendisk *disk;
499+ struct blkcg_gq *blkg;
500+ unsigned int major, minor;
501+ unsigned long long v;
502+ int part, ret;
1298503
1299− if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
504+ if (sscanf(input, "%u:%u %llu", &major, &minor, &v) != 3)
1300505 return -EINVAL;
1301506
1302− spin_lock(&blkio_list_lock);
1303− spin_lock_irq(&blkcg->lock);
1304− blkcg->weight = (unsigned int)val;
1305−
1306− hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
1307− pn = blkio_policy_search_node(blkcg, blkg->dev,
1308− BLKIO_POLICY_PROP, BLKIO_PROP_weight_device);
1309− if (pn)
1310− continue;
1311−
1312− blkio_update_group_weight(blkg, blkcg->weight);
1313− }
1314− spin_unlock_irq(&blkcg->lock);
1315− spin_unlock(&blkio_list_lock);
1316− return 0;
1317−}
507+ disk = get_gendisk(MKDEV(major, minor), &part);
508+ if (!disk || part)
509+ return -EINVAL;
1318510
1319−static u64 blkiocg_file_read_u64 (struct cgroup *cgrp, struct cftype *cft) {
1320− struct blkio_cgroup *blkcg;
1321− enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1322− int name = BLKIOFILE_ATTR(cft->private);
511+ rcu_read_lock();
512+ spin_lock_irq(disk->queue->queue_lock);
1323513
1324− blkcg = cgroup_to_blkio_cgroup(cgrp);
514+ if (blkcg_policy_enabled(disk->queue, pol))
515+ blkg = blkg_lookup_create(blkcg, disk->queue);
516+ else
517+ blkg = ERR_PTR(-EINVAL);
1325518
1326− switch(plid) {
1327− case BLKIO_POLICY_PROP:
1328− switch(name) {
1329− case BLKIO_PROP_weight:
1330− return (u64)blkcg->weight;
519+ if (IS_ERR(blkg)) {
520+ ret = PTR_ERR(blkg);
521+ rcu_read_unlock();
522+ spin_unlock_irq(disk->queue->queue_lock);
523+ put_disk(disk);
524+ /*
525+ * If queue was bypassing, we should retry. Do so after a
526+ * short msleep(). It isn't strictly necessary but queue
527+ * can be bypassing for some time and it's always nice to
528+ * avoid busy looping.
529+ */
530+ if (ret == -EBUSY) {
531+ msleep(10);
532+ ret = restart_syscall();
1331533 }
1332− break;
1333− default:
1334− BUG();
534+ return ret;
1335535 }
536+
537+ ctx->disk = disk;
538+ ctx->blkg = blkg;
539+ ctx->v = v;
1336540 return 0;
1337541 }
542+EXPORT_SYMBOL_GPL(blkg_conf_prep);
1338543
1339−static int
1340−blkiocg_file_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
544+/**
545+ * blkg_conf_finish - finish up per-blkg config update
546+ * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
547+ *
548+ * Finish up after per-blkg config update. This function must be paired
549+ * with blkg_conf_prep().
550+ */
551+void blkg_conf_finish(struct blkg_conf_ctx *ctx)
552+ __releases(ctx->disk->queue->queue_lock) __releases(rcu)
1341553 {
1342− struct blkio_cgroup *blkcg;
1343− enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1344− int name = BLKIOFILE_ATTR(cft->private);
1345−
1346− blkcg = cgroup_to_blkio_cgroup(cgrp);
1347−
1348− switch(plid) {
1349− case BLKIO_POLICY_PROP:
1350− switch(name) {
1351− case BLKIO_PROP_weight:
1352− return blkio_weight_write(blkcg, val);
1353− }
1354− break;
1355− default:
1356− BUG();
1357− }
1358−
1359− return 0;
554+ spin_unlock_irq(ctx->disk->queue->queue_lock);
555+ rcu_read_unlock();
556+ put_disk(ctx->disk);
1360557 }
558+EXPORT_SYMBOL_GPL(blkg_conf_finish);
1361559
1362−struct cftype blkio_files[] = {
1363− {
1364− .name = "weight_device",
1365− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1366− BLKIO_PROP_weight_device),
1367− .read_seq_string = blkiocg_file_read,
1368− .write_string = blkiocg_file_write,
1369− .max_write_len = 256,
1370− },
1371− {
1372− .name = "weight",
1373− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1374− BLKIO_PROP_weight),
1375− .read_u64 = blkiocg_file_read_u64,
1376− .write_u64 = blkiocg_file_write_u64,
1377− },
1378− {
1379− .name = "time",
1380− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1381− BLKIO_PROP_time),
1382− .read_map = blkiocg_file_read_map,
1383− },
1384− {
1385− .name = "sectors",
1386− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1387− BLKIO_PROP_sectors),
1388− .read_map = blkiocg_file_read_map,
1389− },
1390− {
1391− .name = "io_service_bytes",
1392− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1393− BLKIO_PROP_io_service_bytes),
1394− .read_map = blkiocg_file_read_map,
1395− },
1396− {
1397− .name = "io_serviced",
1398− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1399− BLKIO_PROP_io_serviced),
1400− .read_map = blkiocg_file_read_map,
1401− },
1402− {
1403− .name = "io_service_time",
1404− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1405− BLKIO_PROP_io_service_time),
1406− .read_map = blkiocg_file_read_map,
1407− },
1408− {
1409− .name = "io_wait_time",
1410− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1411− BLKIO_PROP_io_wait_time),
1412− .read_map = blkiocg_file_read_map,
1413− },
1414− {
1415− .name = "io_merged",
1416− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1417− BLKIO_PROP_io_merged),
1418− .read_map = blkiocg_file_read_map,
1419− },
1420− {
1421− .name = "io_queued",
1422− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1423− BLKIO_PROP_io_queued),
1424− .read_map = blkiocg_file_read_map,
1425− },
560+struct cftype blkcg_files[] = {
1426561 {
1427562 .name = "reset_stats",
1428− .write_u64 = blkiocg_reset_stats,
1429− },
1430−#ifdef CONFIG_BLK_DEV_THROTTLING
1431− {
1432− .name = "throttle.read_bps_device",
1433− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1434− BLKIO_THROTL_read_bps_device),
1435− .read_seq_string = blkiocg_file_read,
1436− .write_string = blkiocg_file_write,
1437− .max_write_len = 256,
1438− },
1439−
1440− {
1441− .name = "throttle.write_bps_device",
1442− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1443− BLKIO_THROTL_write_bps_device),
1444− .read_seq_string = blkiocg_file_read,
1445− .write_string = blkiocg_file_write,
1446− .max_write_len = 256,
1447− },
1448−
1449− {
1450− .name = "throttle.read_iops_device",
1451− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1452− BLKIO_THROTL_read_iops_device),
1453− .read_seq_string = blkiocg_file_read,
1454− .write_string = blkiocg_file_write,
1455− .max_write_len = 256,
1456− },
1457−
1458− {
1459− .name = "throttle.write_iops_device",
1460− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1461− BLKIO_THROTL_write_iops_device),
1462− .read_seq_string = blkiocg_file_read,
1463− .write_string = blkiocg_file_write,
1464− .max_write_len = 256,
1465− },
1466− {
1467− .name = "throttle.io_service_bytes",
1468− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1469− BLKIO_THROTL_io_service_bytes),
1470− .read_map = blkiocg_file_read_map,
1471− },
1472− {
1473− .name = "throttle.io_serviced",
1474− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1475− BLKIO_THROTL_io_serviced),
1476− .read_map = blkiocg_file_read_map,
1477− },
1478−#endif /* CONFIG_BLK_DEV_THROTTLING */
1479−
1480−#ifdef CONFIG_DEBUG_BLK_CGROUP
1481− {
1482− .name = "avg_queue_size",
1483− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1484− BLKIO_PROP_avg_queue_size),
1485− .read_map = blkiocg_file_read_map,
563+ .write_u64 = blkcg_reset_stats,
1486564 },
1487− {
1488− .name = "group_wait_time",
1489− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1490− BLKIO_PROP_group_wait_time),
1491− .read_map = blkiocg_file_read_map,
1492− },
1493− {
1494− .name = "idle_time",
1495− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1496− BLKIO_PROP_idle_time),
1497− .read_map = blkiocg_file_read_map,
1498− },
1499− {
1500− .name = "empty_time",
1501− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1502− BLKIO_PROP_empty_time),
1503− .read_map = blkiocg_file_read_map,
1504− },
1505− {
1506− .name = "dequeue",
1507− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1508− BLKIO_PROP_dequeue),
1509− .read_map = blkiocg_file_read_map,
1510− },
1511− {
1512− .name = "unaccounted_time",
1513− .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1514− BLKIO_PROP_unaccounted_time),
1515− .read_map = blkiocg_file_read_map,
1516− },
1517−#endif
1518565 { } /* terminate */
1519566 };
1520567
1521−static void blkiocg_destroy(struct cgroup *cgroup)
568+/**
569+ * blkcg_pre_destroy - cgroup pre_destroy callback
570+ * @cgroup: cgroup of interest
571+ *
572+ * This function is called when @cgroup is about to go away and responsible
573+ * for shooting down all blkgs associated with @cgroup. blkgs should be
574+ * removed while holding both q and blkcg locks. As blkcg lock is nested
575+ * inside q lock, this function performs reverse double lock dancing.
576+ *
577+ * This is the blkcg counterpart of ioc_release_fn().
578+ */
579+static int blkcg_pre_destroy(struct cgroup *cgroup)
1522580 {
1523− struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
1524− unsigned long flags;
1525− struct blkio_group *blkg;
1526− void *key;
1527− struct blkio_policy_type *blkiop;
1528− struct blkio_policy_node *pn, *pntmp;
581+ struct blkcg *blkcg = cgroup_to_blkcg(cgroup);
1529582
1530− rcu_read_lock();
1531− do {
1532− spin_lock_irqsave(&blkcg->lock, flags);
583+ spin_lock_irq(&blkcg->lock);
1533584
1534− if (hlist_empty(&blkcg->blkg_list)) {
1535− spin_unlock_irqrestore(&blkcg->lock, flags);
1536− break;
585+ while (!hlist_empty(&blkcg->blkg_list)) {
586+ struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
587+ struct blkcg_gq, blkcg_node);
588+ struct request_queue *q = blkg->q;
589+
590+ if (spin_trylock(q->queue_lock)) {
591+ blkg_destroy(blkg);
592+ spin_unlock(q->queue_lock);
593+ } else {
594+ spin_unlock_irq(&blkcg->lock);
595+ cpu_relax();
596+ spin_lock_irq(&blkcg->lock);
1537597 }
598+ }
1538599
1539− blkg = hlist_entry(blkcg->blkg_list.first, struct blkio_group,
1540− blkcg_node);
1541− key = rcu_dereference(blkg->key);
1542− __blkiocg_del_blkio_group(blkg);
1543−
1544− spin_unlock_irqrestore(&blkcg->lock, flags);
1545−
1546− /*
1547− * This blkio_group is being unlinked as associated cgroup is
1548− * going away. Let all the IO controlling policies know about
1549− * this event.
1550− */
1551− spin_lock(&blkio_list_lock);
1552− list_for_each_entry(blkiop, &blkio_list, list) {
1553− if (blkiop->plid != blkg->plid)
1554− continue;
1555− blkiop->ops.blkio_unlink_group_fn(key, blkg);
1556− }
1557− spin_unlock(&blkio_list_lock);
1558− } while (1);
600+ spin_unlock_irq(&blkcg->lock);
601+ return 0;
602+}
1559603
1560− list_for_each_entry_safe(pn, pntmp, &blkcg->policy_list, node) {
1561− blkio_policy_delete_node(pn);
1562− kfree(pn);
1563− }
604+static void blkcg_destroy(struct cgroup *cgroup)
605+{
606+ struct blkcg *blkcg = cgroup_to_blkcg(cgroup);
1564607
1565− free_css_id(&blkio_subsys, &blkcg->css);
1566− rcu_read_unlock();
1567− if (blkcg != &blkio_root_cgroup)
608+ if (blkcg != &blkcg_root)
1568609 kfree(blkcg);
1569610 }
1570611
1571−static struct cgroup_subsys_state *blkiocg_create(struct cgroup *cgroup)
612+static struct cgroup_subsys_state *blkcg_create(struct cgroup *cgroup)
1572613 {
1573− struct blkio_cgroup *blkcg;
614+ static atomic64_t id_seq = ATOMIC64_INIT(0);
615+ struct blkcg *blkcg;
1574616 struct cgroup *parent = cgroup->parent;
1575617
1576618 if (!parent) {
1577− blkcg = &blkio_root_cgroup;
619+ blkcg = &blkcg_root;
1578620 goto done;
1579621 }
1580622
@@ -1582,22 +624,68 @@ static struct cgroup_subsys_state *blkiocg_create(struct cgroup *cgroup) · 1 unmodified line
1582624 if (!blkcg)
1583625 return ERR_PTR(-ENOMEM);
1584626
1585− blkcg->weight = BLKIO_WEIGHT_DEFAULT;
627+ blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;
628+ blkcg->id = atomic64_inc_return(&id_seq); /* root is 0, start from 1 */
1586629 done:
1587630 spin_lock_init(&blkcg->lock);
631+ INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);
1588632 INIT_HLIST_HEAD(&blkcg->blkg_list);
1589633
1590− INIT_LIST_HEAD(&blkcg->policy_list);
1591634 return &blkcg->css;
1592635 }
1593636
637+/**
638+ * blkcg_init_queue - initialize blkcg part of request queue
639+ * @q: request_queue to initialize
640+ *
641+ * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
642+ * part of new request_queue @q.
643+ *
644+ * RETURNS:
645+ * 0 on success, -errno on failure.
646+ */
647+int blkcg_init_queue(struct request_queue *q)
648+{
649+ might_sleep();
650+
651+ return blk_throtl_init(q);
652+}
653+
654+/**
655+ * blkcg_drain_queue - drain blkcg part of request_queue
656+ * @q: request_queue to drain
657+ *
658+ * Called from blk_drain_queue(). Responsible for draining blkcg part.
659+ */
660+void blkcg_drain_queue(struct request_queue *q)
661+{
662+ lockdep_assert_held(q->queue_lock);
663+
664+ blk_throtl_drain(q);
665+}
666+
667+/**
668+ * blkcg_exit_queue - exit and release blkcg part of request_queue
669+ * @q: request_queue being released
670+ *
671+ * Called from blk_release_queue(). Responsible for exiting blkcg part.
672+ */
673+void blkcg_exit_queue(struct request_queue *q)
674+{
675+ spin_lock_irq(q->queue_lock);
676+ blkg_destroy_all(q);
677+ spin_unlock_irq(q->queue_lock);
678+
679+ blk_throtl_exit(q);
680+}
681+
1594682 /*
1595683 * We cannot support shared io contexts, as we have no mean to support
1596684 * two tasks with the same ioc in two different groups without major rework
1597685 * of the main cic data structures. For now we allow a task to change
1598686 * its cgroup only if it's the only owner of its ioc.
1599687 */
1600−static int blkiocg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
688+static int blkcg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
1601689 {
1602690 struct task_struct *task;
1603691 struct io_context *ioc;
@@ -1616,63 +704,213 @@ static int blkiocg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset) · 12 unmodified lines
1616704 return ret;
1617705 }
1618706
1619−static void blkiocg_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
1620−{
1621− struct task_struct *task;
1622− struct io_context *ioc;
1623−
1624− cgroup_taskset_for_each(task, cgrp, tset) {
1625− /* we don't lose anything even if ioc allocation fails */
1626− ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
1627− if (ioc) {
1628− ioc_cgroup_changed(ioc);
1629− put_io_context(ioc);
1630− }
1631− }
1632−}
1633−
1634707 struct cgroup_subsys blkio_subsys = {
1635708 .name = "blkio",
1636− .create = blkiocg_create,
1637− .can_attach = blkiocg_can_attach,
1638− .attach = blkiocg_attach,
1639− .destroy = blkiocg_destroy,
1640−#ifdef CONFIG_BLK_CGROUP
1641− /* note: blkio_subsys_id is otherwise defined in blk-cgroup.h */
709+ .create = blkcg_create,
710+ .can_attach = blkcg_can_attach,
711+ .pre_destroy = blkcg_pre_destroy,
712+ .destroy = blkcg_destroy,
1642713 .subsys_id = blkio_subsys_id,
1643−#endif
1644− .base_cftypes = blkio_files,
1645− .use_id = 1,
714+ .base_cftypes = blkcg_files,
1646715 .module = THIS_MODULE,
1647716 };
1648717 EXPORT_SYMBOL_GPL(blkio_subsys);
1649718
1650−void blkio_policy_register(struct blkio_policy_type *blkiop)
719+/**
720+ * blkcg_activate_policy - activate a blkcg policy on a request_queue
721+ * @q: request_queue of interest
722+ * @pol: blkcg policy to activate
723+ *
724+ * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
725+ * bypass mode to populate its blkgs with policy_data for @pol.
726+ *
727+ * Activation happens with @q bypassed, so nobody would be accessing blkgs
728+ * from IO path. Update of each blkg is protected by both queue and blkcg
729+ * locks so that holding either lock and testing blkcg_policy_enabled() is
730+ * always enough for dereferencing policy data.
731+ *
732+ * The caller is responsible for synchronizing [de]activations and policy
733+ * [un]registerations. Returns 0 on success, -errno on failure.
734+ */
735+int blkcg_activate_policy(struct request_queue *q,
736+ const struct blkcg_policy *pol)
1651737 {
1652− spin_lock(&blkio_list_lock);
1653− list_add_tail(&blkiop->list, &blkio_list);
1654− spin_unlock(&blkio_list_lock);
738+ LIST_HEAD(pds);
739+ struct blkcg_gq *blkg;
740+ struct blkg_policy_data *pd, *n;
741+ int cnt = 0, ret;
742+
743+ if (blkcg_policy_enabled(q, pol))
744+ return 0;
745+
746+ blk_queue_bypass_start(q);
747+
748+ /* make sure the root blkg exists and count the existing blkgs */
749+ spin_lock_irq(q->queue_lock);
750+
751+ rcu_read_lock();
752+ blkg = __blkg_lookup_create(&blkcg_root, q);
753+ rcu_read_unlock();
754+
755+ if (IS_ERR(blkg)) {
756+ ret = PTR_ERR(blkg);
757+ goto out_unlock;
758+ }
759+ q->root_blkg = blkg;
760+
761+ list_for_each_entry(blkg, &q->blkg_list, q_node)
762+ cnt++;
763+
764+ spin_unlock_irq(q->queue_lock);
765+
766+ /* allocate policy_data for all existing blkgs */
767+ while (cnt--) {
768+ pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
769+ if (!pd) {
770+ ret = -ENOMEM;
771+ goto out_free;
772+ }
773+ list_add_tail(&pd->alloc_node, &pds);
774+ }
775+
776+ /*
777+ * Install the allocated pds. With @q bypassing, no new blkg
778+ * should have been created while the queue lock was dropped.
779+ */
780+ spin_lock_irq(q->queue_lock);
781+
782+ list_for_each_entry(blkg, &q->blkg_list, q_node) {
783+ if (WARN_ON(list_empty(&pds))) {
784+ /* umm... this shouldn't happen, just abort */
785+ ret = -ENOMEM;
786+ goto out_unlock;
787+ }
788+ pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node);
789+ list_del_init(&pd->alloc_node);
790+
791+ /* grab blkcg lock too while installing @pd on @blkg */
792+ spin_lock(&blkg->blkcg->lock);
793+
794+ blkg->pd[pol->plid] = pd;
795+ pd->blkg = blkg;
796+ pol->pd_init_fn(blkg);
797+
798+ spin_unlock(&blkg->blkcg->lock);
799+ }
800+
801+ __set_bit(pol->plid, q->blkcg_pols);
802+ ret = 0;
803+out_unlock:
804+ spin_unlock_irq(q->queue_lock);
805+out_free:
806+ blk_queue_bypass_end(q);
807+ list_for_each_entry_safe(pd, n, &pds, alloc_node)
808+ kfree(pd);
809+ return ret;
1655810 }
1656−EXPORT_SYMBOL_GPL(blkio_policy_register);
811+EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1657812
1658−void blkio_policy_unregister(struct blkio_policy_type *blkiop)
813+/**
814+ * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
815+ * @q: request_queue of interest
816+ * @pol: blkcg policy to deactivate
817+ *
818+ * Deactivate @pol on @q. Follows the same synchronization rules as
819+ * blkcg_activate_policy().
820+ */
821+void blkcg_deactivate_policy(struct request_queue *q,
822+ const struct blkcg_policy *pol)
1659823 {
1660− spin_lock(&blkio_list_lock);
1661− list_del_init(&blkiop->list);
1662− spin_unlock(&blkio_list_lock);
824+ struct blkcg_gq *blkg;
825+
826+ if (!blkcg_policy_enabled(q, pol))
827+ return;
828+
829+ blk_queue_bypass_start(q);
830+ spin_lock_irq(q->queue_lock);
831+
832+ __clear_bit(pol->plid, q->blkcg_pols);
833+
834+ /* if no policy is left, no need for blkgs - shoot them down */
835+ if (bitmap_empty(q->blkcg_pols, BLKCG_MAX_POLS))
836+ blkg_destroy_all(q);
837+
838+ list_for_each_entry(blkg, &q->blkg_list, q_node) {
839+ /* grab blkcg lock too while removing @pd from @blkg */
840+ spin_lock(&blkg->blkcg->lock);
841+
842+ if (pol->pd_exit_fn)
843+ pol->pd_exit_fn(blkg);
844+
845+ kfree(blkg->pd[pol->plid]);
846+ blkg->pd[pol->plid] = NULL;
847+
848+ spin_unlock(&blkg->blkcg->lock);
849+ }
850+
851+ spin_unlock_irq(q->queue_lock);
852+ blk_queue_bypass_end(q);
1663853 }
1664−EXPORT_SYMBOL_GPL(blkio_policy_unregister);
854+EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1665855
1666−static int __init init_cgroup_blkio(void)
856+/**
857+ * blkcg_policy_register - register a blkcg policy
858+ * @pol: blkcg policy to register
859+ *
860+ * Register @pol with blkcg core. Might sleep and @pol may be modified on
861+ * successful registration. Returns 0 on success and -errno on failure.
862+ */
863+int blkcg_policy_register(struct blkcg_policy *pol)
1667864 {
1668− return cgroup_load_subsys(&blkio_subsys);
865+ int i, ret;
866+
867+ if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data)))
868+ return -EINVAL;
869+
870+ mutex_lock(&blkcg_pol_mutex);
871+
872+ /* find an empty slot */
873+ ret = -ENOSPC;
874+ for (i = 0; i < BLKCG_MAX_POLS; i++)
875+ if (!blkcg_policy[i])
876+ break;
877+ if (i >= BLKCG_MAX_POLS)
878+ goto out_unlock;
879+
880+ /* register and update blkgs */
881+ pol->plid = i;
882+ blkcg_policy[i] = pol;
883+
884+ /* everything is in place, add intf files for the new policy */
885+ if (pol->cftypes)
886+ WARN_ON(cgroup_add_cftypes(&blkio_subsys, pol->cftypes));
887+ ret = 0;
888+out_unlock:
889+ mutex_unlock(&blkcg_pol_mutex);
890+ return ret;
1669891 }
892+EXPORT_SYMBOL_GPL(blkcg_policy_register);
1670893
1671−static void __exit exit_cgroup_blkio(void)
894+/**
895+ * blkcg_policy_unregister - unregister a blkcg policy
896+ * @pol: blkcg policy to unregister
897+ *
898+ * Undo blkcg_policy_register(@pol). Might sleep.
899+ */
900+void blkcg_policy_unregister(struct blkcg_policy *pol)
1672901 {
1673− cgroup_unload_subsys(&blkio_subsys);
1674−}
902+ mutex_lock(&blkcg_pol_mutex);
1675903
1676−module_init(init_cgroup_blkio);
1677−module_exit(exit_cgroup_blkio);
1678−MODULE_LICENSE("GPL");
904+ if (WARN_ON(blkcg_policy[pol->plid] != pol))
905+ goto out_unlock;
906+
907+ /* kill the intf files first */
908+ if (pol->cftypes)
909+ cgroup_rm_cftypes(&blkio_subsys, pol->cftypes);
910+
911+ /* unregister and update blkgs */
912+ blkcg_policy[pol->plid] = NULL;
913+out_unlock:
914+ mutex_unlock(&blkcg_pol_mutex);
915+}
916+EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
block/blk-cgroup.h+334−313
@@ -15,350 +15,371 @@ · 14 unmodified lines
1515
1616 #include <linux/cgroup.h>
1717 #include <linux/u64_stats_sync.h>
18−
19−enum blkio_policy_id {
20− BLKIO_POLICY_PROP = 0, /* Proportional Bandwidth division */
21− BLKIO_POLICY_THROTL, /* Throttling */
22−};
18+#include <linux/seq_file.h>
19+#include <linux/radix-tree.h>
2320
2421 /* Max limits for throttle policy */
2522 #define THROTL_IOPS_MAX UINT_MAX
2623
27−#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE)
28−
29−#ifndef CONFIG_BLK_CGROUP
30−/* When blk-cgroup is a module, its subsys_id isn't a compile-time constant */
31−extern struct cgroup_subsys blkio_subsys;
32−#define blkio_subsys_id blkio_subsys.subsys_id
33−#endif
34−
35−enum stat_type {
36− /* Total time spent (in ns) between request dispatch to the driver and
37− * request completion for IOs doen by this cgroup. This may not be
38− * accurate when NCQ is turned on. */
39− BLKIO_STAT_SERVICE_TIME = 0,
40− /* Total time spent waiting in scheduler queue in ns */
41− BLKIO_STAT_WAIT_TIME,
42− /* Number of IOs queued up */
43− BLKIO_STAT_QUEUED,
44− /* All the single valued stats go below this */
45− BLKIO_STAT_TIME,
46−#ifdef CONFIG_DEBUG_BLK_CGROUP
47− /* Time not charged to this cgroup */
48− BLKIO_STAT_UNACCOUNTED_TIME,
49− BLKIO_STAT_AVG_QUEUE_SIZE,
50− BLKIO_STAT_IDLE_TIME,
51− BLKIO_STAT_EMPTY_TIME,
52− BLKIO_STAT_GROUP_WAIT_TIME,
53− BLKIO_STAT_DEQUEUE
54−#endif
55−};
24+/* CFQ specific, out here for blkcg->cfq_weight */
25+#define CFQ_WEIGHT_MIN 10
26+#define CFQ_WEIGHT_MAX 1000
27+#define CFQ_WEIGHT_DEFAULT 500
5628
57−/* Per cpu stats */
58−enum stat_type_cpu {
59− BLKIO_STAT_CPU_SECTORS,
60− /* Total bytes transferred */
61− BLKIO_STAT_CPU_SERVICE_BYTES,
62− /* Total IOs serviced, post merge */
63− BLKIO_STAT_CPU_SERVICED,
64− /* Number of IOs merged */
65− BLKIO_STAT_CPU_MERGED,
66− BLKIO_STAT_CPU_NR
67−};
29+#ifdef CONFIG_BLK_CGROUP
6830
69−enum stat_sub_type {
70− BLKIO_STAT_READ = 0,
71− BLKIO_STAT_WRITE,
72− BLKIO_STAT_SYNC,
73− BLKIO_STAT_ASYNC,
74− BLKIO_STAT_TOTAL
75−};
31+enum blkg_rwstat_type {
32+ BLKG_RWSTAT_READ,
33+ BLKG_RWSTAT_WRITE,
34+ BLKG_RWSTAT_SYNC,
35+ BLKG_RWSTAT_ASYNC,
7636
77−/* blkg state flags */
78−enum blkg_state_flags {
79− BLKG_waiting = 0,
80− BLKG_idling,
81− BLKG_empty,
37+ BLKG_RWSTAT_NR,
38+ BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
8239 };
8340
84−/* cgroup files owned by proportional weight policy */
85−enum blkcg_file_name_prop {
86− BLKIO_PROP_weight = 1,
87− BLKIO_PROP_weight_device,
88− BLKIO_PROP_io_service_bytes,
89− BLKIO_PROP_io_serviced,
90− BLKIO_PROP_time,
91− BLKIO_PROP_sectors,
92− BLKIO_PROP_unaccounted_time,
93− BLKIO_PROP_io_service_time,
94− BLKIO_PROP_io_wait_time,
95− BLKIO_PROP_io_merged,
96− BLKIO_PROP_io_queued,
97− BLKIO_PROP_avg_queue_size,
98− BLKIO_PROP_group_wait_time,
99− BLKIO_PROP_idle_time,
100− BLKIO_PROP_empty_time,
101− BLKIO_PROP_dequeue,
102−};
41+struct blkcg_gq;
10342
104−/* cgroup files owned by throttle policy */
105−enum blkcg_file_name_throtl {
106− BLKIO_THROTL_read_bps_device,
107− BLKIO_THROTL_write_bps_device,
108− BLKIO_THROTL_read_iops_device,
109− BLKIO_THROTL_write_iops_device,
110− BLKIO_THROTL_io_service_bytes,
111− BLKIO_THROTL_io_serviced,
112−};
43+struct blkcg {
44+ struct cgroup_subsys_state css;
45+ spinlock_t lock;
11346
114−struct blkio_cgroup {
115− struct cgroup_subsys_state css;
116− unsigned int weight;
117− spinlock_t lock;
118− struct hlist_head blkg_list;
119− struct list_head policy_list; /* list of blkio_policy_node */
120−};
47+ struct radix_tree_root blkg_tree;
48+ struct blkcg_gq *blkg_hint;
49+ struct hlist_head blkg_list;
50+
51+ /* for policies to test whether associated blkcg has changed */
52+ uint64_t id;
12153
122−struct blkio_group_stats {
123− /* total disk time and nr sectors dispatched by this group */
124− uint64_t time;
125− uint64_t stat_arr[BLKIO_STAT_QUEUED + 1][BLKIO_STAT_TOTAL];
126−#ifdef CONFIG_DEBUG_BLK_CGROUP
127− /* Time not charged to this cgroup */
128− uint64_t unaccounted_time;
129−
130− /* Sum of number of IOs queued across all samples */
131− uint64_t avg_queue_size_sum;
132− /* Count of samples taken for average */
133− uint64_t avg_queue_size_samples;
134− /* How many times this group has been removed from service tree */
135− unsigned long dequeue;
136−
137− /* Total time spent waiting for it to be assigned a timeslice. */
138− uint64_t group_wait_time;
139− uint64_t start_group_wait_time;
140−
141− /* Time spent idling for this blkio_group */
142− uint64_t idle_time;
143− uint64_t start_idle_time;
144− /*
145− * Total time when we have requests queued and do not contain the
146− * current active queue.
147− */
148− uint64_t empty_time;
149− uint64_t start_empty_time;
150− uint16_t flags;
151−#endif
54+ /* TODO: per-policy storage in blkcg */
55+ unsigned int cfq_weight; /* belongs to cfq */
15256 };
15357
154−/* Per cpu blkio group stats */
155−struct blkio_group_stats_cpu {
156− uint64_t sectors;
157− uint64_t stat_arr_cpu[BLKIO_STAT_CPU_NR][BLKIO_STAT_TOTAL];
158− struct u64_stats_sync syncp;
58+struct blkg_stat {
59+ struct u64_stats_sync syncp;
60+ uint64_t cnt;
15961 };
16062
161−struct blkio_group {
162− /* An rcu protected unique identifier for the group */
163− void *key;
164− struct hlist_node blkcg_node;
165− unsigned short blkcg_id;
166− /* Store cgroup path */
167− char path[128];
168− /* The device MKDEV(major, minor), this group has been created for */
169− dev_t dev;
170− /* policy which owns this blk group */
171− enum blkio_policy_id plid;
172−
173− /* Need to serialize the stats in the case of reset/update */
174− spinlock_t stats_lock;
175− struct blkio_group_stats stats;
176− /* Per cpu stats pointer */
177− struct blkio_group_stats_cpu __percpu *stats_cpu;
63+struct blkg_rwstat {
64+ struct u64_stats_sync syncp;
65+ uint64_t cnt[BLKG_RWSTAT_NR];
17866 };
17967
180−struct blkio_policy_node {
181− struct list_head node;
182− dev_t dev;
183− /* This node belongs to max bw policy or porportional weight policy */
184− enum blkio_policy_id plid;
185− /* cgroup file to which this rule belongs to */
186− int fileid;
187−
188− union {
189− unsigned int weight;
190− /*
191− * Rate read/write in terms of bytes per second
192− * Whether this rate represents read or write is determined
193− * by file type "fileid".
194− */
195− u64 bps;
196− unsigned int iops;
197− } val;
68+/*
69+ * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
70+ * request_queue (q). This is used by blkcg policies which need to track
71+ * information per blkcg - q pair.
72+ *
73+ * There can be multiple active blkcg policies and each has its private
74+ * data on each blkg, the size of which is determined by
75+ * blkcg_policy->pd_size. blkcg core allocates and frees such areas
76+ * together with blkg and invokes pd_init/exit_fn() methods.
77+ *
78+ * Such private data must embed struct blkg_policy_data (pd) at the
79+ * beginning and pd_size can't be smaller than pd.
80+ */
81+struct blkg_policy_data {
82+ /* the blkg this per-policy data belongs to */
83+ struct blkcg_gq *blkg;
84+
85+ /* used during policy activation */
86+ struct list_head alloc_node;
19887 };
19988
200−extern unsigned int blkcg_get_weight(struct blkio_cgroup *blkcg,
201− dev_t dev);
202−extern uint64_t blkcg_get_read_bps(struct blkio_cgroup *blkcg,
203− dev_t dev);
204−extern uint64_t blkcg_get_write_bps(struct blkio_cgroup *blkcg,
205− dev_t dev);
206−extern unsigned int blkcg_get_read_iops(struct blkio_cgroup *blkcg,
207− dev_t dev);
208−extern unsigned int blkcg_get_write_iops(struct blkio_cgroup *blkcg,
209− dev_t dev);
210−
211−typedef void (blkio_unlink_group_fn) (void *key, struct blkio_group *blkg);
212−
213−typedef void (blkio_update_group_weight_fn) (void *key,
214− struct blkio_group *blkg, unsigned int weight);
215−typedef void (blkio_update_group_read_bps_fn) (void * key,
216− struct blkio_group *blkg, u64 read_bps);
217−typedef void (blkio_update_group_write_bps_fn) (void *key,
218− struct blkio_group *blkg, u64 write_bps);
219−typedef void (blkio_update_group_read_iops_fn) (void *key,
220− struct blkio_group *blkg, unsigned int read_iops);
221−typedef void (blkio_update_group_write_iops_fn) (void *key,
222− struct blkio_group *blkg, unsigned int write_iops);
223−
224−struct blkio_policy_ops {
225− blkio_unlink_group_fn *blkio_unlink_group_fn;
226− blkio_update_group_weight_fn *blkio_update_group_weight_fn;
227− blkio_update_group_read_bps_fn *blkio_update_group_read_bps_fn;
228− blkio_update_group_write_bps_fn *blkio_update_group_write_bps_fn;
229− blkio_update_group_read_iops_fn *blkio_update_group_read_iops_fn;
230− blkio_update_group_write_iops_fn *blkio_update_group_write_iops_fn;
89+/* association between a blk cgroup and a request queue */
90+struct blkcg_gq {
91+ /* Pointer to the associated request_queue */
92+ struct request_queue *q;
93+ struct list_head q_node;
94+ struct hlist_node blkcg_node;
95+ struct blkcg *blkcg;
96+ /* reference count */
97+ int refcnt;
98+
99+ struct blkg_policy_data *pd[BLKCG_MAX_POLS];
100+
101+ struct rcu_head rcu_head;
231102 };
232103
233−struct blkio_policy_type {
234− struct list_head list;
235− struct blkio_policy_ops ops;
236− enum blkio_policy_id plid;
104+typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg);
105+typedef void (blkcg_pol_exit_pd_fn)(struct blkcg_gq *blkg);
106+typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkcg_gq *blkg);
107+
108+struct blkcg_policy {
109+ int plid;
110+ /* policy specific private data size */
111+ size_t pd_size;
112+ /* cgroup files for the policy */
113+ struct cftype *cftypes;
114+
115+ /* operations */
116+ blkcg_pol_init_pd_fn *pd_init_fn;
117+ blkcg_pol_exit_pd_fn *pd_exit_fn;
118+ blkcg_pol_reset_pd_stats_fn *pd_reset_stats_fn;
237119 };
238120
121+extern struct blkcg blkcg_root;
122+
123+struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup);
124+struct blkcg *bio_blkcg(struct bio *bio);
125+struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q);
126+struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
127+ struct request_queue *q);
128+int blkcg_init_queue(struct request_queue *q);
129+void blkcg_drain_queue(struct request_queue *q);
130+void blkcg_exit_queue(struct request_queue *q);
131+
239132 /* Blkio controller policy registration */
240−extern void blkio_policy_register(struct blkio_policy_type *);
241−extern void blkio_policy_unregister(struct blkio_policy_type *);
133+int blkcg_policy_register(struct blkcg_policy *pol);
134+void blkcg_policy_unregister(struct blkcg_policy *pol);
135+int blkcg_activate_policy(struct request_queue *q,
136+ const struct blkcg_policy *pol);
137+void blkcg_deactivate_policy(struct request_queue *q,
138+ const struct blkcg_policy *pol);
139+
140+void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
141+ u64 (*prfill)(struct seq_file *,
142+ struct blkg_policy_data *, int),
143+ const struct blkcg_policy *pol, int data,
144+ bool show_total);
145+u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
146+u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
147+ const struct blkg_rwstat *rwstat);
148+u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
149+u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
150+ int off);
151+
152+struct blkg_conf_ctx {
153+ struct gendisk *disk;
154+ struct blkcg_gq *blkg;
155+ u64 v;
156+};
157+
158+int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
159+ const char *input, struct blkg_conf_ctx *ctx);
160+void blkg_conf_finish(struct blkg_conf_ctx *ctx);
161+
162+
163+/**
164+ * blkg_to_pdata - get policy private data
165+ * @blkg: blkg of interest
166+ * @pol: policy of interest
167+ *
168+ * Return pointer to private data associated with the @blkg-@pol pair.
169+ */
170+static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
171+ struct blkcg_policy *pol)
172+{
173+ return blkg ? blkg->pd[pol->plid] : NULL;
174+}
175+
176+/**
177+ * pdata_to_blkg - get blkg associated with policy private data
178+ * @pd: policy private data of interest
179+ *
180+ * @pd is policy private data. Determine the blkg it's associated with.
181+ */
182+static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
183+{
184+ return pd ? pd->blkg : NULL;
185+}
186+
187+/**
188+ * blkg_path - format cgroup path of blkg
189+ * @blkg: blkg of interest
190+ * @buf: target buffer
191+ * @buflen: target buffer length
192+ *
193+ * Format the path of the cgroup of @blkg into @buf.
194+ */
195+static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
196+{
197+ int ret;
198+
199+ rcu_read_lock();
200+ ret = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
201+ rcu_read_unlock();
202+ if (ret)
203+ strncpy(buf, "<unavailable>", buflen);
204+ return ret;
205+}
242206
243−static inline char *blkg_path(struct blkio_group *blkg)
207+/**
208+ * blkg_get - get a blkg reference
209+ * @blkg: blkg to get
210+ *
211+ * The caller should be holding queue_lock and an existing reference.
212+ */
213+static inline void blkg_get(struct blkcg_gq *blkg)
244214 {
245− return blkg->path;
215+ lockdep_assert_held(blkg->q->queue_lock);
216+ WARN_ON_ONCE(!blkg->refcnt);
217+ blkg->refcnt++;
246218 }
247219
248−#else
220+void __blkg_release(struct blkcg_gq *blkg);
249221
250−struct blkio_group {
222+/**
223+ * blkg_put - put a blkg reference
224+ * @blkg: blkg to put
225+ *
226+ * The caller should be holding queue_lock.
227+ */
228+static inline void blkg_put(struct blkcg_gq *blkg)
229+{
230+ lockdep_assert_held(blkg->q->queue_lock);
231+ WARN_ON_ONCE(blkg->refcnt <= 0);
232+ if (!--blkg->refcnt)
233+ __blkg_release(blkg);
234+}
235+
236+/**
237+ * blkg_stat_add - add a value to a blkg_stat
238+ * @stat: target blkg_stat
239+ * @val: value to add
240+ *
241+ * Add @val to @stat. The caller is responsible for synchronizing calls to
242+ * this function.
243+ */
244+static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
245+{
246+ u64_stats_update_begin(&stat->syncp);
247+ stat->cnt += val;
248+ u64_stats_update_end(&stat->syncp);
249+}
250+
251+/**
252+ * blkg_stat_read - read the current value of a blkg_stat
253+ * @stat: blkg_stat to read
254+ *
255+ * Read the current value of @stat. This function can be called without
256+ * synchroniztion and takes care of u64 atomicity.
257+ */
258+static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
259+{
260+ unsigned int start;
261+ uint64_t v;
262+
263+ do {
264+ start = u64_stats_fetch_begin(&stat->syncp);
265+ v = stat->cnt;
266+ } while (u64_stats_fetch_retry(&stat->syncp, start));
267+
268+ return v;
269+}
270+
271+/**
272+ * blkg_stat_reset - reset a blkg_stat
273+ * @stat: blkg_stat to reset
274+ */
275+static inline void blkg_stat_reset(struct blkg_stat *stat)
276+{
277+ stat->cnt = 0;
278+}
279+
280+/**
281+ * blkg_rwstat_add - add a value to a blkg_rwstat
282+ * @rwstat: target blkg_rwstat
283+ * @rw: mask of REQ_{WRITE|SYNC}
284+ * @val: value to add
285+ *
286+ * Add @val to @rwstat. The counters are chosen according to @rw. The
287+ * caller is responsible for synchronizing calls to this function.
288+ */
289+static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
290+ int rw, uint64_t val)
291+{
292+ u64_stats_update_begin(&rwstat->syncp);
293+
294+ if (rw & REQ_WRITE)
295+ rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
296+ else
297+ rwstat->cnt[BLKG_RWSTAT_READ] += val;
298+ if (rw & REQ_SYNC)
299+ rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
300+ else
301+ rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;
302+
303+ u64_stats_update_end(&rwstat->syncp);
304+}
305+
306+/**
307+ * blkg_rwstat_read - read the current values of a blkg_rwstat
308+ * @rwstat: blkg_rwstat to read
309+ *
310+ * Read the current snapshot of @rwstat and return it as the return value.
311+ * This function can be called without synchronization and takes care of
312+ * u64 atomicity.
313+ */
314+static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
315+{
316+ unsigned int start;
317+ struct blkg_rwstat tmp;
318+
319+ do {
320+ start = u64_stats_fetch_begin(&rwstat->syncp);
321+ tmp = *rwstat;
322+ } while (u64_stats_fetch_retry(&rwstat->syncp, start));
323+
324+ return tmp;
325+}
326+
327+/**
328+ * blkg_rwstat_sum - read the total count of a blkg_rwstat
329+ * @rwstat: blkg_rwstat to read
330+ *
331+ * Return the total count of @rwstat regardless of the IO direction. This
332+ * function can be called without synchronization and takes care of u64
333+ * atomicity.
334+ */
335+static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat)
336+{
337+ struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);
338+
339+ return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
340+}
341+
342+/**
343+ * blkg_rwstat_reset - reset a blkg_rwstat
344+ * @rwstat: blkg_rwstat to reset
345+ */
346+static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
347+{
348+ memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
349+}
350+
351+#else /* CONFIG_BLK_CGROUP */
352+
353+struct cgroup;
354+
355+struct blkg_policy_data {
251356 };
252357
253−struct blkio_policy_type {
358+struct blkcg_gq {
254359 };
255360
256−static inline void blkio_policy_register(struct blkio_policy_type *blkiop) { }
257−static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { }
258−
259−static inline char *blkg_path(struct blkio_group *blkg) { return NULL; }
260−
261−#endif
262−
263−#define BLKIO_WEIGHT_MIN 10
264−#define BLKIO_WEIGHT_MAX 1000
265−#define BLKIO_WEIGHT_DEFAULT 500
266−
267−#ifdef CONFIG_DEBUG_BLK_CGROUP
268−void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg);
269−void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
270− unsigned long dequeue);
271−void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg);
272−void blkiocg_update_idle_time_stats(struct blkio_group *blkg);
273−void blkiocg_set_start_empty_time(struct blkio_group *blkg);
274−
275−#define BLKG_FLAG_FNS(name) \
276−static inline void blkio_mark_blkg_##name( \
277− struct blkio_group_stats *stats) \
278−{ \
279− stats->flags |= (1 << BLKG_##name); \
280−} \
281−static inline void blkio_clear_blkg_##name( \
282− struct blkio_group_stats *stats) \
283−{ \
284− stats->flags &= ~(1 << BLKG_##name); \
285−} \
286−static inline int blkio_blkg_##name(struct blkio_group_stats *stats) \
287−{ \
288− return (stats->flags & (1 << BLKG_##name)) != 0; \
289−} \
290−
291−BLKG_FLAG_FNS(waiting)
292−BLKG_FLAG_FNS(idling)
293−BLKG_FLAG_FNS(empty)
294−#undef BLKG_FLAG_FNS
295−#else
296−static inline void blkiocg_update_avg_queue_size_stats(
297− struct blkio_group *blkg) {}
298−static inline void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
299− unsigned long dequeue) {}
300−static inline void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg)
301−{}
302−static inline void blkiocg_update_idle_time_stats(struct blkio_group *blkg) {}
303−static inline void blkiocg_set_start_empty_time(struct blkio_group *blkg) {}
304−#endif
305−
306−#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE)
307−extern struct blkio_cgroup blkio_root_cgroup;
308−extern struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup);
309−extern struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk);
310−extern void blkiocg_add_blkio_group(struct blkio_cgroup *blkcg,
311− struct blkio_group *blkg, void *key, dev_t dev,
312− enum blkio_policy_id plid);
313−extern int blkio_alloc_blkg_stats(struct blkio_group *blkg);
314−extern int blkiocg_del_blkio_group(struct blkio_group *blkg);
315−extern struct blkio_group *blkiocg_lookup_group(struct blkio_cgroup *blkcg,
316− void *key);
317−void blkiocg_update_timeslice_used(struct blkio_group *blkg,
318− unsigned long time,
319− unsigned long unaccounted_time);
320−void blkiocg_update_dispatch_stats(struct blkio_group *blkg, uint64_t bytes,
321− bool direction, bool sync);
322−void blkiocg_update_completion_stats(struct blkio_group *blkg,
323− uint64_t start_time, uint64_t io_start_time, bool direction, bool sync);
324−void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction,
325− bool sync);
326−void blkiocg_update_io_add_stats(struct blkio_group *blkg,
327− struct blkio_group *curr_blkg, bool direction, bool sync);
328−void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
329− bool direction, bool sync);
330−#else
331−struct cgroup;
332−static inline struct blkio_cgroup *
333−cgroup_to_blkio_cgroup(struct cgroup *cgroup) { return NULL; }
334−static inline struct blkio_cgroup *
335−task_blkio_cgroup(struct task_struct *tsk) { return NULL; }
336−
337−static inline void blkiocg_add_blkio_group(struct blkio_cgroup *blkcg,
338− struct blkio_group *blkg, void *key, dev_t dev,
339− enum blkio_policy_id plid) {}
340−
341−static inline int blkio_alloc_blkg_stats(struct blkio_group *blkg) { return 0; }
342−
343−static inline int
344−blkiocg_del_blkio_group(struct blkio_group *blkg) { return 0; }
345−
346−static inline struct blkio_group *
347−blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key) { return NULL; }
348−static inline void blkiocg_update_timeslice_used(struct blkio_group *blkg,
349− unsigned long time,
350− unsigned long unaccounted_time)
351−{}
352−static inline void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
353− uint64_t bytes, bool direction, bool sync) {}
354−static inline void blkiocg_update_completion_stats(struct blkio_group *blkg,
355− uint64_t start_time, uint64_t io_start_time, bool direction,
356− bool sync) {}
357−static inline void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
358− bool direction, bool sync) {}
359−static inline void blkiocg_update_io_add_stats(struct blkio_group *blkg,
360− struct blkio_group *curr_blkg, bool direction, bool sync) {}
361−static inline void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
362− bool direction, bool sync) {}
363−#endif
364−#endif /* _BLK_CGROUP_H */
361+struct blkcg_policy {
362+};
363+
364+static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; }
365+static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
366+static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
367+static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
368+static inline void blkcg_drain_queue(struct request_queue *q) { }
369+static inline void blkcg_exit_queue(struct request_queue *q) { }
370+static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
371+static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
372+static inline int blkcg_activate_policy(struct request_queue *q,
373+ const struct blkcg_policy *pol) { return 0; }
374+static inline void blkcg_deactivate_policy(struct request_queue *q,
375+ const struct blkcg_policy *pol) { }
376+
377+static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
378+ struct blkcg_policy *pol) { return NULL; }
379+static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
380+static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
381+static inline void blkg_get(struct blkcg_gq *blkg) { }
382+static inline void blkg_put(struct blkcg_gq *blkg) { }
383+
384+#endif /* CONFIG_BLK_CGROUP */
385+#endif /* _BLK_CGROUP_H */
block/blk-core.c+188−93
@@ -29,11 +29,13 @@ · 28 unmodified lines
2929 #include <linux/fault-inject.h>
3030 #include <linux/list_sort.h>
3131 #include <linux/delay.h>
32+#include <linux/ratelimit.h>
3233
3334 #define CREATE_TRACE_POINTS
3435 #include <trace/events/block.h>
3536
3637 #include "blk.h"
38+#include "blk-cgroup.h"
3739
3840 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
3941 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
@@ -280,7 +282,7 @@ EXPORT_SYMBOL(blk_stop_queue); · 240 unmodified lines
280282 *
281283 * This function does not cancel any asynchronous activity arising
282284 * out of elevator or throttling code. That would require elevaotor_exit()
283− * and blk_throtl_exit() to be called with queue lock initialized.
285+ * and blkcg_exit_queue() to be called with queue lock initialized.
284286 *
285287 */
286288 void blk_sync_queue(struct request_queue *q)
@@ -365,17 +367,23 @@ void blk_drain_queue(struct request_queue *q, bool drain_all) · 78 unmodified lines
365367
366368 spin_lock_irq(q->queue_lock);
367369
368− elv_drain_elevator(q);
369− if (drain_all)
370− blk_throtl_drain(q);
370+ /*
371+ * The caller might be trying to drain @q before its
372+ * elevator is initialized.
373+ */
374+ if (q->elevator)
375+ elv_drain_elevator(q);
376+
377+ blkcg_drain_queue(q);
371378
372379 /*
373380 * This function might be called on a queue which failed
374− * driver init after queue creation. Some drivers
375− * (e.g. fd) get unhappy in such cases. Kick queue iff
376− * dispatch queue has something on it.
381+ * driver init after queue creation or is not yet fully
382+ * active yet. Some drivers (e.g. fd and loop) get unhappy
383+ * in such cases. Kick queue iff dispatch queue has
384+ * something on it and @q has request_fn set.
377385 */
378− if (!list_empty(&q->queue_head))
386+ if (!list_empty(&q->queue_head) && q->request_fn)
379387 __blk_run_queue(q);
380388
381389 drain |= q->rq.elvpriv;
@@ -402,6 +410,49 @@ void blk_drain_queue(struct request_queue *q, bool drain_all) · 20 unmodified lines
402410 }
403411 }
404412
413+/**
414+ * blk_queue_bypass_start - enter queue bypass mode
415+ * @q: queue of interest
416+ *
417+ * In bypass mode, only the dispatch FIFO queue of @q is used. This
418+ * function makes @q enter bypass mode and drains all requests which were
419+ * throttled or issued before. On return, it's guaranteed that no request
420+ * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
421+ * inside queue or RCU read lock.
422+ */
423+void blk_queue_bypass_start(struct request_queue *q)
424+{
425+ bool drain;
426+
427+ spin_lock_irq(q->queue_lock);
428+ drain = !q->bypass_depth++;
429+ queue_flag_set(QUEUE_FLAG_BYPASS, q);
430+ spin_unlock_irq(q->queue_lock);
431+
432+ if (drain) {
433+ blk_drain_queue(q, false);
434+ /* ensure blk_queue_bypass() is %true inside RCU read lock */
435+ synchronize_rcu();
436+ }
437+}
438+EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
439+
440+/**
441+ * blk_queue_bypass_end - leave queue bypass mode
442+ * @q: queue of interest
443+ *
444+ * Leave bypass mode and restore the normal queueing behavior.
445+ */
446+void blk_queue_bypass_end(struct request_queue *q)
447+{
448+ spin_lock_irq(q->queue_lock);
449+ if (!--q->bypass_depth)
450+ queue_flag_clear(QUEUE_FLAG_BYPASS, q);
451+ WARN_ON_ONCE(q->bypass_depth < 0);
452+ spin_unlock_irq(q->queue_lock);
453+}
454+EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
455+
405456 /**
406457 * blk_cleanup_queue - shutdown a request queue
407458 * @q: request queue to shutdown
@@ -418,6 +469,19 @@ void blk_cleanup_queue(struct request_queue *q) · 10 unmodified lines
418469 queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
419470
420471 spin_lock_irq(lock);
472+
473+ /*
474+ * Dead queue is permanently in bypass mode till released. Note
475+ * that, unlike blk_queue_bypass_start(), we aren't performing
476+ * synchronize_rcu() after entering bypass mode to avoid the delay
477+ * as some drivers create and destroy a lot of queues while
478+ * probing. This is still safe because blk_release_queue() will be
479+ * called only after the queue refcnt drops to zero and nothing,
480+ * RCU or not, would be traversing the queue by then.
481+ */
482+ q->bypass_depth++;
483+ queue_flag_set(QUEUE_FLAG_BYPASS, q);
484+
421485 queue_flag_set(QUEUE_FLAG_NOMERGES, q);
422486 queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
423487 queue_flag_set(QUEUE_FLAG_DEAD, q);
@@ -428,13 +492,8 @@ void blk_cleanup_queue(struct request_queue *q) · 4 unmodified lines
428492 spin_unlock_irq(lock);
429493 mutex_unlock(&q->sysfs_lock);
430494
431− /*
432− * Drain all requests queued before DEAD marking. The caller might
433− * be trying to tear down @q before its elevator is initialized, in
434− * which case we don't want to call into draining.
435− */
436− if (q->elevator)
437− blk_drain_queue(q, true);
495+ /* drain all requests queued before DEAD marking */
496+ blk_drain_queue(q, true);
438497
439498 /* @q won't process any more request, flush async actions */
440499 del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
@@ -498,14 +557,15 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) · 57 unmodified lines
498557 if (err)
499558 goto fail_id;
500559
501− if (blk_throtl_init(q))
502− goto fail_id;
503−
504560 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
505561 laptop_mode_timer_fn, (unsigned long) q);
506562 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
563+ INIT_LIST_HEAD(&q->queue_head);
507564 INIT_LIST_HEAD(&q->timeout_list);
508565 INIT_LIST_HEAD(&q->icq_list);
566+#ifdef CONFIG_BLK_CGROUP
567+ INIT_LIST_HEAD(&q->blkg_list);
568+#endif
509569 INIT_LIST_HEAD(&q->flush_queue[0]);
510570 INIT_LIST_HEAD(&q->flush_queue[1]);
511571 INIT_LIST_HEAD(&q->flush_data_in_flight);
@@ -522,6 +582,18 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) · 10 unmodified lines
522582 */
523583 q->queue_lock = &q->__queue_lock;
524584
585+ /*
586+ * A queue starts its life with bypass turned on to avoid
587+ * unnecessary bypass on/off overhead and nasty surprises during
588+ * init. The initial bypass will be finished at the end of
589+ * blk_init_allocated_queue().
590+ */
591+ q->bypass_depth = 1;
592+ __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
593+
594+ if (blkcg_init_queue(q))
595+ goto fail_id;
596+
525597 return q;
526598
527599 fail_id:
@@ -614,15 +686,15 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, · 86 unmodified lines
614686
615687 q->sg_reserved_size = INT_MAX;
616688
617− /*
618− * all done
619− */
620− if (!elevator_init(q, NULL)) {
621− blk_queue_congestion_threshold(q);
622− return q;
623− }
689+ /* init elevator */
690+ if (elevator_init(q, NULL))
691+ return NULL;
624692
625− return NULL;
693+ blk_queue_congestion_threshold(q);
694+
695+ /* all done, end the initial bypass */
696+ blk_queue_bypass_end(q);
697+ return q;
626698 }
627699 EXPORT_SYMBOL(blk_init_allocated_queue);
628700
@@ -648,33 +720,6 @@ static inline void blk_free_request(struct request_queue *q, struct request *rq) · 19 unmodified lines
648720 mempool_free(rq, q->rq.rq_pool);
649721 }
650722
651−static struct request *
652−blk_alloc_request(struct request_queue *q, struct io_cq *icq,
653− unsigned int flags, gfp_t gfp_mask)
654−{
655− struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
656−
657− if (!rq)
658− return NULL;
659−
660− blk_rq_init(q, rq);
661−
662− rq->cmd_flags = flags | REQ_ALLOCED;
663−
664− if (flags & REQ_ELVPRIV) {
665− rq->elv.icq = icq;
666− if (unlikely(elv_set_request(q, rq, gfp_mask))) {
667− mempool_free(rq, q->rq.rq_pool);
668− return NULL;
669− }
670− /* @rq->elv.icq holds on to io_context until @rq is freed */
671− if (icq)
672− get_io_context(icq->ioc);
673− }
674−
675− return rq;
676−}
677−
678723 /*
679724 * ioc_batching returns true if the ioc is a valid batching request and
680725 * should be given priority access to a request.
@@ -762,6 +807,22 @@ static bool blk_rq_should_init_elevator(struct bio *bio) · 81 unmodified lines
762807 return true;
763808 }
764809
810+/**
811+ * rq_ioc - determine io_context for request allocation
812+ * @bio: request being allocated is for this bio (can be %NULL)
813+ *
814+ * Determine io_context to use for request allocation for @bio. May return
815+ * %NULL if %current->io_context doesn't exist.
816+ */
817+static struct io_context *rq_ioc(struct bio *bio)
818+{
819+#ifdef CONFIG_BLK_CGROUP
820+ if (bio && bio->bi_ioc)
821+ return bio->bi_ioc;
822+#endif
823+ return current->io_context;
824+}
825+
765826 /**
766827 * get_request - get a free request
767828 * @q: request_queue to allocate request from
@@ -779,7 +840,7 @@ static bool blk_rq_should_init_elevator(struct bio *bio) · 11 unmodified lines
779840 static struct request *get_request(struct request_queue *q, int rw_flags,
780841 struct bio *bio, gfp_t gfp_mask)
781842 {
782− struct request *rq = NULL;
843+ struct request *rq;
783844 struct request_list *rl = &q->rq;
784845 struct elevator_type *et;
785846 struct io_context *ioc;
@@ -789,7 +850,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags, · 3 unmodified lines
789850 int may_queue;
790851 retry:
791852 et = q->elevator->type;
792− ioc = current->io_context;
853+ ioc = rq_ioc(bio);
793854
794855 if (unlikely(blk_queue_dead(q)))
795856 return NULL;
@@ -808,7 +869,7 @@ retry: · 12 unmodified lines
808869 */
809870 if (!ioc && !retried) {
810871 spin_unlock_irq(q->queue_lock);
811− create_io_context(current, gfp_mask, q->node);
872+ create_io_context(gfp_mask, q->node);
812873 spin_lock_irq(q->queue_lock);
813874 retried = true;
814875 goto retry;
@@ -831,7 +892,7 @@ retry: · 16 unmodified lines
831892 * process is not a "batcher", and not
832893 * exempted by the IO scheduler
833894 */
834− goto out;
895+ return NULL;
835896 }
836897 }
837898 }
@@ -844,7 +905,7 @@ retry: · 6 unmodified lines
844905 * allocated with any setting of ->nr_requests
845906 */
846907 if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
847− goto out;
908+ return NULL;
848909
849910 rl->count[is_sync]++;
850911 rl->starved[is_sync] = 0;
@@ -859,8 +920,7 @@ retry: · 8 unmodified lines
859920 * Also, lookup icq while holding queue_lock. If it doesn't exist,
860921 * it will be created after releasing queue_lock.
861922 */
862− if (blk_rq_should_init_elevator(bio) &&
863− !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags)) {
923+ if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) {
864924 rw_flags |= REQ_ELVPRIV;
865925 rl->elvpriv++;
866926 if (et->icq_cache && ioc)
@@ -871,41 +931,36 @@ retry: · 4 unmodified lines
871931 rw_flags |= REQ_IO_STAT;
872932 spin_unlock_irq(q->queue_lock);
873933
874− /* create icq if missing */
875− if ((rw_flags & REQ_ELVPRIV) && unlikely(et->icq_cache && !icq)) {
876− icq = ioc_create_icq(q, gfp_mask);
877− if (!icq)
878− goto fail_icq;
879− }
880−
881− rq = blk_alloc_request(q, icq, rw_flags, gfp_mask);
934+ /* allocate and init request */
935+ rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
936+ if (!rq)
937+ goto fail_alloc;
882938
883−fail_icq:
884− if (unlikely(!rq)) {
885− /*
886− * Allocation failed presumably due to memory. Undo anything
887− * we might have messed up.
888− *
889− * Allocating task should really be put onto the front of the
890− * wait queue, but this is pretty rare.
891− */
892− spin_lock_irq(q->queue_lock);
893− freed_request(q, rw_flags);
939+ blk_rq_init(q, rq);
940+ rq->cmd_flags = rw_flags | REQ_ALLOCED;
941+
942+ /* init elvpriv */
943+ if (rw_flags & REQ_ELVPRIV) {
944+ if (unlikely(et->icq_cache && !icq)) {
945+ create_io_context(gfp_mask, q->node);
946+ ioc = rq_ioc(bio);
947+ if (!ioc)
948+ goto fail_elvpriv;
949+
950+ icq = ioc_create_icq(ioc, q, gfp_mask);
951+ if (!icq)
952+ goto fail_elvpriv;
953+ }
894954
895− /*
896− * in the very unlikely event that allocation failed and no
897− * requests for this direction was pending, mark us starved
898− * so that freeing of a request in the other direction will
899− * notice us. another possible fix would be to split the
900− * rq mempool into READ and WRITE
901− */
902−rq_starved:
903− if (unlikely(rl->count[is_sync] == 0))
904− rl->starved[is_sync] = 1;
955+ rq->elv.icq = icq;
956+ if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
957+ goto fail_elvpriv;
905958
906− goto out;
959+ /* @rq->elv.icq holds io_context until @rq is freed */
960+ if (icq)
961+ get_io_context(icq->ioc);
907962 }
908−
963+out:
909964 /*
910965 * ioc may be NULL here, and ioc_batching will be false. That's
911966 * OK, if the queue is under the request limit then requests need
@@ -916,8 +971,48 @@ rq_starved: · 4 unmodified lines
916971 ioc->nr_batch_requests--;
917972
918973 trace_block_getrq(q, bio, rw_flags & 1);
919−out:
920974 return rq;
975+
976+fail_elvpriv:
977+ /*
978+ * elvpriv init failed. ioc, icq and elvpriv aren't mempool backed
979+ * and may fail indefinitely under memory pressure and thus
980+ * shouldn't stall IO. Treat this request as !elvpriv. This will
981+ * disturb iosched and blkcg but weird is bettern than dead.
982+ */
983+ printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n",
984+ dev_name(q->backing_dev_info.dev));
985+
986+ rq->cmd_flags &= ~REQ_ELVPRIV;
987+ rq->elv.icq = NULL;
988+
989+ spin_lock_irq(q->queue_lock);
990+ rl->elvpriv--;
991+ spin_unlock_irq(q->queue_lock);
992+ goto out;
993+
994+fail_alloc:
995+ /*
996+ * Allocation failed presumably due to memory. Undo anything we
997+ * might have messed up.
998+ *
999+ * Allocating task should really be put onto the front of the wait
1000+ * queue, but this is pretty rare.
1001+ */
1002+ spin_lock_irq(q->queue_lock);
1003+ freed_request(q, rw_flags);
1004+
1005+ /*
1006+ * in the very unlikely event that allocation failed and no
1007+ * requests for this direction was pending, mark us starved so that
1008+ * freeing of a request in the other direction will notice
1009+ * us. another possible fix would be to split the rq mempool into
1010+ * READ and WRITE
1011+ */
1012+rq_starved:
1013+ if (unlikely(rl->count[is_sync] == 0))
1014+ rl->starved[is_sync] = 1;
1015+ return NULL;
9211016 }
9221017
9231018 /**
@@ -961,7 +1056,7 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags, · 37 unmodified lines
9611056 * up to a big batch of them for a small period time.
9621057 * See ioc_batching, ioc_set_batching
9631058 */
964− create_io_context(current, GFP_NOIO, q->node);
1059+ create_io_context(GFP_NOIO, q->node);
9651060 ioc_set_batching(q, current->io_context);
9661061
9671062 spin_lock_irq(q->queue_lock);
block/blk-ioc.c+35−91
@@ -155,20 +155,20 @@ void put_io_context(struct io_context *ioc) · 154 unmodified lines
155155 }
156156 EXPORT_SYMBOL(put_io_context);
157157
158−/* Called by the exiting task */
159−void exit_io_context(struct task_struct *task)
158+/**
159+ * put_io_context_active - put active reference on ioc
160+ * @ioc: ioc of interest
161+ *
162+ * Undo get_io_context_active(). If active reference reaches zero after
163+ * put, @ioc can never issue further IOs and ioscheds are notified.
164+ */
165+void put_io_context_active(struct io_context *ioc)
160166 {
161− struct io_context *ioc;
162− struct io_cq *icq;
163167 struct hlist_node *n;
164168 unsigned long flags;
169+ struct io_cq *icq;
165170
166− task_lock(task);
167− ioc = task->io_context;
168− task->io_context = NULL;
169− task_unlock(task);
170−
171− if (!atomic_dec_and_test(&ioc->nr_tasks)) {
171+ if (!atomic_dec_and_test(&ioc->active_ref)) {
172172 put_io_context(ioc);
173173 return;
174174 }
@@ -197,6 +197,20 @@ retry: · 22 unmodified lines
197197 put_io_context(ioc);
198198 }
199199
200+/* Called by the exiting task */
201+void exit_io_context(struct task_struct *task)
202+{
203+ struct io_context *ioc;
204+
205+ task_lock(task);
206+ ioc = task->io_context;
207+ task->io_context = NULL;
208+ task_unlock(task);
209+
210+ atomic_dec(&ioc->nr_tasks);
211+ put_io_context_active(ioc);
212+}
213+
200214 /**
201215 * ioc_clear_queue - break any ioc association with the specified queue
202216 * @q: request_queue being cleared
@@ -218,19 +232,18 @@ void ioc_clear_queue(struct request_queue *q) · 15 unmodified lines
218232 }
219233 }
220234
221−void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags,
222− int node)
235+int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node)
223236 {
224237 struct io_context *ioc;
225238
226239 ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
227240 node);
228241 if (unlikely(!ioc))
229− return;
242+ return -ENOMEM;
230243
231244 /* initialize */
232245 atomic_long_set(&ioc->refcount, 1);
233− atomic_set(&ioc->nr_tasks, 1);
246+ atomic_set(&ioc->active_ref, 1);
234247 spin_lock_init(&ioc->lock);
235248 INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH);
236249 INIT_HLIST_HEAD(&ioc->icq_list);
@@ -250,6 +263,8 @@ void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags, · 13 unmodified lines
250263 else
251264 kmem_cache_free(iocontext_cachep, ioc);
252265 task_unlock(task);
266+
267+ return 0;
253268 }
254269
255270 /**
@@ -281,7 +296,7 @@ struct io_context *get_task_io_context(struct task_struct *task, · 25 unmodified lines
281296 return ioc;
282297 }
283298 task_unlock(task);
284− } while (create_io_context(task, gfp_flags, node));
299+ } while (!create_task_io_context(task, gfp_flags, node));
285300
286301 return NULL;
287302 }
@@ -325,26 +340,23 @@ EXPORT_SYMBOL(ioc_lookup_icq); · 37 unmodified lines
325340
326341 /**
327342 * ioc_create_icq - create and link io_cq
343+ * @ioc: io_context of interest
328344 * @q: request_queue of interest
329345 * @gfp_mask: allocation mask
330346 *
331− * Make sure io_cq linking %current->io_context and @q exists. If either
332− * io_context and/or icq don't exist, they will be created using @gfp_mask.
347+ * Make sure io_cq linking @ioc and @q exists. If icq doesn't exist, they
348+ * will be created using @gfp_mask.
333349 *
334350 * The caller is responsible for ensuring @ioc won't go away and @q is
335351 * alive and will stay alive until this function returns.
336352 */
337−struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask)
353+struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
354+ gfp_t gfp_mask)
338355 {
339356 struct elevator_type *et = q->elevator->type;
340− struct io_context *ioc;
341357 struct io_cq *icq;
342358
343359 /* allocate stuff */
344− ioc = create_io_context(current, gfp_mask, q->node);
345− if (!ioc)
346− return NULL;
347−
348360 icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,
349361 q->node);
350362 if (!icq)
@@ -382,74 +394,6 @@ struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask) · 31 unmodified lines
382394 return icq;
383395 }
384396
385−void ioc_set_icq_flags(struct io_context *ioc, unsigned int flags)
386−{
387− struct io_cq *icq;
388− struct hlist_node *n;
389−
390− hlist_for_each_entry(icq, n, &ioc->icq_list, ioc_node)
391− icq->flags |= flags;
392−}
393−
394−/**
395− * ioc_ioprio_changed - notify ioprio change
396− * @ioc: io_context of interest
397− * @ioprio: new ioprio
398− *
399− * @ioc's ioprio has changed to @ioprio. Set %ICQ_IOPRIO_CHANGED for all
400− * icq's. iosched is responsible for checking the bit and applying it on
401− * request issue path.
402− */
403−void ioc_ioprio_changed(struct io_context *ioc, int ioprio)
404−{
405− unsigned long flags;
406−
407− spin_lock_irqsave(&ioc->lock, flags);
408− ioc->ioprio = ioprio;
409− ioc_set_icq_flags(ioc, ICQ_IOPRIO_CHANGED);
410− spin_unlock_irqrestore(&ioc->lock, flags);
411−}
412−
413−/**
414− * ioc_cgroup_changed - notify cgroup change
415− * @ioc: io_context of interest
416− *
417− * @ioc's cgroup has changed. Set %ICQ_CGROUP_CHANGED for all icq's.
418− * iosched is responsible for checking the bit and applying it on request
419− * issue path.
420− */
421−void ioc_cgroup_changed(struct io_context *ioc)
422−{
423− unsigned long flags;
424−
425− spin_lock_irqsave(&ioc->lock, flags);
426− ioc_set_icq_flags(ioc, ICQ_CGROUP_CHANGED);
427− spin_unlock_irqrestore(&ioc->lock, flags);
428−}
429−EXPORT_SYMBOL(ioc_cgroup_changed);
430−
431−/**
432− * icq_get_changed - fetch and clear icq changed mask
433− * @icq: icq of interest
434− *
435− * Fetch and clear ICQ_*_CHANGED bits from @icq. Grabs and releases
436− * @icq->ioc->lock.
437− */
438−unsigned icq_get_changed(struct io_cq *icq)
439−{
440− unsigned int changed = 0;
441− unsigned long flags;
442−
443− if (unlikely(icq->flags & ICQ_CHANGED_MASK)) {
444− spin_lock_irqsave(&icq->ioc->lock, flags);
445− changed = icq->flags & ICQ_CHANGED_MASK;
446− icq->flags &= ~ICQ_CHANGED_MASK;
447− spin_unlock_irqrestore(&icq->ioc->lock, flags);
448− }
449− return changed;
450−}
451−EXPORT_SYMBOL(icq_get_changed);
452−
453397 static int __init blk_ioc_init(void)
454398 {
455399 iocontext_cachep = kmem_cache_create("blkdev_ioc",
block/blk-sysfs.c+3−3
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include <linux/blktrace_api.h>
1010
1111 #include "blk.h"
12+#include "blk-cgroup.h"
1213
1314 struct queue_sysfs_entry {
1415 struct attribute attr;
@@ -479,6 +480,8 @@ static void blk_release_queue(struct kobject *kobj) · 464 unmodified lines
479480
480481 blk_sync_queue(q);
481482
483+ blkcg_exit_queue(q);
484+
482485 if (q->elevator) {
483486 spin_lock_irq(q->queue_lock);
484487 ioc_clear_queue(q);
@@ -486,15 +489,12 @@ static void blk_release_queue(struct kobject *kobj) · 1 unmodified line
486489 elevator_exit(q->elevator);
487490 }
488491
489− blk_throtl_exit(q);
490−
491492 if (rl->rq_pool)
492493 mempool_destroy(rl->rq_pool);
493494
494495 if (q->queue_tags)
495496 __blk_queue_free_tags(q);
496497
497− blk_throtl_release(q);
498498 blk_trace_shutdown(q);
499499
500500 bdi_destroy(&q->backing_dev_info);
block/blk-throttle.c+325−372
@@ -21,6 +21,8 @@ static int throtl_quantum = 32; · 20 unmodified lines
2121 /* Throttling is performed over 100ms slice and after that slice is renewed */
2222 static unsigned long throtl_slice = HZ/10; /* 100 ms */
2323
24+static struct blkcg_policy blkcg_policy_throtl;
25+
2426 /* A workqueue to queue throttle related work */
2527 static struct workqueue_struct *kthrotld_workqueue;
2628 static void throtl_schedule_delayed_work(struct throtl_data *td,
@@ -38,9 +40,17 @@ struct throtl_rb_root { · 11 unmodified lines
3840
3941 #define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
4042
43+/* Per-cpu group stats */
44+struct tg_stats_cpu {
45+ /* total bytes transferred */
46+ struct blkg_rwstat service_bytes;
47+ /* total IOs serviced, post merge */
48+ struct blkg_rwstat serviced;
49+};
50+
4151 struct throtl_grp {
42− /* List of throtl groups on the request queue*/
43− struct hlist_node tg_node;
52+ /* must be the first member */
53+ struct blkg_policy_data pd;
4454
4555 /* active throtl group service_tree member */
4656 struct rb_node rb_node;
@@ -52,8 +62,6 @@ struct throtl_grp { · 5 unmodified lines
5262 */
5363 unsigned long disptime;
5464
55− struct blkio_group blkg;
56− atomic_t ref;
5765 unsigned int flags;
5866
5967 /* Two lists for READ and WRITE */
@@ -80,18 +88,18 @@ struct throtl_grp { · 20 unmodified lines
8088 /* Some throttle limits got updated for the group */
8189 int limits_changed;
8290
83− struct rcu_head rcu_head;
91+ /* Per cpu stats pointer */
92+ struct tg_stats_cpu __percpu *stats_cpu;
93+
94+ /* List of tgs waiting for per cpu stats memory to be allocated */
95+ struct list_head stats_alloc_node;
8496 };
8597
8698 struct throtl_data
8799 {
88− /* List of throtl groups */
89− struct hlist_head tg_list;
90−
91100 /* service tree for active throtl groups */
92101 struct throtl_rb_root tg_service_tree;
93102
94− struct throtl_grp *root_tg;
95103 struct request_queue *queue;
96104
97105 /* Total Number of queued bios on READ and WRITE lists */
@@ -108,6 +116,33 @@ struct throtl_data · 10 unmodified lines
108116 int limits_changed;
109117 };
110118
119+/* list and work item to allocate percpu group stats */
120+static DEFINE_SPINLOCK(tg_stats_alloc_lock);
121+static LIST_HEAD(tg_stats_alloc_list);
122+
123+static void tg_stats_alloc_fn(struct work_struct *);
124+static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
125+
126+static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
127+{
128+ return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
129+}
130+
131+static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
132+{
133+ return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
134+}
135+
136+static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg)
137+{
138+ return pd_to_blkg(&tg->pd);
139+}
140+
141+static inline struct throtl_grp *td_root_tg(struct throtl_data *td)
142+{
143+ return blkg_to_tg(td->queue->root_blkg);
144+}
145+
111146 enum tg_state_flags {
112147 THROTL_TG_FLAG_on_rr = 0, /* on round-robin busy list */
113148 };
@@ -128,244 +163,150 @@ static inline int throtl_tg_##name(const struct throtl_grp *tg) \ · 14 unmodified lines
128163
129164 THROTL_TG_FNS(on_rr);
130165
131−#define throtl_log_tg(td, tg, fmt, args...) \
132− blk_add_trace_msg((td)->queue, "throtl %s " fmt, \
133− blkg_path(&(tg)->blkg), ##args); \
166+#define throtl_log_tg(td, tg, fmt, args...) do { \
167+ char __pbuf[128]; \
168+ \
169+ blkg_path(tg_to_blkg(tg), __pbuf, sizeof(__pbuf)); \
170+ blk_add_trace_msg((td)->queue, "throtl %s " fmt, __pbuf, ##args); \
171+} while (0)
134172
135173 #define throtl_log(td, fmt, args...) \
136174 blk_add_trace_msg((td)->queue, "throtl " fmt, ##args)
137175
138−static inline struct throtl_grp *tg_of_blkg(struct blkio_group *blkg)
139−{
140− if (blkg)
141− return container_of(blkg, struct throtl_grp, blkg);
142−
143− return NULL;
144−}
145−
146176 static inline unsigned int total_nr_queued(struct throtl_data *td)
147177 {
148178 return td->nr_queued[0] + td->nr_queued[1];
149179 }
150180
151−static inline struct throtl_grp *throtl_ref_get_tg(struct throtl_grp *tg)
152−{
153− atomic_inc(&tg->ref);
154− return tg;
155−}
156−
157−static void throtl_free_tg(struct rcu_head *head)
181+/*
182+ * Worker for allocating per cpu stat for tgs. This is scheduled on the
183+ * system_nrt_wq once there are some groups on the alloc_list waiting for
184+ * allocation.
185+ */
186+static void tg_stats_alloc_fn(struct work_struct *work)
158187 {
159− struct throtl_grp *tg;
188+ static struct tg_stats_cpu *stats_cpu; /* this fn is non-reentrant */
189+ struct delayed_work *dwork = to_delayed_work(work);
190+ bool empty = false;
191+
192+alloc_stats:
193+ if (!stats_cpu) {
194+ stats_cpu = alloc_percpu(struct tg_stats_cpu);
195+ if (!stats_cpu) {
196+ /* allocation failed, try again after some time */
197+ queue_delayed_work(system_nrt_wq, dwork,
198+ msecs_to_jiffies(10));
199+ return;
200+ }
201+ }
160202
161− tg = container_of(head, struct throtl_grp, rcu_head);
162− free_percpu(tg->blkg.stats_cpu);
163− kfree(tg);
164−}
203+ spin_lock_irq(&tg_stats_alloc_lock);
165204
166−static void throtl_put_tg(struct throtl_grp *tg)
167−{
168− BUG_ON(atomic_read(&tg->ref) <= 0);
169− if (!atomic_dec_and_test(&tg->ref))
170− return;
205+ if (!list_empty(&tg_stats_alloc_list)) {
206+ struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list,
207+ struct throtl_grp,
208+ stats_alloc_node);
209+ swap(tg->stats_cpu, stats_cpu);
210+ list_del_init(&tg->stats_alloc_node);
211+ }
171212
172− /*
173− * A group is freed in rcu manner. But having an rcu lock does not
174− * mean that one can access all the fields of blkg and assume these
175− * are valid. For example, don't try to follow throtl_data and
176− * request queue links.
177− *
178− * Having a reference to blkg under an rcu allows acess to only
179− * values local to groups like group stats and group rate limits
180− */
181− call_rcu(&tg->rcu_head, throtl_free_tg);
213+ empty = list_empty(&tg_stats_alloc_list);
214+ spin_unlock_irq(&tg_stats_alloc_lock);
215+ if (!empty)
216+ goto alloc_stats;
182217 }
183218
184−static void throtl_init_group(struct throtl_grp *tg)
219+static void throtl_pd_init(struct blkcg_gq *blkg)
185220 {
186− INIT_HLIST_NODE(&tg->tg_node);
221+ struct throtl_grp *tg = blkg_to_tg(blkg);
222+ unsigned long flags;
223+
187224 RB_CLEAR_NODE(&tg->rb_node);
188225 bio_list_init(&tg->bio_lists[0]);
189226 bio_list_init(&tg->bio_lists[1]);
190227 tg->limits_changed = false;
191228
192− /* Practically unlimited BW */
193− tg->bps[0] = tg->bps[1] = -1;
194− tg->iops[0] = tg->iops[1] = -1;
229+ tg->bps[READ] = -1;
230+ tg->bps[WRITE] = -1;
231+ tg->iops[READ] = -1;
232+ tg->iops[WRITE] = -1;
195233
196234 /*
197− * Take the initial reference that will be released on destroy
198− * This can be thought of a joint reference by cgroup and
199− * request queue which will be dropped by either request queue
200− * exit or cgroup deletion path depending on who is exiting first.
235+ * Ugh... We need to perform per-cpu allocation for tg->stats_cpu
236+ * but percpu allocator can't be called from IO path. Queue tg on
237+ * tg_stats_alloc_list and allocate from work item.
201238 */
202− atomic_set(&tg->ref, 1);
239+ spin_lock_irqsave(&tg_stats_alloc_lock, flags);
240+ list_add(&tg->stats_alloc_node, &tg_stats_alloc_list);
241+ queue_delayed_work(system_nrt_wq, &tg_stats_alloc_work, 0);
242+ spin_unlock_irqrestore(&tg_stats_alloc_lock, flags);
203243 }
204244
205−/* Should be called with rcu read lock held (needed for blkcg) */
206−static void
207−throtl_add_group_to_td_list(struct throtl_data *td, struct throtl_grp *tg)
245+static void throtl_pd_exit(struct blkcg_gq *blkg)
208246 {
209− hlist_add_head(&tg->tg_node, &td->tg_list);
210− td->nr_undestroyed_grps++;
211−}
212−
213−static void
214−__throtl_tg_fill_dev_details(struct throtl_data *td, struct throtl_grp *tg)
215−{
216− struct backing_dev_info *bdi = &td->queue->backing_dev_info;
217− unsigned int major, minor;
218−
219− if (!tg || tg->blkg.dev)
220− return;
221−
222− /*
223− * Fill in device details for a group which might not have been
224− * filled at group creation time as queue was being instantiated
225− * and driver had not attached a device yet
226− */
227− if (bdi->dev && dev_name(bdi->dev)) {
228− sscanf(dev_name(bdi->dev), "%u:%u", &major, &minor);
229− tg->blkg.dev = MKDEV(major, minor);
230− }
231−}
232−
233−/*
234− * Should be called with without queue lock held. Here queue lock will be
235− * taken rarely. It will be taken only once during life time of a group
236− * if need be
237− */
238−static void
239−throtl_tg_fill_dev_details(struct throtl_data *td, struct throtl_grp *tg)
240−{
241− if (!tg || tg->blkg.dev)
242− return;
243−
244− spin_lock_irq(td->queue->queue_lock);
245− __throtl_tg_fill_dev_details(td, tg);
246− spin_unlock_irq(td->queue->queue_lock);
247−}
248−
249−static void throtl_init_add_tg_lists(struct throtl_data *td,
250− struct throtl_grp *tg, struct blkio_cgroup *blkcg)
251−{
252− __throtl_tg_fill_dev_details(td, tg);
253−
254− /* Add group onto cgroup list */
255− blkiocg_add_blkio_group(blkcg, &tg->blkg, (void *)td,
256− tg->blkg.dev, BLKIO_POLICY_THROTL);
247+ struct throtl_grp *tg = blkg_to_tg(blkg);
248+ unsigned long flags;
257249
258− tg->bps[READ] = blkcg_get_read_bps(blkcg, tg->blkg.dev);
259− tg->bps[WRITE] = blkcg_get_write_bps(blkcg, tg->blkg.dev);
260− tg->iops[READ] = blkcg_get_read_iops(blkcg, tg->blkg.dev);
261− tg->iops[WRITE] = blkcg_get_write_iops(blkcg, tg->blkg.dev);
250+ spin_lock_irqsave(&tg_stats_alloc_lock, flags);
251+ list_del_init(&tg->stats_alloc_node);
252+ spin_unlock_irqrestore(&tg_stats_alloc_lock, flags);
262253
263− throtl_add_group_to_td_list(td, tg);
254+ free_percpu(tg->stats_cpu);
264255 }
265256
266−/* Should be called without queue lock and outside of rcu period */
267−static struct throtl_grp *throtl_alloc_tg(struct throtl_data *td)
257+static void throtl_pd_reset_stats(struct blkcg_gq *blkg)
268258 {
269− struct throtl_grp *tg = NULL;
270− int ret;
259+ struct throtl_grp *tg = blkg_to_tg(blkg);
260+ int cpu;
271261
272− tg = kzalloc_node(sizeof(*tg), GFP_ATOMIC, td->queue->node);
273− if (!tg)
274− return NULL;
262+ if (tg->stats_cpu == NULL)
263+ return;
275264
276− ret = blkio_alloc_blkg_stats(&tg->blkg);
265+ for_each_possible_cpu(cpu) {
266+ struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
277267
278− if (ret) {
279− kfree(tg);
280− return NULL;
268+ blkg_rwstat_reset(&sc->service_bytes);
269+ blkg_rwstat_reset(&sc->serviced);
281270 }
282−
283− throtl_init_group(tg);
284− return tg;
285271 }
286272
287−static struct
288−throtl_grp *throtl_find_tg(struct throtl_data *td, struct blkio_cgroup *blkcg)
273+static struct throtl_grp *throtl_lookup_tg(struct throtl_data *td,
274+ struct blkcg *blkcg)
289275 {
290− struct throtl_grp *tg = NULL;
291− void *key = td;
292−
293276 /*
294− * This is the common case when there are no blkio cgroups.
295− * Avoid lookup in this case
296− */
297− if (blkcg == &blkio_root_cgroup)
298− tg = td->root_tg;
299− else
300− tg = tg_of_blkg(blkiocg_lookup_group(blkcg, key));
277+ * This is the common case when there are no blkcgs. Avoid lookup
278+ * in this case
279+ */
280+ if (blkcg == &blkcg_root)
281+ return td_root_tg(td);
301282
302− __throtl_tg_fill_dev_details(td, tg);
303− return tg;
283+ return blkg_to_tg(blkg_lookup(blkcg, td->queue));
304284 }
305285
306−static struct throtl_grp * throtl_get_tg(struct throtl_data *td)
286+static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td,
287+ struct blkcg *blkcg)
307288 {
308− struct throtl_grp *tg = NULL, *__tg = NULL;
309− struct blkio_cgroup *blkcg;
310289 struct request_queue *q = td->queue;
311−
312− /* no throttling for dead queue */
313− if (unlikely(blk_queue_dead(q)))
314− return NULL;
315−
316− rcu_read_lock();
317− blkcg = task_blkio_cgroup(current);
318− tg = throtl_find_tg(td, blkcg);
319− if (tg) {
320− rcu_read_unlock();
321− return tg;
322− }
323−
324− /*
325− * Need to allocate a group. Allocation of group also needs allocation
326− * of per cpu stats which in-turn takes a mutex() and can block. Hence
327− * we need to drop rcu lock and queue_lock before we call alloc.
328− */
329− rcu_read_unlock();
330− spin_unlock_irq(q->queue_lock);
331−
332− tg = throtl_alloc_tg(td);
333−
334− /* Group allocated and queue is still alive. take the lock */
335− spin_lock_irq(q->queue_lock);
336−
337− /* Make sure @q is still alive */
338− if (unlikely(blk_queue_dead(q))) {
339− kfree(tg);
340− return NULL;
341− }
342−
343− /*
344− * Initialize the new group. After sleeping, read the blkcg again.
345− */
346− rcu_read_lock();
347− blkcg = task_blkio_cgroup(current);
290+ struct throtl_grp *tg = NULL;
348291
349292 /*
350− * If some other thread already allocated the group while we were
351− * not holding queue lock, free up the group
293+ * This is the common case when there are no blkcgs. Avoid lookup
294+ * in this case
352295 */
353− __tg = throtl_find_tg(td, blkcg);
354−
355− if (__tg) {
356− kfree(tg);
357− rcu_read_unlock();
358− return __tg;
359− }
360−
361− /* Group allocation failed. Account the IO to root group */
362− if (!tg) {
363− tg = td->root_tg;
364− return tg;
296+ if (blkcg == &blkcg_root) {
297+ tg = td_root_tg(td);
298+ } else {
299+ struct blkcg_gq *blkg;
300+
301+ blkg = blkg_lookup_create(blkcg, q);
302+
303+ /* if %NULL and @q is alive, fall back to root_tg */
304+ if (!IS_ERR(blkg))
305+ tg = blkg_to_tg(blkg);
306+ else if (!blk_queue_dead(q))
307+ tg = td_root_tg(td);
365308 }
366309
367− throtl_init_add_tg_lists(td, tg, blkcg);
368− rcu_read_unlock();
369310 return tg;
370311 }
371312
@@ -734,16 +675,41 @@ static bool tg_may_dispatch(struct throtl_data *td, struct throtl_grp *tg, · 362 unmodified lines
734675 return 0;
735676 }
736677
678+static void throtl_update_dispatch_stats(struct blkcg_gq *blkg, u64 bytes,
679+ int rw)
680+{
681+ struct throtl_grp *tg = blkg_to_tg(blkg);
682+ struct tg_stats_cpu *stats_cpu;
683+ unsigned long flags;
684+
685+ /* If per cpu stats are not allocated yet, don't do any accounting. */
686+ if (tg->stats_cpu == NULL)
687+ return;
688+
689+ /*
690+ * Disabling interrupts to provide mutual exclusion between two
691+ * writes on same cpu. It probably is not needed for 64bit. Not
692+ * optimizing that case yet.
693+ */
694+ local_irq_save(flags);
695+
696+ stats_cpu = this_cpu_ptr(tg->stats_cpu);
697+
698+ blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
699+ blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
700+
701+ local_irq_restore(flags);
702+}
703+
737704 static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
738705 {
739706 bool rw = bio_data_dir(bio);
740− bool sync = rw_is_sync(bio->bi_rw);
741707
742708 /* Charge the bio to the group */
743709 tg->bytes_disp[rw] += bio->bi_size;
744710 tg->io_disp[rw]++;
745711
746− blkiocg_update_dispatch_stats(&tg->blkg, bio->bi_size, rw, sync);
712+ throtl_update_dispatch_stats(tg_to_blkg(tg), bio->bi_size, bio->bi_rw);
747713 }
748714
749715 static void throtl_add_bio_tg(struct throtl_data *td, struct throtl_grp *tg,
@@ -753,7 +719,7 @@ static void throtl_add_bio_tg(struct throtl_data *td, struct throtl_grp *tg, · 3 unmodified lines
753719
754720 bio_list_add(&tg->bio_lists[rw], bio);
755721 /* Take a bio reference on tg */
756− throtl_ref_get_tg(tg);
722+ blkg_get(tg_to_blkg(tg));
757723 tg->nr_queued[rw]++;
758724 td->nr_queued[rw]++;
759725 throtl_enqueue_tg(td, tg);
@@ -786,8 +752,8 @@ static void tg_dispatch_one_bio(struct throtl_data *td, struct throtl_grp *tg, · 26 unmodified lines
786752
787753 bio = bio_list_pop(&tg->bio_lists[rw]);
788754 tg->nr_queued[rw]--;
789− /* Drop bio reference on tg */
790− throtl_put_tg(tg);
755+ /* Drop bio reference on blkg */
756+ blkg_put(tg_to_blkg(tg));
791757
792758 BUG_ON(td->nr_queued[rw] <= 0);
793759 td->nr_queued[rw]--;
@@ -865,8 +831,8 @@ static int throtl_select_dispatch(struct throtl_data *td, struct bio_list *bl) · 71 unmodified lines
865831
866832 static void throtl_process_limit_change(struct throtl_data *td)
867833 {
868− struct throtl_grp *tg;
869− struct hlist_node *pos, *n;
834+ struct request_queue *q = td->queue;
835+ struct blkcg_gq *blkg, *n;
870836
871837 if (!td->limits_changed)
872838 return;
@@ -875,7 +841,9 @@ static void throtl_process_limit_change(struct throtl_data *td) · 2 unmodified lines
875841
876842 throtl_log(td, "limits changed");
877843
878− hlist_for_each_entry_safe(tg, pos, n, &td->tg_list, tg_node) {
844+ list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
845+ struct throtl_grp *tg = blkg_to_tg(blkg);
846+
879847 if (!tg->limits_changed)
880848 continue;
881849
@@ -973,120 +941,159 @@ throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay) · 91 unmodified lines
973941 }
974942 }
975943
976−static void
977−throtl_destroy_tg(struct throtl_data *td, struct throtl_grp *tg)
944+static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
945+ struct blkg_policy_data *pd, int off)
978946 {
979− /* Something wrong if we are trying to remove same group twice */
980− BUG_ON(hlist_unhashed(&tg->tg_node));
947+ struct throtl_grp *tg = pd_to_tg(pd);
948+ struct blkg_rwstat rwstat = { }, tmp;
949+ int i, cpu;
981950
982− hlist_del_init(&tg->tg_node);
951+ for_each_possible_cpu(cpu) {
952+ struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
983953
984− /*
985− * Put the reference taken at the time of creation so that when all
986− * queues are gone, group can be destroyed.
987− */
988− throtl_put_tg(tg);
989− td->nr_undestroyed_grps--;
954+ tmp = blkg_rwstat_read((void *)sc + off);
955+ for (i = 0; i < BLKG_RWSTAT_NR; i++)
956+ rwstat.cnt[i] += tmp.cnt[i];
957+ }
958+
959+ return __blkg_prfill_rwstat(sf, pd, &rwstat);
990960 }
991961
992−static void throtl_release_tgs(struct throtl_data *td)
962+static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
963+ struct seq_file *sf)
993964 {
994− struct hlist_node *pos, *n;
995− struct throtl_grp *tg;
965+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
996966
997− hlist_for_each_entry_safe(tg, pos, n, &td->tg_list, tg_node) {
998− /*
999− * If cgroup removal path got to blk_group first and removed
1000− * it from cgroup list, then it will take care of destroying
1001− * cfqg also.
1002− */
1003− if (!blkiocg_del_blkio_group(&tg->blkg))
1004− throtl_destroy_tg(td, tg);
1005− }
967+ blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat, &blkcg_policy_throtl,
968+ cft->private, true);
969+ return 0;
1006970 }
1007971
1008−/*
1009− * Blk cgroup controller notification saying that blkio_group object is being
1010− * delinked as associated cgroup object is going away. That also means that
1011− * no new IO will come in this group. So get rid of this group as soon as
1012− * any pending IO in the group is finished.
1013− *
1014− * This function is called under rcu_read_lock(). key is the rcu protected
1015− * pointer. That means "key" is a valid throtl_data pointer as long as we are
1016− * rcu read lock.
1017− *
1018− * "key" was fetched from blkio_group under blkio_cgroup->lock. That means
1019− * it should not be NULL as even if queue was going away, cgroup deltion
1020− * path got to it first.
1021− */
1022−void throtl_unlink_blkio_group(void *key, struct blkio_group *blkg)
972+static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd,
973+ int off)
1023974 {
1024− unsigned long flags;
1025− struct throtl_data *td = key;
975+ struct throtl_grp *tg = pd_to_tg(pd);
976+ u64 v = *(u64 *)((void *)tg + off);
1026977
1027− spin_lock_irqsave(td->queue->queue_lock, flags);
1028− throtl_destroy_tg(td, tg_of_blkg(blkg));
1029− spin_unlock_irqrestore(td->queue->queue_lock, flags);
978+ if (v == -1)
979+ return 0;
980+ return __blkg_prfill_u64(sf, pd, v);
1030981 }
1031982
1032−static void throtl_update_blkio_group_common(struct throtl_data *td,
1033− struct throtl_grp *tg)
983+static u64 tg_prfill_conf_uint(struct seq_file *sf, struct blkg_policy_data *pd,
984+ int off)
1034985 {
1035− xchg(&tg->limits_changed, true);
1036− xchg(&td->limits_changed, true);
1037− /* Schedule a work now to process the limit change */
1038− throtl_schedule_delayed_work(td, 0);
986+ struct throtl_grp *tg = pd_to_tg(pd);
987+ unsigned int v = *(unsigned int *)((void *)tg + off);
988+
989+ if (v == -1)
990+ return 0;
991+ return __blkg_prfill_u64(sf, pd, v);
1039992 }
1040993
1041−/*
1042− * For all update functions, key should be a valid pointer because these
1043− * update functions are called under blkcg_lock, that means, blkg is
1044− * valid and in turn key is valid. queue exit path can not race because
1045− * of blkcg_lock
1046− *
1047− * Can not take queue lock in update functions as queue lock under blkcg_lock
1048− * is not allowed. Under other paths we take blkcg_lock under queue_lock.
1049− */
1050−static void throtl_update_blkio_group_read_bps(void *key,
1051− struct blkio_group *blkg, u64 read_bps)
994+static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft,
995+ struct seq_file *sf)
1052996 {
1053− struct throtl_data *td = key;
1054− struct throtl_grp *tg = tg_of_blkg(blkg);
1055−
1056− tg->bps[READ] = read_bps;
1057− throtl_update_blkio_group_common(td, tg);
997+ blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_u64,
998+ &blkcg_policy_throtl, cft->private, false);
999+ return 0;
10581000 }
10591001
1060−static void throtl_update_blkio_group_write_bps(void *key,
1061− struct blkio_group *blkg, u64 write_bps)
1002+static int tg_print_conf_uint(struct cgroup *cgrp, struct cftype *cft,
1003+ struct seq_file *sf)
10621004 {
1063− struct throtl_data *td = key;
1064− struct throtl_grp *tg = tg_of_blkg(blkg);
1065−
1066− tg->bps[WRITE] = write_bps;
1067− throtl_update_blkio_group_common(td, tg);
1005+ blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_uint,
1006+ &blkcg_policy_throtl, cft->private, false);
1007+ return 0;
10681008 }
10691009
1070−static void throtl_update_blkio_group_read_iops(void *key,
1071− struct blkio_group *blkg, unsigned int read_iops)
1010+static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
1011+ bool is_u64)
10721012 {
1073− struct throtl_data *td = key;
1074− struct throtl_grp *tg = tg_of_blkg(blkg);
1013+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
1014+ struct blkg_conf_ctx ctx;
1015+ struct throtl_grp *tg;
1016+ struct throtl_data *td;
1017+ int ret;
1018+
1019+ ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);
1020+ if (ret)
1021+ return ret;
1022+
1023+ tg = blkg_to_tg(ctx.blkg);
1024+ td = ctx.blkg->q->td;
1025+
1026+ if (!ctx.v)
1027+ ctx.v = -1;
1028+
1029+ if (is_u64)
1030+ *(u64 *)((void *)tg + cft->private) = ctx.v;
1031+ else
1032+ *(unsigned int *)((void *)tg + cft->private) = ctx.v;
1033+
1034+ /* XXX: we don't need the following deferred processing */
1035+ xchg(&tg->limits_changed, true);
1036+ xchg(&td->limits_changed, true);
1037+ throtl_schedule_delayed_work(td, 0);
10751038
1076− tg->iops[READ] = read_iops;
1077− throtl_update_blkio_group_common(td, tg);
1039+ blkg_conf_finish(&ctx);
1040+ return 0;
10781041 }
10791042
1080−static void throtl_update_blkio_group_write_iops(void *key,
1081− struct blkio_group *blkg, unsigned int write_iops)
1043+static int tg_set_conf_u64(struct cgroup *cgrp, struct cftype *cft,
1044+ const char *buf)
10821045 {
1083− struct throtl_data *td = key;
1084− struct throtl_grp *tg = tg_of_blkg(blkg);
1046+ return tg_set_conf(cgrp, cft, buf, true);
1047+}
10851048
1086− tg->iops[WRITE] = write_iops;
1087− throtl_update_blkio_group_common(td, tg);
1049+static int tg_set_conf_uint(struct cgroup *cgrp, struct cftype *cft,
1050+ const char *buf)
1051+{
1052+ return tg_set_conf(cgrp, cft, buf, false);
10881053 }
10891054
1055+static struct cftype throtl_files[] = {
1056+ {
1057+ .name = "throttle.read_bps_device",
1058+ .private = offsetof(struct throtl_grp, bps[READ]),
1059+ .read_seq_string = tg_print_conf_u64,
1060+ .write_string = tg_set_conf_u64,
1061+ .max_write_len = 256,
1062+ },
1063+ {
1064+ .name = "throttle.write_bps_device",
1065+ .private = offsetof(struct throtl_grp, bps[WRITE]),
1066+ .read_seq_string = tg_print_conf_u64,
1067+ .write_string = tg_set_conf_u64,
1068+ .max_write_len = 256,
1069+ },
1070+ {
1071+ .name = "throttle.read_iops_device",
1072+ .private = offsetof(struct throtl_grp, iops[READ]),
1073+ .read_seq_string = tg_print_conf_uint,
1074+ .write_string = tg_set_conf_uint,
1075+ .max_write_len = 256,
1076+ },
1077+ {
1078+ .name = "throttle.write_iops_device",
1079+ .private = offsetof(struct throtl_grp, iops[WRITE]),
1080+ .read_seq_string = tg_print_conf_uint,
1081+ .write_string = tg_set_conf_uint,
1082+ .max_write_len = 256,
1083+ },
1084+ {
1085+ .name = "throttle.io_service_bytes",
1086+ .private = offsetof(struct tg_stats_cpu, service_bytes),
1087+ .read_seq_string = tg_print_cpu_rwstat,
1088+ },
1089+ {
1090+ .name = "throttle.io_serviced",
1091+ .private = offsetof(struct tg_stats_cpu, serviced),
1092+ .read_seq_string = tg_print_cpu_rwstat,
1093+ },
1094+ { } /* terminate */
1095+};
1096+
10901097 static void throtl_shutdown_wq(struct request_queue *q)
10911098 {
10921099 struct throtl_data *td = q->td;
@@ -1094,19 +1101,13 @@ static void throtl_shutdown_wq(struct request_queue *q) · 1 unmodified line
10941101 cancel_delayed_work_sync(&td->throtl_work);
10951102 }
10961103
1097−static struct blkio_policy_type blkio_policy_throtl = {
1098− .ops = {
1099− .blkio_unlink_group_fn = throtl_unlink_blkio_group,
1100− .blkio_update_group_read_bps_fn =
1101− throtl_update_blkio_group_read_bps,
1102− .blkio_update_group_write_bps_fn =
1103− throtl_update_blkio_group_write_bps,
1104− .blkio_update_group_read_iops_fn =
1105− throtl_update_blkio_group_read_iops,
1106− .blkio_update_group_write_iops_fn =
1107− throtl_update_blkio_group_write_iops,
1108− },
1109− .plid = BLKIO_POLICY_THROTL,
1104+static struct blkcg_policy blkcg_policy_throtl = {
1105+ .pd_size = sizeof(struct throtl_grp),
1106+ .cftypes = throtl_files,
1107+
1108+ .pd_init_fn = throtl_pd_init,
1109+ .pd_exit_fn = throtl_pd_exit,
1110+ .pd_reset_stats_fn = throtl_pd_reset_stats,
11101111 };
11111112
11121113 bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
@@ -1114,7 +1115,7 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio) · 1 unmodified line
11141115 struct throtl_data *td = q->td;
11151116 struct throtl_grp *tg;
11161117 bool rw = bio_data_dir(bio), update_disptime = true;
1117− struct blkio_cgroup *blkcg;
1118+ struct blkcg *blkcg;
11181119 bool throttled = false;
11191120
11201121 if (bio->bi_rw & REQ_THROTTLED) {
@@ -1122,33 +1123,31 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio) · 1 unmodified line
11221123 goto out;
11231124 }
11241125
1126+ /* bio_associate_current() needs ioc, try creating */
1127+ create_io_context(GFP_ATOMIC, q->node);
1128+
11251129 /*
11261130 * A throtl_grp pointer retrieved under rcu can be used to access
11271131 * basic fields like stats and io rates. If a group has no rules,
11281132 * just update the dispatch stats in lockless manner and return.
11291133 */
1130−
11311134 rcu_read_lock();
1132− blkcg = task_blkio_cgroup(current);
1133− tg = throtl_find_tg(td, blkcg);
1135+ blkcg = bio_blkcg(bio);
1136+ tg = throtl_lookup_tg(td, blkcg);
11341137 if (tg) {
1135− throtl_tg_fill_dev_details(td, tg);
1136−
11371138 if (tg_no_rule_group(tg, rw)) {
1138− blkiocg_update_dispatch_stats(&tg->blkg, bio->bi_size,
1139− rw, rw_is_sync(bio->bi_rw));
1140− rcu_read_unlock();
1141− goto out;
1139+ throtl_update_dispatch_stats(tg_to_blkg(tg),
1140+ bio->bi_size, bio->bi_rw);
1141+ goto out_unlock_rcu;
11421142 }
11431143 }
1144− rcu_read_unlock();
11451144
11461145 /*
11471146 * Either group has not been allocated yet or it is not an unlimited
11481147 * IO group
11491148 */
11501149 spin_lock_irq(q->queue_lock);
1151− tg = throtl_get_tg(td);
1150+ tg = throtl_lookup_create_tg(td, blkcg);
11521151 if (unlikely(!tg))
11531152 goto out_unlock;
11541153
@@ -1189,6 +1188,7 @@ queue_bio: · 34 unmodified lines
11891188 tg->io_disp[rw], tg->iops[rw],
11901189 tg->nr_queued[READ], tg->nr_queued[WRITE]);
11911190
1191+ bio_associate_current(bio);
11921192 throtl_add_bio_tg(q->td, tg, bio);
11931193 throttled = true;
11941194
@@ -1199,6 +1199,8 @@ queue_bio: · 4 unmodified lines
11991199
12001200 out_unlock:
12011201 spin_unlock_irq(q->queue_lock);
1202+out_unlock_rcu:
1203+ rcu_read_unlock();
12021204 out:
12031205 return throttled;
12041206 }
@@ -1241,79 +1243,31 @@ void blk_throtl_drain(struct request_queue *q) · 36 unmodified lines
12411243 int blk_throtl_init(struct request_queue *q)
12421244 {
12431245 struct throtl_data *td;
1244− struct throtl_grp *tg;
1246+ int ret;
12451247
12461248 td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node);
12471249 if (!td)
12481250 return -ENOMEM;
12491251
1250− INIT_HLIST_HEAD(&td->tg_list);
12511252 td->tg_service_tree = THROTL_RB_ROOT;
12521253 td->limits_changed = false;
12531254 INIT_DELAYED_WORK(&td->throtl_work, blk_throtl_work);
12541255
1255− /* alloc and Init root group. */
1256+ q->td = td;
12561257 td->queue = q;
1257− tg = throtl_alloc_tg(td);
12581258
1259− if (!tg) {
1259+ /* activate policy */
1260+ ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
1261+ if (ret)
12601262 kfree(td);
1261− return -ENOMEM;
1262− }
1263−
1264− td->root_tg = tg;
1265−
1266− rcu_read_lock();
1267− throtl_init_add_tg_lists(td, tg, &blkio_root_cgroup);
1268− rcu_read_unlock();
1269−
1270− /* Attach throtl data to request queue */
1271− q->td = td;
1272− return 0;
1263+ return ret;
12731264 }
12741265
12751266 void blk_throtl_exit(struct request_queue *q)
12761267 {
1277− struct throtl_data *td = q->td;
1278− bool wait = false;
1279−
1280− BUG_ON(!td);
1281−
1282− throtl_shutdown_wq(q);
1283−
1284− spin_lock_irq(q->queue_lock);
1285− throtl_release_tgs(td);
1286−
1287− /* If there are other groups */
1288− if (td->nr_undestroyed_grps > 0)
1289− wait = true;
1290−
1291− spin_unlock_irq(q->queue_lock);
1292−
1293− /*
1294− * Wait for tg->blkg->key accessors to exit their grace periods.
1295− * Do this wait only if there are other undestroyed groups out
1296− * there (other than root group). This can happen if cgroup deletion
1297− * path claimed the responsibility of cleaning up a group before
1298− * queue cleanup code get to the group.
1299− *
1300− * Do not call synchronize_rcu() unconditionally as there are drivers
1301− * which create/delete request queue hundreds of times during scan/boot
1302− * and synchronize_rcu() can take significant time and slow down boot.
1303− */
1304− if (wait)
1305− synchronize_rcu();
1306−
1307− /*
1308− * Just being safe to make sure after previous flush if some body did
1309− * update limits through cgroup and another work got queued, cancel
1310− * it.
1311− */
1268+ BUG_ON(!q->td);
13121269 throtl_shutdown_wq(q);
1313−}
1314−
1315−void blk_throtl_release(struct request_queue *q)
1316−{
1270+ blkcg_deactivate_policy(q, &blkcg_policy_throtl);
13171271 kfree(q->td);
13181272 }
13191273
@@ -1323,8 +1277,7 @@ static int __init throtl_init(void) · 3 unmodified lines
13231277 if (!kthrotld_workqueue)
13241278 panic("Failed to create kthrotld\n");
13251279
1326− blkio_policy_register(&blkio_policy_throtl);
1327− return 0;
1280+ return blkcg_policy_register(&blkcg_policy_throtl);
13281281 }
13291282
13301283 module_init(throtl_init);
block/blk.h+13−19
@@ -23,7 +23,8 @@ void blk_rq_bio_prep(struct request_queue *q, struct request *rq, · 22 unmodified lines
2323 struct bio *bio);
2424 int blk_rq_append_bio(struct request_queue *q, struct request *rq,
2525 struct bio *bio);
26−void blk_drain_queue(struct request_queue *q, bool drain_all);
26+void blk_queue_bypass_start(struct request_queue *q);
27+void blk_queue_bypass_end(struct request_queue *q);
2728 void blk_dequeue_request(struct request *rq);
2829 void __blk_queue_free_tags(struct request_queue *q);
2930 bool __blk_end_bidi_request(struct request *rq, int error,
@@ -144,9 +145,6 @@ void blk_queue_congestion_threshold(struct request_queue *q); · 114 unmodified lines
144145
145146 int blk_dev_init(void);
146147
147−void elv_quiesce_start(struct request_queue *q);
148−void elv_quiesce_end(struct request_queue *q);
149−
150148
151149 /*
152150 * Return the threshold (number of used requests) at which the queue is
@@ -186,32 +184,30 @@ static inline int blk_do_io_stat(struct request *rq) · 33 unmodified lines
186184 */
187185 void get_io_context(struct io_context *ioc);
188186 struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q);
189−struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask);
187+struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
188+ gfp_t gfp_mask);
190189 void ioc_clear_queue(struct request_queue *q);
191190
192−void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_mask,
193− int node);
191+int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node);
194192
195193 /**
196194 * create_io_context - try to create task->io_context
197− * @task: target task
198195 * @gfp_mask: allocation mask
199196 * @node: allocation node
200197 *
201− * If @task->io_context is %NULL, allocate a new io_context and install it.
202− * Returns the current @task->io_context which may be %NULL if allocation
203− * failed.
198+ * If %current->io_context is %NULL, allocate a new io_context and install
199+ * it. Returns the current %current->io_context which may be %NULL if
200+ * allocation failed.
204201 *
205202 * Note that this function can't be called with IRQ disabled because
206− * task_lock which protects @task->io_context is IRQ-unsafe.
203+ * task_lock which protects %current->io_context is IRQ-unsafe.
207204 */
208−static inline struct io_context *create_io_context(struct task_struct *task,
209− gfp_t gfp_mask, int node)
205+static inline struct io_context *create_io_context(gfp_t gfp_mask, int node)
210206 {
211207 WARN_ON_ONCE(irqs_disabled());
212− if (unlikely(!task->io_context))
213− create_io_context_slowpath(task, gfp_mask, node);
214− return task->io_context;
208+ if (unlikely(!current->io_context))
209+ create_task_io_context(current, gfp_mask, node);
210+ return current->io_context;
215211 }
216212
217213 /*
@@ -222,7 +218,6 @@ extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio); · 4 unmodified lines
222218 extern void blk_throtl_drain(struct request_queue *q);
223219 extern int blk_throtl_init(struct request_queue *q);
224220 extern void blk_throtl_exit(struct request_queue *q);
225−extern void blk_throtl_release(struct request_queue *q);
226221 #else /* CONFIG_BLK_DEV_THROTTLING */
227222 static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
228223 {
@@ -231,7 +226,6 @@ static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio) · 2 unmodified lines
231226 static inline void blk_throtl_drain(struct request_queue *q) { }
232227 static inline int blk_throtl_init(struct request_queue *q) { return 0; }
233228 static inline void blk_throtl_exit(struct request_queue *q) { }
234−static inline void blk_throtl_release(struct request_queue *q) { }
235229 #endif /* CONFIG_BLK_DEV_THROTTLING */
236230
237231 #endif /* BLK_INTERNAL_H */
block/cfq-iosched.c+685−387
@@ -15,7 +15,9 @@ · 14 unmodified lines
1515 #include <linux/ioprio.h>
1616 #include <linux/blktrace_api.h>
1717 #include "blk.h"
18−#include "cfq.h"
18+#include "blk-cgroup.h"
19+
20+static struct blkcg_policy blkcg_policy_cfq __maybe_unused;
1921
2022 /*
2123 * tunables
@@ -171,8 +173,53 @@ enum wl_type_t { · 149 unmodified lines
171173 SYNC_WORKLOAD = 2
172174 };
173175
176+struct cfqg_stats {
177+#ifdef CONFIG_CFQ_GROUP_IOSCHED
178+ /* total bytes transferred */
179+ struct blkg_rwstat service_bytes;
180+ /* total IOs serviced, post merge */
181+ struct blkg_rwstat serviced;
182+ /* number of ios merged */
183+ struct blkg_rwstat merged;
184+ /* total time spent on device in ns, may not be accurate w/ queueing */
185+ struct blkg_rwstat service_time;
186+ /* total time spent waiting in scheduler queue in ns */
187+ struct blkg_rwstat wait_time;
188+ /* number of IOs queued up */
189+ struct blkg_rwstat queued;
190+ /* total sectors transferred */
191+ struct blkg_stat sectors;
192+ /* total disk time and nr sectors dispatched by this group */
193+ struct blkg_stat time;
194+#ifdef CONFIG_DEBUG_BLK_CGROUP
195+ /* time not charged to this cgroup */
196+ struct blkg_stat unaccounted_time;
197+ /* sum of number of ios queued across all samples */
198+ struct blkg_stat avg_queue_size_sum;
199+ /* count of samples taken for average */
200+ struct blkg_stat avg_queue_size_samples;
201+ /* how many times this group has been removed from service tree */
202+ struct blkg_stat dequeue;
203+ /* total time spent waiting for it to be assigned a timeslice. */
204+ struct blkg_stat group_wait_time;
205+ /* time spent idling for this blkcg_gq */
206+ struct blkg_stat idle_time;
207+ /* total time with empty current active q with other requests queued */
208+ struct blkg_stat empty_time;
209+ /* fields after this shouldn't be cleared on stat reset */
210+ uint64_t start_group_wait_time;
211+ uint64_t start_idle_time;
212+ uint64_t start_empty_time;
213+ uint16_t flags;
214+#endif /* CONFIG_DEBUG_BLK_CGROUP */
215+#endif /* CONFIG_CFQ_GROUP_IOSCHED */
216+};
217+
174218 /* This is per cgroup per device grouping structure */
175219 struct cfq_group {
220+ /* must be the first member */
221+ struct blkg_policy_data pd;
222+
176223 /* group service_tree member */
177224 struct rb_node rb_node;
178225
@@ -180,7 +227,7 @@ struct cfq_group { · 1 unmodified line
180227 u64 vdisktime;
181228 unsigned int weight;
182229 unsigned int new_weight;
183− bool needs_update;
230+ unsigned int dev_weight;
184231
185232 /* number of cfqq currently on this group */
186233 int nr_cfqq;
@@ -206,20 +253,21 @@ struct cfq_group { · 19 unmodified lines
206253 unsigned long saved_workload_slice;
207254 enum wl_type_t saved_workload;
208255 enum wl_prio_t saved_serving_prio;
209− struct blkio_group blkg;
210−#ifdef CONFIG_CFQ_GROUP_IOSCHED
211− struct hlist_node cfqd_node;
212− int ref;
213−#endif
256+
214257 /* number of requests that are on the dispatch list or inside driver */
215258 int dispatched;
216259 struct cfq_ttime ttime;
260+ struct cfqg_stats stats;
217261 };
218262
219263 struct cfq_io_cq {
220264 struct io_cq icq; /* must be the first member */
221265 struct cfq_queue *cfqq[2];
222266 struct cfq_ttime ttime;
267+ int ioprio; /* the current ioprio */
268+#ifdef CONFIG_CFQ_GROUP_IOSCHED
269+ uint64_t blkcg_id; /* the current blkcg ID */
270+#endif
223271 };
224272
225273 /*
@@ -229,7 +277,7 @@ struct cfq_data { · 3 unmodified lines
229277 struct request_queue *queue;
230278 /* Root service tree for cfq_groups */
231279 struct cfq_rb_root grp_service_tree;
232− struct cfq_group root_group;
280+ struct cfq_group *root_group;
233281
234282 /*
235283 * The priority currently being served
@@ -303,12 +351,6 @@ struct cfq_data { · 67 unmodified lines
303351 struct cfq_queue oom_cfqq;
304352
305353 unsigned long last_delayed_sync;
306−
307− /* List of cfq groups being managed on this device*/
308− struct hlist_head cfqg_list;
309−
310− /* Number of groups which are on blkcg->blkg_list */
311− unsigned int nr_blkcg_linked_grps;
312354 };
313355
314356 static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd);
@@ -371,21 +413,284 @@ CFQ_CFQQ_FNS(deep); · 56 unmodified lines
371413 CFQ_CFQQ_FNS(wait_busy);
372414 #undef CFQ_CFQQ_FNS
373415
416+static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
417+{
418+ return pd ? container_of(pd, struct cfq_group, pd) : NULL;
419+}
420+
421+static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
422+{
423+ return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
424+}
425+
426+static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
427+{
428+ return pd_to_blkg(&cfqg->pd);
429+}
430+
431+#if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
432+
433+/* cfqg stats flags */
434+enum cfqg_stats_flags {
435+ CFQG_stats_waiting = 0,
436+ CFQG_stats_idling,
437+ CFQG_stats_empty,
438+};
439+
440+#define CFQG_FLAG_FNS(name) \
441+static inline void cfqg_stats_mark_##name(struct cfqg_stats *stats) \
442+{ \
443+ stats->flags |= (1 << CFQG_stats_##name); \
444+} \
445+static inline void cfqg_stats_clear_##name(struct cfqg_stats *stats) \
446+{ \
447+ stats->flags &= ~(1 << CFQG_stats_##name); \
448+} \
449+static inline int cfqg_stats_##name(struct cfqg_stats *stats) \
450+{ \
451+ return (stats->flags & (1 << CFQG_stats_##name)) != 0; \
452+} \
453+
454+CFQG_FLAG_FNS(waiting)
455+CFQG_FLAG_FNS(idling)
456+CFQG_FLAG_FNS(empty)
457+#undef CFQG_FLAG_FNS
458+
459+/* This should be called with the queue_lock held. */
460+static void cfqg_stats_update_group_wait_time(struct cfqg_stats *stats)
461+{
462+ unsigned long long now;
463+
464+ if (!cfqg_stats_waiting(stats))
465+ return;
466+
467+ now = sched_clock();
468+ if (time_after64(now, stats->start_group_wait_time))
469+ blkg_stat_add(&stats->group_wait_time,
470+ now - stats->start_group_wait_time);
471+ cfqg_stats_clear_waiting(stats);
472+}
473+
474+/* This should be called with the queue_lock held. */
475+static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,
476+ struct cfq_group *curr_cfqg)
477+{
478+ struct cfqg_stats *stats = &cfqg->stats;
479+
480+ if (cfqg_stats_waiting(stats))
481+ return;
482+ if (cfqg == curr_cfqg)
483+ return;
484+ stats->start_group_wait_time = sched_clock();
485+ cfqg_stats_mark_waiting(stats);
486+}
487+
488+/* This should be called with the queue_lock held. */
489+static void cfqg_stats_end_empty_time(struct cfqg_stats *stats)
490+{
491+ unsigned long long now;
492+
493+ if (!cfqg_stats_empty(stats))
494+ return;
495+
496+ now = sched_clock();
497+ if (time_after64(now, stats->start_empty_time))
498+ blkg_stat_add(&stats->empty_time,
499+ now - stats->start_empty_time);
500+ cfqg_stats_clear_empty(stats);
501+}
502+
503+static void cfqg_stats_update_dequeue(struct cfq_group *cfqg)
504+{
505+ blkg_stat_add(&cfqg->stats.dequeue, 1);
506+}
507+
508+static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg)
509+{
510+ struct cfqg_stats *stats = &cfqg->stats;
511+
512+ if (blkg_rwstat_sum(&stats->queued))
513+ return;
514+
515+ /*
516+ * group is already marked empty. This can happen if cfqq got new
517+ * request in parent group and moved to this group while being added
518+ * to service tree. Just ignore the event and move on.
519+ */
520+ if (cfqg_stats_empty(stats))
521+ return;
522+
523+ stats->start_empty_time = sched_clock();
524+ cfqg_stats_mark_empty(stats);
525+}
526+
527+static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)
528+{
529+ struct cfqg_stats *stats = &cfqg->stats;
530+
531+ if (cfqg_stats_idling(stats)) {
532+ unsigned long long now = sched_clock();
533+
534+ if (time_after64(now, stats->start_idle_time))
535+ blkg_stat_add(&stats->idle_time,
536+ now - stats->start_idle_time);
537+ cfqg_stats_clear_idling(stats);
538+ }
539+}
540+
541+static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)
542+{
543+ struct cfqg_stats *stats = &cfqg->stats;
544+
545+ BUG_ON(cfqg_stats_idling(stats));
546+
547+ stats->start_idle_time = sched_clock();
548+ cfqg_stats_mark_idling(stats);
549+}
550+
551+static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg)
552+{
553+ struct cfqg_stats *stats = &cfqg->stats;
554+
555+ blkg_stat_add(&stats->avg_queue_size_sum,
556+ blkg_rwstat_sum(&stats->queued));
557+ blkg_stat_add(&stats->avg_queue_size_samples, 1);
558+ cfqg_stats_update_group_wait_time(stats);
559+}
560+
561+#else /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
562+
563+static inline void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg, struct cfq_group *curr_cfqg) { }
564+static inline void cfqg_stats_end_empty_time(struct cfqg_stats *stats) { }
565+static inline void cfqg_stats_update_dequeue(struct cfq_group *cfqg) { }
566+static inline void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg) { }
567+static inline void cfqg_stats_update_idle_time(struct cfq_group *cfqg) { }
568+static inline void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg) { }
569+static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
570+
571+#endif /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
572+
374573 #ifdef CONFIG_CFQ_GROUP_IOSCHED
375−#define cfq_log_cfqq(cfqd, cfqq, fmt, args...) \
574+
575+static inline void cfqg_get(struct cfq_group *cfqg)
576+{
577+ return blkg_get(cfqg_to_blkg(cfqg));
578+}
579+
580+static inline void cfqg_put(struct cfq_group *cfqg)
581+{
582+ return blkg_put(cfqg_to_blkg(cfqg));
583+}
584+
585+#define cfq_log_cfqq(cfqd, cfqq, fmt, args...) do { \
586+ char __pbuf[128]; \
587+ \
588+ blkg_path(cfqg_to_blkg((cfqq)->cfqg), __pbuf, sizeof(__pbuf)); \
376589 blk_add_trace_msg((cfqd)->queue, "cfq%d%c %s " fmt, (cfqq)->pid, \
377− cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
378− blkg_path(&(cfqq)->cfqg->blkg), ##args)
590+ cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
591+ __pbuf, ##args); \
592+} while (0)
379593
380−#define cfq_log_cfqg(cfqd, cfqg, fmt, args...) \
381− blk_add_trace_msg((cfqd)->queue, "%s " fmt, \
382− blkg_path(&(cfqg)->blkg), ##args) \
594+#define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do { \
595+ char __pbuf[128]; \
596+ \
597+ blkg_path(cfqg_to_blkg(cfqg), __pbuf, sizeof(__pbuf)); \
598+ blk_add_trace_msg((cfqd)->queue, "%s " fmt, __pbuf, ##args); \
599+} while (0)
600+
601+static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
602+ struct cfq_group *curr_cfqg, int rw)
603+{
604+ blkg_rwstat_add(&cfqg->stats.queued, rw, 1);
605+ cfqg_stats_end_empty_time(&cfqg->stats);
606+ cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg);
607+}
608+
609+static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
610+ unsigned long time, unsigned long unaccounted_time)
611+{
612+ blkg_stat_add(&cfqg->stats.time, time);
613+#ifdef CONFIG_DEBUG_BLK_CGROUP
614+ blkg_stat_add(&cfqg->stats.unaccounted_time, unaccounted_time);
615+#endif
616+}
617+
618+static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw)
619+{
620+ blkg_rwstat_add(&cfqg->stats.queued, rw, -1);
621+}
622+
623+static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw)
624+{
625+ blkg_rwstat_add(&cfqg->stats.merged, rw, 1);
626+}
627+
628+static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg,
629+ uint64_t bytes, int rw)
630+{
631+ blkg_stat_add(&cfqg->stats.sectors, bytes >> 9);
632+ blkg_rwstat_add(&cfqg->stats.serviced, rw, 1);
633+ blkg_rwstat_add(&cfqg->stats.service_bytes, rw, bytes);
634+}
635+
636+static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
637+ uint64_t start_time, uint64_t io_start_time, int rw)
638+{
639+ struct cfqg_stats *stats = &cfqg->stats;
640+ unsigned long long now = sched_clock();
641+
642+ if (time_after64(now, io_start_time))
643+ blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
644+ if (time_after64(io_start_time, start_time))
645+ blkg_rwstat_add(&stats->wait_time, rw,
646+ io_start_time - start_time);
647+}
648+
649+static void cfq_pd_reset_stats(struct blkcg_gq *blkg)
650+{
651+ struct cfq_group *cfqg = blkg_to_cfqg(blkg);
652+ struct cfqg_stats *stats = &cfqg->stats;
653+
654+ /* queued stats shouldn't be cleared */
655+ blkg_rwstat_reset(&stats->service_bytes);
656+ blkg_rwstat_reset(&stats->serviced);
657+ blkg_rwstat_reset(&stats->merged);
658+ blkg_rwstat_reset(&stats->service_time);
659+ blkg_rwstat_reset(&stats->wait_time);
660+ blkg_stat_reset(&stats->time);
661+#ifdef CONFIG_DEBUG_BLK_CGROUP
662+ blkg_stat_reset(&stats->unaccounted_time);
663+ blkg_stat_reset(&stats->avg_queue_size_sum);
664+ blkg_stat_reset(&stats->avg_queue_size_samples);
665+ blkg_stat_reset(&stats->dequeue);
666+ blkg_stat_reset(&stats->group_wait_time);
667+ blkg_stat_reset(&stats->idle_time);
668+ blkg_stat_reset(&stats->empty_time);
669+#endif
670+}
671+
672+#else /* CONFIG_CFQ_GROUP_IOSCHED */
673+
674+static inline void cfqg_get(struct cfq_group *cfqg) { }
675+static inline void cfqg_put(struct cfq_group *cfqg) { }
383676
384−#else
385677 #define cfq_log_cfqq(cfqd, cfqq, fmt, args...) \
386678 blk_add_trace_msg((cfqd)->queue, "cfq%d " fmt, (cfqq)->pid, ##args)
387679 #define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do {} while (0)
388−#endif
680+
681+static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
682+ struct cfq_group *curr_cfqg, int rw) { }
683+static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
684+ unsigned long time, unsigned long unaccounted_time) { }
685+static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { }
686+static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { }
687+static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg,
688+ uint64_t bytes, int rw) { }
689+static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
690+ uint64_t start_time, uint64_t io_start_time, int rw) { }
691+
692+#endif /* CONFIG_CFQ_GROUP_IOSCHED */
693+
389694 #define cfq_log(cfqd, fmt, args...) \
390695 blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args)
391696
@@ -466,8 +771,9 @@ static inline int cfqg_busy_async_queues(struct cfq_data *cfqd, · 74 unmodified lines
466771 }
467772
468773 static void cfq_dispatch_insert(struct request_queue *, struct request *);
469−static struct cfq_queue *cfq_get_queue(struct cfq_data *, bool,
470− struct io_context *, gfp_t);
774+static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync,
775+ struct cfq_io_cq *cic, struct bio *bio,
776+ gfp_t gfp_mask);
471777
472778 static inline struct cfq_io_cq *icq_to_cic(struct io_cq *icq)
473779 {
@@ -545,7 +851,7 @@ static inline u64 cfq_scale_slice(unsigned long delta, struct cfq_group *cfqg) · 71 unmodified lines
545851 {
546852 u64 d = delta << CFQ_SERVICE_SHIFT;
547853
548− d = d * BLKIO_WEIGHT_DEFAULT;
854+ d = d * CFQ_WEIGHT_DEFAULT;
549855 do_div(d, cfqg->weight);
550856 return d;
551857 }
@@ -872,9 +1178,9 @@ static void · 320 unmodified lines
8721178 cfq_update_group_weight(struct cfq_group *cfqg)
8731179 {
8741180 BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node));
875− if (cfqg->needs_update) {
1181+ if (cfqg->new_weight) {
8761182 cfqg->weight = cfqg->new_weight;
877− cfqg->needs_update = false;
1183+ cfqg->new_weight = 0;
8781184 }
8791185 }
8801186
@@ -936,7 +1242,7 @@ cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg) · 55 unmodified lines
9361242 cfq_log_cfqg(cfqd, cfqg, "del_from_rr group");
9371243 cfq_group_service_tree_del(st, cfqg);
9381244 cfqg->saved_workload_slice = 0;
939− cfq_blkiocg_update_dequeue_stats(&cfqg->blkg, 1);
1245+ cfqg_stats_update_dequeue(cfqg);
9401246 }
9411247
9421248 static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq,
@@ -1008,178 +1314,59 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, · 65 unmodified lines
10081314 "sl_used=%u disp=%u charge=%u iops=%u sect=%lu",
10091315 used_sl, cfqq->slice_dispatch, charge,
10101316 iops_mode(cfqd), cfqq->nr_sectors);
1011− cfq_blkiocg_update_timeslice_used(&cfqg->blkg, used_sl,
1012− unaccounted_sl);
1013− cfq_blkiocg_set_start_empty_time(&cfqg->blkg);
1317+ cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl);
1318+ cfqg_stats_set_start_empty_time(cfqg);
10141319 }
10151320
1016−#ifdef CONFIG_CFQ_GROUP_IOSCHED
1017−static inline struct cfq_group *cfqg_of_blkg(struct blkio_group *blkg)
1018−{
1019− if (blkg)
1020− return container_of(blkg, struct cfq_group, blkg);
1021− return NULL;
1022−}
1023−
1024−static void cfq_update_blkio_group_weight(void *key, struct blkio_group *blkg,
1025− unsigned int weight)
1026−{
1027− struct cfq_group *cfqg = cfqg_of_blkg(blkg);
1028− cfqg->new_weight = weight;
1029− cfqg->needs_update = true;
1030−}
1031−
1032−static void cfq_init_add_cfqg_lists(struct cfq_data *cfqd,
1033− struct cfq_group *cfqg, struct blkio_cgroup *blkcg)
1034−{
1035− struct backing_dev_info *bdi = &cfqd->queue->backing_dev_info;
1036− unsigned int major, minor;
1037−
1038− /*
1039− * Add group onto cgroup list. It might happen that bdi->dev is
1040− * not initialized yet. Initialize this new group without major
1041− * and minor info and this info will be filled in once a new thread
1042− * comes for IO.
1043− */
1044− if (bdi->dev) {
1045− sscanf(dev_name(bdi->dev), "%u:%u", &major, &minor);
1046− cfq_blkiocg_add_blkio_group(blkcg, &cfqg->blkg,
1047− (void *)cfqd, MKDEV(major, minor));
1048− } else
1049− cfq_blkiocg_add_blkio_group(blkcg, &cfqg->blkg,
1050− (void *)cfqd, 0);
1051−
1052− cfqd->nr_blkcg_linked_grps++;
1053− cfqg->weight = blkcg_get_weight(blkcg, cfqg->blkg.dev);
1054−
1055− /* Add group on cfqd list */
1056− hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list);
1057−}
1058−
1059−/*
1060− * Should be called from sleepable context. No request queue lock as per
1061− * cpu stats are allocated dynamically and alloc_percpu needs to be called
1062− * from sleepable context.
1321+/**
1322+ * cfq_init_cfqg_base - initialize base part of a cfq_group
1323+ * @cfqg: cfq_group to initialize
1324+ *
1325+ * Initialize the base part which is used whether %CONFIG_CFQ_GROUP_IOSCHED
1326+ * is enabled or not.
10631327 */
1064−static struct cfq_group * cfq_alloc_cfqg(struct cfq_data *cfqd)
1328+static void cfq_init_cfqg_base(struct cfq_group *cfqg)
10651329 {
1066− struct cfq_group *cfqg = NULL;
1067− int i, j, ret;
10681330 struct cfq_rb_root *st;
1069−
1070− cfqg = kzalloc_node(sizeof(*cfqg), GFP_ATOMIC, cfqd->queue->node);
1071− if (!cfqg)
1072− return NULL;
1331+ int i, j;
10731332
10741333 for_each_cfqg_st(cfqg, i, j, st)
10751334 *st = CFQ_RB_ROOT;
10761335 RB_CLEAR_NODE(&cfqg->rb_node);
10771336
10781337 cfqg->ttime.last_end_request = jiffies;
1079−
1080− /*
1081− * Take the initial reference that will be released on destroy
1082− * This can be thought of a joint reference by cgroup and
1083− * elevator which will be dropped by either elevator exit
1084− * or cgroup deletion path depending on who is exiting first.
1085− */
1086− cfqg->ref = 1;
1087−
1088− ret = blkio_alloc_blkg_stats(&cfqg->blkg);
1089− if (ret) {
1090− kfree(cfqg);
1091− return NULL;
1092− }
1093−
1094− return cfqg;
10951338 }
10961339
1097−static struct cfq_group *
1098−cfq_find_cfqg(struct cfq_data *cfqd, struct blkio_cgroup *blkcg)
1340+#ifdef CONFIG_CFQ_GROUP_IOSCHED
1341+static void cfq_pd_init(struct blkcg_gq *blkg)
10991342 {
1100− struct cfq_group *cfqg = NULL;
1101− void *key = cfqd;
1102− struct backing_dev_info *bdi = &cfqd->queue->backing_dev_info;
1103− unsigned int major, minor;
1104−
1105− /*
1106− * This is the common case when there are no blkio cgroups.
1107− * Avoid lookup in this case
1108− */
1109− if (blkcg == &blkio_root_cgroup)
1110− cfqg = &cfqd->root_group;
1111− else
1112− cfqg = cfqg_of_blkg(blkiocg_lookup_group(blkcg, key));
1113−
1114− if (cfqg && !cfqg->blkg.dev && bdi->dev && dev_name(bdi->dev)) {
1115− sscanf(dev_name(bdi->dev), "%u:%u", &major, &minor);
1116− cfqg->blkg.dev = MKDEV(major, minor);
1117− }
1343+ struct cfq_group *cfqg = blkg_to_cfqg(blkg);
11181344
1119− return cfqg;
1345+ cfq_init_cfqg_base(cfqg);
1346+ cfqg->weight = blkg->blkcg->cfq_weight;
11201347 }
11211348
11221349 /*
11231350 * Search for the cfq group current task belongs to. request_queue lock must
11241351 * be held.
11251352 */
1126−static struct cfq_group *cfq_get_cfqg(struct cfq_data *cfqd)
1353+static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd,
1354+ struct blkcg *blkcg)
11271355 {
1128− struct blkio_cgroup *blkcg;
1129− struct cfq_group *cfqg = NULL, *__cfqg = NULL;
11301356 struct request_queue *q = cfqd->queue;
1357+ struct cfq_group *cfqg = NULL;
11311358
1132− rcu_read_lock();
1133− blkcg = task_blkio_cgroup(current);
1134− cfqg = cfq_find_cfqg(cfqd, blkcg);
1135− if (cfqg) {
1136− rcu_read_unlock();
1137− return cfqg;
1138− }
1139−
1140− /*
1141− * Need to allocate a group. Allocation of group also needs allocation
1142− * of per cpu stats which in-turn takes a mutex() and can block. Hence
1143− * we need to drop rcu lock and queue_lock before we call alloc.
1144− *
1145− * Not taking any queue reference here and assuming that queue is
1146− * around by the time we return. CFQ queue allocation code does
1147− * the same. It might be racy though.
1148− */
1149−
1150− rcu_read_unlock();
1151− spin_unlock_irq(q->queue_lock);
1152−
1153− cfqg = cfq_alloc_cfqg(cfqd);
1154−
1155− spin_lock_irq(q->queue_lock);
1156−
1157− rcu_read_lock();
1158− blkcg = task_blkio_cgroup(current);
1159−
1160− /*
1161− * If some other thread already allocated the group while we were
1162− * not holding queue lock, free up the group
1163− */
1164− __cfqg = cfq_find_cfqg(cfqd, blkcg);
1359+ /* avoid lookup for the common case where there's no blkcg */
1360+ if (blkcg == &blkcg_root) {
1361+ cfqg = cfqd->root_group;
1362+ } else {
1363+ struct blkcg_gq *blkg;
11651364
1166− if (__cfqg) {
1167− kfree(cfqg);
1168− rcu_read_unlock();
1169− return __cfqg;
1365+ blkg = blkg_lookup_create(blkcg, q);
1366+ if (!IS_ERR(blkg))
1367+ cfqg = blkg_to_cfqg(blkg);
11701368 }
11711369
1172− if (!cfqg)
1173− cfqg = &cfqd->root_group;
1174−
1175− cfq_init_add_cfqg_lists(cfqd, cfqg, blkcg);
1176− rcu_read_unlock();
1177− return cfqg;
1178−}
1179−
1180−static inline struct cfq_group *cfq_ref_get_cfqg(struct cfq_group *cfqg)
1181−{
1182− cfqg->ref++;
11831370 return cfqg;
11841371 }
11851372
@@ -1187,94 +1374,224 @@ static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) · 1 unmodified line
11871374 {
11881375 /* Currently, all async queues are mapped to root group */
11891376 if (!cfq_cfqq_sync(cfqq))
1190− cfqg = &cfqq->cfqd->root_group;
1377+ cfqg = cfqq->cfqd->root_group;
11911378
11921379 cfqq->cfqg = cfqg;
11931380 /* cfqq reference on cfqg */
1194− cfqq->cfqg->ref++;
1381+ cfqg_get(cfqg);
11951382 }
11961383
1197−static void cfq_put_cfqg(struct cfq_group *cfqg)
1384+static u64 cfqg_prfill_weight_device(struct seq_file *sf,
1385+ struct blkg_policy_data *pd, int off)
11981386 {
1199− struct cfq_rb_root *st;
1200− int i, j;
1387+ struct cfq_group *cfqg = pd_to_cfqg(pd);
12011388
1202− BUG_ON(cfqg->ref <= 0);
1203− cfqg->ref--;
1204− if (cfqg->ref)
1205− return;
1206− for_each_cfqg_st(cfqg, i, j, st)
1207− BUG_ON(!RB_EMPTY_ROOT(&st->rb));
1208− free_percpu(cfqg->blkg.stats_cpu);
1209− kfree(cfqg);
1389+ if (!cfqg->dev_weight)
1390+ return 0;
1391+ return __blkg_prfill_u64(sf, pd, cfqg->dev_weight);
12101392 }
12111393
1212−static void cfq_destroy_cfqg(struct cfq_data *cfqd, struct cfq_group *cfqg)
1394+static int cfqg_print_weight_device(struct cgroup *cgrp, struct cftype *cft,
1395+ struct seq_file *sf)
12131396 {
1214− /* Something wrong if we are trying to remove same group twice */
1215− BUG_ON(hlist_unhashed(&cfqg->cfqd_node));
1397+ blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp),
1398+ cfqg_prfill_weight_device, &blkcg_policy_cfq, 0,
1399+ false);
1400+ return 0;
1401+}
12161402
1217− hlist_del_init(&cfqg->cfqd_node);
1403+static int cfq_print_weight(struct cgroup *cgrp, struct cftype *cft,
1404+ struct seq_file *sf)
1405+{
1406+ seq_printf(sf, "%u\n", cgroup_to_blkcg(cgrp)->cfq_weight);
1407+ return 0;
1408+}
12181409
1219− BUG_ON(cfqd->nr_blkcg_linked_grps <= 0);
1220− cfqd->nr_blkcg_linked_grps--;
1410+static int cfqg_set_weight_device(struct cgroup *cgrp, struct cftype *cft,
1411+ const char *buf)
1412+{
1413+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
1414+ struct blkg_conf_ctx ctx;
1415+ struct cfq_group *cfqg;
1416+ int ret;
12211417
1222− /*
1223− * Put the reference taken at the time of creation so that when all
1224− * queues are gone, group can be destroyed.
1225− */
1226− cfq_put_cfqg(cfqg);
1418+ ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx);
1419+ if (ret)
1420+ return ret;
1421+
1422+ ret = -EINVAL;
1423+ cfqg = blkg_to_cfqg(ctx.blkg);
1424+ if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) {
1425+ cfqg->dev_weight = ctx.v;
1426+ cfqg->new_weight = cfqg->dev_weight ?: blkcg->cfq_weight;
1427+ ret = 0;
1428+ }
1429+
1430+ blkg_conf_finish(&ctx);
1431+ return ret;
12271432 }
12281433
1229−static void cfq_release_cfq_groups(struct cfq_data *cfqd)
1434+static int cfq_set_weight(struct cgroup *cgrp, struct cftype *cft, u64 val)
12301435 {
1231− struct hlist_node *pos, *n;
1232− struct cfq_group *cfqg;
1436+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
1437+ struct blkcg_gq *blkg;
1438+ struct hlist_node *n;
12331439
1234− hlist_for_each_entry_safe(cfqg, pos, n, &cfqd->cfqg_list, cfqd_node) {
1235− /*
1236− * If cgroup removal path got to blk_group first and removed
1237− * it from cgroup list, then it will take care of destroying
1238− * cfqg also.
1239− */
1240− if (!cfq_blkiocg_del_blkio_group(&cfqg->blkg))
1241− cfq_destroy_cfqg(cfqd, cfqg);
1440+ if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX)
1441+ return -EINVAL;
1442+
1443+ spin_lock_irq(&blkcg->lock);
1444+ blkcg->cfq_weight = (unsigned int)val;
1445+
1446+ hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
1447+ struct cfq_group *cfqg = blkg_to_cfqg(blkg);
1448+
1449+ if (cfqg && !cfqg->dev_weight)
1450+ cfqg->new_weight = blkcg->cfq_weight;
12421451 }
1452+
1453+ spin_unlock_irq(&blkcg->lock);
1454+ return 0;
12431455 }
12441456
1245−/*
1246− * Blk cgroup controller notification saying that blkio_group object is being
1247− * delinked as associated cgroup object is going away. That also means that
1248− * no new IO will come in this group. So get rid of this group as soon as
1249− * any pending IO in the group is finished.
1250− *
1251− * This function is called under rcu_read_lock(). key is the rcu protected
1252− * pointer. That means "key" is a valid cfq_data pointer as long as we are rcu
1253− * read lock.
1254− *
1255− * "key" was fetched from blkio_group under blkio_cgroup->lock. That means
1256− * it should not be NULL as even if elevator was exiting, cgroup deltion
1257− * path got to it first.
1258− */
1259−static void cfq_unlink_blkio_group(void *key, struct blkio_group *blkg)
1457+static int cfqg_print_stat(struct cgroup *cgrp, struct cftype *cft,
1458+ struct seq_file *sf)
12601459 {
1261− unsigned long flags;
1262− struct cfq_data *cfqd = key;
1460+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
12631461
1264− spin_lock_irqsave(cfqd->queue->queue_lock, flags);
1265− cfq_destroy_cfqg(cfqd, cfqg_of_blkg(blkg));
1266− spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
1462+ blkcg_print_blkgs(sf, blkcg, blkg_prfill_stat, &blkcg_policy_cfq,
1463+ cft->private, false);
1464+ return 0;
12671465 }
12681466
1269−#else /* GROUP_IOSCHED */
1270−static struct cfq_group *cfq_get_cfqg(struct cfq_data *cfqd)
1467+static int cfqg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
1468+ struct seq_file *sf)
12711469 {
1272− return &cfqd->root_group;
1470+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
1471+
1472+ blkcg_print_blkgs(sf, blkcg, blkg_prfill_rwstat, &blkcg_policy_cfq,
1473+ cft->private, true);
1474+ return 0;
12731475 }
12741476
1275−static inline struct cfq_group *cfq_ref_get_cfqg(struct cfq_group *cfqg)
1477+#ifdef CONFIG_DEBUG_BLK_CGROUP
1478+static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf,
1479+ struct blkg_policy_data *pd, int off)
12761480 {
1277− return cfqg;
1481+ struct cfq_group *cfqg = pd_to_cfqg(pd);
1482+ u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);
1483+ u64 v = 0;
1484+
1485+ if (samples) {
1486+ v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);
1487+ do_div(v, samples);
1488+ }
1489+ __blkg_prfill_u64(sf, pd, v);
1490+ return 0;
1491+}
1492+
1493+/* print avg_queue_size */
1494+static int cfqg_print_avg_queue_size(struct cgroup *cgrp, struct cftype *cft,
1495+ struct seq_file *sf)
1496+{
1497+ struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
1498+
1499+ blkcg_print_blkgs(sf, blkcg, cfqg_prfill_avg_queue_size,
1500+ &blkcg_policy_cfq, 0, false);
1501+ return 0;
1502+}
1503+#endif /* CONFIG_DEBUG_BLK_CGROUP */
1504+
1505+static struct cftype cfq_blkcg_files[] = {
1506+ {
1507+ .name = "weight_device",
1508+ .read_seq_string = cfqg_print_weight_device,
1509+ .write_string = cfqg_set_weight_device,
1510+ .max_write_len = 256,
1511+ },
1512+ {
1513+ .name = "weight",
1514+ .read_seq_string = cfq_print_weight,
1515+ .write_u64 = cfq_set_weight,
1516+ },
1517+ {
1518+ .name = "time",
1519+ .private = offsetof(struct cfq_group, stats.time),
1520+ .read_seq_string = cfqg_print_stat,
1521+ },
1522+ {
1523+ .name = "sectors",
1524+ .private = offsetof(struct cfq_group, stats.sectors),
1525+ .read_seq_string = cfqg_print_stat,
1526+ },
1527+ {
1528+ .name = "io_service_bytes",
1529+ .private = offsetof(struct cfq_group, stats.service_bytes),
1530+ .read_seq_string = cfqg_print_rwstat,
1531+ },
1532+ {
1533+ .name = "io_serviced",
1534+ .private = offsetof(struct cfq_group, stats.serviced),
1535+ .read_seq_string = cfqg_print_rwstat,
1536+ },
1537+ {
1538+ .name = "io_service_time",
1539+ .private = offsetof(struct cfq_group, stats.service_time),
1540+ .read_seq_string = cfqg_print_rwstat,
1541+ },
1542+ {
1543+ .name = "io_wait_time",
1544+ .private = offsetof(struct cfq_group, stats.wait_time),
1545+ .read_seq_string = cfqg_print_rwstat,
1546+ },
1547+ {
1548+ .name = "io_merged",
1549+ .private = offsetof(struct cfq_group, stats.merged),
1550+ .read_seq_string = cfqg_print_rwstat,
1551+ },
1552+ {
1553+ .name = "io_queued",
1554+ .private = offsetof(struct cfq_group, stats.queued),
1555+ .read_seq_string = cfqg_print_rwstat,
1556+ },
1557+#ifdef CONFIG_DEBUG_BLK_CGROUP
1558+ {
1559+ .name = "avg_queue_size",
1560+ .read_seq_string = cfqg_print_avg_queue_size,
1561+ },
1562+ {
1563+ .name = "group_wait_time",
1564+ .private = offsetof(struct cfq_group, stats.group_wait_time),
1565+ .read_seq_string = cfqg_print_stat,
1566+ },
1567+ {
1568+ .name = "idle_time",
1569+ .private = offsetof(struct cfq_group, stats.idle_time),
1570+ .read_seq_string = cfqg_print_stat,
1571+ },
1572+ {
1573+ .name = "empty_time",
1574+ .private = offsetof(struct cfq_group, stats.empty_time),
1575+ .read_seq_string = cfqg_print_stat,
1576+ },
1577+ {
1578+ .name = "dequeue",
1579+ .private = offsetof(struct cfq_group, stats.dequeue),
1580+ .read_seq_string = cfqg_print_stat,
1581+ },
1582+ {
1583+ .name = "unaccounted_time",
1584+ .private = offsetof(struct cfq_group, stats.unaccounted_time),
1585+ .read_seq_string = cfqg_print_stat,
1586+ },
1587+#endif /* CONFIG_DEBUG_BLK_CGROUP */
1588+ { } /* terminate */
1589+};
1590+#else /* GROUP_IOSCHED */
1591+static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd,
1592+ struct blkcg *blkcg)
1593+{
1594+ return cfqd->root_group;
12781595 }
12791596
12801597 static inline void
@@ -1282,9 +1599,6 @@ cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) { · 1 unmodified line
12821599 cfqq->cfqg = cfqg;
12831600 }
12841601
1285−static void cfq_release_cfq_groups(struct cfq_data *cfqd) {}
1286−static inline void cfq_put_cfqg(struct cfq_group *cfqg) {}
1287−
12881602 #endif /* GROUP_IOSCHED */
12891603
12901604 /*
@@ -1551,12 +1865,10 @@ static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) · 260 unmodified lines
15511865 {
15521866 elv_rb_del(&cfqq->sort_list, rq);
15531867 cfqq->queued[rq_is_sync(rq)]--;
1554− cfq_blkiocg_update_io_remove_stats(&(RQ_CFQG(rq))->blkg,
1555− rq_data_dir(rq), rq_is_sync(rq));
1868+ cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
15561869 cfq_add_rq_rb(rq);
1557− cfq_blkiocg_update_io_add_stats(&(RQ_CFQG(rq))->blkg,
1558− &cfqq->cfqd->serving_group->blkg, rq_data_dir(rq),
1559− rq_is_sync(rq));
1870+ cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group,
1871+ rq->cmd_flags);
15601872 }
15611873
15621874 static struct request *
@@ -1612,8 +1924,7 @@ static void cfq_remove_request(struct request *rq) · 49 unmodified lines
16121924 cfq_del_rq_rb(rq);
16131925
16141926 cfqq->cfqd->rq_queued--;
1615− cfq_blkiocg_update_io_remove_stats(&(RQ_CFQG(rq))->blkg,
1616− rq_data_dir(rq), rq_is_sync(rq));
1927+ cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
16171928 if (rq->cmd_flags & REQ_PRIO) {
16181929 WARN_ON(!cfqq->prio_pending);
16191930 cfqq->prio_pending--;
@@ -1648,8 +1959,7 @@ static void cfq_merged_request(struct request_queue *q, struct request *req, · 28 unmodified lines
16481959 static void cfq_bio_merged(struct request_queue *q, struct request *req,
16491960 struct bio *bio)
16501961 {
1651− cfq_blkiocg_update_io_merged_stats(&(RQ_CFQG(req))->blkg,
1652− bio_data_dir(bio), cfq_bio_sync(bio));
1962+ cfqg_stats_update_io_merged(RQ_CFQG(req), bio->bi_rw);
16531963 }
16541964
16551965 static void
@@ -1671,8 +1981,7 @@ cfq_merged_requests(struct request_queue *q, struct request *rq, · 15 unmodified lines
16711981 if (cfqq->next_rq == next)
16721982 cfqq->next_rq = rq;
16731983 cfq_remove_request(next);
1674− cfq_blkiocg_update_io_merged_stats(&(RQ_CFQG(rq))->blkg,
1675− rq_data_dir(next), rq_is_sync(next));
1984+ cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags);
16761985
16771986 cfqq = RQ_CFQQ(next);
16781987 /*
@@ -1713,7 +2022,7 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq, · 34 unmodified lines
17132022 static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
17142023 {
17152024 del_timer(&cfqd->idle_slice_timer);
1716− cfq_blkiocg_update_idle_time_stats(&cfqq->cfqg->blkg);
2025+ cfqg_stats_update_idle_time(cfqq->cfqg);
17172026 }
17182027
17192028 static void __cfq_set_active_queue(struct cfq_data *cfqd,
@@ -1722,7 +2031,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd, · 2 unmodified lines
17222031 if (cfqq) {
17232032 cfq_log_cfqq(cfqd, cfqq, "set_active wl_prio:%d wl_type:%d",
17242033 cfqd->serving_prio, cfqd->serving_type);
1725− cfq_blkiocg_update_avg_queue_size_stats(&cfqq->cfqg->blkg);
2034+ cfqg_stats_update_avg_queue_size(cfqq->cfqg);
17262035 cfqq->slice_start = 0;
17272036 cfqq->dispatch_start = jiffies;
17282037 cfqq->allocated_slice = 0;
@@ -2043,7 +2352,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) · 314 unmodified lines
20432352 * task has exited, don't wait
20442353 */
20452354 cic = cfqd->active_cic;
2046− if (!cic || !atomic_read(&cic->icq.ioc->nr_tasks))
2355+ if (!cic || !atomic_read(&cic->icq.ioc->active_ref))
20472356 return;
20482357
20492358 /*
@@ -2070,7 +2379,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) · 20 unmodified lines
20702379 sl = cfqd->cfq_slice_idle;
20712380
20722381 mod_timer(&cfqd->idle_slice_timer, jiffies + sl);
2073− cfq_blkiocg_update_set_idle_time_stats(&cfqq->cfqg->blkg);
2382+ cfqg_stats_set_start_idle_time(cfqq->cfqg);
20742383 cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl,
20752384 group_idle ? 1 : 0);
20762385 }
@@ -2093,8 +2402,7 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq) · 16 unmodified lines
20932402
20942403 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++;
20952404 cfqq->nr_sectors += blk_rq_sectors(rq);
2096− cfq_blkiocg_update_dispatch_stats(&cfqq->cfqg->blkg, blk_rq_bytes(rq),
2097− rq_data_dir(rq), rq_is_sync(rq));
2405+ cfqg_stats_update_dispatch(cfqq->cfqg, blk_rq_bytes(rq), rq->cmd_flags);
20982406 }
20992407
21002408 /*
@@ -2677,7 +2985,7 @@ static void cfq_put_queue(struct cfq_queue *cfqq) · 576 unmodified lines
26772985
26782986 BUG_ON(cfq_cfqq_on_rr(cfqq));
26792987 kmem_cache_free(cfq_pool, cfqq);
2680− cfq_put_cfqg(cfqg);
2988+ cfqg_put(cfqg);
26812989 }
26822990
26832991 static void cfq_put_cooperator(struct cfq_queue *cfqq)
@@ -2736,7 +3044,7 @@ static void cfq_exit_icq(struct io_cq *icq) · 52 unmodified lines
27363044 }
27373045 }
27383046
2739−static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc)
3047+static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)
27403048 {
27413049 struct task_struct *tsk = current;
27423050 int ioprio_class;
@@ -2744,7 +3052,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc) · 1 unmodified line
27443052 if (!cfq_cfqq_prio_changed(cfqq))
27453053 return;
27463054
2747− ioprio_class = IOPRIO_PRIO_CLASS(ioc->ioprio);
3055+ ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);
27483056 switch (ioprio_class) {
27493057 default:
27503058 printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class);
@@ -2756,11 +3064,11 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc) · 5 unmodified lines
27563064 cfqq->ioprio_class = task_nice_ioclass(tsk);
27573065 break;
27583066 case IOPRIO_CLASS_RT:
2759− cfqq->ioprio = task_ioprio(ioc);
3067+ cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
27603068 cfqq->ioprio_class = IOPRIO_CLASS_RT;
27613069 break;
27623070 case IOPRIO_CLASS_BE:
2763− cfqq->ioprio = task_ioprio(ioc);
3071+ cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
27643072 cfqq->ioprio_class = IOPRIO_CLASS_BE;
27653073 break;
27663074 case IOPRIO_CLASS_IDLE:
@@ -2778,19 +3086,24 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc) · 11 unmodified lines
27783086 cfq_clear_cfqq_prio_changed(cfqq);
27793087 }
27803088
2781−static void changed_ioprio(struct cfq_io_cq *cic)
3089+static void check_ioprio_changed(struct cfq_io_cq *cic, struct bio *bio)
27823090 {
3091+ int ioprio = cic->icq.ioc->ioprio;
27833092 struct cfq_data *cfqd = cic_to_cfqd(cic);
27843093 struct cfq_queue *cfqq;
27853094
2786− if (unlikely(!cfqd))
3095+ /*
3096+ * Check whether ioprio has changed. The condition may trigger
3097+ * spuriously on a newly created cic but there's no harm.
3098+ */
3099+ if (unlikely(!cfqd) || likely(cic->ioprio == ioprio))
27873100 return;
27883101
27893102 cfqq = cic->cfqq[BLK_RW_ASYNC];
27903103 if (cfqq) {
27913104 struct cfq_queue *new_cfqq;
2792− new_cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic->icq.ioc,
2793− GFP_ATOMIC);
3105+ new_cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio,
3106+ GFP_ATOMIC);
27943107 if (new_cfqq) {
27953108 cic->cfqq[BLK_RW_ASYNC] = new_cfqq;
27963109 cfq_put_queue(cfqq);
@@ -2800,6 +3113,8 @@ static void changed_ioprio(struct cfq_io_cq *cic) · 3 unmodified lines
28003113 cfqq = cic->cfqq[BLK_RW_SYNC];
28013114 if (cfqq)
28023115 cfq_mark_cfqq_prio_changed(cfqq);
3116+
3117+ cic->ioprio = ioprio;
28033118 }
28043119
28053120 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
@@ -2823,17 +3138,24 @@ static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq, · 17 unmodified lines
28233138 }
28243139
28253140 #ifdef CONFIG_CFQ_GROUP_IOSCHED
2826−static void changed_cgroup(struct cfq_io_cq *cic)
3141+static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)
28273142 {
2828− struct cfq_queue *sync_cfqq = cic_to_cfqq(cic, 1);
28293143 struct cfq_data *cfqd = cic_to_cfqd(cic);
2830− struct request_queue *q;
3144+ struct cfq_queue *sync_cfqq;
3145+ uint64_t id;
28313146
2832− if (unlikely(!cfqd))
2833− return;
3147+ rcu_read_lock();
3148+ id = bio_blkcg(bio)->id;
3149+ rcu_read_unlock();
28343150
2835− q = cfqd->queue;
3151+ /*
3152+ * Check whether blkcg has changed. The condition may trigger
3153+ * spuriously on a newly created cic but there's no harm.
3154+ */
3155+ if (unlikely(!cfqd) || likely(cic->blkcg_id == id))
3156+ return;
28363157
3158+ sync_cfqq = cic_to_cfqq(cic, 1);
28373159 if (sync_cfqq) {
28383160 /*
28393161 * Drop reference to sync queue. A new sync queue will be
@@ -2843,21 +3165,26 @@ static void changed_cgroup(struct cfq_io_cq *cic) · 3 unmodified lines
28433165 cic_set_cfqq(cic, NULL, 1);
28443166 cfq_put_queue(sync_cfqq);
28453167 }
3168+
3169+ cic->blkcg_id = id;
28463170 }
3171+#else
3172+static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { }
28473173 #endif /* CONFIG_CFQ_GROUP_IOSCHED */
28483174
28493175 static struct cfq_queue *
2850−cfq_find_alloc_queue(struct cfq_data *cfqd, bool is_sync,
2851− struct io_context *ioc, gfp_t gfp_mask)
3176+cfq_find_alloc_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,
3177+ struct bio *bio, gfp_t gfp_mask)
28523178 {
3179+ struct blkcg *blkcg;
28533180 struct cfq_queue *cfqq, *new_cfqq = NULL;
2854− struct cfq_io_cq *cic;
28553181 struct cfq_group *cfqg;
28563182
28573183 retry:
2858− cfqg = cfq_get_cfqg(cfqd);
2859− cic = cfq_cic_lookup(cfqd, ioc);
2860− /* cic always exists here */
3184+ rcu_read_lock();
3185+
3186+ blkcg = bio_blkcg(bio);
3187+ cfqg = cfq_lookup_create_cfqg(cfqd, blkcg);
28613188 cfqq = cic_to_cfqq(cic, is_sync);
28623189
28633190 /*
@@ -2870,6 +3197,7 @@ retry: · 6 unmodified lines
28703197 cfqq = new_cfqq;
28713198 new_cfqq = NULL;
28723199 } else if (gfp_mask & __GFP_WAIT) {
3200+ rcu_read_unlock();
28733201 spin_unlock_irq(cfqd->queue->queue_lock);
28743202 new_cfqq = kmem_cache_alloc_node(cfq_pool,
28753203 gfp_mask | __GFP_ZERO,
@@ -2885,7 +3213,7 @@ retry: · 9 unmodified lines
28853213
28863214 if (cfqq) {
28873215 cfq_init_cfqq(cfqd, cfqq, current->pid, is_sync);
2888− cfq_init_prio_data(cfqq, ioc);
3216+ cfq_init_prio_data(cfqq, cic);
28893217 cfq_link_cfqq_cfqg(cfqq, cfqg);
28903218 cfq_log_cfqq(cfqd, cfqq, "alloced");
28913219 } else
@@ -2895,6 +3223,7 @@ retry: · 3 unmodified lines
28953223 if (new_cfqq)
28963224 kmem_cache_free(cfq_pool, new_cfqq);
28973225
3226+ rcu_read_unlock();
28983227 return cfqq;
28993228 }
29003229
@@ -2904,6 +3233,9 @@ cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio) · 3 unmodified lines
29043233 switch (ioprio_class) {
29053234 case IOPRIO_CLASS_RT:
29063235 return &cfqd->async_cfqq[0][ioprio];
3236+ case IOPRIO_CLASS_NONE:
3237+ ioprio = IOPRIO_NORM;
3238+ /* fall through */
29073239 case IOPRIO_CLASS_BE:
29083240 return &cfqd->async_cfqq[1][ioprio];
29093241 case IOPRIO_CLASS_IDLE:
@@ -2914,11 +3246,11 @@ cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio) · 4 unmodified lines
29143246 }
29153247
29163248 static struct cfq_queue *
2917−cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct io_context *ioc,
2918− gfp_t gfp_mask)
3249+cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,
3250+ struct bio *bio, gfp_t gfp_mask)
29193251 {
2920− const int ioprio = task_ioprio(ioc);
2921− const int ioprio_class = task_ioprio_class(ioc);
3252+ const int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);
3253+ const int ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
29223254 struct cfq_queue **async_cfqq = NULL;
29233255 struct cfq_queue *cfqq = NULL;
29243256
@@ -2928,7 +3260,7 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct io_context *ioc, · 3 unmodified lines
29283260 }
29293261
29303262 if (!cfqq)
2931− cfqq = cfq_find_alloc_queue(cfqd, is_sync, ioc, gfp_mask);
3263+ cfqq = cfq_find_alloc_queue(cfqd, is_sync, cic, bio, gfp_mask);
29323264
29333265 /*
29343266 * pin the queue now that it's allocated, scheduler exit will prune it
@@ -3010,7 +3342,7 @@ cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq, · 75 unmodified lines
30103342
30113343 if (cfqq->next_rq && (cfqq->next_rq->cmd_flags & REQ_NOIDLE))
30123344 enable_idle = 0;
3013− else if (!atomic_read(&cic->icq.ioc->nr_tasks) ||
3345+ else if (!atomic_read(&cic->icq.ioc->active_ref) ||
30143346 !cfqd->cfq_slice_idle ||
30153347 (!cfq_cfqq_deep(cfqq) && CFQQ_SEEKY(cfqq)))
30163348 enable_idle = 0;
@@ -3174,8 +3506,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq, · 157 unmodified lines
31743506 cfq_clear_cfqq_wait_request(cfqq);
31753507 __blk_run_queue(cfqd->queue);
31763508 } else {
3177− cfq_blkiocg_update_idle_time_stats(
3178− &cfqq->cfqg->blkg);
3509+ cfqg_stats_update_idle_time(cfqq->cfqg);
31793510 cfq_mark_cfqq_must_dispatch(cfqq);
31803511 }
31813512 }
@@ -3197,14 +3528,13 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq) · 15 unmodified lines
31973528 struct cfq_queue *cfqq = RQ_CFQQ(rq);
31983529
31993530 cfq_log_cfqq(cfqd, cfqq, "insert_request");
3200− cfq_init_prio_data(cfqq, RQ_CIC(rq)->icq.ioc);
3531+ cfq_init_prio_data(cfqq, RQ_CIC(rq));
32013532
32023533 rq_set_fifo_time(rq, jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]);
32033534 list_add_tail(&rq->queuelist, &cfqq->fifo);
32043535 cfq_add_rq_rb(rq);
3205− cfq_blkiocg_update_io_add_stats(&(RQ_CFQG(rq))->blkg,
3206− &cfqd->serving_group->blkg, rq_data_dir(rq),
3207− rq_is_sync(rq));
3536+ cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,
3537+ rq->cmd_flags);
32083538 cfq_rq_enqueued(cfqd, cfqq, rq);
32093539 }
32103540
@@ -3300,9 +3630,8 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) · 89 unmodified lines
33003630 cfqd->rq_in_driver--;
33013631 cfqq->dispatched--;
33023632 (RQ_CFQG(rq))->dispatched--;
3303− cfq_blkiocg_update_completion_stats(&cfqq->cfqg->blkg,
3304− rq_start_time_ns(rq), rq_io_start_time_ns(rq),
3305− rq_data_dir(rq), rq_is_sync(rq));
3633+ cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq),
3634+ rq_io_start_time_ns(rq), rq->cmd_flags);
33063635
33073636 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
33083637
@@ -3399,7 +3728,7 @@ static int cfq_may_queue(struct request_queue *q, int rw) · 90 unmodified lines
33993728
34003729 cfqq = cic_to_cfqq(cic, rw_is_sync(rw));
34013730 if (cfqq) {
3402− cfq_init_prio_data(cfqq, cic->icq.ioc);
3731+ cfq_init_prio_data(cfqq, cic);
34033732
34043733 return __cfq_may_queue(cfqq);
34053734 }
@@ -3421,7 +3750,7 @@ static void cfq_put_request(struct request *rq) · 15 unmodified lines
34213750 cfqq->allocated[rw]--;
34223751
34233752 /* Put down rq reference on cfqg */
3424− cfq_put_cfqg(RQ_CFQG(rq));
3753+ cfqg_put(RQ_CFQG(rq));
34253754 rq->elv.priv[0] = NULL;
34263755 rq->elv.priv[1] = NULL;
34273756
@@ -3465,32 +3794,25 @@ split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq) · 37 unmodified lines
34653794 * Allocate cfq data structures associated with this request.
34663795 */
34673796 static int
3468−cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
3797+cfq_set_request(struct request_queue *q, struct request *rq, struct bio *bio,
3798+ gfp_t gfp_mask)
34693799 {
34703800 struct cfq_data *cfqd = q->elevator->elevator_data;
34713801 struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);
34723802 const int rw = rq_data_dir(rq);
34733803 const bool is_sync = rq_is_sync(rq);
34743804 struct cfq_queue *cfqq;
3475− unsigned int changed;
34763805
34773806 might_sleep_if(gfp_mask & __GFP_WAIT);
34783807
34793808 spin_lock_irq(q->queue_lock);
34803809
3481− /* handle changed notifications */
3482− changed = icq_get_changed(&cic->icq);
3483− if (unlikely(changed & ICQ_IOPRIO_CHANGED))
3484− changed_ioprio(cic);
3485−#ifdef CONFIG_CFQ_GROUP_IOSCHED
3486− if (unlikely(changed & ICQ_CGROUP_CHANGED))
3487− changed_cgroup(cic);
3488−#endif
3489−
3810+ check_ioprio_changed(cic, bio);
3811+ check_blkcg_changed(cic, bio);
34903812 new_queue:
34913813 cfqq = cic_to_cfqq(cic, is_sync);
34923814 if (!cfqq || cfqq == &cfqd->oom_cfqq) {
3493− cfqq = cfq_get_queue(cfqd, is_sync, cic->icq.ioc, gfp_mask);
3815+ cfqq = cfq_get_queue(cfqd, is_sync, cic, bio, gfp_mask);
34943816 cic_set_cfqq(cic, cfqq, is_sync);
34953817 } else {
34963818 /*
@@ -3516,8 +3838,9 @@ new_queue: · 19 unmodified lines
35163838 cfqq->allocated[rw]++;
35173839
35183840 cfqq->ref++;
3841+ cfqg_get(cfqq->cfqg);
35193842 rq->elv.priv[0] = cfqq;
3520− rq->elv.priv[1] = cfq_ref_get_cfqg(cfqq->cfqg);
3843+ rq->elv.priv[1] = cfqq->cfqg;
35213844 spin_unlock_irq(q->queue_lock);
35223845 return 0;
35233846 }
@@ -3614,7 +3937,6 @@ static void cfq_exit_queue(struct elevator_queue *e) · 90 unmodified lines
36143937 {
36153938 struct cfq_data *cfqd = e->elevator_data;
36163939 struct request_queue *q = cfqd->queue;
3617− bool wait = false;
36183940
36193941 cfq_shutdown_timer_wq(cfqd);
36203942
@@ -3624,89 +3946,52 @@ static void cfq_exit_queue(struct elevator_queue *e) · 3 unmodified lines
36243946 __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
36253947
36263948 cfq_put_async_queues(cfqd);
3627− cfq_release_cfq_groups(cfqd);
3628−
3629− /*
3630− * If there are groups which we could not unlink from blkcg list,
3631− * wait for a rcu period for them to be freed.
3632− */
3633− if (cfqd->nr_blkcg_linked_grps)
3634− wait = true;
36353949
36363950 spin_unlock_irq(q->queue_lock);
36373951
36383952 cfq_shutdown_timer_wq(cfqd);
36393953
3640− /*
3641− * Wait for cfqg->blkg->key accessors to exit their grace periods.
3642− * Do this wait only if there are other unlinked groups out
3643− * there. This can happen if cgroup deletion path claimed the
3644− * responsibility of cleaning up a group before queue cleanup code
3645− * get to the group.
3646− *
3647− * Do not call synchronize_rcu() unconditionally as there are drivers
3648− * which create/delete request queue hundreds of times during scan/boot
3649− * and synchronize_rcu() can take significant time and slow down boot.
3650− */
3651− if (wait)
3652− synchronize_rcu();
3653−
3654−#ifdef CONFIG_CFQ_GROUP_IOSCHED
3655− /* Free up per cpu stats for root group */
3656− free_percpu(cfqd->root_group.blkg.stats_cpu);
3954+#ifndef CONFIG_CFQ_GROUP_IOSCHED
3955+ kfree(cfqd->root_group);
36573956 #endif
3957+ blkcg_deactivate_policy(q, &blkcg_policy_cfq);
36583958 kfree(cfqd);
36593959 }
36603960
3661−static void *cfq_init_queue(struct request_queue *q)
3961+static int cfq_init_queue(struct request_queue *q)
36623962 {
36633963 struct cfq_data *cfqd;
3664− int i, j;
3665− struct cfq_group *cfqg;
3666− struct cfq_rb_root *st;
3964+ struct blkcg_gq *blkg __maybe_unused;
3965+ int i, ret;
36673966
36683967 cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL | __GFP_ZERO, q->node);
36693968 if (!cfqd)
3670− return NULL;
3969+ return -ENOMEM;
3970+
3971+ cfqd->queue = q;
3972+ q->elevator->elevator_data = cfqd;
36713973
36723974 /* Init root service tree */
36733975 cfqd->grp_service_tree = CFQ_RB_ROOT;
36743976
3675− /* Init root group */
3676− cfqg = &cfqd->root_group;
3677− for_each_cfqg_st(cfqg, i, j, st)
3678− *st = CFQ_RB_ROOT;
3679− RB_CLEAR_NODE(&cfqg->rb_node);
3680−
3681− /* Give preference to root group over other groups */
3682− cfqg->weight = 2*BLKIO_WEIGHT_DEFAULT;
3683−
3977+ /* Init root group and prefer root group over other groups by default */
36843978 #ifdef CONFIG_CFQ_GROUP_IOSCHED
3685− /*
3686− * Set root group reference to 2. One reference will be dropped when
3687− * all groups on cfqd->cfqg_list are being deleted during queue exit.
3688− * Other reference will remain there as we don't want to delete this
3689− * group as it is statically allocated and gets destroyed when
3690− * throtl_data goes away.
3691− */
3692− cfqg->ref = 2;
3693−
3694− if (blkio_alloc_blkg_stats(&cfqg->blkg)) {
3695− kfree(cfqg);
3696− kfree(cfqd);
3697− return NULL;
3698− }
3699−
3700− rcu_read_lock();
3979+ ret = blkcg_activate_policy(q, &blkcg_policy_cfq);
3980+ if (ret)
3981+ goto out_free;
37013982
3702− cfq_blkiocg_add_blkio_group(&blkio_root_cgroup, &cfqg->blkg,
3703− (void *)cfqd, 0);
3704− rcu_read_unlock();
3705− cfqd->nr_blkcg_linked_grps++;
3983+ cfqd->root_group = blkg_to_cfqg(q->root_blkg);
3984+#else
3985+ ret = -ENOMEM;
3986+ cfqd->root_group = kzalloc_node(sizeof(*cfqd->root_group),
3987+ GFP_KERNEL, cfqd->queue->node);
3988+ if (!cfqd->root_group)
3989+ goto out_free;
37063990
3707− /* Add group on cfqd->cfqg_list */
3708− hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list);
3991+ cfq_init_cfqg_base(cfqd->root_group);
37093992 #endif
3993+ cfqd->root_group->weight = 2 * CFQ_WEIGHT_DEFAULT;
3994+
37103995 /*
37113996 * Not strictly needed (since RB_ROOT just clears the node and we
37123997 * zeroed cfqd on alloc), but better be safe in case someone decides
@@ -3718,13 +4003,17 @@ static void *cfq_init_queue(struct request_queue *q) · 5 unmodified lines
37184003 /*
37194004 * Our fallback cfqq if cfq_find_alloc_queue() runs into OOM issues.
37204005 * Grab a permanent reference to it, so that the normal code flow
3721− * will not attempt to free it.
4006+ * will not attempt to free it. oom_cfqq is linked to root_group
4007+ * but shouldn't hold a reference as it'll never be unlinked. Lose
4008+ * the reference from linking right away.
37224009 */
37234010 cfq_init_cfqq(cfqd, &cfqd->oom_cfqq, 1, 0);
37244011 cfqd->oom_cfqq.ref++;
3725− cfq_link_cfqq_cfqg(&cfqd->oom_cfqq, &cfqd->root_group);
37264012
3727− cfqd->queue = q;
4013+ spin_lock_irq(q->queue_lock);
4014+ cfq_link_cfqq_cfqg(&cfqd->oom_cfqq, cfqd->root_group);
4015+ cfqg_put(cfqd->root_group);
4016+ spin_unlock_irq(q->queue_lock);
37284017
37294018 init_timer(&cfqd->idle_slice_timer);
37304019 cfqd->idle_slice_timer.function = cfq_idle_slice_timer;
@@ -3750,7 +4039,11 @@ static void *cfq_init_queue(struct request_queue *q) · 19 unmodified lines
37504039 * second, in order to have larger depth for async operations.
37514040 */
37524041 cfqd->last_delayed_sync = jiffies - HZ;
3753− return cfqd;
4042+ return 0;
4043+
4044+out_free:
4045+ kfree(cfqd);
4046+ return ret;
37544047 }
37554048
37564049 /*
@@ -3877,15 +4170,13 @@ static struct elevator_type iosched_cfq = { · 120 unmodified lines
38774170 };
38784171
38794172 #ifdef CONFIG_CFQ_GROUP_IOSCHED
3880−static struct blkio_policy_type blkio_policy_cfq = {
3881− .ops = {
3882− .blkio_unlink_group_fn = cfq_unlink_blkio_group,
3883− .blkio_update_group_weight_fn = cfq_update_blkio_group_weight,
3884− },
3885− .plid = BLKIO_POLICY_PROP,
4173+static struct blkcg_policy blkcg_policy_cfq = {
4174+ .pd_size = sizeof(struct cfq_group),
4175+ .cftypes = cfq_blkcg_files,
4176+
4177+ .pd_init_fn = cfq_pd_init,
4178+ .pd_reset_stats_fn = cfq_pd_reset_stats,
38864179 };
3887−#else
3888−static struct blkio_policy_type blkio_policy_cfq;
38894180 #endif
38904181
38914182 static int __init cfq_init(void)
@@ -3906,24 +4197,31 @@ static int __init cfq_init(void) · 14 unmodified lines
39064197 #else
39074198 cfq_group_idle = 0;
39084199 #endif
4200+
4201+ ret = blkcg_policy_register(&blkcg_policy_cfq);
4202+ if (ret)
4203+ return ret;
4204+
39094205 cfq_pool = KMEM_CACHE(cfq_queue, 0);
39104206 if (!cfq_pool)
3911− return -ENOMEM;
4207+ goto err_pol_unreg;
39124208
39134209 ret = elv_register(&iosched_cfq);
3914− if (ret) {
3915− kmem_cache_destroy(cfq_pool);
3916− return ret;
3917− }
3918−
3919− blkio_policy_register(&blkio_policy_cfq);
4210+ if (ret)
4211+ goto err_free_pool;
39204212
39214213 return 0;
4214+
4215+err_free_pool:
4216+ kmem_cache_destroy(cfq_pool);
4217+err_pol_unreg:
4218+ blkcg_policy_unregister(&blkcg_policy_cfq);
4219+ return ret;
39224220 }
39234221
39244222 static void __exit cfq_exit(void)
39254223 {
3926− blkio_policy_unregister(&blkio_policy_cfq);
4224+ blkcg_policy_unregister(&blkcg_policy_cfq);
39274225 elv_unregister(&iosched_cfq);
39284226 kmem_cache_destroy(cfq_pool);
39294227 }
block/cfq.h+0−115
@@ -1,115 +0,0 @@
1−#ifndef _CFQ_H
2−#define _CFQ_H
3−#include "blk-cgroup.h"
4−
5−#ifdef CONFIG_CFQ_GROUP_IOSCHED
6−static inline void cfq_blkiocg_update_io_add_stats(struct blkio_group *blkg,
7− struct blkio_group *curr_blkg, bool direction, bool sync)
8−{
9− blkiocg_update_io_add_stats(blkg, curr_blkg, direction, sync);
10−}
11−
12−static inline void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
13− unsigned long dequeue)
14−{
15− blkiocg_update_dequeue_stats(blkg, dequeue);
16−}
17−
18−static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg,
19− unsigned long time, unsigned long unaccounted_time)
20−{
21− blkiocg_update_timeslice_used(blkg, time, unaccounted_time);
22−}
23−
24−static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg)
25−{
26− blkiocg_set_start_empty_time(blkg);
27−}
28−
29−static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg,
30− bool direction, bool sync)
31−{
32− blkiocg_update_io_remove_stats(blkg, direction, sync);
33−}
34−
35−static inline void cfq_blkiocg_update_io_merged_stats(struct blkio_group *blkg,
36− bool direction, bool sync)
37−{
38− blkiocg_update_io_merged_stats(blkg, direction, sync);
39−}
40−
41−static inline void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg)
42−{
43− blkiocg_update_idle_time_stats(blkg);
44−}
45−
46−static inline void
47−cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg)
48−{
49− blkiocg_update_avg_queue_size_stats(blkg);
50−}
51−
52−static inline void
53−cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg)
54−{
55− blkiocg_update_set_idle_time_stats(blkg);
56−}
57−
58−static inline void cfq_blkiocg_update_dispatch_stats(struct blkio_group *blkg,
59− uint64_t bytes, bool direction, bool sync)
60−{
61− blkiocg_update_dispatch_stats(blkg, bytes, direction, sync);
62−}
63−
64−static inline void cfq_blkiocg_update_completion_stats(struct blkio_group *blkg, uint64_t start_time, uint64_t io_start_time, bool direction, bool sync)
65−{
66− blkiocg_update_completion_stats(blkg, start_time, io_start_time,
67− direction, sync);
68−}
69−
70−static inline void cfq_blkiocg_add_blkio_group(struct blkio_cgroup *blkcg,
71− struct blkio_group *blkg, void *key, dev_t dev) {
72− blkiocg_add_blkio_group(blkcg, blkg, key, dev, BLKIO_POLICY_PROP);
73−}
74−
75−static inline int cfq_blkiocg_del_blkio_group(struct blkio_group *blkg)
76−{
77− return blkiocg_del_blkio_group(blkg);
78−}
79−
80−#else /* CFQ_GROUP_IOSCHED */
81−static inline void cfq_blkiocg_update_io_add_stats(struct blkio_group *blkg,
82− struct blkio_group *curr_blkg, bool direction, bool sync) {}
83−
84−static inline void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg,
85− unsigned long dequeue) {}
86−
87−static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg,
88− unsigned long time, unsigned long unaccounted_time) {}
89−static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg) {}
90−static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg,
91− bool direction, bool sync) {}
92−static inline void cfq_blkiocg_update_io_merged_stats(struct blkio_group *blkg,
93− bool direction, bool sync) {}
94−static inline void cfq_blkiocg_update_idle_time_stats(struct blkio_group *blkg)
95−{
96−}
97−static inline void
98−cfq_blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg) {}
99−
100−static inline void
101−cfq_blkiocg_update_set_idle_time_stats(struct blkio_group *blkg) {}
102−
103−static inline void cfq_blkiocg_update_dispatch_stats(struct blkio_group *blkg,
104− uint64_t bytes, bool direction, bool sync) {}
105−static inline void cfq_blkiocg_update_completion_stats(struct blkio_group *blkg, uint64_t start_time, uint64_t io_start_time, bool direction, bool sync) {}
106−
107−static inline void cfq_blkiocg_add_blkio_group(struct blkio_cgroup *blkcg,
108− struct blkio_group *blkg, void *key, dev_t dev) {}
109−static inline int cfq_blkiocg_del_blkio_group(struct blkio_group *blkg)
110−{
111− return 0;
112−}
113−
114−#endif /* CFQ_GROUP_IOSCHED */
115−#endif
block/deadline-iosched.c+5−3
@@ -337,13 +337,13 @@ static void deadline_exit_queue(struct elevator_queue *e) · 336 unmodified lines
337337 /*
338338 * initialize elevator private data (deadline_data).
339339 */
340−static void *deadline_init_queue(struct request_queue *q)
340+static int deadline_init_queue(struct request_queue *q)
341341 {
342342 struct deadline_data *dd;
343343
344344 dd = kmalloc_node(sizeof(*dd), GFP_KERNEL | __GFP_ZERO, q->node);
345345 if (!dd)
346− return NULL;
346+ return -ENOMEM;
347347
348348 INIT_LIST_HEAD(&dd->fifo_list[READ]);
349349 INIT_LIST_HEAD(&dd->fifo_list[WRITE]);
@@ -354,7 +354,9 @@ static void *deadline_init_queue(struct request_queue *q) · 4 unmodified lines
354354 dd->writes_starved = writes_starved;
355355 dd->front_merges = 1;
356356 dd->fifo_batch = fifo_batch;
357− return dd;
357+
358+ q->elevator->elevator_data = dd;
359+ return 0;
358360 }
359361
360362 /*
block/elevator.c+48−73
@@ -38,6 +38,7 @@ · 37 unmodified lines
3838 #include <trace/events/block.h>
3939
4040 #include "blk.h"
41+#include "blk-cgroup.h"
4142
4243 static DEFINE_SPINLOCK(elv_list_lock);
4344 static LIST_HEAD(elv_list);
@@ -121,15 +122,6 @@ static struct elevator_type *elevator_get(const char *name) · 77 unmodified lines
121122 return e;
122123 }
123124
124−static int elevator_init_queue(struct request_queue *q,
125− struct elevator_queue *eq)
126−{
127− eq->elevator_data = eq->type->ops.elevator_init_fn(q);
128− if (eq->elevator_data)
129− return 0;
130− return -ENOMEM;
131−}
132−
133125 static char chosen_elevator[ELV_NAME_MAX];
134126
135127 static int __init elevator_setup(char *str)
@@ -188,7 +180,6 @@ static void elevator_release(struct kobject *kobj) · 52 unmodified lines
188180 int elevator_init(struct request_queue *q, char *name)
189181 {
190182 struct elevator_type *e = NULL;
191− struct elevator_queue *eq;
192183 int err;
193184
194185 if (unlikely(q->elevator))
@@ -222,17 +213,16 @@ int elevator_init(struct request_queue *q, char *name) · 27 unmodified lines
222213 }
223214 }
224215
225− eq = elevator_alloc(q, e);
226− if (!eq)
216+ q->elevator = elevator_alloc(q, e);
217+ if (!q->elevator)
227218 return -ENOMEM;
228219
229− err = elevator_init_queue(q, eq);
220+ err = e->ops.elevator_init_fn(q);
230221 if (err) {
231− kobject_put(&eq->kobj);
222+ kobject_put(&q->elevator->kobj);
232223 return err;
233224 }
234225
235− q->elevator = eq;
236226 return 0;
237227 }
238228 EXPORT_SYMBOL(elevator_init);
@@ -564,25 +554,6 @@ void elv_drain_elevator(struct request_queue *q) · 325 unmodified lines
564554 }
565555 }
566556
567−void elv_quiesce_start(struct request_queue *q)
568−{
569− if (!q->elevator)
570− return;
571−
572− spin_lock_irq(q->queue_lock);
573− queue_flag_set(QUEUE_FLAG_ELVSWITCH, q);
574− spin_unlock_irq(q->queue_lock);
575−
576− blk_drain_queue(q, false);
577−}
578−
579−void elv_quiesce_end(struct request_queue *q)
580−{
581− spin_lock_irq(q->queue_lock);
582− queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q);
583− spin_unlock_irq(q->queue_lock);
584−}
585−
586557 void __elv_add_request(struct request_queue *q, struct request *rq, int where)
587558 {
588559 trace_block_rq_insert(q, rq);
@@ -692,12 +663,13 @@ struct request *elv_former_request(struct request_queue *q, struct request *rq) · 103 unmodified lines
692663 return NULL;
693664 }
694665
695−int elv_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
666+int elv_set_request(struct request_queue *q, struct request *rq,
667+ struct bio *bio, gfp_t gfp_mask)
696668 {
697669 struct elevator_queue *e = q->elevator;
698670
699671 if (e->type->ops.elevator_set_req_fn)
700− return e->type->ops.elevator_set_req_fn(q, rq, gfp_mask);
672+ return e->type->ops.elevator_set_req_fn(q, rq, bio, gfp_mask);
701673 return 0;
702674 }
703675
@@ -801,8 +773,9 @@ static struct kobj_type elv_ktype = { · 97 unmodified lines
801773 .release = elevator_release,
802774 };
803775
804−int __elv_register_queue(struct request_queue *q, struct elevator_queue *e)
776+int elv_register_queue(struct request_queue *q)
805777 {
778+ struct elevator_queue *e = q->elevator;
806779 int error;
807780
808781 error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
@@ -820,11 +793,6 @@ int __elv_register_queue(struct request_queue *q, struct elevator_queue *e) · 11 unmodified lines
820793 }
821794 return error;
822795 }
823−
824−int elv_register_queue(struct request_queue *q)
825−{
826− return __elv_register_queue(q, q->elevator);
827−}
828796 EXPORT_SYMBOL(elv_register_queue);
829797
830798 void elv_unregister_queue(struct request_queue *q)
@@ -907,53 +875,60 @@ EXPORT_SYMBOL_GPL(elv_unregister); · 76 unmodified lines
907875 */
908876 static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
909877 {
910− struct elevator_queue *old_elevator, *e;
878+ struct elevator_queue *old = q->elevator;
879+ bool registered = old->registered;
911880 int err;
912881
913− /* allocate new elevator */
914− e = elevator_alloc(q, new_e);
915− if (!e)
916− return -ENOMEM;
882+ /*
883+ * Turn on BYPASS and drain all requests w/ elevator private data.
884+ * Block layer doesn't call into a quiesced elevator - all requests
885+ * are directly put on the dispatch list without elevator data
886+ * using INSERT_BACK. All requests have SOFTBARRIER set and no
887+ * merge happens either.
888+ */
889+ blk_queue_bypass_start(q);
890+
891+ /* unregister and clear all auxiliary data of the old elevator */
892+ if (registered)
893+ elv_unregister_queue(q);
894+
895+ spin_lock_irq(q->queue_lock);
896+ ioc_clear_queue(q);
897+ spin_unlock_irq(q->queue_lock);
917898
918− err = elevator_init_queue(q, e);
899+ /* allocate, init and register new elevator */
900+ err = -ENOMEM;
901+ q->elevator = elevator_alloc(q, new_e);
902+ if (!q->elevator)
903+ goto fail_init;
904+
905+ err = new_e->ops.elevator_init_fn(q);
919906 if (err) {
920− kobject_put(&e->kobj);
921− return err;
907+ kobject_put(&q->elevator->kobj);
908+ goto fail_init;
922909 }
923910
924− /* turn on BYPASS and drain all requests w/ elevator private data */
925− elv_quiesce_start(q);
926−
927− /* unregister old queue, register new one and kill old elevator */
928− if (q->elevator->registered) {
929− elv_unregister_queue(q);
930− err = __elv_register_queue(q, e);
911+ if (registered) {
912+ err = elv_register_queue(q);
931913 if (err)
932914 goto fail_register;
933915 }
934916
935− /* done, clear io_cq's, switch elevators and turn off BYPASS */
936− spin_lock_irq(q->queue_lock);
937− ioc_clear_queue(q);
938− old_elevator = q->elevator;
939− q->elevator = e;
940− spin_unlock_irq(q->queue_lock);
941−
942− elevator_exit(old_elevator);
943− elv_quiesce_end(q);
917+ /* done, kill the old one and finish */
918+ elevator_exit(old);
919+ blk_queue_bypass_end(q);
944920
945− blk_add_trace_msg(q, "elv switch: %s", e->type->elevator_name);
921+ blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
946922
947923 return 0;
948924
949925 fail_register:
950− /*
951− * switch failed, exit the new io scheduler and reattach the old
952− * one again (along with re-adding the sysfs dir)
953− */
954− elevator_exit(e);
926+ elevator_exit(q->elevator);
927+fail_init:
928+ /* switch failed, restore and re-register old elevator */
929+ q->elevator = old;
955930 elv_register_queue(q);
956− elv_quiesce_end(q);
931+ blk_queue_bypass_end(q);
957932
958933 return err;
959934 }
block/noop-iosched.c+5−3
@@ -59,15 +59,17 @@ noop_latter_request(struct request_queue *q, struct request *rq) · 58 unmodified lines
5959 return list_entry(rq->queuelist.next, struct request, queuelist);
6060 }
6161
62−static void *noop_init_queue(struct request_queue *q)
62+static int noop_init_queue(struct request_queue *q)
6363 {
6464 struct noop_data *nd;
6565
6666 nd = kmalloc_node(sizeof(*nd), GFP_KERNEL, q->node);
6767 if (!nd)
68− return NULL;
68+ return -ENOMEM;
69+
6970 INIT_LIST_HEAD(&nd->queue);
70− return nd;
71+ q->elevator->elevator_data = nd;
72+ return 0;
7173 }
7274
7375 static void noop_exit_queue(struct elevator_queue *e)
fs/bio.c+61−0
@@ -19,12 +19,14 @@ · 18 unmodified lines
1919 #include <linux/swap.h>
2020 #include <linux/bio.h>
2121 #include <linux/blkdev.h>
22+#include <linux/iocontext.h>
2223 #include <linux/slab.h>
2324 #include <linux/init.h>
2425 #include <linux/kernel.h>
2526 #include <linux/export.h>
2627 #include <linux/mempool.h>
2728 #include <linux/workqueue.h>
29+#include <linux/cgroup.h>
2830 #include <scsi/sg.h> /* for struct sg_iovec */
2931
3032 #include <trace/events/block.h>
@@ -418,6 +420,7 @@ void bio_put(struct bio *bio) · 387 unmodified lines
418420 * last put frees it
419421 */
420422 if (atomic_dec_and_test(&bio->bi_cnt)) {
423+ bio_disassociate_task(bio);
421424 bio->bi_next = NULL;
422425 bio->bi_destructor(bio);
423426 }
@@ -1646,6 +1649,64 @@ bad: · 1222 unmodified lines
16461649 }
16471650 EXPORT_SYMBOL(bioset_create);
16481651
1652+#ifdef CONFIG_BLK_CGROUP
1653+/**
1654+ * bio_associate_current - associate a bio with %current
1655+ * @bio: target bio
1656+ *
1657+ * Associate @bio with %current if it hasn't been associated yet. Block
1658+ * layer will treat @bio as if it were issued by %current no matter which
1659+ * task actually issues it.
1660+ *
1661+ * This function takes an extra reference of @task's io_context and blkcg
1662+ * which will be put when @bio is released. The caller must own @bio,
1663+ * ensure %current->io_context exists, and is responsible for synchronizing
1664+ * calls to this function.
1665+ */
1666+int bio_associate_current(struct bio *bio)
1667+{
1668+ struct io_context *ioc;
1669+ struct cgroup_subsys_state *css;
1670+
1671+ if (bio->bi_ioc)
1672+ return -EBUSY;
1673+
1674+ ioc = current->io_context;
1675+ if (!ioc)
1676+ return -ENOENT;
1677+
1678+ /* acquire active ref on @ioc and associate */
1679+ get_io_context_active(ioc);
1680+ bio->bi_ioc = ioc;
1681+
1682+ /* associate blkcg if exists */
1683+ rcu_read_lock();
1684+ css = task_subsys_state(current, blkio_subsys_id);
1685+ if (css && css_tryget(css))
1686+ bio->bi_css = css;
1687+ rcu_read_unlock();
1688+
1689+ return 0;
1690+}
1691+
1692+/**
1693+ * bio_disassociate_task - undo bio_associate_current()
1694+ * @bio: target bio
1695+ */
1696+void bio_disassociate_task(struct bio *bio)
1697+{
1698+ if (bio->bi_ioc) {
1699+ put_io_context(bio->bi_ioc);
1700+ bio->bi_ioc = NULL;
1701+ }
1702+ if (bio->bi_css) {
1703+ css_put(bio->bi_css);
1704+ bio->bi_css = NULL;
1705+ }
1706+}
1707+
1708+#endif /* CONFIG_BLK_CGROUP */
1709+
16491710 static void __init biovec_init_slabs(void)
16501711 {
16511712 int i;
fs/ioprio.c+1−1
@@ -50,7 +50,7 @@ int set_task_ioprio(struct task_struct *task, int ioprio) · 49 unmodified lines
5050
5151 ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
5252 if (ioc) {
53− ioc_ioprio_changed(ioc, ioprio);
53+ ioc->ioprio = ioprio;
5454 put_io_context(ioc);
5555 }
5656
fs/splice.c+2−2
@@ -1388,7 +1388,7 @@ static long do_splice(struct file *in, loff_t __user *off_in, · 1387 unmodified lines
13881388 */
13891389 static int get_iovec_page_array(const struct iovec __user *iov,
13901390 unsigned int nr_vecs, struct page **pages,
1391− struct partial_page *partial, int aligned,
1391+ struct partial_page *partial, bool aligned,
13921392 unsigned int pipe_buffers)
13931393 {
13941394 int buffers = 0, error = 0;
@@ -1626,7 +1626,7 @@ static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov, · 231 unmodified lines
16261626 return -ENOMEM;
16271627
16281628 spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages,
1629− spd.partial, flags & SPLICE_F_GIFT,
1629+ spd.partial, false,
16301630 pipe->buffers);
16311631 if (spd.nr_pages <= 0)
16321632 ret = spd.nr_pages;
include/linux/bio.h+8−0
@@ -269,6 +269,14 @@ extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set · 268 unmodified lines
269269 extern void bvec_free_bs(struct bio_set *, struct bio_vec *, unsigned int);
270270 extern unsigned int bvec_nr_vecs(unsigned short idx);
271271
272+#ifdef CONFIG_BLK_CGROUP
273+int bio_associate_current(struct bio *bio);
274+void bio_disassociate_task(struct bio *bio);
275+#else /* CONFIG_BLK_CGROUP */
276+static inline int bio_associate_current(struct bio *bio) { return -ENOENT; }
277+static inline void bio_disassociate_task(struct bio *bio) { }
278+#endif /* CONFIG_BLK_CGROUP */
279+
272280 /*
273281 * bio_set is used to allow other portions of the IO system to
274282 * allocate their own private memory pools for bio and iovec structures.
include/linux/blk_types.h+10−0
@@ -14,6 +14,8 @@ struct bio; · 13 unmodified lines
1414 struct bio_integrity_payload;
1515 struct page;
1616 struct block_device;
17+struct io_context;
18+struct cgroup_subsys_state;
1719 typedef void (bio_end_io_t) (struct bio *, int);
1820 typedef void (bio_destructor_t) (struct bio *);
1921
@@ -66,6 +68,14 @@ struct bio { · 46 unmodified lines
6668 bio_end_io_t *bi_end_io;
6769
6870 void *bi_private;
71+#ifdef CONFIG_BLK_CGROUP
72+ /*
73+ * Optional ioc and css associated with this bio. Put on bio
74+ * release. Read comment on top of bio_associate_current().
75+ */
76+ struct io_context *bi_ioc;
77+ struct cgroup_subsys_state *bi_css;
78+#endif
6979 #if defined(CONFIG_BLK_DEV_INTEGRITY)
7080 struct bio_integrity_payload *bi_integrity; /* data integrity */
7181 #endif
include/linux/blkdev.h+19−1
@@ -32,10 +32,17 @@ struct blk_trace; · 31 unmodified lines
3232 struct request;
3333 struct sg_io_hdr;
3434 struct bsg_job;
35+struct blkcg_gq;
3536
3637 #define BLKDEV_MIN_RQ 4
3738 #define BLKDEV_MAX_RQ 128 /* Default maximum */
3839
40+/*
41+ * Maximum number of blkcg policies allowed to be registered concurrently.
42+ * Defined here to simplify include dependency.
43+ */
44+#define BLKCG_MAX_POLS 2
45+
3946 struct request;
4047 typedef void (rq_end_io_fn)(struct request *, int);
4148
@@ -363,6 +370,11 @@ struct request_queue { · 321 unmodified lines
363370 struct list_head timeout_list;
364371
365372 struct list_head icq_list;
373+#ifdef CONFIG_BLK_CGROUP
374+ DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
375+ struct blkcg_gq *root_blkg;
376+ struct list_head blkg_list;
377+#endif
366378
367379 struct queue_limits limits;
368380
@@ -390,12 +402,17 @@ struct request_queue { · 21 unmodified lines
390402
391403 struct mutex sysfs_lock;
392404
405+ int bypass_depth;
406+
393407 #if defined(CONFIG_BLK_DEV_BSG)
394408 bsg_job_fn *bsg_job_fn;
395409 int bsg_job_size;
396410 struct bsg_class_device bsg_dev;
397411 #endif
398412
413+#ifdef CONFIG_BLK_CGROUP
414+ struct list_head all_q_node;
415+#endif
399416 #ifdef CONFIG_BLK_DEV_THROTTLING
400417 /* Throttle data */
401418 struct throtl_data *td;
@@ -407,7 +424,7 @@ struct request_queue { · 5 unmodified lines
407424 #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */
408425 #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */
409426 #define QUEUE_FLAG_DEAD 5 /* queue being torn down */
410−#define QUEUE_FLAG_ELVSWITCH 6 /* don't use elevator, just do FIFO */
427+#define QUEUE_FLAG_BYPASS 6 /* act as dumb FIFO queue */
411428 #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */
412429 #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */
413430 #define QUEUE_FLAG_SAME_COMP 9 /* complete on same CPU-group */
@@ -491,6 +508,7 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) · 77 unmodified lines
491508 #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
492509 #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
493510 #define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
511+#define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
494512 #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
495513 #define blk_queue_noxmerges(q) \
496514 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
include/linux/elevator.h+5−3
@@ -28,12 +28,13 @@ typedef int (elevator_may_queue_fn) (struct request_queue *, int); · 27 unmodified lines
2828
2929 typedef void (elevator_init_icq_fn) (struct io_cq *);
3030 typedef void (elevator_exit_icq_fn) (struct io_cq *);
31−typedef int (elevator_set_req_fn) (struct request_queue *, struct request *, gfp_t);
31+typedef int (elevator_set_req_fn) (struct request_queue *, struct request *,
32+ struct bio *, gfp_t);
3233 typedef void (elevator_put_req_fn) (struct request *);
3334 typedef void (elevator_activate_req_fn) (struct request_queue *, struct request *);
3435 typedef void (elevator_deactivate_req_fn) (struct request_queue *, struct request *);
3536
36−typedef void *(elevator_init_fn) (struct request_queue *);
37+typedef int (elevator_init_fn) (struct request_queue *);
3738 typedef void (elevator_exit_fn) (struct elevator_queue *);
3839
3940 struct elevator_ops
@@ -129,7 +130,8 @@ extern void elv_unregister_queue(struct request_queue *q); · 89 unmodified lines
129130 extern int elv_may_queue(struct request_queue *, int);
130131 extern void elv_abort_queue(struct request_queue *);
131132 extern void elv_completed_request(struct request_queue *, struct request *);
132−extern int elv_set_request(struct request_queue *, struct request *, gfp_t);
133+extern int elv_set_request(struct request_queue *q, struct request *rq,
134+ struct bio *bio, gfp_t gfp_mask);
133135 extern void elv_put_request(struct request_queue *, struct request *);
134136 extern void elv_drain_elevator(struct request_queue *);
135137
include/linux/iocontext.h+22−17
@@ -6,11 +6,7 @@ · 5 unmodified lines
66 #include <linux/workqueue.h>
77
88 enum {
9− ICQ_IOPRIO_CHANGED = 1 << 0,
10− ICQ_CGROUP_CHANGED = 1 << 1,
119 ICQ_EXITED = 1 << 2,
12−
13− ICQ_CHANGED_MASK = ICQ_IOPRIO_CHANGED | ICQ_CGROUP_CHANGED,
1410 };
1511
1612 /*
@@ -100,6 +96,7 @@ struct io_cq { · 83 unmodified lines
10096 */
10197 struct io_context {
10298 atomic_long_t refcount;
99+ atomic_t active_ref;
103100 atomic_t nr_tasks;
104101
105102 /* all the fields below are protected by this lock */
@@ -120,29 +117,37 @@ struct io_context { · 14 unmodified lines
120117 struct work_struct release_work;
121118 };
122119
123−static inline struct io_context *ioc_task_link(struct io_context *ioc)
120+/**
121+ * get_io_context_active - get active reference on ioc
122+ * @ioc: ioc of interest
123+ *
124+ * Only iocs with active reference can issue new IOs. This function
125+ * acquires an active reference on @ioc. The caller must already have an
126+ * active reference on @ioc.
127+ */
128+static inline void get_io_context_active(struct io_context *ioc)
124129 {
125− /*
126− * if ref count is zero, don't allow sharing (ioc is going away, it's
127− * a race).
128− */
129− if (ioc && atomic_long_inc_not_zero(&ioc->refcount)) {
130− atomic_inc(&ioc->nr_tasks);
131− return ioc;
132− }
130+ WARN_ON_ONCE(atomic_long_read(&ioc->refcount) <= 0);
131+ WARN_ON_ONCE(atomic_read(&ioc->active_ref) <= 0);
132+ atomic_long_inc(&ioc->refcount);
133+ atomic_inc(&ioc->active_ref);
134+}
135+
136+static inline void ioc_task_link(struct io_context *ioc)
137+{
138+ get_io_context_active(ioc);
133139
134− return NULL;
140+ WARN_ON_ONCE(atomic_read(&ioc->nr_tasks) <= 0);
141+ atomic_inc(&ioc->nr_tasks);
135142 }
136143
137144 struct task_struct;
138145 #ifdef CONFIG_BLOCK
139146 void put_io_context(struct io_context *ioc);
147+void put_io_context_active(struct io_context *ioc);
140148 void exit_io_context(struct task_struct *task);
141149 struct io_context *get_task_io_context(struct task_struct *task,
142150 gfp_t gfp_flags, int node);
143−void ioc_ioprio_changed(struct io_context *ioc, int ioprio);
144−void ioc_cgroup_changed(struct io_context *ioc);
145−unsigned int icq_get_changed(struct io_cq *icq);
146151 #else
147152 struct io_context;
148153 static inline void put_io_context(struct io_context *ioc) { }
include/linux/ioprio.h+5−17
@@ -42,26 +42,14 @@ enum { · 41 unmodified lines
4242 };
4343
4444 /*
45− * if process has set io priority explicitly, use that. if not, convert
46− * the cpu scheduler nice value to an io priority
45+ * Fallback BE priority
4746 */
4847 #define IOPRIO_NORM (4)
49−static inline int task_ioprio(struct io_context *ioc)
50−{
51− if (ioprio_valid(ioc->ioprio))
52− return IOPRIO_PRIO_DATA(ioc->ioprio);
53−
54− return IOPRIO_NORM;
55−}
56−
57−static inline int task_ioprio_class(struct io_context *ioc)
58−{
59− if (ioprio_valid(ioc->ioprio))
60− return IOPRIO_PRIO_CLASS(ioc->ioprio);
61−
62− return IOPRIO_CLASS_BE;
63−}
6448
49+/*
50+ * if process has set io priority explicitly, use that. if not, convert
51+ * the cpu scheduler nice value to an io priority
52+ */
6553 static inline int task_nice_ioprio(struct task_struct *task)
6654 {
6755 return (task_nice(task) + 20) / 5;
init/Kconfig+1−1
@@ -803,7 +803,7 @@ config RT_GROUP_SCHED · 802 unmodified lines
803803 endif #CGROUP_SCHED
804804
805805 config BLK_CGROUP
806− tristate "Block IO controller"
806+ bool "Block IO controller"
807807 depends on BLOCK
808808 default n
809809 ---help---
kernel/fork.c+2−3
@@ -976,9 +976,8 @@ static int copy_io(unsigned long clone_flags, struct task_struct *tsk) · 975 unmodified lines
976976 * Share io context with parent, if CLONE_IO is set
977977 */
978978 if (clone_flags & CLONE_IO) {
979− tsk->io_context = ioc_task_link(ioc);
980− if (unlikely(!tsk->io_context))
981− return -ENOMEM;
979+ ioc_task_link(ioc);
980+ tsk->io_context = ioc;
982981 } else if (ioprio_valid(ioc->ioprio)) {
983982 new_ioc = get_task_io_context(tsk, GFP_KERNEL, NUMA_NO_NODE);
984983 if (unlikely(!new_ioc))