History

[PATCH] [BLOCK] cfq-iosched: change cfq io context linking from list to tree

On setups with many disks, we spend a considerable amount of time looking up the process-disk mapping on each queue of io. Testing with a NULL based block driver, this costs 40-50% reduction in throughput for 1000 disks.

Jens Axboe committed 20 years ago

Commit e2d74ac1 parent 329b10b3 files · +114 −124Browse files

Unified · Split

3 files
  1. block/cfq-iosched.c +95 −110
  2. block/ll_rw_blk.c +13 −6
  3. include/linux/blkdev.h +6 −8
block/cfq-iosched.c+95−110
@@ -1190,19 +1190,19 @@ cfq_find_cfq_hash(struct cfq_data *cfqd, unsigned int key, unsigned short prio) · 1189 unmodified lines
11901190 return __cfq_find_cfq_hash(cfqd, key, prio, hash_long(key, CFQ_QHASH_SHIFT));
11911191 }
11921192
1193−static void cfq_free_io_context(struct cfq_io_context *cic)
1193+static void cfq_free_io_context(struct io_context *ioc)
11941194 {
11951195 struct cfq_io_context *__cic;
1196− struct list_head *entry, *next;
1197− int freed = 1;
1196+ struct rb_node *n;
1197+ int freed = 0;
11981198
1199− list_for_each_safe(entry, next, &cic->list) {
1200− __cic = list_entry(entry, struct cfq_io_context, list);
1199+ while ((n = rb_first(&ioc->cic_root)) != NULL) {
1200+ __cic = rb_entry(n, struct cfq_io_context, rb_node);
1201+ rb_erase(&__cic->rb_node, &ioc->cic_root);
12011202 kmem_cache_free(cfq_ioc_pool, __cic);
12021203 freed++;
12031204 }
12041205
1205− kmem_cache_free(cfq_ioc_pool, cic);
12061206 if (atomic_sub_and_test(freed, &ioc_count) && ioc_gone)
12071207 complete(ioc_gone);
12081208 }
@@ -1210,8 +1210,7 @@ static void cfq_free_io_context(struct cfq_io_context *cic) · 1 unmodified line
12101210 static void cfq_trim(struct io_context *ioc)
12111211 {
12121212 ioc->set_ioprio = NULL;
1213− if (ioc->cic)
1214− cfq_free_io_context(ioc->cic);
1213+ cfq_free_io_context(ioc);
12151214 }
12161215
12171216 /*
@@ -1250,26 +1249,26 @@ static void cfq_exit_single_io_context(struct cfq_io_context *cic) · 32 unmodified lines
12501249 spin_unlock(q->queue_lock);
12511250 }
12521251
1253−static void cfq_exit_io_context(struct cfq_io_context *cic)
1252+static void cfq_exit_io_context(struct io_context *ioc)
12541253 {
12551254 struct cfq_io_context *__cic;
1256− struct list_head *entry;
12571255 unsigned long flags;
1258−
1259− local_irq_save(flags);
1256+ struct rb_node *n;
12601257
12611258 /*
12621259 * put the reference this task is holding to the various queues
12631260 */
1264− read_lock(&cfq_exit_lock);
1265− list_for_each(entry, &cic->list) {
1266− __cic = list_entry(entry, struct cfq_io_context, list);
1261+ read_lock_irqsave(&cfq_exit_lock, flags);
1262+
1263+ n = rb_first(&ioc->cic_root);
1264+ while (n != NULL) {
1265+ __cic = rb_entry(n, struct cfq_io_context, rb_node);
1266+
12671267 cfq_exit_single_io_context(__cic);
1268+ n = rb_next(n);
12681269 }
12691270
1270− cfq_exit_single_io_context(cic);
1271− read_unlock(&cfq_exit_lock);
1272− local_irq_restore(flags);
1271+ read_unlock_irqrestore(&cfq_exit_lock, flags);
12731272 }
12741273
12751274 static struct cfq_io_context *
@@ -1278,10 +1277,10 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask) · 2 unmodified lines
12781277 struct cfq_io_context *cic = kmem_cache_alloc(cfq_ioc_pool, gfp_mask);
12791278
12801279 if (cic) {
1281− INIT_LIST_HEAD(&cic->list);
1280+ RB_CLEAR(&cic->rb_node);
1281+ cic->key = NULL;
12821282 cic->cfqq[ASYNC] = NULL;
12831283 cic->cfqq[SYNC] = NULL;
1284− cic->key = NULL;
12851284 cic->last_end_request = jiffies;
12861285 cic->ttime_total = 0;
12871286 cic->ttime_samples = 0;
@@ -1373,15 +1372,17 @@ static inline void changed_ioprio(struct cfq_io_context *cic) · 85 unmodified lines
13731372 static int cfq_ioc_set_ioprio(struct io_context *ioc, unsigned int ioprio)
13741373 {
13751374 struct cfq_io_context *cic;
1375+ struct rb_node *n;
13761376
13771377 write_lock(&cfq_exit_lock);
13781378
1379− cic = ioc->cic;
1380−
1381− changed_ioprio(cic);
1382−
1383− list_for_each_entry(cic, &cic->list, list)
1379+ n = rb_first(&ioc->cic_root);
1380+ while (n != NULL) {
1381+ cic = rb_entry(n, struct cfq_io_context, rb_node);
1382+
13841383 changed_ioprio(cic);
1384+ n = rb_next(n);
1385+ }
13851386
13861387 write_unlock(&cfq_exit_lock);
13871388
@@ -1445,14 +1446,67 @@ out: · 57 unmodified lines
14451446 return cfqq;
14461447 }
14471448
1449+static struct cfq_io_context *
1450+cfq_cic_rb_lookup(struct cfq_data *cfqd, struct io_context *ioc)
1451+{
1452+ struct rb_node *n = ioc->cic_root.rb_node;
1453+ struct cfq_io_context *cic;
1454+ void *key = cfqd;
1455+
1456+ while (n) {
1457+ cic = rb_entry(n, struct cfq_io_context, rb_node);
1458+
1459+ if (key < cic->key)
1460+ n = n->rb_left;
1461+ else if (key > cic->key)
1462+ n = n->rb_right;
1463+ else
1464+ return cic;
1465+ }
1466+
1467+ return NULL;
1468+}
1469+
1470+static inline void
1471+cfq_cic_link(struct cfq_data *cfqd, struct io_context *ioc,
1472+ struct cfq_io_context *cic)
1473+{
1474+ struct rb_node **p = &ioc->cic_root.rb_node;
1475+ struct rb_node *parent = NULL;
1476+ struct cfq_io_context *__cic;
1477+
1478+ read_lock(&cfq_exit_lock);
1479+
1480+ cic->ioc = ioc;
1481+ cic->key = cfqd;
1482+
1483+ ioc->set_ioprio = cfq_ioc_set_ioprio;
1484+
1485+ while (*p) {
1486+ parent = *p;
1487+ __cic = rb_entry(parent, struct cfq_io_context, rb_node);
1488+
1489+ if (cic->key < __cic->key)
1490+ p = &(*p)->rb_left;
1491+ else if (cic->key > __cic->key)
1492+ p = &(*p)->rb_right;
1493+ else
1494+ BUG();
1495+ }
1496+
1497+ rb_link_node(&cic->rb_node, parent, p);
1498+ rb_insert_color(&cic->rb_node, &ioc->cic_root);
1499+ list_add(&cic->queue_list, &cfqd->cic_list);
1500+ read_unlock(&cfq_exit_lock);
1501+}
1502+
14481503 /*
14491504 * Setup general io context and cfq io context. There can be several cfq
14501505 * io contexts per general io context, if this process is doing io to more
1451− * than one device managed by cfq. Note that caller is holding a reference to
1452− * cfqq, so we don't need to worry about it disappearing
1506+ * than one device managed by cfq.
14531507 */
14541508 static struct cfq_io_context *
1455−cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
1509+cfq_get_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
14561510 {
14571511 struct io_context *ioc = NULL;
14581512 struct cfq_io_context *cic;
@@ -1463,88 +1517,15 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask) · 4 unmodified lines
14631517 if (!ioc)
14641518 return NULL;
14651519
1466−restart:
1467− if ((cic = ioc->cic) == NULL) {
1468− cic = cfq_alloc_io_context(cfqd, gfp_mask);
1469−
1470− if (cic == NULL)
1471− goto err;
1472−
1473− /*
1474− * manually increment generic io_context usage count, it
1475− * cannot go away since we are already holding one ref to it
1476− */
1477− cic->ioc = ioc;
1478− cic->key = cfqd;
1479− read_lock(&cfq_exit_lock);
1480− ioc->set_ioprio = cfq_ioc_set_ioprio;
1481− ioc->cic = cic;
1482− list_add(&cic->queue_list, &cfqd->cic_list);
1483− read_unlock(&cfq_exit_lock);
1484− } else {
1485− struct cfq_io_context *__cic;
1486−
1487− /*
1488− * the first cic on the list is actually the head itself
1489− */
1490− if (cic->key == cfqd)
1491− goto out;
1492−
1493− if (unlikely(!cic->key)) {
1494− read_lock(&cfq_exit_lock);
1495− if (list_empty(&cic->list))
1496− ioc->cic = NULL;
1497− else
1498− ioc->cic = list_entry(cic->list.next,
1499− struct cfq_io_context,
1500− list);
1501− read_unlock(&cfq_exit_lock);
1502− kmem_cache_free(cfq_ioc_pool, cic);
1503− atomic_dec(&ioc_count);
1504− goto restart;
1505− }
1506−
1507− /*
1508− * cic exists, check if we already are there. linear search
1509− * should be ok here, the list will usually not be more than
1510− * 1 or a few entries long
1511− */
1512− list_for_each_entry(__cic, &cic->list, list) {
1513− /*
1514− * this process is already holding a reference to
1515− * this queue, so no need to get one more
1516− */
1517− if (__cic->key == cfqd) {
1518− cic = __cic;
1519− goto out;
1520− }
1521− if (unlikely(!__cic->key)) {
1522− read_lock(&cfq_exit_lock);
1523− list_del(&__cic->list);
1524− read_unlock(&cfq_exit_lock);
1525− kmem_cache_free(cfq_ioc_pool, __cic);
1526− atomic_dec(&ioc_count);
1527− goto restart;
1528− }
1529− }
1520+ cic = cfq_cic_rb_lookup(cfqd, ioc);
1521+ if (cic)
1522+ goto out;
15301523
1531− /*
1532− * nope, process doesn't have a cic assoicated with this
1533− * cfqq yet. get a new one and add to list
1534− */
1535− __cic = cfq_alloc_io_context(cfqd, gfp_mask);
1536− if (__cic == NULL)
1537− goto err;
1538−
1539− __cic->ioc = ioc;
1540− __cic->key = cfqd;
1541− read_lock(&cfq_exit_lock);
1542− list_add(&__cic->list, &cic->list);
1543− list_add(&__cic->queue_list, &cfqd->cic_list);
1544− read_unlock(&cfq_exit_lock);
1545− cic = __cic;
1546− }
1524+ cic = cfq_alloc_io_context(cfqd, gfp_mask);
1525+ if (cic == NULL)
1526+ goto err;
15471527
1528+ cfq_cic_link(cfqd, ioc, cic);
15481529 out:
15491530 return cic;
15501531 err:
@@ -1965,7 +1946,7 @@ cfq_set_request(request_queue_t *q, struct request *rq, struct bio *bio, · 414 unmodified lines
19651946
19661947 might_sleep_if(gfp_mask & __GFP_WAIT);
19671948
1968− cic = cfq_get_io_context(cfqd, key, gfp_mask);
1949+ cic = cfq_get_io_context(cfqd, gfp_mask);
19691950
19701951 spin_lock_irqsave(q->queue_lock, flags);
19711952
@@ -2133,11 +2114,14 @@ static void cfq_exit_queue(elevator_t *e) · 161 unmodified lines
21332114 request_queue_t *q = cfqd->queue;
21342115
21352116 cfq_shutdown_timer_wq(cfqd);
2117+
21362118 write_lock(&cfq_exit_lock);
21372119 spin_lock_irq(q->queue_lock);
2120+
21382121 if (cfqd->active_queue)
21392122 __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
2140− while(!list_empty(&cfqd->cic_list)) {
2123+
2124+ while (!list_empty(&cfqd->cic_list)) {
21412125 struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,
21422126 struct cfq_io_context,
21432127 queue_list);
@@ -2152,6 +2136,7 @@ static void cfq_exit_queue(elevator_t *e) · 8 unmodified lines
21522136 cic->key = NULL;
21532137 list_del_init(&cic->queue_list);
21542138 }
2139+
21552140 spin_unlock_irq(q->queue_lock);
21562141 write_unlock(&cfq_exit_lock);
21572142
block/ll_rw_blk.c+13−6
@@ -3539,11 +3539,15 @@ void put_io_context(struct io_context *ioc) · 3538 unmodified lines
35393539 BUG_ON(atomic_read(&ioc->refcount) == 0);
35403540
35413541 if (atomic_dec_and_test(&ioc->refcount)) {
3542+ struct cfq_io_context *cic;
3543+
35423544 rcu_read_lock();
35433545 if (ioc->aic && ioc->aic->dtor)
35443546 ioc->aic->dtor(ioc->aic);
3545− if (ioc->cic && ioc->cic->dtor)
3546− ioc->cic->dtor(ioc->cic);
3547+ if (ioc->cic_root.rb_node != NULL) {
3548+ cic = rb_entry(rb_first(&ioc->cic_root), struct cfq_io_context, rb_node);
3549+ cic->dtor(ioc);
3550+ }
35473551 rcu_read_unlock();
35483552
35493553 kmem_cache_free(iocontext_cachep, ioc);
@@ -3556,6 +3560,7 @@ void exit_io_context(void) · 6 unmodified lines
35563560 {
35573561 unsigned long flags;
35583562 struct io_context *ioc;
3563+ struct cfq_io_context *cic;
35593564
35603565 local_irq_save(flags);
35613566 task_lock(current);
@@ -3567,9 +3572,11 @@ void exit_io_context(void) · 5 unmodified lines
35673572
35683573 if (ioc->aic && ioc->aic->exit)
35693574 ioc->aic->exit(ioc->aic);
3570− if (ioc->cic && ioc->cic->exit)
3571− ioc->cic->exit(ioc->cic);
3572−
3575+ if (ioc->cic_root.rb_node != NULL) {
3576+ cic = rb_entry(rb_first(&ioc->cic_root), struct cfq_io_context, rb_node);
3577+ cic->exit(ioc);
3578+ }
3579+
35733580 put_io_context(ioc);
35743581 }
35753582
@@ -3598,7 +3605,7 @@ struct io_context *current_io_context(gfp_t gfp_flags) · 22 unmodified lines
35983605 ret->last_waited = jiffies; /* doesn't matter... */
35993606 ret->nr_batch_requests = 0; /* because this is 0 */
36003607 ret->aic = NULL;
3601− ret->cic = NULL;
3608+ ret->cic_root.rb_node = NULL;
36023609 tsk->io_context = ret;
36033610 }
36043611
include/linux/blkdev.h+6−8
@@ -55,13 +55,11 @@ struct as_io_context { · 54 unmodified lines
5555
5656 struct cfq_queue;
5757 struct cfq_io_context {
58− /*
59− * circular list of cfq_io_contexts belonging to a process io context
60− */
61− struct list_head list;
62− struct cfq_queue *cfqq[2];
58+ struct rb_node rb_node;
6359 void *key;
6460
61+ struct cfq_queue *cfqq[2];
62+
6563 struct io_context *ioc;
6664
6765 unsigned long last_end_request;
@@ -72,8 +70,8 @@ struct cfq_io_context { · 4 unmodified lines
7270
7371 struct list_head queue_list;
7472
75− void (*dtor)(struct cfq_io_context *);
76− void (*exit)(struct cfq_io_context *);
73+ void (*dtor)(struct io_context *); /* destructor */
74+ void (*exit)(struct io_context *); /* called on task exit */
7775 };
7876
7977 /*
@@ -94,7 +92,7 @@ struct io_context { · 14 unmodified lines
9492 int nr_batch_requests; /* Number of requests left in the batch */
9593
9694 struct as_io_context *aic;
97− struct cfq_io_context *cic;
95+ struct rb_root cic_root;
9896 };
9997
10098 void put_io_context(struct io_context *ioc);