History

workqueue: implement worker states

Implement worker states. After created, a worker is STARTED. While a worker isn't processing a work, it's IDLE and chained on gcwq->idle_list. While processing a work, a worker is BUSY and chained on gcwq->busy_hash. Also, gcwq now counts the number of all workers and idle ones. worker_thread() is restructured to reflect state transitions. cwq->more_work is removed and waking up a worker makes it check for events. A worker is killed by setting DIE flag while it's IDLE and waking it up. This gives gcwq better visibility of what's going on and allows it to find out whether a work is executing quickly which is necessary to have multiple workers processing the same cwq.

Tejun Heo committed 16 years ago

Commit c8e55f31 parent 8b03ae31 file · +173 −41Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+173−41
@@ -35,6 +35,17 @@ · 34 unmodified lines
35#include <linux/lockdep.h>35#include <linux/lockdep.h>
36#include <linux/idr.h>36#include <linux/idr.h>
37 37
38enum {
39 /* worker flags */
40 WORKER_STARTED = 1 << 0, /* started */
41 WORKER_DIE = 1 << 1, /* die die die */
42 WORKER_IDLE = 1 << 2, /* is idle */
43
44 BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
45 BUSY_WORKER_HASH_SIZE = 1 << BUSY_WORKER_HASH_ORDER,
46 BUSY_WORKER_HASH_MASK = BUSY_WORKER_HASH_SIZE - 1,
47};
48
38/*49/*
39 * Structure fields follow one of the following exclusion rules.50 * Structure fields follow one of the following exclusion rules.
40 *51 *
@@ -51,11 +62,18 @@ struct global_cwq; · 10 unmodified lines
51struct cpu_workqueue_struct;62struct cpu_workqueue_struct;
52 63
53struct worker {64struct worker {
65 /* on idle list while idle, on busy hash table while busy */
66 union {
67 struct list_head entry; /* L: while idle */
68 struct hlist_node hentry; /* L: while busy */
69 };
70
54 struct work_struct *current_work; /* L: work being processed */71 struct work_struct *current_work; /* L: work being processed */
55 struct list_head scheduled; /* L: scheduled works */72 struct list_head scheduled; /* L: scheduled works */
56 struct task_struct *task; /* I: worker task */73 struct task_struct *task; /* I: worker task */
57 struct global_cwq *gcwq; /* I: the associated gcwq */74 struct global_cwq *gcwq; /* I: the associated gcwq */
58 struct cpu_workqueue_struct *cwq; /* I: the associated cwq */75 struct cpu_workqueue_struct *cwq; /* I: the associated cwq */
76 unsigned int flags; /* L: flags */
59 int id; /* I: worker id */77 int id; /* I: worker id */
60};78};
61 79
@@ -65,6 +83,15 @@ struct worker { · 3 unmodified lines
65struct global_cwq {83struct global_cwq {
66 spinlock_t lock; /* the gcwq lock */84 spinlock_t lock; /* the gcwq lock */
67 unsigned int cpu; /* I: the associated cpu */85 unsigned int cpu; /* I: the associated cpu */
86
87 int nr_workers; /* L: total number of workers */
88 int nr_idle; /* L: currently idle ones */
89
90 /* workers are chained either in the idle_list or busy_hash */
91 struct list_head idle_list; /* L: list of idle workers */
92 struct hlist_head busy_hash[BUSY_WORKER_HASH_SIZE];
93 /* L: hash of busy workers */
94
68 struct ida worker_ida; /* L: for worker IDs */95 struct ida worker_ida; /* L: for worker IDs */
69} ____cacheline_aligned_in_smp;96} ____cacheline_aligned_in_smp;
70 97
@@ -77,7 +104,6 @@ struct global_cwq { · 6 unmodified lines
77struct cpu_workqueue_struct {104struct cpu_workqueue_struct {
78 struct global_cwq *gcwq; /* I: the associated gcwq */105 struct global_cwq *gcwq; /* I: the associated gcwq */
79 struct list_head worklist;106 struct list_head worklist;
80 wait_queue_head_t more_work;
81 struct worker *worker;107 struct worker *worker;
82 struct workqueue_struct *wq; /* I: the owning workqueue */108 struct workqueue_struct *wq; /* I: the owning workqueue */
83 int work_color; /* L: current color */109 int work_color; /* L: current color */
@@ -306,6 +332,33 @@ static inline struct cpu_workqueue_struct *get_wq_data(struct work_struct *work) · 222 unmodified lines
306 WORK_STRUCT_WQ_DATA_MASK);332 WORK_STRUCT_WQ_DATA_MASK);
307}333}
308 334
335/**
336 * busy_worker_head - return the busy hash head for a work
337 * @gcwq: gcwq of interest
338 * @work: work to be hashed
339 *
340 * Return hash head of @gcwq for @work.
341 *
342 * CONTEXT:
343 * spin_lock_irq(gcwq->lock).
344 *
345 * RETURNS:
346 * Pointer to the hash head.
347 */
348static struct hlist_head *busy_worker_head(struct global_cwq *gcwq,
349 struct work_struct *work)
350{
351 const int base_shift = ilog2(sizeof(struct work_struct));
352 unsigned long v = (unsigned long)work;
353
354 /* simple shift and fold hash, do we need something better? */
355 v >>= base_shift;
356 v += v >> BUSY_WORKER_HASH_ORDER;
357 v &= BUSY_WORKER_HASH_MASK;
358
359 return &gcwq->busy_hash[v];
360}
361
309/**362/**
310 * insert_work - insert a work into cwq363 * insert_work - insert a work into cwq
311 * @cwq: cwq @work belongs to364 * @cwq: cwq @work belongs to
@@ -332,7 +385,7 @@ static void insert_work(struct cpu_workqueue_struct *cwq, · 20 unmodified lines
332 smp_wmb();385 smp_wmb();
333 386
334 list_add_tail(&work->entry, head);387 list_add_tail(&work->entry, head);
335 wake_up(&cwq->more_work);388 wake_up_process(cwq->worker->task);
336}389}
337 390
338static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,391static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
@@ -470,13 +523,59 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, · 131 unmodified lines
470}523}
471EXPORT_SYMBOL_GPL(queue_delayed_work_on);524EXPORT_SYMBOL_GPL(queue_delayed_work_on);
472 525
526/**
527 * worker_enter_idle - enter idle state
528 * @worker: worker which is entering idle state
529 *
530 * @worker is entering idle state. Update stats and idle timer if
531 * necessary.
532 *
533 * LOCKING:
534 * spin_lock_irq(gcwq->lock).
535 */
536static void worker_enter_idle(struct worker *worker)
537{
538 struct global_cwq *gcwq = worker->gcwq;
539
540 BUG_ON(worker->flags & WORKER_IDLE);
541 BUG_ON(!list_empty(&worker->entry) &&
542 (worker->hentry.next || worker->hentry.pprev));
543
544 worker->flags |= WORKER_IDLE;
545 gcwq->nr_idle++;
546
547 /* idle_list is LIFO */
548 list_add(&worker->entry, &gcwq->idle_list);
549}
550
551/**
552 * worker_leave_idle - leave idle state
553 * @worker: worker which is leaving idle state
554 *
555 * @worker is leaving idle state. Update stats.
556 *
557 * LOCKING:
558 * spin_lock_irq(gcwq->lock).
559 */
560static void worker_leave_idle(struct worker *worker)
561{
562 struct global_cwq *gcwq = worker->gcwq;
563
564 BUG_ON(!(worker->flags & WORKER_IDLE));
565 worker->flags &= ~WORKER_IDLE;
566 gcwq->nr_idle--;
567 list_del_init(&worker->entry);
568}
569
473static struct worker *alloc_worker(void)570static struct worker *alloc_worker(void)
474{571{
475 struct worker *worker;572 struct worker *worker;
476 573
477 worker = kzalloc(sizeof(*worker), GFP_KERNEL);574 worker = kzalloc(sizeof(*worker), GFP_KERNEL);
478 if (worker)575 if (worker) {
576 INIT_LIST_HEAD(&worker->entry);
479 INIT_LIST_HEAD(&worker->scheduled);577 INIT_LIST_HEAD(&worker->scheduled);
578 }
480 return worker;579 return worker;
481}580}
482 581
@@ -541,13 +640,16 @@ fail: · 58 unmodified lines
541 * start_worker - start a newly created worker640 * start_worker - start a newly created worker
542 * @worker: worker to start641 * @worker: worker to start
543 *642 *
544 * Start @worker.643 * Make the gcwq aware of @worker and start it.
545 *644 *
546 * CONTEXT:645 * CONTEXT:
547 * spin_lock_irq(gcwq->lock).646 * spin_lock_irq(gcwq->lock).
548 */647 */
549static void start_worker(struct worker *worker)648static void start_worker(struct worker *worker)
550{649{
650 worker->flags |= WORKER_STARTED;
651 worker->gcwq->nr_workers++;
652 worker_enter_idle(worker);
551 wake_up_process(worker->task);653 wake_up_process(worker->task);
552}654}
553 655
@@ -555,7 +657,10 @@ static void start_worker(struct worker *worker) · 1 unmodified line
555 * destroy_worker - destroy a workqueue worker657 * destroy_worker - destroy a workqueue worker
556 * @worker: worker to be destroyed658 * @worker: worker to be destroyed
557 *659 *
558 * Destroy @worker.660 * Destroy @worker and adjust @gcwq stats accordingly.
661 *
662 * CONTEXT:
663 * spin_lock_irq(gcwq->lock) which is released and regrabbed.
559 */664 */
560static void destroy_worker(struct worker *worker)665static void destroy_worker(struct worker *worker)
561{666{
@@ -566,12 +671,21 @@ static void destroy_worker(struct worker *worker) · 4 unmodified lines
566 BUG_ON(worker->current_work);671 BUG_ON(worker->current_work);
567 BUG_ON(!list_empty(&worker->scheduled));672 BUG_ON(!list_empty(&worker->scheduled));
568 673
674 if (worker->flags & WORKER_STARTED)
675 gcwq->nr_workers--;
676 if (worker->flags & WORKER_IDLE)
677 gcwq->nr_idle--;
678
679 list_del_init(&worker->entry);
680 worker->flags |= WORKER_DIE;
681
682 spin_unlock_irq(&gcwq->lock);
683
569 kthread_stop(worker->task);684 kthread_stop(worker->task);
570 kfree(worker);685 kfree(worker);
571 686
572 spin_lock_irq(&gcwq->lock);687 spin_lock_irq(&gcwq->lock);
573 ida_remove(&gcwq->worker_ida, id);688 ida_remove(&gcwq->worker_ida, id);
574 spin_unlock_irq(&gcwq->lock);
575}689}
576 690
577/**691/**
@@ -686,6 +800,7 @@ static void process_one_work(struct worker *worker, struct work_struct *work) · 108 unmodified lines
686{800{
687 struct cpu_workqueue_struct *cwq = worker->cwq;801 struct cpu_workqueue_struct *cwq = worker->cwq;
688 struct global_cwq *gcwq = cwq->gcwq;802 struct global_cwq *gcwq = cwq->gcwq;
803 struct hlist_head *bwh = busy_worker_head(gcwq, work);
689 work_func_t f = work->func;804 work_func_t f = work->func;
690 int work_color;805 int work_color;
691#ifdef CONFIG_LOCKDEP806#ifdef CONFIG_LOCKDEP
@@ -700,6 +815,7 @@ static void process_one_work(struct worker *worker, struct work_struct *work) · 8 unmodified lines
700#endif815#endif
701 /* claim and process */816 /* claim and process */
702 debug_work_deactivate(work);817 debug_work_deactivate(work);
818 hlist_add_head(&worker->hentry, bwh);
703 worker->current_work = work;819 worker->current_work = work;
704 work_color = get_work_color(work);820 work_color = get_work_color(work);
705 list_del_init(&work->entry);821 list_del_init(&work->entry);
@@ -727,6 +843,7 @@ static void process_one_work(struct worker *worker, struct work_struct *work) · 21 unmodified lines
727 spin_lock_irq(&gcwq->lock);843 spin_lock_irq(&gcwq->lock);
728 844
729 /* we're done with it, release */845 /* we're done with it, release */
846 hlist_del_init(&worker->hentry);
730 worker->current_work = NULL;847 worker->current_work = NULL;
731 cwq_dec_nr_in_flight(cwq, work_color);848 cwq_dec_nr_in_flight(cwq, work_color);
732}849}
@@ -763,47 +880,56 @@ static int worker_thread(void *__worker) · 30 unmodified lines
763 struct worker *worker = __worker;880 struct worker *worker = __worker;
764 struct global_cwq *gcwq = worker->gcwq;881 struct global_cwq *gcwq = worker->gcwq;
765 struct cpu_workqueue_struct *cwq = worker->cwq;882 struct cpu_workqueue_struct *cwq = worker->cwq;
766 DEFINE_WAIT(wait);
767 883
768 for (;;) {884woke_up:
769 prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);885 if (unlikely(!cpumask_equal(&worker->task->cpus_allowed,
770 if (!kthread_should_stop() &&886 get_cpu_mask(gcwq->cpu))))
771 list_empty(&cwq->worklist))887 set_cpus_allowed_ptr(worker->task, get_cpu_mask(gcwq->cpu));
772 schedule();
773 finish_wait(&cwq->more_work, &wait);
774 888
775 if (kthread_should_stop())889 spin_lock_irq(&gcwq->lock);
776 break;
777 890
778 if (unlikely(!cpumask_equal(&worker->task->cpus_allowed,891 /* DIE can be set only while we're idle, checking here is enough */
779 get_cpu_mask(gcwq->cpu))))892 if (worker->flags & WORKER_DIE) {
780 set_cpus_allowed_ptr(worker->task,893 spin_unlock_irq(&gcwq->lock);
781 get_cpu_mask(gcwq->cpu));894 return 0;
895 }
782 896
783 spin_lock_irq(&gcwq->lock);897 worker_leave_idle(worker);
784 898
785 while (!list_empty(&cwq->worklist)) {899 /*
786 struct work_struct *work =900 * ->scheduled list can only be filled while a worker is
787 list_first_entry(&cwq->worklist,901 * preparing to process a work or actually processing it.
788 struct work_struct, entry);902 * Make sure nobody diddled with it while I was sleeping.
789 903 */
790 if (likely(!(*work_data_bits(work) &904 BUG_ON(!list_empty(&worker->scheduled));
791 WORK_STRUCT_LINKED))) {905
792 /* optimization path, not strictly necessary */906 while (!list_empty(&cwq->worklist)) {
793 process_one_work(worker, work);907 struct work_struct *work =
794 if (unlikely(!list_empty(&worker->scheduled)))908 list_first_entry(&cwq->worklist,
795 process_scheduled_works(worker);909 struct work_struct, entry);
796 } else {910
797 move_linked_works(work, &worker->scheduled,911 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
798 NULL);912 /* optimization path, not strictly necessary */
913 process_one_work(worker, work);
914 if (unlikely(!list_empty(&worker->scheduled)))
799 process_scheduled_works(worker);915 process_scheduled_works(worker);
800 }916 } else {
917 move_linked_works(work, &worker->scheduled, NULL);
918 process_scheduled_works(worker);
801 }919 }
802
803 spin_unlock_irq(&gcwq->lock);
804 }920 }
805 921
806 return 0;922 /*
923 * gcwq->lock is held and there's no work to process, sleep.
924 * Workers are woken up only while holding gcwq->lock, so
925 * setting the current state before releasing gcwq->lock is
926 * enough to prevent losing any event.
927 */
928 worker_enter_idle(worker);
929 __set_current_state(TASK_INTERRUPTIBLE);
930 spin_unlock_irq(&gcwq->lock);
931 schedule();
932 goto woke_up;
807}933}
808 934
809struct wq_barrier {935struct wq_barrier {
@@ -1600,7 +1726,6 @@ struct workqueue_struct *__create_workqueue_key(const char *name, · 790 unmodified lines
1600 cwq->max_active = max_active;1726 cwq->max_active = max_active;
1601 INIT_LIST_HEAD(&cwq->worklist);1727 INIT_LIST_HEAD(&cwq->worklist);
1602 INIT_LIST_HEAD(&cwq->delayed_works);1728 INIT_LIST_HEAD(&cwq->delayed_works);
1603 init_waitqueue_head(&cwq->more_work);
1604 1729
1605 if (failed)1730 if (failed)
1606 continue;1731 continue;
@@ -1651,7 +1776,7 @@ EXPORT_SYMBOL_GPL(__create_workqueue_key); · 44 unmodified lines
1651 */1776 */
1652void destroy_workqueue(struct workqueue_struct *wq)1777void destroy_workqueue(struct workqueue_struct *wq)
1653{1778{
1654 int cpu;1779 unsigned int cpu;
1655 1780
1656 flush_workqueue(wq);1781 flush_workqueue(wq);
1657 1782
@@ -1670,8 +1795,10 @@ void destroy_workqueue(struct workqueue_struct *wq) · 12 unmodified lines
1670 int i;1795 int i;
1671 1796
1672 if (cwq->worker) {1797 if (cwq->worker) {
1798 spin_lock_irq(&cwq->gcwq->lock);
1673 destroy_worker(cwq->worker);1799 destroy_worker(cwq->worker);
1674 cwq->worker = NULL;1800 cwq->worker = NULL;
1801 spin_unlock_irq(&cwq->gcwq->lock);
1675 }1802 }
1676 1803
1677 for (i = 0; i < WORK_NR_COLORS; i++)1804 for (i = 0; i < WORK_NR_COLORS; i++)
@@ -1881,7 +2008,7 @@ void thaw_workqueues(void) · 203 unmodified lines
1881 cwq->nr_active < cwq->max_active)2008 cwq->nr_active < cwq->max_active)
1882 cwq_activate_first_delayed(cwq);2009 cwq_activate_first_delayed(cwq);
1883 2010
1884 wake_up(&cwq->more_work);2011 wake_up_process(cwq->worker->task);
1885 }2012 }
1886 2013
1887 spin_unlock_irq(&gcwq->lock);2014 spin_unlock_irq(&gcwq->lock);
@@ -1896,6 +2023,7 @@ out_unlock: · 8 unmodified lines
1896void __init init_workqueues(void)2023void __init init_workqueues(void)
1897{2024{
1898 unsigned int cpu;2025 unsigned int cpu;
2026 int i;
1899 2027
1900 singlethread_cpu = cpumask_first(cpu_possible_mask);2028 singlethread_cpu = cpumask_first(cpu_possible_mask);
1901 hotcpu_notifier(workqueue_cpu_callback, 0);2029 hotcpu_notifier(workqueue_cpu_callback, 0);
@@ -1907,6 +2035,10 @@ void __init init_workqueues(void) · 5 unmodified lines
1907 spin_lock_init(&gcwq->lock);2035 spin_lock_init(&gcwq->lock);
1908 gcwq->cpu = cpu;2036 gcwq->cpu = cpu;
1909 2037
2038 INIT_LIST_HEAD(&gcwq->idle_list);
2039 for (i = 0; i < BUSY_WORKER_HASH_SIZE; i++)
2040 INIT_HLIST_HEAD(&gcwq->busy_hash[i]);
2041
1910 ida_init(&gcwq->worker_ida);2042 ida_init(&gcwq->worker_ida);
1911 }2043 }
1912 2044