History

workqueue: introduce worker

Separate out worker thread related information to struct worker from struct cpu_workqueue_struct and implement helper functions to deal with the new struct worker. The only change which is visible outside is that now workqueue worker are all named "kworker/CPUID:WORKERID" where WORKERID is allocated from per-cpu ida. This is in preparation of concurrency managed workqueue where shared multiple workers would be available per cpu.

Tejun Heo committed 16 years ago

Commit c34056a1 parent 73f53c41 file · +150 −61Browse files

Showing the changes to kernel/workqueue.c (1 of 1 file). Show all

Unified · Split

kernel/workqueue.c+150−61
@@ -33,6 +33,7 @@ · 32 unmodified lines
3333 #include <linux/kallsyms.h>
3434 #include <linux/debug_locks.h>
3535 #include <linux/lockdep.h>
36+#include <linux/idr.h>
3637
3738 /*
3839 * Structure fields follow one of the following exclusion rules.
@@ -46,6 +47,15 @@ · 7 unmodified lines
4647 * W: workqueue_lock protected.
4748 */
4849
50+struct cpu_workqueue_struct;
51+
52+struct worker {
53+ struct work_struct *current_work; /* L: work being processed */
54+ struct task_struct *task; /* I: worker task */
55+ struct cpu_workqueue_struct *cwq; /* I: the associated cwq */
56+ int id; /* I: worker id */
57+};
58+
4959 /*
5060 * The per-CPU workqueue (if single thread, we always use the first
5161 * possible cpu). The lower WORK_STRUCT_FLAG_BITS of
@@ -58,15 +68,14 @@ struct cpu_workqueue_struct { · 6 unmodified lines
5868
5969 struct list_head worklist;
6070 wait_queue_head_t more_work;
61− struct work_struct *current_work;
6271 unsigned int cpu;
72+ struct worker *worker;
6373
6474 struct workqueue_struct *wq; /* I: the owning workqueue */
6575 int work_color; /* L: current color */
6676 int flush_color; /* L: flushing color */
6777 int nr_in_flight[WORK_NR_COLORS];
6878 /* L: nr of in_flight works */
69− struct task_struct *thread;
7079 };
7180
7281 /*
@@ -214,6 +223,9 @@ static inline void debug_work_deactivate(struct work_struct *work) { } · 141 unmodified lines
214223 /* Serializes the accesses to the list of workqueues. */
215224 static DEFINE_SPINLOCK(workqueue_lock);
216225 static LIST_HEAD(workqueues);
226+static DEFINE_PER_CPU(struct ida, worker_ida);
227+
228+static int worker_thread(void *__worker);
217229
218230 static int singlethread_cpu __read_mostly;
219231
@@ -428,6 +440,105 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, · 208 unmodified lines
428440 }
429441 EXPORT_SYMBOL_GPL(queue_delayed_work_on);
430442
443+static struct worker *alloc_worker(void)
444+{
445+ struct worker *worker;
446+
447+ worker = kzalloc(sizeof(*worker), GFP_KERNEL);
448+ return worker;
449+}
450+
451+/**
452+ * create_worker - create a new workqueue worker
453+ * @cwq: cwq the new worker will belong to
454+ * @bind: whether to set affinity to @cpu or not
455+ *
456+ * Create a new worker which is bound to @cwq. The returned worker
457+ * can be started by calling start_worker() or destroyed using
458+ * destroy_worker().
459+ *
460+ * CONTEXT:
461+ * Might sleep. Does GFP_KERNEL allocations.
462+ *
463+ * RETURNS:
464+ * Pointer to the newly created worker.
465+ */
466+static struct worker *create_worker(struct cpu_workqueue_struct *cwq, bool bind)
467+{
468+ int id = -1;
469+ struct worker *worker = NULL;
470+
471+ spin_lock(&workqueue_lock);
472+ while (ida_get_new(&per_cpu(worker_ida, cwq->cpu), &id)) {
473+ spin_unlock(&workqueue_lock);
474+ if (!ida_pre_get(&per_cpu(worker_ida, cwq->cpu), GFP_KERNEL))
475+ goto fail;
476+ spin_lock(&workqueue_lock);
477+ }
478+ spin_unlock(&workqueue_lock);
479+
480+ worker = alloc_worker();
481+ if (!worker)
482+ goto fail;
483+
484+ worker->cwq = cwq;
485+ worker->id = id;
486+
487+ worker->task = kthread_create(worker_thread, worker, "kworker/%u:%d",
488+ cwq->cpu, id);
489+ if (IS_ERR(worker->task))
490+ goto fail;
491+
492+ if (bind)
493+ kthread_bind(worker->task, cwq->cpu);
494+
495+ return worker;
496+fail:
497+ if (id >= 0) {
498+ spin_lock(&workqueue_lock);
499+ ida_remove(&per_cpu(worker_ida, cwq->cpu), id);
500+ spin_unlock(&workqueue_lock);
501+ }
502+ kfree(worker);
503+ return NULL;
504+}
505+
506+/**
507+ * start_worker - start a newly created worker
508+ * @worker: worker to start
509+ *
510+ * Start @worker.
511+ *
512+ * CONTEXT:
513+ * spin_lock_irq(cwq->lock).
514+ */
515+static void start_worker(struct worker *worker)
516+{
517+ wake_up_process(worker->task);
518+}
519+
520+/**
521+ * destroy_worker - destroy a workqueue worker
522+ * @worker: worker to be destroyed
523+ *
524+ * Destroy @worker.
525+ */
526+static void destroy_worker(struct worker *worker)
527+{
528+ int cpu = worker->cwq->cpu;
529+ int id = worker->id;
530+
531+ /* sanity check frenzy */
532+ BUG_ON(worker->current_work);
533+
534+ kthread_stop(worker->task);
535+ kfree(worker);
536+
537+ spin_lock(&workqueue_lock);
538+ ida_remove(&per_cpu(worker_ida, cpu), id);
539+ spin_unlock(&workqueue_lock);
540+}
541+
431542 /**
432543 * cwq_dec_nr_in_flight - decrement cwq's nr_in_flight
433544 * @cwq: cwq of interest
@@ -468,7 +579,7 @@ static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color) · 34 unmodified lines
468579
469580 /**
470581 * process_one_work - process single work
471− * @cwq: cwq to process work for
582+ * @worker: self
472583 * @work: work to process
473584 *
474585 * Process @work. This function contains all the logics necessary to
@@ -480,9 +591,9 @@ static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color) · 5 unmodified lines
480591 * CONTEXT:
481592 * spin_lock_irq(cwq->lock) which is released and regrabbed.
482593 */
483−static void process_one_work(struct cpu_workqueue_struct *cwq,
484− struct work_struct *work)
594+static void process_one_work(struct worker *worker, struct work_struct *work)
485595 {
596+ struct cpu_workqueue_struct *cwq = worker->cwq;
486597 work_func_t f = work->func;
487598 int work_color;
488599 #ifdef CONFIG_LOCKDEP
@@ -497,7 +608,7 @@ static void process_one_work(struct cpu_workqueue_struct *cwq, · 8 unmodified lines
497608 #endif
498609 /* claim and process */
499610 debug_work_deactivate(work);
500− cwq->current_work = work;
611+ worker->current_work = work;
501612 work_color = get_work_color(work);
502613 list_del_init(&work->entry);
503614
@@ -524,30 +635,33 @@ static void process_one_work(struct cpu_workqueue_struct *cwq, · 20 unmodified lines
524635 spin_lock_irq(&cwq->lock);
525636
526637 /* we're done with it, release */
527− cwq->current_work = NULL;
638+ worker->current_work = NULL;
528639 cwq_dec_nr_in_flight(cwq, work_color);
529640 }
530641
531−static void run_workqueue(struct cpu_workqueue_struct *cwq)
642+static void run_workqueue(struct worker *worker)
532643 {
644+ struct cpu_workqueue_struct *cwq = worker->cwq;
645+
533646 spin_lock_irq(&cwq->lock);
534647 while (!list_empty(&cwq->worklist)) {
535648 struct work_struct *work = list_entry(cwq->worklist.next,
536649 struct work_struct, entry);
537− process_one_work(cwq, work);
650+ process_one_work(worker, work);
538651 }
539652 spin_unlock_irq(&cwq->lock);
540653 }
541654
542655 /**
543656 * worker_thread - the worker thread function
544− * @__cwq: cwq to serve
657+ * @__worker: self
545658 *
546659 * The cwq worker thread function.
547660 */
548−static int worker_thread(void *__cwq)
661+static int worker_thread(void *__worker)
549662 {
550− struct cpu_workqueue_struct *cwq = __cwq;
663+ struct worker *worker = __worker;
664+ struct cpu_workqueue_struct *cwq = worker->cwq;
551665 DEFINE_WAIT(wait);
552666
553667 if (cwq->wq->flags & WQ_FREEZEABLE)
@@ -566,11 +680,11 @@ static int worker_thread(void *__cwq) · 12 unmodified lines
566680 if (kthread_should_stop())
567681 break;
568682
569− if (unlikely(!cpumask_equal(&cwq->thread->cpus_allowed,
683+ if (unlikely(!cpumask_equal(&worker->task->cpus_allowed,
570684 get_cpu_mask(cwq->cpu))))
571− set_cpus_allowed_ptr(cwq->thread,
685+ set_cpus_allowed_ptr(worker->task,
572686 get_cpu_mask(cwq->cpu));
573− run_workqueue(cwq);
687+ run_workqueue(worker);
574688 }
575689
576690 return 0;
@@ -873,7 +987,7 @@ int flush_work(struct work_struct *work) · 296 unmodified lines
873987 goto already_gone;
874988 prev = &work->entry;
875989 } else {
876− if (cwq->current_work != work)
990+ if (!cwq->worker || cwq->worker->current_work != work)
877991 goto already_gone;
878992 prev = &cwq->worklist;
879993 }
@@ -937,7 +1051,7 @@ static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq, · 57 unmodified lines
9371051 int running = 0;
9381052
9391053 spin_lock_irq(&cwq->lock);
940− if (unlikely(cwq->current_work == work)) {
1054+ if (unlikely(cwq->worker && cwq->worker->current_work == work)) {
9411055 insert_wq_barrier(cwq, &barr, cwq->worklist.next);
9421056 running = 1;
9431057 }
@@ -1225,7 +1339,7 @@ int current_is_keventd(void) · 281 unmodified lines
12251339 BUG_ON(!keventd_wq);
12261340
12271341 cwq = get_cwq(cpu, keventd_wq);
1228− if (current == cwq->thread)
1342+ if (current == cwq->worker->task)
12291343 ret = 1;
12301344
12311345 return ret;
@@ -1279,38 +1393,6 @@ static void free_cwqs(struct cpu_workqueue_struct *cwqs) · 47 unmodified lines
12791393 #endif
12801394 }
12811395
1282−static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
1283−{
1284− struct workqueue_struct *wq = cwq->wq;
1285− struct task_struct *p;
1286−
1287− p = kthread_create(worker_thread, cwq, "%s/%d", wq->name, cpu);
1288− /*
1289− * Nobody can add the work_struct to this cwq,
1290− * if (caller is __create_workqueue)
1291− * nobody should see this wq
1292− * else // caller is CPU_UP_PREPARE
1293− * cpu is not on cpu_online_map
1294− * so we can abort safely.
1295− */
1296− if (IS_ERR(p))
1297− return PTR_ERR(p);
1298− cwq->thread = p;
1299−
1300− return 0;
1301−}
1302−
1303−static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
1304−{
1305− struct task_struct *p = cwq->thread;
1306−
1307− if (p != NULL) {
1308− if (cpu >= 0)
1309− kthread_bind(p, cpu);
1310− wake_up_process(p);
1311− }
1312−}
1313−
13141396 struct workqueue_struct *__create_workqueue_key(const char *name,
13151397 unsigned int flags,
13161398 struct lock_class_key *key,
@@ -1318,7 +1400,8 @@ struct workqueue_struct *__create_workqueue_key(const char *name, · 1 unmodified line
13181400 {
13191401 bool singlethread = flags & WQ_SINGLE_THREAD;
13201402 struct workqueue_struct *wq;
1321− int err = 0, cpu;
1403+ bool failed = false;
1404+ unsigned int cpu;
13221405
13231406 wq = kzalloc(sizeof(*wq), GFP_KERNEL);
13241407 if (!wq)
@@ -1348,20 +1431,21 @@ struct workqueue_struct *__create_workqueue_key(const char *name, · 23 unmodified lines
13481431 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
13491432
13501433 BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK);
1351− cwq->wq = wq;
13521434 cwq->cpu = cpu;
1435+ cwq->wq = wq;
13531436 cwq->flush_color = -1;
13541437 spin_lock_init(&cwq->lock);
13551438 INIT_LIST_HEAD(&cwq->worklist);
13561439 init_waitqueue_head(&cwq->more_work);
13571440
1358− if (err)
1441+ if (failed)
13591442 continue;
1360− err = create_workqueue_thread(cwq, cpu);
1361− if (cpu_online(cpu) && !singlethread)
1362− start_workqueue_thread(cwq, cpu);
1443+ cwq->worker = create_worker(cwq,
1444+ cpu_online(cpu) && !singlethread);
1445+ if (cwq->worker)
1446+ start_worker(cwq->worker);
13631447 else
1364− start_workqueue_thread(cwq, -1);
1448+ failed = true;
13651449 }
13661450
13671451 spin_lock(&workqueue_lock);
@@ -1370,7 +1454,7 @@ struct workqueue_struct *__create_workqueue_key(const char *name, · 2 unmodified lines
13701454
13711455 cpu_maps_update_done();
13721456
1373− if (err) {
1457+ if (failed) {
13741458 destroy_workqueue(wq);
13751459 wq = NULL;
13761460 }
@@ -1406,9 +1490,9 @@ void destroy_workqueue(struct workqueue_struct *wq) · 29 unmodified lines
14061490 struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
14071491 int i;
14081492
1409− if (cwq->thread) {
1410− kthread_stop(cwq->thread);
1411− cwq->thread = NULL;
1493+ if (cwq->worker) {
1494+ destroy_worker(cwq->worker);
1495+ cwq->worker = NULL;
14121496 }
14131497
14141498 for (i = 0; i < WORK_NR_COLORS; i++)
@@ -1495,6 +1579,11 @@ EXPORT_SYMBOL_GPL(work_on_cpu); · 80 unmodified lines
14951579
14961580 void __init init_workqueues(void)
14971581 {
1582+ unsigned int cpu;
1583+
1584+ for_each_possible_cpu(cpu)
1585+ ida_init(&per_cpu(worker_ida, cpu));
1586+
14981587 singlethread_cpu = cpumask_first(cpu_possible_mask);
14991588 hotcpu_notifier(workqueue_cpu_callback, 0);
15001589 keventd_wq = create_workqueue("events");