tasklet_unlock_wait() spin waits for TASKLET_STATE_RUN to be cleared. This is wasting CPU cycles in a tight loop which is especially painful in a guest when the CPU running the tasklet is scheduled out. tasklet_unlock_wait() is invoked from tasklet_kill() which is used in teardown paths and not performance critical at all. Replace the spin wait with wait_var_event(). There are no users of tasklet_unlock_wait() which are invoked from atomic contexts. The usage in tasklet_disable() has been replaced temporarily with the spin waiting variant until the atomic users are fixed up and will be converted to the sleep wait variant later.
Unified · Split
| @@ -664,17 +664,8 @@ static inline int tasklet_trylock(struct tasklet_struct *t) · 663 unmodified lines | |||
| 664 | return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); | 664 | return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); |
| 665 | } | 665 | } |
| 666 | 666 | ||
| 667 | static inline void tasklet_unlock(struct tasklet_struct *t) | 667 | void tasklet_unlock(struct tasklet_struct *t); |
| 668 | { | 668 | void tasklet_unlock_wait(struct tasklet_struct *t); |
| 669 | smp_mb__before_atomic(); | ||
| 670 | clear_bit(TASKLET_STATE_RUN, &(t)->state); | ||
| 671 | } | ||
| 672 | |||
| 673 | static inline void tasklet_unlock_wait(struct tasklet_struct *t) | ||
| 674 | { | ||
| 675 | while (test_bit(TASKLET_STATE_RUN, &t->state)) | ||
| 676 | cpu_relax(); | ||
| 677 | } | ||
| 678 | 669 | ||
| 679 | /* | 670 | /* |
| 680 | * Do not use in new code. Waiting for tasklets from atomic contexts is | 671 | * Do not use in new code. Waiting for tasklets from atomic contexts is |
| @@ -25,6 +25,7 @@ · 24 unmodified lines | |||
| 25 | #include <linux/smpboot.h> | 25 | #include <linux/smpboot.h> |
| 26 | #include <linux/tick.h> | 26 | #include <linux/tick.h> |
| 27 | #include <linux/irq.h> | 27 | #include <linux/irq.h> |
| 28 | #include <linux/wait_bit.h> | ||
| 28 | 29 | ||
| 29 | #include <asm/softirq_stack.h> | 30 | #include <asm/softirq_stack.h> |
| 30 | 31 | ||
| @@ -632,6 +633,23 @@ void tasklet_kill(struct tasklet_struct *t) · 601 unmodified lines | |||
| 632 | } | 633 | } |
| 633 | EXPORT_SYMBOL(tasklet_kill); | 634 | EXPORT_SYMBOL(tasklet_kill); |
| 634 | 635 | ||
| 636 | #ifdef CONFIG_SMP | ||
| 637 | void tasklet_unlock(struct tasklet_struct *t) | ||
| 638 | { | ||
| 639 | smp_mb__before_atomic(); | ||
| 640 | clear_bit(TASKLET_STATE_RUN, &t->state); | ||
| 641 | smp_mb__after_atomic(); | ||
| 642 | wake_up_var(&t->state); | ||
| 643 | } | ||
| 644 | EXPORT_SYMBOL_GPL(tasklet_unlock); | ||
| 645 | |||
| 646 | void tasklet_unlock_wait(struct tasklet_struct *t) | ||
| 647 | { | ||
| 648 | wait_var_event(&t->state, !test_bit(TASKLET_STATE_RUN, &t->state)); | ||
| 649 | } | ||
| 650 | EXPORT_SYMBOL_GPL(tasklet_unlock_wait); | ||
| 651 | #endif | ||
| 652 | |||
| 635 | void __init softirq_init(void) | 653 | void __init softirq_init(void) |
| 636 | { | 654 | { |
| 637 | int cpu; | 655 | int cpu; |