tasklet_unlock_wait() spin waits for TASKLET_STATE_RUN to be cleared. This is wasting CPU cycles in a tight loop which is especially painful in a guest when the CPU running the tasklet is scheduled out. tasklet_unlock_wait() is invoked from tasklet_kill() which is used in teardown paths and not performance critical at all. Replace the spin wait with wait_var_event(). There are no users of tasklet_unlock_wait() which are invoked from atomic contexts. The usage in tasklet_disable() has been replaced temporarily with the spin waiting variant until the atomic users are fixed up and will be converted to the sleep wait variant later.
Unified · Split
| @@ -664,17 +664,8 @@ static inline int tasklet_trylock(struct tasklet_struct *t) · 663 unmodified lines | ||
| 664 | 664 | return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); |
| 665 | 665 | } |
| 666 | 666 | |
| 667 | −static inline void tasklet_unlock(struct tasklet_struct *t) | |
| 668 | −{ | |
| 669 | − smp_mb__before_atomic(); | |
| 670 | − clear_bit(TASKLET_STATE_RUN, &(t)->state); | |
| 671 | −} | |
| 672 | − | |
| 673 | −static inline void tasklet_unlock_wait(struct tasklet_struct *t) | |
| 674 | −{ | |
| 675 | − while (test_bit(TASKLET_STATE_RUN, &t->state)) | |
| 676 | − cpu_relax(); | |
| 677 | −} | |
| 667 | +void tasklet_unlock(struct tasklet_struct *t); | |
| 668 | +void tasklet_unlock_wait(struct tasklet_struct *t); | |
| 678 | 669 | |
| 679 | 670 | /* |
| 680 | 671 | * Do not use in new code. Waiting for tasklets from atomic contexts is |
| @@ -25,6 +25,7 @@ · 24 unmodified lines | ||
| 25 | 25 | #include <linux/smpboot.h> |
| 26 | 26 | #include <linux/tick.h> |
| 27 | 27 | #include <linux/irq.h> |
| 28 | +#include <linux/wait_bit.h> | |
| 28 | 29 | |
| 29 | 30 | #include <asm/softirq_stack.h> |
| 30 | 31 | |
| @@ -632,6 +633,23 @@ void tasklet_kill(struct tasklet_struct *t) · 601 unmodified lines | ||
| 632 | 633 | } |
| 633 | 634 | EXPORT_SYMBOL(tasklet_kill); |
| 634 | 635 | |
| 636 | +#ifdef CONFIG_SMP | |
| 637 | +void tasklet_unlock(struct tasklet_struct *t) | |
| 638 | +{ | |
| 639 | + smp_mb__before_atomic(); | |
| 640 | + clear_bit(TASKLET_STATE_RUN, &t->state); | |
| 641 | + smp_mb__after_atomic(); | |
| 642 | + wake_up_var(&t->state); | |
| 643 | +} | |
| 644 | +EXPORT_SYMBOL_GPL(tasklet_unlock); | |
| 645 | + | |
| 646 | +void tasklet_unlock_wait(struct tasklet_struct *t) | |
| 647 | +{ | |
| 648 | + wait_var_event(&t->state, !test_bit(TASKLET_STATE_RUN, &t->state)); | |
| 649 | +} | |
| 650 | +EXPORT_SYMBOL_GPL(tasklet_unlock_wait); | |
| 651 | +#endif | |
| 652 | + | |
| 635 | 653 | void __init softirq_init(void) |
| 636 | 654 | { |
| 637 | 655 | int cpu; |