Setting LBF_ALL_PINNED during active load balance is only valid when there is only 1 running task on the rq otherwise this ends up increasing the balance interval whereas other tasks could migrate after the next interval once they become cache-cold as an example. LBF_ALL_PINNED flag is now always set it by default. It is then cleared when we find one task that can be pulled when calling detach_tasks() or during active migration.
Unified · Split
| @@ -9639,6 +9639,8 @@ redo: · 9638 unmodified lines | |||
| 9639 | env.src_rq = busiest; | 9639 | env.src_rq = busiest; |
| 9640 | 9640 | ||
| 9641 | ld_moved = 0; | 9641 | ld_moved = 0; |
| 9642 | /* Clear this flag as soon as we find a pullable task */ | ||
| 9643 | env.flags |= LBF_ALL_PINNED; | ||
| 9642 | if (busiest->nr_running > 1) { | 9644 | if (busiest->nr_running > 1) { |
| 9643 | /* | 9645 | /* |
| 9644 | * Attempt to move tasks. If find_busiest_group has found | 9646 | * Attempt to move tasks. If find_busiest_group has found |
| @@ -9646,7 +9648,6 @@ redo: · 1 unmodified line | |||
| 9646 | * still unbalanced. ld_moved simply stays zero, so it is | 9648 | * still unbalanced. ld_moved simply stays zero, so it is |
| 9647 | * correctly treated as an imbalance. | 9649 | * correctly treated as an imbalance. |
| 9648 | */ | 9650 | */ |
| 9649 | env.flags |= LBF_ALL_PINNED; | ||
| 9650 | env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); | 9651 | env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); |
| 9651 | 9652 | ||
| 9652 | more_balance: | 9653 | more_balance: |
| @@ -9772,10 +9773,12 @@ more_balance: · 119 unmodified lines | |||
| 9772 | if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { | 9773 | if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { |
| 9773 | raw_spin_unlock_irqrestore(&busiest->lock, | 9774 | raw_spin_unlock_irqrestore(&busiest->lock, |
| 9774 | flags); | 9775 | flags); |
| 9775 | env.flags |= LBF_ALL_PINNED; | ||
| 9776 | goto out_one_pinned; | 9776 | goto out_one_pinned; |
| 9777 | } | 9777 | } |
| 9778 | 9778 | ||
| 9779 | /* Record that we found at least one task that could run on this_cpu */ | ||
| 9780 | env.flags &= ~LBF_ALL_PINNED; | ||
| 9781 | |||
| 9779 | /* | 9782 | /* |
| 9780 | * ->active_balance synchronizes accesses to | 9783 | * ->active_balance synchronizes accesses to |
| 9781 | * ->active_balance_work. Once set, it's cleared | 9784 | * ->active_balance_work. Once set, it's cleared |