With commit: 'b7031a02ec75 ("sched/fair: Add NOHZ_STATS_KICK")' rebalance_domains of the local cfs_rq happens before others idle cpus have updated nohz.next_balance and its value is overwritten. Move the update of nohz.next_balance for other idles cpus before balancing and updating the next_balance of local cfs_rq. Also, the nohz.next_balance is now updated only if all idle cpus got a chance to rebalance their domains and the idle balance has not been aborted because of new activities on the CPU. In case of need_resched, the idle load balance will be kick the next jiffie in order to address remaining ilb.
Unified · Split
| @@ -10022,7 +10022,12 @@ static void kick_ilb(unsigned int flags) · 10021 unmodified lines | ||
| 10022 | 10022 | { |
| 10023 | 10023 | int ilb_cpu; |
| 10024 | 10024 | |
| 10025 | − nohz.next_balance++; | |
| 10025 | + /* | |
| 10026 | + * Increase nohz.next_balance only when if full ilb is triggered but | |
| 10027 | + * not if we only update stats. | |
| 10028 | + */ | |
| 10029 | + if (flags & NOHZ_BALANCE_KICK) | |
| 10030 | + nohz.next_balance = jiffies+1; | |
| 10026 | 10031 | |
| 10027 | 10032 | ilb_cpu = find_new_ilb(); |
| 10028 | 10033 | |
| @@ -10343,6 +10348,14 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, · 314 unmodified lines | ||
| 10343 | 10348 | } |
| 10344 | 10349 | } |
| 10345 | 10350 | |
| 10351 | + /* | |
| 10352 | + * next_balance will be updated only when there is a need. | |
| 10353 | + * When the CPU is attached to null domain for ex, it will not be | |
| 10354 | + * updated. | |
| 10355 | + */ | |
| 10356 | + if (likely(update_next_balance)) | |
| 10357 | + nohz.next_balance = next_balance; | |
| 10358 | + | |
| 10346 | 10359 | /* Newly idle CPU doesn't need an update */ |
| 10347 | 10360 | if (idle != CPU_NEWLY_IDLE) { |
| 10348 | 10361 | update_blocked_averages(this_cpu); |
| @@ -10363,14 +10376,6 @@ abort: · 14 unmodified lines | ||
| 10363 | 10376 | if (has_blocked_load) |
| 10364 | 10377 | WRITE_ONCE(nohz.has_blocked, 1); |
| 10365 | 10378 | |
| 10366 | − /* | |
| 10367 | − * next_balance will be updated only when there is a need. | |
| 10368 | − * When the CPU is attached to null domain for ex, it will not be | |
| 10369 | − * updated. | |
| 10370 | − */ | |
| 10371 | − if (likely(update_next_balance)) | |
| 10372 | − nohz.next_balance = next_balance; | |
| 10373 | − | |
| 10374 | 10379 | return ret; |
| 10375 | 10380 | } |
| 10376 | 10381 | |