With commit: 'b7031a02ec75 ("sched/fair: Add NOHZ_STATS_KICK")' rebalance_domains of the local cfs_rq happens before others idle cpus have updated nohz.next_balance and its value is overwritten. Move the update of nohz.next_balance for other idles cpus before balancing and updating the next_balance of local cfs_rq. Also, the nohz.next_balance is now updated only if all idle cpus got a chance to rebalance their domains and the idle balance has not been aborted because of new activities on the CPU. In case of need_resched, the idle load balance will be kick the next jiffie in order to address remaining ilb.
Unified · Split
| @@ -10022,7 +10022,12 @@ static void kick_ilb(unsigned int flags) · 10021 unmodified lines | |||
| 10022 | { | 10022 | { |
| 10023 | int ilb_cpu; | 10023 | int ilb_cpu; |
| 10024 | 10024 | ||
| 10025 | nohz.next_balance++; | 10025 | /* |
| 10026 | * Increase nohz.next_balance only when if full ilb is triggered but | ||
| 10027 | * not if we only update stats. | ||
| 10028 | */ | ||
| 10029 | if (flags & NOHZ_BALANCE_KICK) | ||
| 10030 | nohz.next_balance = jiffies+1; | ||
| 10026 | 10031 | ||
| 10027 | ilb_cpu = find_new_ilb(); | 10032 | ilb_cpu = find_new_ilb(); |
| 10028 | 10033 | ||
| @@ -10343,6 +10348,14 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, · 314 unmodified lines | |||
| 10343 | } | 10348 | } |
| 10344 | } | 10349 | } |
| 10345 | 10350 | ||
| 10351 | /* | ||
| 10352 | * next_balance will be updated only when there is a need. | ||
| 10353 | * When the CPU is attached to null domain for ex, it will not be | ||
| 10354 | * updated. | ||
| 10355 | */ | ||
| 10356 | if (likely(update_next_balance)) | ||
| 10357 | nohz.next_balance = next_balance; | ||
| 10358 | |||
| 10346 | /* Newly idle CPU doesn't need an update */ | 10359 | /* Newly idle CPU doesn't need an update */ |
| 10347 | if (idle != CPU_NEWLY_IDLE) { | 10360 | if (idle != CPU_NEWLY_IDLE) { |
| 10348 | update_blocked_averages(this_cpu); | 10361 | update_blocked_averages(this_cpu); |
| @@ -10363,14 +10376,6 @@ abort: · 14 unmodified lines | |||
| 10363 | if (has_blocked_load) | 10376 | if (has_blocked_load) |
| 10364 | WRITE_ONCE(nohz.has_blocked, 1); | 10377 | WRITE_ONCE(nohz.has_blocked, 1); |
| 10365 | 10378 | ||
| 10366 | /* | ||
| 10367 | * next_balance will be updated only when there is a need. | ||
| 10368 | * When the CPU is attached to null domain for ex, it will not be | ||
| 10369 | * updated. | ||
| 10370 | */ | ||
| 10371 | if (likely(update_next_balance)) | ||
| 10372 | nohz.next_balance = next_balance; | ||
| 10373 | |||
| 10374 | return ret; | 10379 | return ret; |
| 10375 | } | 10380 | } |
| 10376 | 10381 | ||