sched/core.c uses update_avg() for rq->avg_idle and sched/fair.c uses an open-coded version (with the exact same decay factor) for rq->avg_scan_cost. On top of that, select_idle_cpu() expects to be able to compare these two fields. The only difference between the two is that rq->avg_scan_cost is computed using a pure division rather than a shift. Turns out it actually matters, first of all because the shifted value can be negative, and the standard has this to say about it: """ The result of E1 >> E2 is E1 right-shifted E2 bit positions. [...] If E1 has a signed type and a negative value, the resulting value is implementation-defined. """ Not only this, but (arithmetic) right shifting a negative value (using 2's complement) is *not* equivalent to dividing it by the corresponding power of 2. Let's look at a few examples: -4 -> 0xF..FC -4 >> 3 -> 0xF..FF == -1 != -4 / 8 -8 -> 0xF..F8 -8 >> 3 -> 0xF..FF == -1 == -8 / 8 -9 -> 0xF..F7 -9 >> 3 -> 0xF..FE == -2 != -9 / 8 Make update_avg() use a division, and export it to the private scheduler header to reuse it where relevant. Note that this still lets compilers use a shift here, but should prevent any unwanted surprise. The disassembly of select_idle_cpu() remains unchanged on arm64, and ttwu_do_wakeup() gains 2 instructions; the diff sort of looks like this: - sub x1, x1, x0 + subs x1, x1, x0 // set condition codes + add x0, x1, #0x7 + csel x0, x0, x1, mi // x0 = x1 < 0 ? x0 : x1 add x0, x3, x0, asr #3 which does the right thing (i.e. gives us the expected result while still using an arithmetic shift)
Showing the changes to kernel (3 of 3 files). Show all
Unified · Split
| @@ -2119,12 +2119,6 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) · 2118 unmodified lines | ||
| 2119 | 2119 | return cpu; |
| 2120 | 2120 | } |
| 2121 | 2121 | |
| 2122 | −static void update_avg(u64 *avg, u64 sample) | |
| 2123 | −{ | |
| 2124 | − s64 diff = sample - *avg; | |
| 2125 | − *avg += diff >> 3; | |
| 2126 | −} | |
| 2127 | − | |
| 2128 | 2122 | void sched_set_stop_task(int cpu, struct task_struct *stop) |
| 2129 | 2123 | { |
| 2130 | 2124 | struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; |
| @@ -6080,8 +6080,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · 6079 unmodified lines | ||
| 6080 | 6080 | struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); |
| 6081 | 6081 | struct sched_domain *this_sd; |
| 6082 | 6082 | u64 avg_cost, avg_idle; |
| 6083 | − u64 time, cost; | |
| 6084 | − s64 delta; | |
| 6083 | + u64 time; | |
| 6085 | 6084 | int this = smp_processor_id(); |
| 6086 | 6085 | int cpu, nr = INT_MAX; |
| 6087 | 6086 | |
| @@ -6119,9 +6118,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t · 31 unmodified lines | ||
| 6119 | 6118 | } |
| 6120 | 6119 | |
| 6121 | 6120 | time = cpu_clock(this) - time; |
| 6122 | − cost = this_sd->avg_scan_cost; | |
| 6123 | − delta = (s64)(time - cost) / 8; | |
| 6124 | − this_sd->avg_scan_cost += delta; | |
| 6121 | + update_avg(&this_sd->avg_scan_cost, time); | |
| 6125 | 6122 | |
| 6126 | 6123 | return cpu; |
| 6127 | 6124 | } |
| @@ -195,6 +195,12 @@ static inline int task_has_dl_policy(struct task_struct *p) · 194 unmodified lines | ||
| 195 | 195 | |
| 196 | 196 | #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) |
| 197 | 197 | |
| 198 | +static inline void update_avg(u64 *avg, u64 sample) | |
| 199 | +{ | |
| 200 | + s64 diff = sample - *avg; | |
| 201 | + *avg += diff / 8; | |
| 202 | +} | |
| 203 | + | |
| 198 | 204 | /* |
| 199 | 205 | * !! For sched_setattr_nocheck() (kernel) only !! |
| 200 | 206 | * |