Passing large sets of similar arguments all around the load-balancer gets tiresom when you want to modify something. Stick them all in a helper structure and pass the structure around.
Unified · Split
| @@ -3135,13 +3135,25 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) · 3134 unmodified lines | ||
| 3135 | 3135 | #define LBF_HAD_BREAKS 0x0C /* count HAD_BREAKs overflows into ABORT */ |
| 3136 | 3136 | #define LBF_ABORT 0x10 |
| 3137 | 3137 | |
| 3138 | +struct lb_env { | |
| 3139 | + struct sched_domain *sd; | |
| 3140 | + | |
| 3141 | + int this_cpu; | |
| 3142 | + struct rq *this_rq; | |
| 3143 | + | |
| 3144 | + struct rq *busiest_rq; | |
| 3145 | + struct cfs_rq *busiest_cfs_rq; | |
| 3146 | + | |
| 3147 | + enum cpu_idle_type idle; | |
| 3148 | + unsigned long max_load_move; | |
| 3149 | + unsigned int flags; | |
| 3150 | +}; | |
| 3151 | + | |
| 3138 | 3152 | /* |
| 3139 | 3153 | * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? |
| 3140 | 3154 | */ |
| 3141 | 3155 | static |
| 3142 | −int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, | |
| 3143 | − struct sched_domain *sd, enum cpu_idle_type idle, | |
| 3144 | − int *lb_flags) | |
| 3156 | +int can_migrate_task(struct task_struct *p, struct lb_env *env) | |
| 3145 | 3157 | { |
| 3146 | 3158 | int tsk_cache_hot = 0; |
| 3147 | 3159 | /* |
| @@ -3150,13 +3162,13 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, · 2 unmodified lines | ||
| 3150 | 3162 | * 2) cannot be migrated to this CPU due to cpus_allowed, or |
| 3151 | 3163 | * 3) are cache-hot on their current CPU. |
| 3152 | 3164 | */ |
| 3153 | − if (!cpumask_test_cpu(this_cpu, tsk_cpus_allowed(p))) { | |
| 3165 | + if (!cpumask_test_cpu(env->this_cpu, tsk_cpus_allowed(p))) { | |
| 3154 | 3166 | schedstat_inc(p, se.statistics.nr_failed_migrations_affine); |
| 3155 | 3167 | return 0; |
| 3156 | 3168 | } |
| 3157 | − *lb_flags &= ~LBF_ALL_PINNED; | |
| 3169 | + env->flags &= ~LBF_ALL_PINNED; | |
| 3158 | 3170 | |
| 3159 | − if (task_running(rq, p)) { | |
| 3171 | + if (task_running(env->busiest_rq, p)) { | |
| 3160 | 3172 | schedstat_inc(p, se.statistics.nr_failed_migrations_running); |
| 3161 | 3173 | return 0; |
| 3162 | 3174 | } |
| @@ -3167,12 +3179,12 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, · 4 unmodified lines | ||
| 3167 | 3179 | * 2) too many balance attempts have failed. |
| 3168 | 3180 | */ |
| 3169 | 3181 | |
| 3170 | − tsk_cache_hot = task_hot(p, rq->clock_task, sd); | |
| 3182 | + tsk_cache_hot = task_hot(p, env->busiest_rq->clock_task, env->sd); | |
| 3171 | 3183 | if (!tsk_cache_hot || |
| 3172 | − sd->nr_balance_failed > sd->cache_nice_tries) { | |
| 3184 | + env->sd->nr_balance_failed > env->sd->cache_nice_tries) { | |
| 3173 | 3185 | #ifdef CONFIG_SCHEDSTATS |
| 3174 | 3186 | if (tsk_cache_hot) { |
| 3175 | − schedstat_inc(sd, lb_hot_gained[idle]); | |
| 3187 | + schedstat_inc(env->sd, lb_hot_gained[env->idle]); | |
| 3176 | 3188 | schedstat_inc(p, se.statistics.nr_forced_migrations); |
| 3177 | 3189 | } |
| 3178 | 3190 | #endif |
| @@ -3193,31 +3205,27 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, · 14 unmodified lines | ||
| 3193 | 3205 | * |
| 3194 | 3206 | * Called with both runqueues locked. |
| 3195 | 3207 | */ |
| 3196 | −static int | |
| 3197 | −move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest, | |
| 3198 | − struct sched_domain *sd, enum cpu_idle_type idle) | |
| 3208 | +static int move_one_task(struct lb_env *env) | |
| 3199 | 3209 | { |
| 3200 | 3210 | struct task_struct *p, *n; |
| 3201 | 3211 | struct cfs_rq *cfs_rq; |
| 3202 | − int pinned = 0; | |
| 3203 | 3212 | |
| 3204 | − for_each_leaf_cfs_rq(busiest, cfs_rq) { | |
| 3213 | + for_each_leaf_cfs_rq(env->busiest_rq, cfs_rq) { | |
| 3205 | 3214 | list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) { |
| 3206 | 3215 | if (throttled_lb_pair(task_group(p), |
| 3207 | − busiest->cpu, this_cpu)) | |
| 3216 | + env->busiest_rq->cpu, env->this_cpu)) | |
| 3208 | 3217 | break; |
| 3209 | 3218 | |
| 3210 | − if (!can_migrate_task(p, busiest, this_cpu, | |
| 3211 | − sd, idle, &pinned)) | |
| 3219 | + if (!can_migrate_task(p, env)) | |
| 3212 | 3220 | continue; |
| 3213 | 3221 | |
| 3214 | − pull_task(busiest, p, this_rq, this_cpu); | |
| 3222 | + pull_task(env->busiest_rq, p, env->this_rq, env->this_cpu); | |
| 3215 | 3223 | /* |
| 3216 | 3224 | * Right now, this is only the second place pull_task() |
| 3217 | 3225 | * is called, so we can safely collect pull_task() |
| 3218 | 3226 | * stats here rather than inside pull_task(). |
| 3219 | 3227 | */ |
| 3220 | − schedstat_inc(sd, lb_gained[idle]); | |
| 3228 | + schedstat_inc(env->sd, lb_gained[env->idle]); | |
| 3221 | 3229 | return 1; |
| 3222 | 3230 | } |
| 3223 | 3231 | } |
| @@ -3225,31 +3233,26 @@ move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest, · 1 unmodified line | ||
| 3225 | 3233 | return 0; |
| 3226 | 3234 | } |
| 3227 | 3235 | |
| 3228 | −static unsigned long | |
| 3229 | −balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, | |
| 3230 | − unsigned long max_load_move, struct sched_domain *sd, | |
| 3231 | − enum cpu_idle_type idle, int *lb_flags, | |
| 3232 | − struct cfs_rq *busiest_cfs_rq) | |
| 3236 | +static unsigned long balance_tasks(struct lb_env *env) | |
| 3233 | 3237 | { |
| 3234 | 3238 | int loops = 0, pulled = 0; |
| 3235 | − long rem_load_move = max_load_move; | |
| 3239 | + long rem_load_move = env->max_load_move; | |
| 3236 | 3240 | struct task_struct *p, *n; |
| 3237 | 3241 | |
| 3238 | − if (max_load_move == 0) | |
| 3242 | + if (env->max_load_move == 0) | |
| 3239 | 3243 | goto out; |
| 3240 | 3244 | |
| 3241 | − list_for_each_entry_safe(p, n, &busiest_cfs_rq->tasks, se.group_node) { | |
| 3245 | + list_for_each_entry_safe(p, n, &env->busiest_cfs_rq->tasks, se.group_node) { | |
| 3242 | 3246 | if (loops++ > sysctl_sched_nr_migrate) { |
| 3243 | − *lb_flags |= LBF_NEED_BREAK; | |
| 3247 | + env->flags |= LBF_NEED_BREAK; | |
| 3244 | 3248 | break; |
| 3245 | 3249 | } |
| 3246 | 3250 | |
| 3247 | 3251 | if ((p->se.load.weight >> 1) > rem_load_move || |
| 3248 | − !can_migrate_task(p, busiest, this_cpu, sd, idle, | |
| 3249 | − lb_flags)) | |
| 3252 | + !can_migrate_task(p, env)) | |
| 3250 | 3253 | continue; |
| 3251 | 3254 | |
| 3252 | − pull_task(busiest, p, this_rq, this_cpu); | |
| 3255 | + pull_task(env->busiest_rq, p, env->this_rq, env->this_cpu); | |
| 3253 | 3256 | pulled++; |
| 3254 | 3257 | rem_load_move -= p->se.load.weight; |
| 3255 | 3258 | |
| @@ -3259,8 +3262,8 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, · 3 unmodified lines | ||
| 3259 | 3262 | * kernels will stop after the first task is pulled to minimize |
| 3260 | 3263 | * the critical section. |
| 3261 | 3264 | */ |
| 3262 | − if (idle == CPU_NEWLY_IDLE) { | |
| 3263 | − *lb_flags |= LBF_ABORT; | |
| 3265 | + if (env->idle == CPU_NEWLY_IDLE) { | |
| 3266 | + env->flags |= LBF_ABORT; | |
| 3264 | 3267 | break; |
| 3265 | 3268 | } |
| 3266 | 3269 | #endif |
| @@ -3278,9 +3281,9 @@ out: · 11 unmodified lines | ||
| 3278 | 3281 | * so we can safely collect pull_task() stats here rather than |
| 3279 | 3282 | * inside pull_task(). |
| 3280 | 3283 | */ |
| 3281 | − schedstat_add(sd, lb_gained[idle], pulled); | |
| 3284 | + schedstat_add(env->sd, lb_gained[env->idle], pulled); | |
| 3282 | 3285 | |
| 3283 | − return max_load_move - rem_load_move; | |
| 3286 | + return env->max_load_move - rem_load_move; | |
| 3284 | 3287 | } |
| 3285 | 3288 | |
| 3286 | 3289 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| @@ -3363,40 +3366,39 @@ static void update_h_load(long cpu) · 76 unmodified lines | ||
| 3363 | 3366 | walk_tg_tree(tg_load_down, tg_nop, (void *)cpu); |
| 3364 | 3367 | } |
| 3365 | 3368 | |
| 3366 | −static unsigned long | |
| 3367 | −load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, | |
| 3368 | − unsigned long max_load_move, | |
| 3369 | − struct sched_domain *sd, enum cpu_idle_type idle, | |
| 3370 | − int *lb_flags) | |
| 3369 | +static unsigned long load_balance_fair(struct lb_env *env) | |
| 3371 | 3370 | { |
| 3372 | − long rem_load_move = max_load_move; | |
| 3373 | − struct cfs_rq *busiest_cfs_rq; | |
| 3371 | + unsigned long max_load_move = env->max_load_move; | |
| 3372 | + long rem_load_move = env->max_load_move; | |
| 3374 | 3373 | |
| 3375 | 3374 | rcu_read_lock(); |
| 3376 | − update_h_load(cpu_of(busiest)); | |
| 3375 | + update_h_load(cpu_of(env->busiest_rq)); | |
| 3377 | 3376 | |
| 3378 | − for_each_leaf_cfs_rq(busiest, busiest_cfs_rq) { | |
| 3379 | − unsigned long busiest_h_load = busiest_cfs_rq->h_load; | |
| 3380 | − unsigned long busiest_weight = busiest_cfs_rq->load.weight; | |
| 3377 | + for_each_leaf_cfs_rq(env->busiest_rq, env->busiest_cfs_rq) { | |
| 3378 | + unsigned long busiest_h_load = env->busiest_cfs_rq->h_load; | |
| 3379 | + unsigned long busiest_weight = env->busiest_cfs_rq->load.weight; | |
| 3381 | 3380 | u64 rem_load, moved_load; |
| 3382 | 3381 | |
| 3383 | − if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT)) | |
| 3382 | + if (env->flags & (LBF_NEED_BREAK|LBF_ABORT)) | |
| 3384 | 3383 | break; |
| 3385 | 3384 | |
| 3386 | 3385 | /* |
| 3387 | 3386 | * empty group or part of a throttled hierarchy |
| 3388 | 3387 | */ |
| 3389 | − if (!busiest_cfs_rq->task_weight || | |
| 3390 | − throttled_lb_pair(busiest_cfs_rq->tg, cpu_of(busiest), this_cpu)) | |
| 3388 | + if (!env->busiest_cfs_rq->task_weight) | |
| 3389 | + continue; | |
| 3390 | + | |
| 3391 | + if (throttled_lb_pair(env->busiest_cfs_rq->tg, | |
| 3392 | + cpu_of(env->busiest_rq), | |
| 3393 | + env->this_cpu)) | |
| 3391 | 3394 | continue; |
| 3392 | 3395 | |
| 3393 | 3396 | rem_load = (u64)rem_load_move * busiest_weight; |
| 3394 | 3397 | rem_load = div_u64(rem_load, busiest_h_load + 1); |
| 3395 | 3398 | |
| 3396 | − moved_load = balance_tasks(this_rq, this_cpu, busiest, | |
| 3397 | − rem_load, sd, idle, lb_flags, | |
| 3398 | − busiest_cfs_rq); | |
| 3399 | + env->max_load_move = rem_load; | |
| 3399 | 3400 | |
| 3401 | + moved_load = balance_tasks(env); | |
| 3400 | 3402 | if (!moved_load) |
| 3401 | 3403 | continue; |
| 3402 | 3404 | |
| @@ -3416,15 +3418,10 @@ static inline void update_shares(int cpu) · 13 unmodified lines | ||
| 3416 | 3418 | { |
| 3417 | 3419 | } |
| 3418 | 3420 | |
| 3419 | −static unsigned long | |
| 3420 | −load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, | |
| 3421 | − unsigned long max_load_move, | |
| 3422 | − struct sched_domain *sd, enum cpu_idle_type idle, | |
| 3423 | − int *lb_flags) | |
| 3421 | +static unsigned long load_balance_fair(struct lb_env *env) | |
| 3424 | 3422 | { |
| 3425 | − return balance_tasks(this_rq, this_cpu, busiest, | |
| 3426 | − max_load_move, sd, idle, lb_flags, | |
| 3427 | − &busiest->cfs); | |
| 3423 | + env->busiest_cfs_rq = &env->busiest_rq->cfs; | |
| 3424 | + return balance_tasks(env); | |
| 3428 | 3425 | } |
| 3429 | 3426 | #endif |
| 3430 | 3427 | |
| @@ -3435,21 +3432,17 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, · 4 unmodified lines | ||
| 3435 | 3432 | * |
| 3436 | 3433 | * Called with both runqueues locked. |
| 3437 | 3434 | */ |
| 3438 | −static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, | |
| 3439 | − unsigned long max_load_move, | |
| 3440 | − struct sched_domain *sd, enum cpu_idle_type idle, | |
| 3441 | − int *lb_flags) | |
| 3435 | +static int move_tasks(struct lb_env *env) | |
| 3442 | 3436 | { |
| 3437 | + unsigned long max_load_move = env->max_load_move; | |
| 3443 | 3438 | unsigned long total_load_moved = 0, load_moved; |
| 3444 | 3439 | |
| 3445 | 3440 | do { |
| 3446 | − load_moved = load_balance_fair(this_rq, this_cpu, busiest, | |
| 3447 | − max_load_move - total_load_moved, | |
| 3448 | − sd, idle, lb_flags); | |
| 3449 | − | |
| 3441 | + env->max_load_move = max_load_move - total_load_moved; | |
| 3442 | + load_moved = load_balance_fair(env); | |
| 3450 | 3443 | total_load_moved += load_moved; |
| 3451 | 3444 | |
| 3452 | − if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT)) | |
| 3445 | + if (env->flags & (LBF_NEED_BREAK|LBF_ABORT)) | |
| 3453 | 3446 | break; |
| 3454 | 3447 | |
| 3455 | 3448 | #ifdef CONFIG_PREEMPT |
| @@ -3458,8 +3451,8 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, · 2 unmodified lines | ||
| 3458 | 3451 | * kernels will stop after the first task is pulled to minimize |
| 3459 | 3452 | * the critical section. |
| 3460 | 3453 | */ |
| 3461 | − if (idle == CPU_NEWLY_IDLE && this_rq->nr_running) { | |
| 3462 | − *lb_flags |= LBF_ABORT; | |
| 3454 | + if (env->idle == CPU_NEWLY_IDLE && env->this_rq->nr_running) { | |
| 3455 | + env->flags |= LBF_ABORT; | |
| 3463 | 3456 | break; |
| 3464 | 3457 | } |
| 3465 | 3458 | #endif |
| @@ -4459,13 +4452,20 @@ static int load_balance(int this_cpu, struct rq *this_rq, · 993 unmodified lines | ||
| 4459 | 4452 | struct sched_domain *sd, enum cpu_idle_type idle, |
| 4460 | 4453 | int *balance) |
| 4461 | 4454 | { |
| 4462 | − int ld_moved, lb_flags = 0, active_balance = 0; | |
| 4455 | + int ld_moved, active_balance = 0; | |
| 4463 | 4456 | struct sched_group *group; |
| 4464 | 4457 | unsigned long imbalance; |
| 4465 | 4458 | struct rq *busiest; |
| 4466 | 4459 | unsigned long flags; |
| 4467 | 4460 | struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); |
| 4468 | 4461 | |
| 4462 | + struct lb_env env = { | |
| 4463 | + .sd = sd, | |
| 4464 | + .this_cpu = this_cpu, | |
| 4465 | + .this_rq = this_rq, | |
| 4466 | + .idle = idle, | |
| 4467 | + }; | |
| 4468 | + | |
| 4469 | 4469 | cpumask_copy(cpus, cpu_active_mask); |
| 4470 | 4470 | |
| 4471 | 4471 | schedstat_inc(sd, lb_count[idle]); |
| @@ -4500,11 +4500,13 @@ redo: · 28 unmodified lines | ||
| 4500 | 4500 | * still unbalanced. ld_moved simply stays zero, so it is |
| 4501 | 4501 | * correctly treated as an imbalance. |
| 4502 | 4502 | */ |
| 4503 | − lb_flags |= LBF_ALL_PINNED; | |
| 4503 | + env.flags |= LBF_ALL_PINNED; | |
| 4504 | + env.max_load_move = imbalance; | |
| 4505 | + env.busiest_rq = busiest; | |
| 4506 | + | |
| 4504 | 4507 | local_irq_save(flags); |
| 4505 | 4508 | double_rq_lock(this_rq, busiest); |
| 4506 | − ld_moved = move_tasks(this_rq, this_cpu, busiest, | |
| 4507 | − imbalance, sd, idle, &lb_flags); | |
| 4509 | + ld_moved = move_tasks(&env); | |
| 4508 | 4510 | double_rq_unlock(this_rq, busiest); |
| 4509 | 4511 | local_irq_restore(flags); |
| 4510 | 4512 | |
| @@ -4514,18 +4516,18 @@ redo: · 3 unmodified lines | ||
| 4514 | 4516 | if (ld_moved && this_cpu != smp_processor_id()) |
| 4515 | 4517 | resched_cpu(this_cpu); |
| 4516 | 4518 | |
| 4517 | − if (lb_flags & LBF_ABORT) | |
| 4519 | + if (env.flags & LBF_ABORT) | |
| 4518 | 4520 | goto out_balanced; |
| 4519 | 4521 | |
| 4520 | − if (lb_flags & LBF_NEED_BREAK) { | |
| 4521 | − lb_flags += LBF_HAD_BREAK - LBF_NEED_BREAK; | |
| 4522 | − if (lb_flags & LBF_ABORT) | |
| 4522 | + if (env.flags & LBF_NEED_BREAK) { | |
| 4523 | + env.flags += LBF_HAD_BREAK - LBF_NEED_BREAK; | |
| 4524 | + if (env.flags & LBF_ABORT) | |
| 4523 | 4525 | goto out_balanced; |
| 4524 | 4526 | goto redo; |
| 4525 | 4527 | } |
| 4526 | 4528 | |
| 4527 | 4529 | /* All tasks on this runqueue were pinned by CPU affinity */ |
| 4528 | − if (unlikely(lb_flags & LBF_ALL_PINNED)) { | |
| 4530 | + if (unlikely(env.flags & LBF_ALL_PINNED)) { | |
| 4529 | 4531 | cpumask_clear_cpu(cpu_of(busiest), cpus); |
| 4530 | 4532 | if (!cpumask_empty(cpus)) |
| 4531 | 4533 | goto redo; |
| @@ -4555,7 +4557,7 @@ redo: · 23 unmodified lines | ||
| 4555 | 4557 | tsk_cpus_allowed(busiest->curr))) { |
| 4556 | 4558 | raw_spin_unlock_irqrestore(&busiest->lock, |
| 4557 | 4559 | flags); |
| 4558 | − lb_flags |= LBF_ALL_PINNED; | |
| 4560 | + env.flags |= LBF_ALL_PINNED; | |
| 4559 | 4561 | goto out_one_pinned; |
| 4560 | 4562 | } |
| 4561 | 4563 | |
| @@ -4608,7 +4610,7 @@ out_balanced: · 46 unmodified lines | ||
| 4608 | 4610 | |
| 4609 | 4611 | out_one_pinned: |
| 4610 | 4612 | /* tune up the balancing interval */ |
| 4611 | − if (((lb_flags & LBF_ALL_PINNED) && | |
| 4613 | + if (((env.flags & LBF_ALL_PINNED) && | |
| 4612 | 4614 | sd->balance_interval < MAX_PINNED_INTERVAL) || |
| 4613 | 4615 | (sd->balance_interval < sd->max_interval)) |
| 4614 | 4616 | sd->balance_interval *= 2; |
| @@ -4718,10 +4720,17 @@ static int active_load_balance_cpu_stop(void *data) · 103 unmodified lines | ||
| 4718 | 4720 | } |
| 4719 | 4721 | |
| 4720 | 4722 | if (likely(sd)) { |
| 4723 | + struct lb_env env = { | |
| 4724 | + .sd = sd, | |
| 4725 | + .this_cpu = target_cpu, | |
| 4726 | + .this_rq = target_rq, | |
| 4727 | + .busiest_rq = busiest_rq, | |
| 4728 | + .idle = CPU_IDLE, | |
| 4729 | + }; | |
| 4730 | + | |
| 4721 | 4731 | schedstat_inc(sd, alb_count); |
| 4722 | 4732 | |
| 4723 | − if (move_one_task(target_rq, target_cpu, busiest_rq, | |
| 4724 | − sd, CPU_IDLE)) | |
| 4733 | + if (move_one_task(&env)) | |
| 4725 | 4734 | schedstat_inc(sd, alb_pushed); |
| 4726 | 4735 | else |
| 4727 | 4736 | schedstat_inc(sd, alb_failed); |