History

sched: Move load-balancing arguments into helper struct

Passing large sets of similar arguments all around the load-balancer gets tiresom when you want to modify something. Stick them all in a helper structure and pass the structure around.

Peter Zijlstra committed 14 years ago

Commit 8e45cb51 parent 3c7d5181 file · +93 −84Browse files

Showing the changes to kernel (1 of 1 file). Show all

Unified · Split

kernel/sched/fair.c+93−84
@@ -3135,13 +3135,25 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) · 3134 unmodified lines
31353135 #define LBF_HAD_BREAKS 0x0C /* count HAD_BREAKs overflows into ABORT */
31363136 #define LBF_ABORT 0x10
31373137
3138+struct lb_env {
3139+ struct sched_domain *sd;
3140+
3141+ int this_cpu;
3142+ struct rq *this_rq;
3143+
3144+ struct rq *busiest_rq;
3145+ struct cfs_rq *busiest_cfs_rq;
3146+
3147+ enum cpu_idle_type idle;
3148+ unsigned long max_load_move;
3149+ unsigned int flags;
3150+};
3151+
31383152 /*
31393153 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
31403154 */
31413155 static
3142−int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
3143− struct sched_domain *sd, enum cpu_idle_type idle,
3144− int *lb_flags)
3156+int can_migrate_task(struct task_struct *p, struct lb_env *env)
31453157 {
31463158 int tsk_cache_hot = 0;
31473159 /*
@@ -3150,13 +3162,13 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, · 2 unmodified lines
31503162 * 2) cannot be migrated to this CPU due to cpus_allowed, or
31513163 * 3) are cache-hot on their current CPU.
31523164 */
3153− if (!cpumask_test_cpu(this_cpu, tsk_cpus_allowed(p))) {
3165+ if (!cpumask_test_cpu(env->this_cpu, tsk_cpus_allowed(p))) {
31543166 schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
31553167 return 0;
31563168 }
3157− *lb_flags &= ~LBF_ALL_PINNED;
3169+ env->flags &= ~LBF_ALL_PINNED;
31583170
3159− if (task_running(rq, p)) {
3171+ if (task_running(env->busiest_rq, p)) {
31603172 schedstat_inc(p, se.statistics.nr_failed_migrations_running);
31613173 return 0;
31623174 }
@@ -3167,12 +3179,12 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, · 4 unmodified lines
31673179 * 2) too many balance attempts have failed.
31683180 */
31693181
3170− tsk_cache_hot = task_hot(p, rq->clock_task, sd);
3182+ tsk_cache_hot = task_hot(p, env->busiest_rq->clock_task, env->sd);
31713183 if (!tsk_cache_hot ||
3172− sd->nr_balance_failed > sd->cache_nice_tries) {
3184+ env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
31733185 #ifdef CONFIG_SCHEDSTATS
31743186 if (tsk_cache_hot) {
3175− schedstat_inc(sd, lb_hot_gained[idle]);
3187+ schedstat_inc(env->sd, lb_hot_gained[env->idle]);
31763188 schedstat_inc(p, se.statistics.nr_forced_migrations);
31773189 }
31783190 #endif
@@ -3193,31 +3205,27 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, · 14 unmodified lines
31933205 *
31943206 * Called with both runqueues locked.
31953207 */
3196−static int
3197−move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3198− struct sched_domain *sd, enum cpu_idle_type idle)
3208+static int move_one_task(struct lb_env *env)
31993209 {
32003210 struct task_struct *p, *n;
32013211 struct cfs_rq *cfs_rq;
3202− int pinned = 0;
32033212
3204− for_each_leaf_cfs_rq(busiest, cfs_rq) {
3213+ for_each_leaf_cfs_rq(env->busiest_rq, cfs_rq) {
32053214 list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) {
32063215 if (throttled_lb_pair(task_group(p),
3207− busiest->cpu, this_cpu))
3216+ env->busiest_rq->cpu, env->this_cpu))
32083217 break;
32093218
3210− if (!can_migrate_task(p, busiest, this_cpu,
3211− sd, idle, &pinned))
3219+ if (!can_migrate_task(p, env))
32123220 continue;
32133221
3214− pull_task(busiest, p, this_rq, this_cpu);
3222+ pull_task(env->busiest_rq, p, env->this_rq, env->this_cpu);
32153223 /*
32163224 * Right now, this is only the second place pull_task()
32173225 * is called, so we can safely collect pull_task()
32183226 * stats here rather than inside pull_task().
32193227 */
3220− schedstat_inc(sd, lb_gained[idle]);
3228+ schedstat_inc(env->sd, lb_gained[env->idle]);
32213229 return 1;
32223230 }
32233231 }
@@ -3225,31 +3233,26 @@ move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest, · 1 unmodified line
32253233 return 0;
32263234 }
32273235
3228−static unsigned long
3229−balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
3230− unsigned long max_load_move, struct sched_domain *sd,
3231− enum cpu_idle_type idle, int *lb_flags,
3232− struct cfs_rq *busiest_cfs_rq)
3236+static unsigned long balance_tasks(struct lb_env *env)
32333237 {
32343238 int loops = 0, pulled = 0;
3235− long rem_load_move = max_load_move;
3239+ long rem_load_move = env->max_load_move;
32363240 struct task_struct *p, *n;
32373241
3238− if (max_load_move == 0)
3242+ if (env->max_load_move == 0)
32393243 goto out;
32403244
3241− list_for_each_entry_safe(p, n, &busiest_cfs_rq->tasks, se.group_node) {
3245+ list_for_each_entry_safe(p, n, &env->busiest_cfs_rq->tasks, se.group_node) {
32423246 if (loops++ > sysctl_sched_nr_migrate) {
3243− *lb_flags |= LBF_NEED_BREAK;
3247+ env->flags |= LBF_NEED_BREAK;
32443248 break;
32453249 }
32463250
32473251 if ((p->se.load.weight >> 1) > rem_load_move ||
3248− !can_migrate_task(p, busiest, this_cpu, sd, idle,
3249− lb_flags))
3252+ !can_migrate_task(p, env))
32503253 continue;
32513254
3252− pull_task(busiest, p, this_rq, this_cpu);
3255+ pull_task(env->busiest_rq, p, env->this_rq, env->this_cpu);
32533256 pulled++;
32543257 rem_load_move -= p->se.load.weight;
32553258
@@ -3259,8 +3262,8 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, · 3 unmodified lines
32593262 * kernels will stop after the first task is pulled to minimize
32603263 * the critical section.
32613264 */
3262− if (idle == CPU_NEWLY_IDLE) {
3263− *lb_flags |= LBF_ABORT;
3265+ if (env->idle == CPU_NEWLY_IDLE) {
3266+ env->flags |= LBF_ABORT;
32643267 break;
32653268 }
32663269 #endif
@@ -3278,9 +3281,9 @@ out: · 11 unmodified lines
32783281 * so we can safely collect pull_task() stats here rather than
32793282 * inside pull_task().
32803283 */
3281− schedstat_add(sd, lb_gained[idle], pulled);
3284+ schedstat_add(env->sd, lb_gained[env->idle], pulled);
32823285
3283− return max_load_move - rem_load_move;
3286+ return env->max_load_move - rem_load_move;
32843287 }
32853288
32863289 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -3363,40 +3366,39 @@ static void update_h_load(long cpu) · 76 unmodified lines
33633366 walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
33643367 }
33653368
3366−static unsigned long
3367−load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
3368− unsigned long max_load_move,
3369− struct sched_domain *sd, enum cpu_idle_type idle,
3370− int *lb_flags)
3369+static unsigned long load_balance_fair(struct lb_env *env)
33713370 {
3372− long rem_load_move = max_load_move;
3373− struct cfs_rq *busiest_cfs_rq;
3371+ unsigned long max_load_move = env->max_load_move;
3372+ long rem_load_move = env->max_load_move;
33743373
33753374 rcu_read_lock();
3376− update_h_load(cpu_of(busiest));
3375+ update_h_load(cpu_of(env->busiest_rq));
33773376
3378− for_each_leaf_cfs_rq(busiest, busiest_cfs_rq) {
3379− unsigned long busiest_h_load = busiest_cfs_rq->h_load;
3380− unsigned long busiest_weight = busiest_cfs_rq->load.weight;
3377+ for_each_leaf_cfs_rq(env->busiest_rq, env->busiest_cfs_rq) {
3378+ unsigned long busiest_h_load = env->busiest_cfs_rq->h_load;
3379+ unsigned long busiest_weight = env->busiest_cfs_rq->load.weight;
33813380 u64 rem_load, moved_load;
33823381
3383− if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT))
3382+ if (env->flags & (LBF_NEED_BREAK|LBF_ABORT))
33843383 break;
33853384
33863385 /*
33873386 * empty group or part of a throttled hierarchy
33883387 */
3389− if (!busiest_cfs_rq->task_weight ||
3390− throttled_lb_pair(busiest_cfs_rq->tg, cpu_of(busiest), this_cpu))
3388+ if (!env->busiest_cfs_rq->task_weight)
3389+ continue;
3390+
3391+ if (throttled_lb_pair(env->busiest_cfs_rq->tg,
3392+ cpu_of(env->busiest_rq),
3393+ env->this_cpu))
33913394 continue;
33923395
33933396 rem_load = (u64)rem_load_move * busiest_weight;
33943397 rem_load = div_u64(rem_load, busiest_h_load + 1);
33953398
3396− moved_load = balance_tasks(this_rq, this_cpu, busiest,
3397− rem_load, sd, idle, lb_flags,
3398− busiest_cfs_rq);
3399+ env->max_load_move = rem_load;
33993400
3401+ moved_load = balance_tasks(env);
34003402 if (!moved_load)
34013403 continue;
34023404
@@ -3416,15 +3418,10 @@ static inline void update_shares(int cpu) · 13 unmodified lines
34163418 {
34173419 }
34183420
3419−static unsigned long
3420−load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
3421− unsigned long max_load_move,
3422− struct sched_domain *sd, enum cpu_idle_type idle,
3423− int *lb_flags)
3421+static unsigned long load_balance_fair(struct lb_env *env)
34243422 {
3425− return balance_tasks(this_rq, this_cpu, busiest,
3426− max_load_move, sd, idle, lb_flags,
3427− &busiest->cfs);
3423+ env->busiest_cfs_rq = &env->busiest_rq->cfs;
3424+ return balance_tasks(env);
34283425 }
34293426 #endif
34303427
@@ -3435,21 +3432,17 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, · 4 unmodified lines
34353432 *
34363433 * Called with both runqueues locked.
34373434 */
3438−static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
3439− unsigned long max_load_move,
3440− struct sched_domain *sd, enum cpu_idle_type idle,
3441− int *lb_flags)
3435+static int move_tasks(struct lb_env *env)
34423436 {
3437+ unsigned long max_load_move = env->max_load_move;
34433438 unsigned long total_load_moved = 0, load_moved;
34443439
34453440 do {
3446− load_moved = load_balance_fair(this_rq, this_cpu, busiest,
3447− max_load_move - total_load_moved,
3448− sd, idle, lb_flags);
3449−
3441+ env->max_load_move = max_load_move - total_load_moved;
3442+ load_moved = load_balance_fair(env);
34503443 total_load_moved += load_moved;
34513444
3452− if (*lb_flags & (LBF_NEED_BREAK|LBF_ABORT))
3445+ if (env->flags & (LBF_NEED_BREAK|LBF_ABORT))
34533446 break;
34543447
34553448 #ifdef CONFIG_PREEMPT
@@ -3458,8 +3451,8 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, · 2 unmodified lines
34583451 * kernels will stop after the first task is pulled to minimize
34593452 * the critical section.
34603453 */
3461− if (idle == CPU_NEWLY_IDLE && this_rq->nr_running) {
3462− *lb_flags |= LBF_ABORT;
3454+ if (env->idle == CPU_NEWLY_IDLE && env->this_rq->nr_running) {
3455+ env->flags |= LBF_ABORT;
34633456 break;
34643457 }
34653458 #endif
@@ -4459,13 +4452,20 @@ static int load_balance(int this_cpu, struct rq *this_rq, · 993 unmodified lines
44594452 struct sched_domain *sd, enum cpu_idle_type idle,
44604453 int *balance)
44614454 {
4462− int ld_moved, lb_flags = 0, active_balance = 0;
4455+ int ld_moved, active_balance = 0;
44634456 struct sched_group *group;
44644457 unsigned long imbalance;
44654458 struct rq *busiest;
44664459 unsigned long flags;
44674460 struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
44684461
4462+ struct lb_env env = {
4463+ .sd = sd,
4464+ .this_cpu = this_cpu,
4465+ .this_rq = this_rq,
4466+ .idle = idle,
4467+ };
4468+
44694469 cpumask_copy(cpus, cpu_active_mask);
44704470
44714471 schedstat_inc(sd, lb_count[idle]);
@@ -4500,11 +4500,13 @@ redo: · 28 unmodified lines
45004500 * still unbalanced. ld_moved simply stays zero, so it is
45014501 * correctly treated as an imbalance.
45024502 */
4503− lb_flags |= LBF_ALL_PINNED;
4503+ env.flags |= LBF_ALL_PINNED;
4504+ env.max_load_move = imbalance;
4505+ env.busiest_rq = busiest;
4506+
45044507 local_irq_save(flags);
45054508 double_rq_lock(this_rq, busiest);
4506− ld_moved = move_tasks(this_rq, this_cpu, busiest,
4507− imbalance, sd, idle, &lb_flags);
4509+ ld_moved = move_tasks(&env);
45084510 double_rq_unlock(this_rq, busiest);
45094511 local_irq_restore(flags);
45104512
@@ -4514,18 +4516,18 @@ redo: · 3 unmodified lines
45144516 if (ld_moved && this_cpu != smp_processor_id())
45154517 resched_cpu(this_cpu);
45164518
4517− if (lb_flags & LBF_ABORT)
4519+ if (env.flags & LBF_ABORT)
45184520 goto out_balanced;
45194521
4520− if (lb_flags & LBF_NEED_BREAK) {
4521− lb_flags += LBF_HAD_BREAK - LBF_NEED_BREAK;
4522− if (lb_flags & LBF_ABORT)
4522+ if (env.flags & LBF_NEED_BREAK) {
4523+ env.flags += LBF_HAD_BREAK - LBF_NEED_BREAK;
4524+ if (env.flags & LBF_ABORT)
45234525 goto out_balanced;
45244526 goto redo;
45254527 }
45264528
45274529 /* All tasks on this runqueue were pinned by CPU affinity */
4528− if (unlikely(lb_flags & LBF_ALL_PINNED)) {
4530+ if (unlikely(env.flags & LBF_ALL_PINNED)) {
45294531 cpumask_clear_cpu(cpu_of(busiest), cpus);
45304532 if (!cpumask_empty(cpus))
45314533 goto redo;
@@ -4555,7 +4557,7 @@ redo: · 23 unmodified lines
45554557 tsk_cpus_allowed(busiest->curr))) {
45564558 raw_spin_unlock_irqrestore(&busiest->lock,
45574559 flags);
4558− lb_flags |= LBF_ALL_PINNED;
4560+ env.flags |= LBF_ALL_PINNED;
45594561 goto out_one_pinned;
45604562 }
45614563
@@ -4608,7 +4610,7 @@ out_balanced: · 46 unmodified lines
46084610
46094611 out_one_pinned:
46104612 /* tune up the balancing interval */
4611− if (((lb_flags & LBF_ALL_PINNED) &&
4613+ if (((env.flags & LBF_ALL_PINNED) &&
46124614 sd->balance_interval < MAX_PINNED_INTERVAL) ||
46134615 (sd->balance_interval < sd->max_interval))
46144616 sd->balance_interval *= 2;
@@ -4718,10 +4720,17 @@ static int active_load_balance_cpu_stop(void *data) · 103 unmodified lines
47184720 }
47194721
47204722 if (likely(sd)) {
4723+ struct lb_env env = {
4724+ .sd = sd,
4725+ .this_cpu = target_cpu,
4726+ .this_rq = target_rq,
4727+ .busiest_rq = busiest_rq,
4728+ .idle = CPU_IDLE,
4729+ };
4730+
47214731 schedstat_inc(sd, alb_count);
47224732
4723− if (move_one_task(target_rq, target_cpu, busiest_rq,
4724− sd, CPU_IDLE))
4733+ if (move_one_task(&env))
47254734 schedstat_inc(sd, alb_pushed);
47264735 else
47274736 schedstat_inc(sd, alb_failed);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t