History

sched/fair: Remove task_group->se pointer array

Now that struct sched_entity is co-located with struct cfs_rq for non-root task groups, the task_group->se pointer array is redundant. The associated sched_entity can be loaded directly from the cfs_rq. This patch performs the access conversion with the helpers: - is_root_task_group(tg): checks if a task group is the root task group. It compares the task group's address with the global root_task_group variable. - tg_se(tg, cpu): retrieves the cfs_rq and returns the address of the co-located se. This function checks if tg is the root task group to ensure behaving the same of previous tg->se[cpu]. Replaces all accesses that use the tg->se[cpu] pointer array with calls to the new tg_se(tg, cpu) accessor. - cfs_rq_se(cfs_rq): simplifies access paths like cfs_rq->tg->se[...] to use the co-located sched_entity. This function also checks if tg is the root task group to ensure same behavior. Since tg_se is not in very hot code paths, and the branch is a register comparison with an immediate value (`&root_task_group`), the performance impact is expected to be negligible.

Zecheng Li committed 5 months ago

Commit 89e1f671 parent dfcfc974 files · +38 −27Browse files

Unified · Split

4 files
  1. kernel/sched/core.c +2 −5
  2. kernel/sched/debug.c +1 −1
  3. kernel/sched/fair.c +9 −16
  4. kernel/sched/sched.h +26 −5
kernel/sched/core.c+2−5
@@ -8923,7 +8923,7 @@ void __init sched_init(void) · 8922 unmodified lines
8923 wait_bit_init();8923 wait_bit_init();
8924 8924
8925#ifdef CONFIG_FAIR_GROUP_SCHED8925#ifdef CONFIG_FAIR_GROUP_SCHED
8926 ptr += 2 * nr_cpu_ids * sizeof(void **);8926 ptr += nr_cpu_ids * sizeof(void **);
8927#endif8927#endif
8928#ifdef CONFIG_RT_GROUP_SCHED8928#ifdef CONFIG_RT_GROUP_SCHED
8929 ptr += 2 * nr_cpu_ids * sizeof(void **);8929 ptr += 2 * nr_cpu_ids * sizeof(void **);
@@ -8932,9 +8932,6 @@ void __init sched_init(void) · 2 unmodified lines
8932 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);8932 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
8933 8933
8934#ifdef CONFIG_FAIR_GROUP_SCHED8934#ifdef CONFIG_FAIR_GROUP_SCHED
8935 root_task_group.se = (struct sched_entity **)ptr;
8936 ptr += nr_cpu_ids * sizeof(void **);
8937
8938 root_task_group.cfs_rq = (struct cfs_rq **)ptr;8935 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
8939 ptr += nr_cpu_ids * sizeof(void **);8936 ptr += nr_cpu_ids * sizeof(void **);
8940 8937
@@ -10016,7 +10013,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v) · 1075 unmodified lines
10016 int i;10013 int i;
10017 10014
10018 for_each_possible_cpu(i) {10015 for_each_possible_cpu(i) {
10019 stats = __schedstats_from_se(tg->se[i]);10016 stats = __schedstats_from_se(tg_se(tg, i));
10020 ws += schedstat_val(stats->wait_sum);10017 ws += schedstat_val(stats->wait_sum);
10021 }10018 }
10022 10019
kernel/sched/debug.c+1−1
@@ -808,7 +808,7 @@ void dirty_sched_domain_sysctl(int cpu) · 807 unmodified lines
808#ifdef CONFIG_FAIR_GROUP_SCHED808#ifdef CONFIG_FAIR_GROUP_SCHED
809static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)809static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
810{810{
811 struct sched_entity *se = tg->se[cpu];811 struct sched_entity *se = tg_se(tg, cpu);
812 812
813#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)813#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
814#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \814#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \
kernel/sched/fair.c+9−16
@@ -6876,7 +6876,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 6875 unmodified lines
6876{6876{
6877 struct rq *rq = rq_of(cfs_rq);6877 struct rq *rq = rq_of(cfs_rq);
6878 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);6878 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
6879 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];6879 struct sched_entity *se = cfs_rq_se(cfs_rq);
6880 6880
6881 /*6881 /*
6882 * It's possible we are called with runtime_remaining < 0 due to things6882 * It's possible we are called with runtime_remaining < 0 due to things
@@ -11102,7 +11102,6 @@ static bool __update_blocked_fair(struct rq *rq, bool *done) · 4219 unmodified lines
11102{11102{
11103 struct cfs_rq *cfs_rq, *pos;11103 struct cfs_rq *cfs_rq, *pos;
11104 bool decayed = false;11104 bool decayed = false;
11105 int cpu = cpu_of(rq);
11106 11105
11107 /*11106 /*
11108 * Iterates the task_group tree in a bottom up fashion, see11107 * Iterates the task_group tree in a bottom up fashion, see
@@ -11122,7 +11121,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done) · 13 unmodified lines
11122 }11121 }
11123 11122
11124 /* Propagate pending load changes to the parent, if any: */11123 /* Propagate pending load changes to the parent, if any: */
11125 se = cfs_rq->tg->se[cpu];11124 se = cfs_rq_se(cfs_rq);
11126 if (se && !skip_blocked_update(se))11125 if (se && !skip_blocked_update(se))
11127 update_load_avg(cfs_rq_of(se), se, UPDATE_TG);11126 update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
11128 11127
@@ -11148,8 +11147,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done) · 19 unmodified lines
11148 */11147 */
11149static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)11148static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
11150{11149{
11151 struct rq *rq = rq_of(cfs_rq);11150 struct sched_entity *se = cfs_rq_se(cfs_rq);
11152 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
11153 unsigned long now = jiffies;11151 unsigned long now = jiffies;
11154 unsigned long load;11152 unsigned long load;
11155 11153
@@ -15086,7 +15084,6 @@ void free_fair_sched_group(struct task_group *tg) · 3930 unmodified lines
15086 }15084 }
15087 15085
15088 kfree(tg->cfs_rq);15086 kfree(tg->cfs_rq);
15089 kfree(tg->se);
15090}15087}
15091 15088
15092int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)15089int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
@@ -15099,9 +15096,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 6 unmodified lines
15099 tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids);15096 tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids);
15100 if (!tg->cfs_rq)15097 if (!tg->cfs_rq)
15101 goto err;15098 goto err;
15102 tg->se = kzalloc_objs(se, nr_cpu_ids);
15103 if (!tg->se)
15104 goto err;
15105 15099
15106 tg->shares = NICE_0_LOAD;15100 tg->shares = NICE_0_LOAD;
15107 15101
@@ -15116,7 +15110,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 8 unmodified lines
15116 cfs_rq = &state->cfs_rq;15110 cfs_rq = &state->cfs_rq;
15117 se = &state->se;15111 se = &state->se;
15118 init_cfs_rq(cfs_rq);15112 init_cfs_rq(cfs_rq);
15119 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);15113 init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
15120 init_entity_runnable_average(se);15114 init_entity_runnable_average(se);
15121 }15115 }
15122 15116
@@ -15135,7 +15129,7 @@ void online_fair_sched_group(struct task_group *tg) · 12 unmodified lines
15135 15129
15136 for_each_possible_cpu(i) {15130 for_each_possible_cpu(i) {
15137 rq = cpu_rq(i);15131 rq = cpu_rq(i);
15138 se = tg->se[i];15132 se = tg_se(tg, i);
15139 rq_lock_irq(rq, &rf);15133 rq_lock_irq(rq, &rf);
15140 update_rq_clock(rq);15134 update_rq_clock(rq);
15141 attach_entity_cfs_rq(se);15135 attach_entity_cfs_rq(se);
@@ -15152,7 +15146,7 @@ void unregister_fair_sched_group(struct task_group *tg) · 10 unmodified lines
15152 15146
15153 for_each_possible_cpu(cpu) {15147 for_each_possible_cpu(cpu) {
15154 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];15148 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
15155 struct sched_entity *se = tg->se[cpu];15149 struct sched_entity *se = tg_se(tg, cpu);
15156 struct rq *rq = cpu_rq(cpu);15150 struct rq *rq = cpu_rq(cpu);
15157 15151
15158 if (se) {15152 if (se) {
@@ -15189,7 +15183,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, · 30 unmodified lines
15189 init_cfs_rq_runtime(cfs_rq);15183 init_cfs_rq_runtime(cfs_rq);
15190 15184
15191 tg->cfs_rq[cpu] = cfs_rq;15185 tg->cfs_rq[cpu] = cfs_rq;
15192 tg->se[cpu] = se;
15193 15186
15194 /* se could be NULL for root_task_group */15187 /* se could be NULL for root_task_group */
15195 if (!se)15188 if (!se)
@@ -15220,7 +15213,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) · 24 unmodified lines
15220 /*15213 /*
15221 * We can't change the weight of the root cgroup.15214 * We can't change the weight of the root cgroup.
15222 */15215 */
15223 if (!tg->se[0])15216 if (is_root_task_group(tg))
15224 return -EINVAL;15217 return -EINVAL;
15225 15218
15226 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));15219 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
@@ -15231,7 +15224,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) · 4 unmodified lines
15231 tg->shares = shares;15224 tg->shares = shares;
15232 for_each_possible_cpu(i) {15225 for_each_possible_cpu(i) {
15233 struct rq *rq = cpu_rq(i);15226 struct rq *rq = cpu_rq(i);
15234 struct sched_entity *se = tg->se[i];15227 struct sched_entity *se = tg_se(tg, i);
15235 struct rq_flags rf;15228 struct rq_flags rf;
15236 15229
15237 /* Propagate contribution to hierarchy */15230 /* Propagate contribution to hierarchy */
@@ -15282,7 +15275,7 @@ int sched_group_set_idle(struct task_group *tg, long idle) · 44 unmodified lines
15282 15275
15283 for_each_possible_cpu(i) {15276 for_each_possible_cpu(i) {
15284 struct rq *rq = cpu_rq(i);15277 struct rq *rq = cpu_rq(i);
15285 struct sched_entity *se = tg->se[i];15278 struct sched_entity *se = tg_se(tg, i);
15286 struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];15279 struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];
15287 bool was_idle = cfs_rq_is_idle(grp_cfs_rq);15280 bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
15288 long idle_task_delta;15281 long idle_task_delta;
kernel/sched/sched.h+26−5
@@ -484,8 +484,6 @@ struct task_group { · 483 unmodified lines
484#endif484#endif
485 485
486#ifdef CONFIG_FAIR_GROUP_SCHED486#ifdef CONFIG_FAIR_GROUP_SCHED
487 /* schedulable entities of this group on each CPU */
488 struct sched_entity **se;
489 /* runqueue "owned" by this group on each CPU */487 /* runqueue "owned" by this group on each CPU */
490 struct cfs_rq **cfs_rq;488 struct cfs_rq **cfs_rq;
491 unsigned long shares;489 unsigned long shares;
@@ -934,7 +932,8 @@ struct dl_rq { · 442 unmodified lines
934};932};
935 933
936#ifdef CONFIG_FAIR_GROUP_SCHED934#ifdef CONFIG_FAIR_GROUP_SCHED
937 935/* Check whether a task group is root tg */
936#define is_root_task_group(tg) ((tg) == &root_task_group)
938/* An entity is a task if it doesn't "own" a runqueue */937/* An entity is a task if it doesn't "own" a runqueue */
939#define entity_is_task(se) (!se->my_q)938#define entity_is_task(se) (!se->my_q)
940 939
@@ -2304,6 +2303,28 @@ struct cfs_tg_state { · 1363 unmodified lines
2304 struct sched_entity se;2303 struct sched_entity se;
2305 struct sched_statistics stats;2304 struct sched_statistics stats;
2306} __no_randomize_layout;2305} __no_randomize_layout;
2306
2307static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
2308{
2309 struct cfs_tg_state *state;
2310
2311 if (is_root_task_group(tg))
2312 return NULL;
2313
2314 state = container_of(tg->cfs_rq[cpu], struct cfs_tg_state, cfs_rq);
2315 return &state->se;
2316}
2317
2318static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq)
2319{
2320 struct cfs_tg_state *state;
2321
2322 if (is_root_task_group(cfs_rq->tg))
2323 return NULL;
2324
2325 state = container_of(cfs_rq, struct cfs_tg_state, cfs_rq);
2326 return &state->se;
2327}
2307#endif2328#endif
2308 2329
2309/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */2330/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
@@ -2316,8 +2337,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) · 6 unmodified lines
2316#ifdef CONFIG_FAIR_GROUP_SCHED2337#ifdef CONFIG_FAIR_GROUP_SCHED
2317 set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);2338 set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
2318 p->se.cfs_rq = tg->cfs_rq[cpu];2339 p->se.cfs_rq = tg->cfs_rq[cpu];
2319 p->se.parent = tg->se[cpu];2340 p->se.parent = tg_se(tg, cpu);
2320 p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0;2341 p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
2321#endif2342#endif
2322 2343
2323#ifdef CONFIG_RT_GROUP_SCHED2344#ifdef CONFIG_RT_GROUP_SCHED

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t