Improve data locality and reduce pointer chasing by allocating struct cfs_rq and struct sched_entity together for non-root task groups. This is achieved by introducing a new combined struct cfs_tg_state that holds both objects in a single allocation. This patch: - Introduces struct cfs_tg_state that embeds cfs_rq, sched_entity, and sched_statistics together in a single structure. - Updates __schedstats_from_se() in stats.h to use cfs_tg_state for accessing sched_statistics from a group sched_entity. - Modifies alloc_fair_sched_group() and free_fair_sched_group() to allocate and free the new struct as a single unit. - Modifies the per-CPU pointers in task_group->se and task_group->cfs_rq to point to the members in the new combined structure.
Unified · Split
| @@ -15083,8 +15083,6 @@ void free_fair_sched_group(struct task_group *tg) · 15082 unmodified lines | ||
| 15083 | 15083 | for_each_possible_cpu(i) { |
| 15084 | 15084 | if (tg->cfs_rq) |
| 15085 | 15085 | kfree(tg->cfs_rq[i]); |
| 15086 | − if (tg->se) | |
| 15087 | − kfree(tg->se[i]); | |
| 15088 | 15086 | } |
| 15089 | 15087 | |
| 15090 | 15088 | kfree(tg->cfs_rq); |
| @@ -15093,6 +15091,7 @@ void free_fair_sched_group(struct task_group *tg) · 2 unmodified lines | ||
| 15093 | 15091 | |
| 15094 | 15092 | int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) |
| 15095 | 15093 | { |
| 15094 | + struct cfs_tg_state *state; | |
| 15096 | 15095 | struct sched_entity *se; |
| 15097 | 15096 | struct cfs_rq *cfs_rq; |
| 15098 | 15097 | int i; |
| @@ -15109,16 +15108,13 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 10 unmodified lines | ||
| 15109 | 15108 | init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); |
| 15110 | 15109 | |
| 15111 | 15110 | for_each_possible_cpu(i) { |
| 15112 | − cfs_rq = kzalloc_node(sizeof(struct cfs_rq), | |
| 15113 | − GFP_KERNEL, cpu_to_node(i)); | |
| 15114 | − if (!cfs_rq) | |
| 15111 | + state = kzalloc_node(sizeof(*state), | |
| 15112 | + GFP_KERNEL, cpu_to_node(i)); | |
| 15113 | + if (!state) | |
| 15115 | 15114 | goto err; |
| 15116 | 15115 | |
| 15117 | − se = kzalloc_node(sizeof(struct sched_entity_stats), | |
| 15118 | − GFP_KERNEL, cpu_to_node(i)); | |
| 15119 | − if (!se) | |
| 15120 | − goto err_free_rq; | |
| 15121 | − | |
| 15116 | + cfs_rq = &state->cfs_rq; | |
| 15117 | + se = &state->se; | |
| 15122 | 15118 | init_cfs_rq(cfs_rq); |
| 15123 | 15119 | init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); |
| 15124 | 15120 | init_entity_runnable_average(se); |
| @@ -15126,8 +15122,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 1 unmodified line | ||
| 15126 | 15122 | |
| 15127 | 15123 | return 1; |
| 15128 | 15124 | |
| 15129 | −err_free_rq: | |
| 15130 | − kfree(cfs_rq); | |
| 15131 | 15125 | err: |
| 15132 | 15126 | return 0; |
| 15133 | 15127 | } |
| @@ -2294,6 +2294,18 @@ static inline struct task_group *task_group(struct task_struct *p) · 2293 unmodified lines | ||
| 2294 | 2294 | return p->sched_task_group; |
| 2295 | 2295 | } |
| 2296 | 2296 | |
| 2297 | +#ifdef CONFIG_FAIR_GROUP_SCHED | |
| 2298 | +/* | |
| 2299 | + * Defined here to be available before stats.h is included, since | |
| 2300 | + * stats.h has dependencies on things defined later in this file. | |
| 2301 | + */ | |
| 2302 | +struct cfs_tg_state { | |
| 2303 | + struct cfs_rq cfs_rq; | |
| 2304 | + struct sched_entity se; | |
| 2305 | + struct sched_statistics stats; | |
| 2306 | +} __no_randomize_layout; | |
| 2307 | +#endif | |
| 2308 | + | |
| 2297 | 2309 | /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ |
| 2298 | 2310 | static inline void set_task_rq(struct task_struct *p, unsigned int cpu) |
| 2299 | 2311 | { |
| @@ -89,19 +89,12 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt · 88 unmodified lines | ||
| 89 | 89 | |
| 90 | 90 | #endif /* CONFIG_SCHEDSTATS */ |
| 91 | 91 | |
| 92 | −#ifdef CONFIG_FAIR_GROUP_SCHED | |
| 93 | −struct sched_entity_stats { | |
| 94 | − struct sched_entity se; | |
| 95 | − struct sched_statistics stats; | |
| 96 | −} __no_randomize_layout; | |
| 97 | −#endif | |
| 98 | − | |
| 99 | 92 | static inline struct sched_statistics * |
| 100 | 93 | __schedstats_from_se(struct sched_entity *se) |
| 101 | 94 | { |
| 102 | 95 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 103 | 96 | if (!entity_is_task(se)) |
| 104 | − return &container_of(se, struct sched_entity_stats, se)->stats; | |
| 97 | + return &container_of(se, struct cfs_tg_state, se)->stats; | |
| 105 | 98 | #endif |
| 106 | 99 | return &task_of(se)->stats; |
| 107 | 100 | } |