History

sched/fair: Co-locate cfs_rq and sched_entity in cfs_tg_state

Improve data locality and reduce pointer chasing by allocating struct cfs_rq and struct sched_entity together for non-root task groups. This is achieved by introducing a new combined struct cfs_tg_state that holds both objects in a single allocation. This patch: - Introduces struct cfs_tg_state that embeds cfs_rq, sched_entity, and sched_statistics together in a single structure. - Updates __schedstats_from_se() in stats.h to use cfs_tg_state for accessing sched_statistics from a group sched_entity. - Modifies alloc_fair_sched_group() and free_fair_sched_group() to allocate and free the new struct as a single unit. - Modifies the per-CPU pointers in task_group->se and task_group->cfs_rq to point to the members in the new combined structure.

Zecheng Li committed 5 months ago

Commit dfcfc971 parent 63c1a123 files · +19 −20Browse files

Unified · Split

3 files
  1. kernel/sched/fair.c +6 −12
  2. kernel/sched/sched.h +12 −0
  3. kernel/sched/stats.h +1 −8
kernel/sched/fair.c+6−12
@@ -15083,8 +15083,6 @@ void free_fair_sched_group(struct task_group *tg) · 15082 unmodified lines
15083 for_each_possible_cpu(i) {15083 for_each_possible_cpu(i) {
15084 if (tg->cfs_rq)15084 if (tg->cfs_rq)
15085 kfree(tg->cfs_rq[i]);15085 kfree(tg->cfs_rq[i]);
15086 if (tg->se)
15087 kfree(tg->se[i]);
15088 }15086 }
15089 15087
15090 kfree(tg->cfs_rq);15088 kfree(tg->cfs_rq);
@@ -15093,6 +15091,7 @@ void free_fair_sched_group(struct task_group *tg) · 2 unmodified lines
15093 15091
15094int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)15092int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
15095{15093{
15094 struct cfs_tg_state *state;
15096 struct sched_entity *se;15095 struct sched_entity *se;
15097 struct cfs_rq *cfs_rq;15096 struct cfs_rq *cfs_rq;
15098 int i;15097 int i;
@@ -15109,16 +15108,13 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 10 unmodified lines
15109 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));15108 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
15110 15109
15111 for_each_possible_cpu(i) {15110 for_each_possible_cpu(i) {
15112 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),15111 state = kzalloc_node(sizeof(*state),
15113 GFP_KERNEL, cpu_to_node(i));15112 GFP_KERNEL, cpu_to_node(i));
15114 if (!cfs_rq)15113 if (!state)
15115 goto err;15114 goto err;
15116 15115
15117 se = kzalloc_node(sizeof(struct sched_entity_stats),15116 cfs_rq = &state->cfs_rq;
15118 GFP_KERNEL, cpu_to_node(i));15117 se = &state->se;
15119 if (!se)
15120 goto err_free_rq;
15121
15122 init_cfs_rq(cfs_rq);15118 init_cfs_rq(cfs_rq);
15123 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);15119 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
15124 init_entity_runnable_average(se);15120 init_entity_runnable_average(se);
@@ -15126,8 +15122,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 1 unmodified line
15126 15122
15127 return 1;15123 return 1;
15128 15124
15129err_free_rq:
15130 kfree(cfs_rq);
15131err:15125err:
15132 return 0;15126 return 0;
15133}15127}
kernel/sched/sched.h+12−0
@@ -2294,6 +2294,18 @@ static inline struct task_group *task_group(struct task_struct *p) · 2293 unmodified lines
2294 return p->sched_task_group;2294 return p->sched_task_group;
2295}2295}
2296 2296
2297#ifdef CONFIG_FAIR_GROUP_SCHED
2298/*
2299 * Defined here to be available before stats.h is included, since
2300 * stats.h has dependencies on things defined later in this file.
2301 */
2302struct cfs_tg_state {
2303 struct cfs_rq cfs_rq;
2304 struct sched_entity se;
2305 struct sched_statistics stats;
2306} __no_randomize_layout;
2307#endif
2308
2297/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */2309/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
2298static inline void set_task_rq(struct task_struct *p, unsigned int cpu)2310static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
2299{2311{
kernel/sched/stats.h+1−8
@@ -89,19 +89,12 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt · 88 unmodified lines
89 89
90#endif /* CONFIG_SCHEDSTATS */90#endif /* CONFIG_SCHEDSTATS */
91 91
92#ifdef CONFIG_FAIR_GROUP_SCHED
93struct sched_entity_stats {
94 struct sched_entity se;
95 struct sched_statistics stats;
96} __no_randomize_layout;
97#endif
98
99static inline struct sched_statistics *92static inline struct sched_statistics *
100__schedstats_from_se(struct sched_entity *se)93__schedstats_from_se(struct sched_entity *se)
101{94{
102#ifdef CONFIG_FAIR_GROUP_SCHED95#ifdef CONFIG_FAIR_GROUP_SCHED
103 if (!entity_is_task(se))96 if (!entity_is_task(se))
104 return &container_of(se, struct sched_entity_stats, se)->stats;97 return &container_of(se, struct cfs_tg_state, se)->stats;
105#endif98#endif
106 return &task_of(se)->stats;99 return &task_of(se)->stats;
107}100}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t