History

sched/fair: Co-locate cfs_rq and sched_entity in cfs_tg_state

Improve data locality and reduce pointer chasing by allocating struct cfs_rq and struct sched_entity together for non-root task groups. This is achieved by introducing a new combined struct cfs_tg_state that holds both objects in a single allocation. This patch: - Introduces struct cfs_tg_state that embeds cfs_rq, sched_entity, and sched_statistics together in a single structure. - Updates __schedstats_from_se() in stats.h to use cfs_tg_state for accessing sched_statistics from a group sched_entity. - Modifies alloc_fair_sched_group() and free_fair_sched_group() to allocate and free the new struct as a single unit. - Modifies the per-CPU pointers in task_group->se and task_group->cfs_rq to point to the members in the new combined structure.

Zecheng Li committed 5 months ago

Commit dfcfc971 parent 63c1a123 files · +19 −20Browse files

Unified · Split

3 files
  1. kernel/sched/fair.c +6 −12
  2. kernel/sched/sched.h +12 −0
  3. kernel/sched/stats.h +1 −8
kernel/sched/fair.c+6−12
@@ -15083,8 +15083,6 @@ void free_fair_sched_group(struct task_group *tg) · 15082 unmodified lines
1508315083 for_each_possible_cpu(i) {
1508415084 if (tg->cfs_rq)
1508515085 kfree(tg->cfs_rq[i]);
15086− if (tg->se)
15087− kfree(tg->se[i]);
1508815086 }
1508915087
1509015088 kfree(tg->cfs_rq);
@@ -15093,6 +15091,7 @@ void free_fair_sched_group(struct task_group *tg) · 2 unmodified lines
1509315091
1509415092 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
1509515093 {
15094+ struct cfs_tg_state *state;
1509615095 struct sched_entity *se;
1509715096 struct cfs_rq *cfs_rq;
1509815097 int i;
@@ -15109,16 +15108,13 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 10 unmodified lines
1510915108 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
1511015109
1511115110 for_each_possible_cpu(i) {
15112− cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
15113− GFP_KERNEL, cpu_to_node(i));
15114− if (!cfs_rq)
15111+ state = kzalloc_node(sizeof(*state),
15112+ GFP_KERNEL, cpu_to_node(i));
15113+ if (!state)
1511515114 goto err;
1511615115
15117− se = kzalloc_node(sizeof(struct sched_entity_stats),
15118− GFP_KERNEL, cpu_to_node(i));
15119− if (!se)
15120− goto err_free_rq;
15121−
15116+ cfs_rq = &state->cfs_rq;
15117+ se = &state->se;
1512215118 init_cfs_rq(cfs_rq);
1512315119 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
1512415120 init_entity_runnable_average(se);
@@ -15126,8 +15122,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) · 1 unmodified line
1512615122
1512715123 return 1;
1512815124
15129−err_free_rq:
15130− kfree(cfs_rq);
1513115125 err:
1513215126 return 0;
1513315127 }
kernel/sched/sched.h+12−0
@@ -2294,6 +2294,18 @@ static inline struct task_group *task_group(struct task_struct *p) · 2293 unmodified lines
22942294 return p->sched_task_group;
22952295 }
22962296
2297+#ifdef CONFIG_FAIR_GROUP_SCHED
2298+/*
2299+ * Defined here to be available before stats.h is included, since
2300+ * stats.h has dependencies on things defined later in this file.
2301+ */
2302+struct cfs_tg_state {
2303+ struct cfs_rq cfs_rq;
2304+ struct sched_entity se;
2305+ struct sched_statistics stats;
2306+} __no_randomize_layout;
2307+#endif
2308+
22972309 /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
22982310 static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
22992311 {
kernel/sched/stats.h+1−8
@@ -89,19 +89,12 @@ static inline void rq_sched_info_depart (struct rq *rq, unsigned long long delt · 88 unmodified lines
8989
9090 #endif /* CONFIG_SCHEDSTATS */
9191
92−#ifdef CONFIG_FAIR_GROUP_SCHED
93−struct sched_entity_stats {
94− struct sched_entity se;
95− struct sched_statistics stats;
96−} __no_randomize_layout;
97−#endif
98−
9992 static inline struct sched_statistics *
10093 __schedstats_from_se(struct sched_entity *se)
10194 {
10295 #ifdef CONFIG_FAIR_GROUP_SCHED
10396 if (!entity_is_task(se))
104− return &container_of(se, struct sched_entity_stats, se)->stats;
97+ return &container_of(se, struct cfs_tg_state, se)->stats;
10598 #endif
10699 return &task_of(se)->stats;
107100 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t