History

sched/fair: Allocate cfs_tg_state with percpu allocator

To remove the cfs_rq pointer array in task_group, allocate the combined cfs_rq and sched_entity using the per-cpu allocator. This patch implements the following: - Changes task_group->cfs_rq from 'struct cfs_rq **' to 'struct cfs_rq __percpu *'. - Updates memory allocation in alloc_fair_sched_group() and free_fair_sched_group() to use alloc_percpu() and free_percpu() respectively. - Uses the inline accessor tg_cfs_rq(tg, cpu) with per_cpu_ptr() to retrieve the pointer to cfs_rq for the given task group and CPU. - Replaces direct accesses tg->cfs_rq[cpu] with calls to the new tg_cfs_rq(tg, cpu) helper. - Handles the root_task_group: since struct rq is already a per-cpu variable (runqueues), its embedded cfs_rq (rq->cfs) is also per-cpu. Therefore, we assign root_task_group.cfs_rq = &runqueues.cfs. - Cleanup the code in initializing the root task group. This change places each CPU's cfs_rq and sched_entity in its local per-cpu memory area to remove the per-task_group pointer arrays.

Zecheng Li committed 5 months ago

Commit b8fea7a1 parent 89e1f673 files · +45 −58Browse files

Unified · Split

3 files
  1. kernel/sched/core.c +13 −22
  2. kernel/sched/fair.c +22 −32
  3. kernel/sched/sched.h +10 −4
kernel/sched/core.c+13−22
@@ -8907,7 +8907,7 @@ static struct kmem_cache *task_group_cache __ro_after_init; · 8906 unmodified lines
89078907
89088908 void __init sched_init(void)
89098909 {
8910− unsigned long ptr = 0;
8910+ unsigned long __maybe_unused ptr = 0;
89118911 int i;
89128912
89138913 /* Make sure the linker didn't screw up */
@@ -8923,33 +8923,24 @@ void __init sched_init(void) · 9 unmodified lines
89238923 wait_bit_init();
89248924
89258925 #ifdef CONFIG_FAIR_GROUP_SCHED
8926− ptr += nr_cpu_ids * sizeof(void **);
8927−#endif
8928−#ifdef CONFIG_RT_GROUP_SCHED
8929− ptr += 2 * nr_cpu_ids * sizeof(void **);
8930−#endif
8931− if (ptr) {
8932− ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
8926+ root_task_group.cfs_rq = &runqueues.cfs;
89338927
8934−#ifdef CONFIG_FAIR_GROUP_SCHED
8935− root_task_group.cfs_rq = (struct cfs_rq **)ptr;
8936− ptr += nr_cpu_ids * sizeof(void **);
8937−
8938− root_task_group.shares = ROOT_TASK_GROUP_LOAD;
8939− init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
8928+ root_task_group.shares = ROOT_TASK_GROUP_LOAD;
8929+ init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL);
89408930 #endif /* CONFIG_FAIR_GROUP_SCHED */
89418931 #ifdef CONFIG_EXT_GROUP_SCHED
8942− scx_tg_init(&root_task_group);
8932+ scx_tg_init(&root_task_group);
89438933 #endif /* CONFIG_EXT_GROUP_SCHED */
89448934 #ifdef CONFIG_RT_GROUP_SCHED
8945− root_task_group.rt_se = (struct sched_rt_entity **)ptr;
8946− ptr += nr_cpu_ids * sizeof(void **);
8935+ ptr += 2 * nr_cpu_ids * sizeof(void **);
8936+ ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
8937+ root_task_group.rt_se = (struct sched_rt_entity **)ptr;
8938+ ptr += nr_cpu_ids * sizeof(void **);
89478939
8948− root_task_group.rt_rq = (struct rt_rq **)ptr;
8949− ptr += nr_cpu_ids * sizeof(void **);
8940+ root_task_group.rt_rq = (struct rt_rq **)ptr;
8941+ ptr += nr_cpu_ids * sizeof(void **);
89508942
89518943 #endif /* CONFIG_RT_GROUP_SCHED */
8952− }
89538944
89548945 init_defrootdomain();
89558946
@@ -9864,7 +9855,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, · 908 unmodified lines
98649855 }
98659856
98669857 for_each_online_cpu(i) {
9867− struct cfs_rq *cfs_rq = tg->cfs_rq[i];
9858+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, i);
98689859 struct rq *rq = cfs_rq->rq;
98699860
98709861 guard(rq_lock_irq)(rq);
@@ -10032,7 +10023,7 @@ static u64 throttled_time_self(struct task_group *tg) · 161 unmodified lines
1003210023 u64 total = 0;
1003310024
1003410025 for_each_possible_cpu(i) {
10035− total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time);
10026+ total += READ_ONCE(tg_cfs_rq(tg, i)->throttled_clock_self_time);
1003610027 }
1003710028
1003810029 return total;
kernel/sched/fair.c+22−32
@@ -334,7 +334,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) · 333 unmodified lines
334334 * to a tree or when we reach the top of the tree
335335 */
336336 if (cfs_rq->tg->parent &&
337− cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
337+ tg_cfs_rq(cfs_rq->tg->parent, cpu)->on_list) {
338338 /*
339339 * If parent is already on the list, we add the child
340340 * just before. Thanks to circular linked property of
@@ -342,7 +342,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) · 1 unmodified line
342342 * of the list that starts by parent.
343343 */
344344 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
345− &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
345+ &(tg_cfs_rq(cfs_rq->tg->parent, cpu)->leaf_cfs_rq_list));
346346 /*
347347 * The branch is now connected to its tree so we can
348348 * reset tmp_alone_branch to the beginning of the
@@ -5037,7 +5037,7 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) · 4688 unmodified lines
50375037
50385038 rcu_read_lock();
50395039 list_for_each_entry_rcu(tg, &task_groups, list) {
5040− struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5040+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
50415041
50425042 clear_tg_load_avg(cfs_rq);
50435043 }
@@ -6594,7 +6594,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) · 1550 unmodified lines
65946594
65956595 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu)
65966596 {
6597− return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]);
6597+ return throttled_hierarchy(tg_cfs_rq(task_group(p), dst_cpu));
65986598 }
65996599
66006600 static inline bool task_is_throttled(struct task_struct *p)
@@ -6740,7 +6740,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags); · 139 unmodified lines
67406740 static int tg_unthrottle_up(struct task_group *tg, void *data)
67416741 {
67426742 struct rq *rq = data;
6743− struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
6743+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
67446744 struct task_struct *p, *tmp;
67456745
67466746 if (--cfs_rq->throttle_count)
@@ -6811,7 +6811,7 @@ static void record_throttle_clock(struct cfs_rq *cfs_rq) · 64 unmodified lines
68116811 static int tg_throttle_down(struct task_group *tg, void *data)
68126812 {
68136813 struct rq *rq = data;
6814− struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
6814+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
68156815
68166816 if (cfs_rq->throttle_count++)
68176817 return 0;
@@ -7285,8 +7285,8 @@ static void sync_throttle(struct task_group *tg, int cpu) · 467 unmodified lines
72857285 if (!tg->parent)
72867286 return;
72877287
7288− cfs_rq = tg->cfs_rq[cpu];
7289− pcfs_rq = tg->parent->cfs_rq[cpu];
7288+ cfs_rq = tg_cfs_rq(tg, cpu);
7289+ pcfs_rq = tg_cfs_rq(tg->parent, cpu);
72907290
72917291 cfs_rq->throttle_count = pcfs_rq->throttle_count;
72927292 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
@@ -7478,7 +7478,7 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq) · 185 unmodified lines
74787478 rcu_read_lock();
74797479 list_for_each_entry_rcu(tg, &task_groups, list) {
74807480 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
7481− struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
7481+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
74827482
74837483 raw_spin_lock(&cfs_b->lock);
74847484 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
@@ -7507,7 +7507,7 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) · 22 unmodified lines
75077507
75087508 rcu_read_lock();
75097509 list_for_each_entry_rcu(tg, &task_groups, list) {
7510− struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
7510+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq));
75117511
75127512 if (!cfs_rq->runtime_enabled)
75137513 continue;
@@ -10382,7 +10382,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_ · 2868 unmodified lines
1038210382 struct cfs_rq *dst_cfs_rq;
1038310383
1038410384 #ifdef CONFIG_FAIR_GROUP_SCHED
10385− dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu];
10385+ dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu);
1038610386 #else
1038710387 dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
1038810388 #endif
@@ -14812,7 +14812,7 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu) · 4423 unmodified lines
1481214812 struct cfs_rq *cfs_rq;
1481314813
1481414814 #ifdef CONFIG_FAIR_GROUP_SCHED
14815− cfs_rq = task_group(p)->cfs_rq[cpu];
14815+ cfs_rq = tg_cfs_rq(task_group(p), cpu);
1481614816 #else
1481714817 cfs_rq = &cpu_rq(cpu)->cfs;
1481814818 #endif
@@ -15076,39 +15076,31 @@ static void task_change_group_fair(struct task_struct *p) · 257 unmodified lines
1507615076
1507715077 void free_fair_sched_group(struct task_group *tg)
1507815078 {
15079− int i;
15080−
15081− for_each_possible_cpu(i) {
15082− if (tg->cfs_rq)
15083− kfree(tg->cfs_rq[i]);
15084− }
15085−
15086− kfree(tg->cfs_rq);
15079+ free_percpu(tg->cfs_rq);
1508715080 }
1508815081
1508915082 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
1509015083 {
15091− struct cfs_tg_state *state;
15084+ struct cfs_tg_state __percpu *state;
1509215085 struct sched_entity *se;
1509315086 struct cfs_rq *cfs_rq;
1509415087 int i;
1509515088
15096− tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids);
15097− if (!tg->cfs_rq)
15089+ state = alloc_percpu_gfp(struct cfs_tg_state, GFP_KERNEL);
15090+ if (!state)
1509815091 goto err;
1509915092
15093+ tg->cfs_rq = &state->cfs_rq;
1510015094 tg->shares = NICE_0_LOAD;
1510115095
1510215096 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));
1510315097
1510415098 for_each_possible_cpu(i) {
15105− state = kzalloc_node(sizeof(*state),
15106− GFP_KERNEL, cpu_to_node(i));
15107− if (!state)
15099+ cfs_rq = tg_cfs_rq(tg, i);
15100+ if (!cfs_rq)
1510815101 goto err;
1510915102
15110− cfs_rq = &state->cfs_rq;
15111− se = &state->se;
15103+ se = tg_se(tg, i);
1511215104 init_cfs_rq(cfs_rq);
1511315105 init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
1511415106 init_entity_runnable_average(se);
@@ -15145,7 +15137,7 @@ void unregister_fair_sched_group(struct task_group *tg) · 30 unmodified lines
1514515137 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
1514615138
1514715139 for_each_possible_cpu(cpu) {
15148− struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
15140+ struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu);
1514915141 struct sched_entity *se = tg_se(tg, cpu);
1515015142 struct rq *rq = cpu_rq(cpu);
1515115143
@@ -15182,8 +15174,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, · 30 unmodified lines
1518215174 cfs_rq->rq = rq;
1518315175 init_cfs_rq_runtime(cfs_rq);
1518415176
15185− tg->cfs_rq[cpu] = cfs_rq;
15186−
1518715177 /* se could be NULL for root_task_group */
1518815178 if (!se)
1518915179 return;
@@ -15276,7 +15266,7 @@ int sched_group_set_idle(struct task_group *tg, long idle) · 86 unmodified lines
1527615266 for_each_possible_cpu(i) {
1527715267 struct rq *rq = cpu_rq(i);
1527815268 struct sched_entity *se = tg_se(tg, i);
15279− struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];
15269+ struct cfs_rq *grp_cfs_rq = tg_cfs_rq(tg, i);
1528015270 bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
1528115271 long idle_task_delta;
1528215272 struct rq_flags rf;
kernel/sched/sched.h+10−4
@@ -485,7 +485,7 @@ struct task_group { · 484 unmodified lines
485485
486486 #ifdef CONFIG_FAIR_GROUP_SCHED
487487 /* runqueue "owned" by this group on each CPU */
488− struct cfs_rq **cfs_rq;
488+ struct cfs_rq __percpu *cfs_rq;
489489 unsigned long shares;
490490 /*
491491 * load_avg can be heavily contended at clock tick time, so put
@@ -2304,6 +2304,12 @@ struct cfs_tg_state { · 1812 unmodified lines
23042304 struct sched_statistics stats;
23052305 } __no_randomize_layout;
23062306
2307+/* Access a specific CPU's cfs_rq from a task group */
2308+static inline struct cfs_rq *tg_cfs_rq(struct task_group *tg, int cpu)
2309+{
2310+ return per_cpu_ptr(tg->cfs_rq, cpu);
2311+}
2312+
23072313 static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
23082314 {
23092315 struct cfs_tg_state *state;
@@ -2311,7 +2317,7 @@ static inline struct sched_entity *tg_se(struct task_group *tg, int cpu) · 1 unmodified line
23112317 if (is_root_task_group(tg))
23122318 return NULL;
23132319
2314− state = container_of(tg->cfs_rq[cpu], struct cfs_tg_state, cfs_rq);
2320+ state = container_of(tg_cfs_rq(tg, cpu), struct cfs_tg_state, cfs_rq);
23152321 return &state->se;
23162322 }
23172323
@@ -2335,8 +2341,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) · 17 unmodified lines
23352341 #endif
23362342
23372343 #ifdef CONFIG_FAIR_GROUP_SCHED
2338− set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
2339− p->se.cfs_rq = tg->cfs_rq[cpu];
2344+ set_task_rq_fair(&p->se, p->se.cfs_rq, tg_cfs_rq(tg, cpu));
2345+ p->se.cfs_rq = tg_cfs_rq(tg, cpu);
23402346 p->se.parent = tg_se(tg, cpu);
23412347 p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
23422348 #endif