History

sched/mmcid: Prevent CID stalls due to concurrent forks

A newly forked task is accounted as MMCID user before the task is visible in the process' thread list and the global task list. This creates the following problem: CPU1 CPU2 fork() sched_mm_cid_fork(tnew1) tnew1->mm.mm_cid_users++; tnew1->mm_cid.cid = getcid() -> preemption fork() sched_mm_cid_fork(tnew2) tnew2->mm.mm_cid_users++; // Reaches the per CPU threshold mm_cid_fixup_tasks_to_cpus() for_each_other(current, p) .... As tnew1 is not visible yet, this fails to fix up the already allocated CID of tnew1. As a consequence a subsequent schedule in might fail to acquire a (transitional) CID and the machine stalls. Move the invocation of sched_mm_cid_fork() after the new task becomes visible in the thread and the task list to prevent this. This also makes it symmetrical vs. exit() where the task is removed as CID user before the task is removed from the thread and task lists.

Thomas Gleixner committed 8 months ago

Commit b2e48c41 parent 1f318b93 files · +15 −11Browse files

Showing the changes to kernel (2 of 3 files). Show all

Unified · Split

2 files
  1. kernel/fork.c +0 −2
  2. kernel/sched/core.c +15 −7
kernel/fork.c+0−2
@@ -1586,7 +1586,6 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) · 1585 unmodified lines
1586 1586
1587 tsk->mm = mm;1587 tsk->mm = mm;
1588 tsk->active_mm = mm;1588 tsk->active_mm = mm;
1589 sched_mm_cid_fork(tsk);
1590 return 0;1589 return 0;
1591}1590}
1592 1591
@@ -2498,7 +2497,6 @@ bad_fork_cleanup_namespaces: · 905 unmodified lines
2498 exit_nsproxy_namespaces(p);2497 exit_nsproxy_namespaces(p);
2499bad_fork_cleanup_mm:2498bad_fork_cleanup_mm:
2500 if (p->mm) {2499 if (p->mm) {
2501 sched_mm_cid_exit(p);
2502 mm_clear_owner(p->mm, p);2500 mm_clear_owner(p->mm, p);
2503 mmput(p->mm);2501 mmput(p->mm);
2504 }2502 }
kernel/sched/core.c+15−7
@@ -4729,8 +4729,11 @@ void sched_cancel_fork(struct task_struct *p) · 4728 unmodified lines
4729 scx_cancel_fork(p);4729 scx_cancel_fork(p);
4730}4730}
4731 4731
4732static void sched_mm_cid_fork(struct task_struct *t);
4733
4732void sched_post_fork(struct task_struct *p)4734void sched_post_fork(struct task_struct *p)
4733{4735{
4736 sched_mm_cid_fork(p);
4734 uclamp_post_fork(p);4737 uclamp_post_fork(p);
4735 scx_post_fork(p);4738 scx_post_fork(p);
4736}4739}
@@ -10646,12 +10649,13 @@ static void mm_cid_do_fixup_tasks_to_cpus(struct mm_struct *mm) · 5909 unmodified lines
10646 * possible switch back to per task mode happens either in the10649 * possible switch back to per task mode happens either in the
10647 * deferred handler function or in the next fork()/exit().10650 * deferred handler function or in the next fork()/exit().
10648 *10651 *
10649 * The caller has already transferred. The newly incoming task is10652 * The caller has already transferred so remove it from the users
10650 * already accounted for, but not yet visible.10653 * count. The incoming task is already visible and has mm_cid.active,
10654 * but has task::mm_cid::cid == UNSET. Still it needs to be accounted
10655 * for. Concurrent fork()s might add more threads, but all of them have
10656 * task::mm_cid::active = 0, so they don't affect the accounting here.
10651 */10657 */
10652 users = mm->mm_cid.users - 2;10658 users = mm->mm_cid.users - 1;
10653 if (!users)
10654 return;
10655 10659
10656 guard(rcu)();10660 guard(rcu)();
10657 for_other_threads(current, t) {10661 for_other_threads(current, t) {
@@ -10688,12 +10692,15 @@ static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm) · 30 unmodified lines
10688 return mm_update_max_cids(mm);10692 return mm_update_max_cids(mm);
10689}10693}
10690 10694
10691void sched_mm_cid_fork(struct task_struct *t)10695static void sched_mm_cid_fork(struct task_struct *t)
10692{10696{
10693 struct mm_struct *mm = t->mm;10697 struct mm_struct *mm = t->mm;
10694 bool percpu;10698 bool percpu;
10695 10699
10696 WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);10700 if (!mm)
10701 return;
10702
10703 WARN_ON_ONCE(t->mm_cid.cid != MM_CID_UNSET);
10697 10704
10698 guard(mutex)(&mm->mm_cid.mutex);10705 guard(mutex)(&mm->mm_cid.mutex);
10699 scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {10706 scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
@@ -10885,6 +10892,7 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p) · 185 unmodified lines
10885}10892}
10886#else /* CONFIG_SCHED_MM_CID */10893#else /* CONFIG_SCHED_MM_CID */
10887static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }10894static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
10895static inline void sched_mm_cid_fork(struct task_struct *t) { }
10888#endif /* !CONFIG_SCHED_MM_CID */10896#endif /* !CONFIG_SCHED_MM_CID */
10889 10897
10890static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx);10898static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t