History

sched/mmcid: Prevent CID stalls due to concurrent forks

A newly forked task is accounted as MMCID user before the task is visible in the process' thread list and the global task list. This creates the following problem: CPU1 CPU2 fork() sched_mm_cid_fork(tnew1) tnew1->mm.mm_cid_users++; tnew1->mm_cid.cid = getcid() -> preemption fork() sched_mm_cid_fork(tnew2) tnew2->mm.mm_cid_users++; // Reaches the per CPU threshold mm_cid_fixup_tasks_to_cpus() for_each_other(current, p) .... As tnew1 is not visible yet, this fails to fix up the already allocated CID of tnew1. As a consequence a subsequent schedule in might fail to acquire a (transitional) CID and the machine stalls. Move the invocation of sched_mm_cid_fork() after the new task becomes visible in the thread and the task list to prevent this. This also makes it symmetrical vs. exit() where the task is removed as CID user before the task is removed from the thread and task lists.

Thomas Gleixner committed 8 months ago

Commit b2e48c41 parent 1f318b93 files · +15 −11Browse files

Showing the changes to kernel (2 of 3 files). Show all

Unified · Split

2 files
  1. kernel/fork.c +0 −2
  2. kernel/sched/core.c +15 −7
kernel/fork.c+0−2
@@ -1586,7 +1586,6 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) · 1585 unmodified lines
15861586
15871587 tsk->mm = mm;
15881588 tsk->active_mm = mm;
1589− sched_mm_cid_fork(tsk);
15901589 return 0;
15911590 }
15921591
@@ -2498,7 +2497,6 @@ bad_fork_cleanup_namespaces: · 905 unmodified lines
24982497 exit_nsproxy_namespaces(p);
24992498 bad_fork_cleanup_mm:
25002499 if (p->mm) {
2501− sched_mm_cid_exit(p);
25022500 mm_clear_owner(p->mm, p);
25032501 mmput(p->mm);
25042502 }
kernel/sched/core.c+15−7
@@ -4729,8 +4729,11 @@ void sched_cancel_fork(struct task_struct *p) · 4728 unmodified lines
47294729 scx_cancel_fork(p);
47304730 }
47314731
4732+static void sched_mm_cid_fork(struct task_struct *t);
4733+
47324734 void sched_post_fork(struct task_struct *p)
47334735 {
4736+ sched_mm_cid_fork(p);
47344737 uclamp_post_fork(p);
47354738 scx_post_fork(p);
47364739 }
@@ -10646,12 +10649,13 @@ static void mm_cid_do_fixup_tasks_to_cpus(struct mm_struct *mm) · 5909 unmodified lines
1064610649 * possible switch back to per task mode happens either in the
1064710650 * deferred handler function or in the next fork()/exit().
1064810651 *
10649− * The caller has already transferred. The newly incoming task is
10650− * already accounted for, but not yet visible.
10652+ * The caller has already transferred so remove it from the users
10653+ * count. The incoming task is already visible and has mm_cid.active,
10654+ * but has task::mm_cid::cid == UNSET. Still it needs to be accounted
10655+ * for. Concurrent fork()s might add more threads, but all of them have
10656+ * task::mm_cid::active = 0, so they don't affect the accounting here.
1065110657 */
10652− users = mm->mm_cid.users - 2;
10653− if (!users)
10654− return;
10658+ users = mm->mm_cid.users - 1;
1065510659
1065610660 guard(rcu)();
1065710661 for_other_threads(current, t) {
@@ -10688,12 +10692,15 @@ static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm) · 30 unmodified lines
1068810692 return mm_update_max_cids(mm);
1068910693 }
1069010694
10691−void sched_mm_cid_fork(struct task_struct *t)
10695+static void sched_mm_cid_fork(struct task_struct *t)
1069210696 {
1069310697 struct mm_struct *mm = t->mm;
1069410698 bool percpu;
1069510699
10696− WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
10700+ if (!mm)
10701+ return;
10702+
10703+ WARN_ON_ONCE(t->mm_cid.cid != MM_CID_UNSET);
1069710704
1069810705 guard(mutex)(&mm->mm_cid.mutex);
1069910706 scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
@@ -10885,6 +10892,7 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p) · 185 unmodified lines
1088510892 }
1088610893 #else /* CONFIG_SCHED_MM_CID */
1088710894 static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
10895+static inline void sched_mm_cid_fork(struct task_struct *t) { }
1088810896 #endif /* !CONFIG_SCHED_MM_CID */
1088910897
1089010898 static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t