History

kthread: Prevent unpark race which puts threads on the wrong cpu

The smpboot threads rely on the park/unpark mechanism which binds per cpu threads on a particular core. Though the functionality is racy: CPU0 CPU1 CPU2 unpark(T) wake_up_process(T) clear(SHOULD_PARK) T runs leave parkme() due to !SHOULD_PARK bind_to(CPU2) BUG_ON(wrong CPU) We cannot let the tasks move themself to the target CPU as one of those tasks is actually the migration thread itself, which requires that it starts running on the target cpu right away. The solution to this problem is to prevent wakeups in park mode which are not from unpark(). That way we can guarantee that the association of the task to the target cpu is working correctly. Add a new task state (TASK_PARKED) which prevents other wakeups and use this state explicitly for the unpark wakeup. Peter noticed: Also, since the task state is visible to userspace and all the parked tasks are still in the PID space, its a good hint in ps and friends that these tasks aren't really there for the moment. The migration thread has another related issue. CPU0 CPU1 Bring up CPU2 create_thread(T) park(T) wait_for_completion() parkme() complete() sched_set_stop_task() schedule(TASK_PARKED) The sched_set_stop_task() call is issued while the task is on the runqueue of CPU1 and that confuses the hell out of the stop_task class on that cpu. So we need the same synchronizaion before sched_set_stop_task().

Thomas Gleixner committed 13 years ago

Commit f2530dc1 parent cfb63ba5 files · +45 −29Browse files

Showing the changes to kernel (2 of 5 files). Show all

Unified · Split

2 files
  1. kernel/kthread.c +28 −24
  2. kernel/smpboot.c +12 −2
kernel/kthread.c+28−24
@@ -124,12 +124,12 @@ void *kthread_data(struct task_struct *task) · 123 unmodified lines
124124
125125 static void __kthread_parkme(struct kthread *self)
126126 {
127− __set_current_state(TASK_INTERRUPTIBLE);
127+ __set_current_state(TASK_PARKED);
128128 while (test_bit(KTHREAD_SHOULD_PARK, &self->flags)) {
129129 if (!test_and_set_bit(KTHREAD_IS_PARKED, &self->flags))
130130 complete(&self->parked);
131131 schedule();
132− __set_current_state(TASK_INTERRUPTIBLE);
132+ __set_current_state(TASK_PARKED);
133133 }
134134 clear_bit(KTHREAD_IS_PARKED, &self->flags);
135135 __set_current_state(TASK_RUNNING);
@@ -256,8 +256,13 @@ struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), · 120 unmodified lines
256256 }
257257 EXPORT_SYMBOL(kthread_create_on_node);
258258
259−static void __kthread_bind(struct task_struct *p, unsigned int cpu)
259+static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state)
260260 {
261+ /* Must have done schedule() in kthread() before we set_task_cpu */
262+ if (!wait_task_inactive(p, state)) {
263+ WARN_ON(1);
264+ return;
265+ }
261266 /* It's safe because the task is inactive. */
262267 do_set_cpus_allowed(p, cpumask_of(cpu));
263268 p->flags |= PF_THREAD_BOUND;
@@ -274,12 +279,7 @@ static void __kthread_bind(struct task_struct *p, unsigned int cpu) · 10 unmodified lines
274279 */
275280 void kthread_bind(struct task_struct *p, unsigned int cpu)
276281 {
277− /* Must have done schedule() in kthread() before we set_task_cpu */
278− if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) {
279− WARN_ON(1);
280− return;
281− }
282− __kthread_bind(p, cpu);
282+ __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
283283 }
284284 EXPORT_SYMBOL(kthread_bind);
285285
@@ -324,6 +324,22 @@ static struct kthread *task_get_live_kthread(struct task_struct *k) · 38 unmodified lines
324324 return NULL;
325325 }
326326
327+static void __kthread_unpark(struct task_struct *k, struct kthread *kthread)
328+{
329+ clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
330+ /*
331+ * We clear the IS_PARKED bit here as we don't wait
332+ * until the task has left the park code. So if we'd
333+ * park before that happens we'd see the IS_PARKED bit
334+ * which might be about to be cleared.
335+ */
336+ if (test_and_clear_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
337+ if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
338+ __kthread_bind(k, kthread->cpu, TASK_PARKED);
339+ wake_up_state(k, TASK_PARKED);
340+ }
341+}
342+
327343 /**
328344 * kthread_unpark - unpark a thread created by kthread_create().
329345 * @k: thread created by kthread_create().
@@ -336,20 +352,8 @@ void kthread_unpark(struct task_struct *k) · 6 unmodified lines
336352 {
337353 struct kthread *kthread = task_get_live_kthread(k);
338354
339− if (kthread) {
340− clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
341− /*
342− * We clear the IS_PARKED bit here as we don't wait
343− * until the task has left the park code. So if we'd
344− * park before that happens we'd see the IS_PARKED bit
345− * which might be about to be cleared.
346− */
347− if (test_and_clear_bit(KTHREAD_IS_PARKED, &kthread->flags)) {
348− if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
349− __kthread_bind(k, kthread->cpu);
350− wake_up_process(k);
351− }
352− }
355+ if (kthread)
356+ __kthread_unpark(k, kthread);
353357 put_task_struct(k);
354358 }
355359
@@ -407,7 +411,7 @@ int kthread_stop(struct task_struct *k) · 51 unmodified lines
407411 trace_sched_kthread_stop(k);
408412 if (kthread) {
409413 set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
410− clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
414+ __kthread_unpark(k, kthread);
411415 wake_up_process(k);
412416 wait_for_completion(&kthread->exited);
413417 }
kernel/smpboot.c+12−2
@@ -185,8 +185,18 @@ __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu) · 184 unmodified lines
185185 }
186186 get_task_struct(tsk);
187187 *per_cpu_ptr(ht->store, cpu) = tsk;
188− if (ht->create)
189− ht->create(cpu);
188+ if (ht->create) {
189+ /*
190+ * Make sure that the task has actually scheduled out
191+ * into park position, before calling the create
192+ * callback. At least the migration thread callback
193+ * requires that the task is off the runqueue.
194+ */
195+ if (!wait_task_inactive(tsk, TASK_PARKED))
196+ WARN_ON(1);
197+ else
198+ ht->create(cpu);
199+ }
190200 return 0;
191201 }
192202