History

task_work: Fix NMI race condition

__schedule() // disable irqs <NMI> task_work_add(current, work, TWA_NMI_CURRENT); </NMI> // current = next; // enable irqs <IRQ> task_work_set_notify_irq() test_and_set_tsk_thread_flag(current, TIF_NOTIFY_RESUME); // wrong task! </IRQ> // original task skips task work on its next return to user (or exit!)

Peter Zijlstra committed 1 year ago

Commit ef1ea981 parent 34976ea1 file · +7 −1Browse files

Showing the changes to kernel/task_work.c (1 of 1 file). Show all

Unified · Split

kernel/task_work.c+7−1
@@ -9,7 +9,12 @@ static struct callback_head work_exited; /* all we need is ->next == NULL */ · 8 unmodified lines
9#ifdef CONFIG_IRQ_WORK9#ifdef CONFIG_IRQ_WORK
10static void task_work_set_notify_irq(struct irq_work *entry)10static void task_work_set_notify_irq(struct irq_work *entry)
11{11{
12 test_and_set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);12 /*
13 * no-op IPI
14 *
15 * TWA_NMI_CURRENT will already have set the TIF flag, all
16 * this interrupt does it tickle the return-to-user path.
17 */
13}18}
14static DEFINE_PER_CPU(struct irq_work, irq_work_NMI_resume) =19static DEFINE_PER_CPU(struct irq_work, irq_work_NMI_resume) =
15 IRQ_WORK_INIT_HARD(task_work_set_notify_irq);20 IRQ_WORK_INIT_HARD(task_work_set_notify_irq);
@@ -86,6 +91,7 @@ int task_work_add(struct task_struct *task, struct callback_head *work, · 70 unmodified lines
86 break;91 break;
87#ifdef CONFIG_IRQ_WORK92#ifdef CONFIG_IRQ_WORK
88 case TWA_NMI_CURRENT:93 case TWA_NMI_CURRENT:
94 set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
89 irq_work_queue(this_cpu_ptr(&irq_work_NMI_resume));95 irq_work_queue(this_cpu_ptr(&irq_work_NMI_resume));
90 break;96 break;
91#endif97#endif