We currently have a race: if we're preempted during syscall exit, we can fail to process syscall return work that is queued up while we're preempted in ret_from_sys_call after checking ti.flags. Fix it by disabling interrupts before checking ti.flags.
Unified · Split
| @@ -364,12 +364,21 @@ system_call_fastpath: · 363 unmodified lines | ||
| 364 | 364 | * Has incomplete stack frame and undefined top of stack. |
| 365 | 365 | */ |
| 366 | 366 | ret_from_sys_call: |
| 367 | − testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) | |
| 368 | − jnz int_ret_from_sys_call_fixup /* Go the the slow path */ | |
| 369 | − | |
| 370 | 367 | LOCKDEP_SYS_EXIT |
| 371 | 368 | DISABLE_INTERRUPTS(CLBR_NONE) |
| 372 | 369 | TRACE_IRQS_OFF |
| 370 | + | |
| 371 | + /* | |
| 372 | + * We must check ti flags with interrupts (or at least preemption) | |
| 373 | + * off because we must *never* return to userspace without | |
| 374 | + * processing exit work that is enqueued if we're preempted here. | |
| 375 | + * In particular, returning to userspace with any of the one-shot | |
| 376 | + * flags (TIF_NOTIFY_RESUME, TIF_USER_RETURN_NOTIFY, etc) set is | |
| 377 | + * very bad. | |
| 378 | + */ | |
| 379 | + testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) | |
| 380 | + jnz int_ret_from_sys_call_fixup /* Go the the slow path */ | |
| 381 | + | |
| 373 | 382 | CFI_REMEMBER_STATE |
| 374 | 383 | /* |
| 375 | 384 | * sysretq will re-enable interrupts: |
| @@ -386,7 +395,7 @@ ret_from_sys_call: · 10 unmodified lines | ||
| 386 | 395 | |
| 387 | 396 | int_ret_from_sys_call_fixup: |
| 388 | 397 | FIXUP_TOP_OF_STACK %r11, -ARGOFFSET |
| 389 | − jmp int_ret_from_sys_call | |
| 398 | + jmp int_ret_from_sys_call_irqs_off | |
| 390 | 399 | |
| 391 | 400 | /* Do syscall tracing */ |
| 392 | 401 | tracesys: |
| @@ -432,6 +441,7 @@ tracesys_phase2: · 39 unmodified lines | ||
| 432 | 441 | GLOBAL(int_ret_from_sys_call) |
| 433 | 442 | DISABLE_INTERRUPTS(CLBR_NONE) |
| 434 | 443 | TRACE_IRQS_OFF |
| 444 | +int_ret_from_sys_call_irqs_off: | |
| 435 | 445 | movl $_TIF_ALLWORK_MASK,%edi |
| 436 | 446 | /* edi: mask to check */ |
| 437 | 447 | GLOBAL(int_with_check) |