History

rseq: Switch to fast path processing on exit to user

Now that all bits and pieces are in place, hook the RSEQ handling fast path function into exit_to_user_mode_prepare() after the TIF work bits have been handled. If case of fast path failure, TIF_NOTIFY_RESUME has been raised and the caller needs to take another turn through the TIF handling slow path. This only works for architectures which use the generic entry code. Architectures who still have their own incomplete hacks are not supported and won't be. This results in the following improvements: Kernel build Before After Reduction exit to user 80692981 80514451 signal checks: 32581 121 99% slowpath runs: 1201408 1.49% 198 0.00% 100% fastpath runs: 675941 0.84% N/A id updates: 1233989 1.53% 50541 0.06% 96% cs checks: 1125366 1.39% 0 0.00% 100% cs cleared: 1125366 100% 0 100% cs fixup: 0 0% 0 RSEQ selftests Before After Reduction exit to user: 386281778 387373750 signal checks: 35661203 0 100% slowpath runs: 140542396 36.38% 100 0.00% 100% fastpath runs: 9509789 2.51% N/A id updates: 176203599 45.62% 9087994 2.35% 95% cs checks: 175587856 45.46% 4728394 1.22% 98% cs cleared: 172359544 98.16% 1319307 27.90% 99% cs fixup: 3228312 1.84% 3409087 72.10% The 'cs cleared' and 'cs fixup' percentages are not relative to the exit to user invocations, they are relative to the actual 'cs check' invocations. While some of this could have been avoided in the original code, like the obvious clearing of CS when it's already clear, the main problem of going through TIF_NOTIFY_RESUME cannot be solved. In some workloads the RSEQ notify handler is invoked more than once before going out to user space. Doing this once when everything has stabilized is the only solution to avoid this. The initial attempt to completely decouple it from the TIF work turned out to be suboptimal for workloads, which do a lot of quick and short system calls. Even if the fast path decision is only 4 instructions (including a conditional branch), this adds up quickly and becomes measurable when the rate for actually having to handle rseq is in the low single digit percentage range of user/kernel transitions.

Thomas Gleixner committed 1 year ago

Commit 3db6b381 parent 05b44ae6 files · +41 −22Browse files

Unified · Split

6 files
  1. include/linux/irq-entry-common.h +2 −5
  2. include/linux/resume_user_mode.h +1 −1
  3. include/linux/rseq.h +12 −6
  4. init/Kconfig +1 −1
  5. kernel/entry/common.c +19 −7
  6. kernel/rseq.c +6 −2
include/linux/irq-entry-common.h+2−5
@@ -197,11 +197,8 @@ static __always_inline void arch_exit_to_user_mode(void) { } · 196 unmodified lines
197 */197 */
198void arch_do_signal_or_restart(struct pt_regs *regs);198void arch_do_signal_or_restart(struct pt_regs *regs);
199 199
200/**200/* Handle pending TIF work */
201 * exit_to_user_mode_loop - do any pending work before leaving to user space201unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work);
202 */
203unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
204 unsigned long ti_work);
205 202
206/**203/**
207 * exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required204 * exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required
include/linux/resume_user_mode.h+1−1
@@ -59,7 +59,7 @@ static inline void resume_user_mode_work(struct pt_regs *regs) · 58 unmodified lines
59 mem_cgroup_handle_over_high(GFP_KERNEL);59 mem_cgroup_handle_over_high(GFP_KERNEL);
60 blkcg_maybe_throttle_current();60 blkcg_maybe_throttle_current();
61 61
62 rseq_handle_notify_resume(regs);62 rseq_handle_slowpath(regs);
63}63}
64 64
65#endif /* LINUX_RESUME_USER_MODE_H */65#endif /* LINUX_RESUME_USER_MODE_H */
include/linux/rseq.h+12−6
@@ -7,13 +7,19 @@ · 6 unmodified lines
7 7
8#include <uapi/linux/rseq.h>8#include <uapi/linux/rseq.h>
9 9
10void __rseq_handle_notify_resume(struct pt_regs *regs);10void __rseq_handle_slowpath(struct pt_regs *regs);
11 11
12static inline void rseq_handle_notify_resume(struct pt_regs *regs)12/* Invoked from resume_user_mode_work() */
13static inline void rseq_handle_slowpath(struct pt_regs *regs)
13{14{
14 /* '&' is intentional to spare one conditional branch */15 if (IS_ENABLED(CONFIG_GENERIC_ENTRY)) {
15 if (current->rseq.event.sched_switch & current->rseq.event.has_rseq)16 if (current->rseq.event.slowpath)
16 __rseq_handle_notify_resume(regs);17 __rseq_handle_slowpath(regs);
18 } else {
19 /* '&' is intentional to spare one conditional branch */
20 if (current->rseq.event.sched_switch & current->rseq.event.has_rseq)
21 __rseq_handle_slowpath(regs);
22 }
17}23}
18 24
19void __rseq_signal_deliver(int sig, struct pt_regs *regs);25void __rseq_signal_deliver(int sig, struct pt_regs *regs);
@@ -152,7 +158,7 @@ static inline void rseq_fork(struct task_struct *t, u64 clone_flags) · 132 unmodified lines
152}158}
153 159
154#else /* CONFIG_RSEQ */160#else /* CONFIG_RSEQ */
155static inline void rseq_handle_notify_resume(struct pt_regs *regs) { }161static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
156static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }162static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
157static inline void rseq_sched_switch_event(struct task_struct *t) { }163static inline void rseq_sched_switch_event(struct task_struct *t) { }
158static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }164static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }
init/Kconfig+1−1
@@ -1941,7 +1941,7 @@ config RSEQ_DEBUG_DEFAULT_ENABLE · 1940 unmodified lines
1941config DEBUG_RSEQ1941config DEBUG_RSEQ
1942 default n1942 default n
1943 bool "Enable debugging of rseq() system call" if EXPERT1943 bool "Enable debugging of rseq() system call" if EXPERT
1944 depends on RSEQ && DEBUG_KERNEL1944 depends on RSEQ && DEBUG_KERNEL && !GENERIC_ENTRY
1945 select RSEQ_DEBUG_DEFAULT_ENABLE1945 select RSEQ_DEBUG_DEFAULT_ENABLE
1946 help1946 help
1947 Enable extra debugging checks for the rseq system call.1947 Enable extra debugging checks for the rseq system call.
kernel/entry/common.c+19−7
@@ -11,13 +11,8 @@ · 10 unmodified lines
11/* Workaround to allow gradual conversion of architecture code */11/* Workaround to allow gradual conversion of architecture code */
12void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }12void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }
13 13
14/**14static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs,
15 * exit_to_user_mode_loop - do any pending work before leaving to user space15 unsigned long ti_work)
16 * @regs: Pointer to pt_regs on entry stack
17 * @ti_work: TIF work flags as read by the caller
18 */
19__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
20 unsigned long ti_work)
21{16{
22 /*17 /*
23 * Before returning to user space ensure that all pending work18 * Before returning to user space ensure that all pending work
@@ -62,6 +57,23 @@ __always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs, · 38 unmodified lines
62 return ti_work;57 return ti_work;
63}58}
64 59
60/**
61 * exit_to_user_mode_loop - do any pending work before leaving to user space
62 * @regs: Pointer to pt_regs on entry stack
63 * @ti_work: TIF work flags as read by the caller
64 */
65__always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
66 unsigned long ti_work)
67{
68 for (;;) {
69 ti_work = __exit_to_user_mode_loop(regs, ti_work);
70
71 if (likely(!rseq_exit_to_user_mode_restart(regs)))
72 return ti_work;
73 ti_work = read_thread_flags();
74 }
75}
76
65noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)77noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
66{78{
67 irqentry_state_t ret = {79 irqentry_state_t ret = {
kernel/rseq.c+6−2
@@ -237,7 +237,11 @@ efault: · 236 unmodified lines
237 237
238static void rseq_slowpath_update_usr(struct pt_regs *regs)238static void rseq_slowpath_update_usr(struct pt_regs *regs)
239{239{
240 /* Preserve rseq state and user_irq state for exit to user */240 /*
241 * Preserve rseq state and user_irq state. The generic entry code
242 * clears user_irq on the way out, the non-generic entry
243 * architectures are not having user_irq.
244 */
241 const struct rseq_event evt_mask = { .has_rseq = true, .user_irq = true, };245 const struct rseq_event evt_mask = { .has_rseq = true, .user_irq = true, };
242 struct task_struct *t = current;246 struct task_struct *t = current;
243 struct rseq_ids ids;247 struct rseq_ids ids;
@@ -289,7 +293,7 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs) · 45 unmodified lines
289 }293 }
290}294}
291 295
292void __rseq_handle_notify_resume(struct pt_regs *regs)296void __rseq_handle_slowpath(struct pt_regs *regs)
293{297{
294 /*298 /*
295 * If invoked from hypervisors before entering the guest via299 * If invoked from hypervisors before entering the guest via