On context switch we are locking the vtime seqcount of the scheduling-out task twice: * On vtime_task_switch_common(), when we flush the pending vtime through vtime_account_system() * On arch_vtime_task_switch() to reset the vtime state. This is pointless as these actions can be performed without the need to unlock/lock in the middle. The reason these steps are separated is to consolidate a very small amount of common code between CONFIG_VIRT_CPU_ACCOUNTING_GEN and CONFIG_VIRT_CPU_ACCOUNTING_NATIVE. Performance in this fast path is definitely a priority over artificial code factorization so split the task switch code between GEN and NATIVE and mutualize the parts than can run under a single seqcount locked block. As a side effect, vtime_account_idle() becomes included in the seqcount protection. This happens to be a welcome preparation in order to properly support kcpustat under vtime in the future and fetch CPUTIME_IDLE without race.
Showing the changes to include/linux (1 of 2 files). Show all
Unified · Split
| @@ -14,8 +14,12 @@ struct task_struct; · 13 unmodified lines | ||
| 14 | 14 | * vtime_accounting_cpu_enabled() definitions/declarations |
| 15 | 15 | */ |
| 16 | 16 | #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) |
| 17 | + | |
| 17 | 18 | static inline bool vtime_accounting_cpu_enabled(void) { return true; } |
| 19 | +extern void vtime_task_switch(struct task_struct *prev); | |
| 20 | + | |
| 18 | 21 | #elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN) |
| 22 | + | |
| 19 | 23 | /* |
| 20 | 24 | * Checks if vtime is enabled on some CPU. Cputime readers want to be careful |
| 21 | 25 | * in that case and compute the tickless cputime. |
| @@ -36,33 +40,29 @@ static inline bool vtime_accounting_cpu_enabled(void) · 14 unmodified lines | ||
| 36 | 40 | |
| 37 | 41 | return false; |
| 38 | 42 | } |
| 43 | + | |
| 44 | +extern void vtime_task_switch_generic(struct task_struct *prev); | |
| 45 | + | |
| 46 | +static inline void vtime_task_switch(struct task_struct *prev) | |
| 47 | +{ | |
| 48 | + if (vtime_accounting_cpu_enabled()) | |
| 49 | + vtime_task_switch_generic(prev); | |
| 50 | +} | |
| 51 | + | |
| 39 | 52 | #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ |
| 53 | + | |
| 40 | 54 | static inline bool vtime_accounting_cpu_enabled(void) { return false; } |
| 41 | −#endif | |
| 55 | +static inline void vtime_task_switch(struct task_struct *prev) { } | |
| 42 | 56 | |
| 57 | +#endif | |
| 43 | 58 | |
| 44 | 59 | /* |
| 45 | 60 | * Common vtime APIs |
| 46 | 61 | */ |
| 47 | 62 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
| 48 | − | |
| 49 | −#ifdef __ARCH_HAS_VTIME_TASK_SWITCH | |
| 50 | −extern void vtime_task_switch(struct task_struct *prev); | |
| 51 | −#else | |
| 52 | −extern void vtime_common_task_switch(struct task_struct *prev); | |
| 53 | −static inline void vtime_task_switch(struct task_struct *prev) | |
| 54 | −{ | |
| 55 | − if (vtime_accounting_cpu_enabled()) | |
| 56 | − vtime_common_task_switch(prev); | |
| 57 | −} | |
| 58 | −#endif /* __ARCH_HAS_VTIME_TASK_SWITCH */ | |
| 59 | − | |
| 60 | 63 | extern void vtime_account_kernel(struct task_struct *tsk); |
| 61 | 64 | extern void vtime_account_idle(struct task_struct *tsk); |
| 62 | − | |
| 63 | 65 | #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ |
| 64 | − | |
| 65 | −static inline void vtime_task_switch(struct task_struct *prev) { } | |
| 66 | 66 | static inline void vtime_account_kernel(struct task_struct *tsk) { } |
| 67 | 67 | #endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ |
| 68 | 68 | |