On context switch we are locking the vtime seqcount of the scheduling-out task twice: * On vtime_task_switch_common(), when we flush the pending vtime through vtime_account_system() * On arch_vtime_task_switch() to reset the vtime state. This is pointless as these actions can be performed without the need to unlock/lock in the middle. The reason these steps are separated is to consolidate a very small amount of common code between CONFIG_VIRT_CPU_ACCOUNTING_GEN and CONFIG_VIRT_CPU_ACCOUNTING_NATIVE. Performance in this fast path is definitely a priority over artificial code factorization so split the task switch code between GEN and NATIVE and mutualize the parts than can run under a single seqcount locked block. As a side effect, vtime_account_idle() becomes included in the seqcount protection. This happens to be a welcome preparation in order to properly support kcpustat under vtime in the future and fetch CPUTIME_IDLE without race.
Showing the changes to include/linux (1 of 2 files). Show all
Unified · Split
| @@ -14,8 +14,12 @@ struct task_struct; · 13 unmodified lines | |||
| 14 | * vtime_accounting_cpu_enabled() definitions/declarations | 14 | * vtime_accounting_cpu_enabled() definitions/declarations |
| 15 | */ | 15 | */ |
| 16 | #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) | 16 | #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) |
| 17 | |||
| 17 | static inline bool vtime_accounting_cpu_enabled(void) { return true; } | 18 | static inline bool vtime_accounting_cpu_enabled(void) { return true; } |
| 19 | extern void vtime_task_switch(struct task_struct *prev); | ||
| 20 | |||
| 18 | #elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN) | 21 | #elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN) |
| 22 | |||
| 19 | /* | 23 | /* |
| 20 | * Checks if vtime is enabled on some CPU. Cputime readers want to be careful | 24 | * Checks if vtime is enabled on some CPU. Cputime readers want to be careful |
| 21 | * in that case and compute the tickless cputime. | 25 | * in that case and compute the tickless cputime. |
| @@ -36,33 +40,29 @@ static inline bool vtime_accounting_cpu_enabled(void) · 14 unmodified lines | |||
| 36 | 40 | ||
| 37 | return false; | 41 | return false; |
| 38 | } | 42 | } |
| 43 | |||
| 44 | extern void vtime_task_switch_generic(struct task_struct *prev); | ||
| 45 | |||
| 46 | static inline void vtime_task_switch(struct task_struct *prev) | ||
| 47 | { | ||
| 48 | if (vtime_accounting_cpu_enabled()) | ||
| 49 | vtime_task_switch_generic(prev); | ||
| 50 | } | ||
| 51 | |||
| 39 | #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ | 52 | #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ |
| 53 | |||
| 40 | static inline bool vtime_accounting_cpu_enabled(void) { return false; } | 54 | static inline bool vtime_accounting_cpu_enabled(void) { return false; } |
| 41 | #endif | 55 | static inline void vtime_task_switch(struct task_struct *prev) { } |
| 42 | 56 | ||
| 57 | #endif | ||
| 43 | 58 | ||
| 44 | /* | 59 | /* |
| 45 | * Common vtime APIs | 60 | * Common vtime APIs |
| 46 | */ | 61 | */ |
| 47 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING | 62 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
| 48 | |||
| 49 | #ifdef __ARCH_HAS_VTIME_TASK_SWITCH | ||
| 50 | extern void vtime_task_switch(struct task_struct *prev); | ||
| 51 | #else | ||
| 52 | extern void vtime_common_task_switch(struct task_struct *prev); | ||
| 53 | static inline void vtime_task_switch(struct task_struct *prev) | ||
| 54 | { | ||
| 55 | if (vtime_accounting_cpu_enabled()) | ||
| 56 | vtime_common_task_switch(prev); | ||
| 57 | } | ||
| 58 | #endif /* __ARCH_HAS_VTIME_TASK_SWITCH */ | ||
| 59 | |||
| 60 | extern void vtime_account_kernel(struct task_struct *tsk); | 63 | extern void vtime_account_kernel(struct task_struct *tsk); |
| 61 | extern void vtime_account_idle(struct task_struct *tsk); | 64 | extern void vtime_account_idle(struct task_struct *tsk); |
| 62 | |||
| 63 | #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ | 65 | #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ |
| 64 | |||
| 65 | static inline void vtime_task_switch(struct task_struct *prev) { } | ||
| 66 | static inline void vtime_account_kernel(struct task_struct *tsk) { } | 66 | static inline void vtime_account_kernel(struct task_struct *tsk) { } |
| 67 | #endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ | 67 | #endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ |
| 68 | 68 | ||