Currently CSD lock debugging can be switched on and off via a kernel config option only. Unfortunately there is at least one problem with CSD lock handling pending for about 2 years now, which has been seen in different environments (mostly when running virtualized under KVM or Xen, at least once on bare metal). Multiple attempts to catch this issue have finally led to introduction of CSD lock debug code, but this code is not in use in most distros as it has some impact on performance. In order to be able to ship kernels with CONFIG_CSD_LOCK_WAIT_DEBUG enabled even for production use, add a boot parameter for switching the debug functionality on. This will reduce any performance impact of the debug coding to a bare minimum when not being used.
Showing the changes to kernel (1 of 2 files). Show all
Unified · Split
| @@ -24,6 +24,7 @@ · 23 unmodified lines | |||
| 24 | #include <linux/sched/clock.h> | 24 | #include <linux/sched/clock.h> |
| 25 | #include <linux/nmi.h> | 25 | #include <linux/nmi.h> |
| 26 | #include <linux/sched/debug.h> | 26 | #include <linux/sched/debug.h> |
| 27 | #include <linux/jump_label.h> | ||
| 27 | 28 | ||
| 28 | #include "smpboot.h" | 29 | #include "smpboot.h" |
| 29 | #include "sched/smp.h" | 30 | #include "sched/smp.h" |
| @@ -102,6 +103,20 @@ void __init call_function_init(void) · 72 unmodified lines | |||
| 102 | 103 | ||
| 103 | #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG | 104 | #ifdef CONFIG_CSD_LOCK_WAIT_DEBUG |
| 104 | 105 | ||
| 106 | static DEFINE_STATIC_KEY_FALSE(csdlock_debug_enabled); | ||
| 107 | |||
| 108 | static int __init csdlock_debug(char *str) | ||
| 109 | { | ||
| 110 | unsigned int val = 0; | ||
| 111 | |||
| 112 | get_option(&str, &val); | ||
| 113 | if (val) | ||
| 114 | static_branch_enable(&csdlock_debug_enabled); | ||
| 115 | |||
| 116 | return 0; | ||
| 117 | } | ||
| 118 | early_param("csdlock_debug", csdlock_debug); | ||
| 119 | |||
| 105 | static DEFINE_PER_CPU(call_single_data_t *, cur_csd); | 120 | static DEFINE_PER_CPU(call_single_data_t *, cur_csd); |
| 106 | static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func); | 121 | static DEFINE_PER_CPU(smp_call_func_t, cur_csd_func); |
| 107 | static DEFINE_PER_CPU(void *, cur_csd_info); | 122 | static DEFINE_PER_CPU(void *, cur_csd_info); |
| @@ -110,7 +125,7 @@ static DEFINE_PER_CPU(void *, cur_csd_info); · 2 unmodified lines | |||
| 110 | static atomic_t csd_bug_count = ATOMIC_INIT(0); | 125 | static atomic_t csd_bug_count = ATOMIC_INIT(0); |
| 111 | 126 | ||
| 112 | /* Record current CSD work for current CPU, NULL to erase. */ | 127 | /* Record current CSD work for current CPU, NULL to erase. */ |
| 113 | static void csd_lock_record(call_single_data_t *csd) | 128 | static void __csd_lock_record(call_single_data_t *csd) |
| 114 | { | 129 | { |
| 115 | if (!csd) { | 130 | if (!csd) { |
| 116 | smp_mb(); /* NULL cur_csd after unlock. */ | 131 | smp_mb(); /* NULL cur_csd after unlock. */ |
| @@ -125,7 +140,13 @@ static void csd_lock_record(call_single_data_t *csd) · 8 unmodified lines | |||
| 125 | /* Or before unlock, as the case may be. */ | 140 | /* Or before unlock, as the case may be. */ |
| 126 | } | 141 | } |
| 127 | 142 | ||
| 128 | static __always_inline int csd_lock_wait_getcpu(call_single_data_t *csd) | 143 | static __always_inline void csd_lock_record(call_single_data_t *csd) |
| 144 | { | ||
| 145 | if (static_branch_unlikely(&csdlock_debug_enabled)) | ||
| 146 | __csd_lock_record(csd); | ||
| 147 | } | ||
| 148 | |||
| 149 | static int csd_lock_wait_getcpu(call_single_data_t *csd) | ||
| 129 | { | 150 | { |
| 130 | unsigned int csd_type; | 151 | unsigned int csd_type; |
| 131 | 152 | ||
| @@ -140,7 +161,7 @@ static __always_inline int csd_lock_wait_getcpu(call_single_data_t *csd) · 8 unmodified lines | |||
| 140 | * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, | 161 | * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, |
| 141 | * so waiting on other types gets much less information. | 162 | * so waiting on other types gets much less information. |
| 142 | */ | 163 | */ |
| 143 | static __always_inline bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id) | 164 | static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id) |
| 144 | { | 165 | { |
| 145 | int cpu = -1; | 166 | int cpu = -1; |
| 146 | int cpux; | 167 | int cpux; |
| @@ -204,7 +225,7 @@ static __always_inline bool csd_lock_wait_toolong(call_single_data_t *csd, u64 t · 57 unmodified lines | |||
| 204 | * previous function call. For multi-cpu calls its even more interesting | 225 | * previous function call. For multi-cpu calls its even more interesting |
| 205 | * as we'll have to ensure no other cpu is observing our csd. | 226 | * as we'll have to ensure no other cpu is observing our csd. |
| 206 | */ | 227 | */ |
| 207 | static __always_inline void csd_lock_wait(call_single_data_t *csd) | 228 | static void __csd_lock_wait(call_single_data_t *csd) |
| 208 | { | 229 | { |
| 209 | int bug_id = 0; | 230 | int bug_id = 0; |
| 210 | u64 ts0, ts1; | 231 | u64 ts0, ts1; |
| @@ -218,6 +239,15 @@ static __always_inline void csd_lock_wait(call_single_data_t *csd) · 7 unmodified lines | |||
| 218 | smp_acquire__after_ctrl_dep(); | 239 | smp_acquire__after_ctrl_dep(); |
| 219 | } | 240 | } |
| 220 | 241 | ||
| 242 | static __always_inline void csd_lock_wait(call_single_data_t *csd) | ||
| 243 | { | ||
| 244 | if (static_branch_unlikely(&csdlock_debug_enabled)) { | ||
| 245 | __csd_lock_wait(csd); | ||
| 246 | return; | ||
| 247 | } | ||
| 248 | |||
| 249 | smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); | ||
| 250 | } | ||
| 221 | #else | 251 | #else |
| 222 | static void csd_lock_record(call_single_data_t *csd) | 252 | static void csd_lock_record(call_single_data_t *csd) |
| 223 | { | 253 | { |