In order to eliminate a function call, make "trace_active" into "disable_stack_tracer" and convert stack_tracer_disable() and friends into static inline functions.
Unified · Split
| @@ -287,8 +287,40 @@ stack_trace_sysctl(struct ctl_table *table, int write, · 286 unmodified lines | ||
| 287 | 287 | void __user *buffer, size_t *lenp, |
| 288 | 288 | loff_t *ppos); |
| 289 | 289 | |
| 290 | −void stack_tracer_disable(void); | |
| 291 | −void stack_tracer_enable(void); | |
| 290 | +/* DO NOT MODIFY THIS VARIABLE DIRECTLY! */ | |
| 291 | +DECLARE_PER_CPU(int, disable_stack_tracer); | |
| 292 | + | |
| 293 | +/** | |
| 294 | + * stack_tracer_disable - temporarily disable the stack tracer | |
| 295 | + * | |
| 296 | + * There's a few locations (namely in RCU) where stack tracing | |
| 297 | + * cannot be executed. This function is used to disable stack | |
| 298 | + * tracing during those critical sections. | |
| 299 | + * | |
| 300 | + * This function must be called with preemption or interrupts | |
| 301 | + * disabled and stack_tracer_enable() must be called shortly after | |
| 302 | + * while preemption or interrupts are still disabled. | |
| 303 | + */ | |
| 304 | +static inline void stack_tracer_disable(void) | |
| 305 | +{ | |
| 306 | + /* Preemption or interupts must be disabled */ | |
| 307 | + if (IS_ENABLED(CONFIG_PREEMPT_DEBUG)) | |
| 308 | + WARN_ON_ONCE(!preempt_count() || !irqs_disabled()); | |
| 309 | + this_cpu_inc(disable_stack_tracer); | |
| 310 | +} | |
| 311 | + | |
| 312 | +/** | |
| 313 | + * stack_tracer_enable - re-enable the stack tracer | |
| 314 | + * | |
| 315 | + * After stack_tracer_disable() is called, stack_tracer_enable() | |
| 316 | + * must be called shortly afterward. | |
| 317 | + */ | |
| 318 | +static inline void stack_tracer_enable(void) | |
| 319 | +{ | |
| 320 | + if (IS_ENABLED(CONFIG_PREEMPT_DEBUG)) | |
| 321 | + WARN_ON_ONCE(!preempt_count() || !irqs_disabled()); | |
| 322 | + this_cpu_dec(disable_stack_tracer); | |
| 323 | +} | |
| 292 | 324 | #else |
| 293 | 325 | static inline void stack_tracer_disable(void) { } |
| 294 | 326 | static inline void stack_tracer_enable(void) { } |
| @@ -35,44 +35,12 @@ unsigned long stack_trace_max_size; · 34 unmodified lines | ||
| 35 | 35 | arch_spinlock_t stack_trace_max_lock = |
| 36 | 36 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
| 37 | 37 | |
| 38 | −static DEFINE_PER_CPU(int, trace_active); | |
| 38 | +DEFINE_PER_CPU(int, disable_stack_tracer); | |
| 39 | 39 | static DEFINE_MUTEX(stack_sysctl_mutex); |
| 40 | 40 | |
| 41 | 41 | int stack_tracer_enabled; |
| 42 | 42 | static int last_stack_tracer_enabled; |
| 43 | 43 | |
| 44 | −/** | |
| 45 | − * stack_tracer_disable - temporarily disable the stack tracer | |
| 46 | − * | |
| 47 | − * There's a few locations (namely in RCU) where stack tracing | |
| 48 | − * cannot be executed. This function is used to disable stack | |
| 49 | − * tracing during those critical sections. | |
| 50 | − * | |
| 51 | − * This function must be called with preemption or interrupts | |
| 52 | − * disabled and stack_tracer_enable() must be called shortly after | |
| 53 | − * while preemption or interrupts are still disabled. | |
| 54 | − */ | |
| 55 | −void stack_tracer_disable(void) | |
| 56 | −{ | |
| 57 | − /* Preemption or interupts must be disabled */ | |
| 58 | − if (IS_ENABLED(CONFIG_PREEMPT_DEBUG)) | |
| 59 | − WARN_ON_ONCE(!preempt_count() || !irqs_disabled()); | |
| 60 | − this_cpu_inc(trace_active); | |
| 61 | −} | |
| 62 | − | |
| 63 | −/** | |
| 64 | − * stack_tracer_enable - re-enable the stack tracer | |
| 65 | − * | |
| 66 | − * After stack_tracer_disable() is called, stack_tracer_enable() | |
| 67 | − * must be called shortly afterward. | |
| 68 | − */ | |
| 69 | −void stack_tracer_enable(void) | |
| 70 | −{ | |
| 71 | − if (IS_ENABLED(CONFIG_PREEMPT_DEBUG)) | |
| 72 | − WARN_ON_ONCE(!preempt_count() || !irqs_disabled()); | |
| 73 | − this_cpu_dec(trace_active); | |
| 74 | −} | |
| 75 | − | |
| 76 | 44 | void stack_trace_print(void) |
| 77 | 45 | { |
| 78 | 46 | long i; |
| @@ -243,8 +211,8 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip, · 164 unmodified lines | ||
| 243 | 211 | preempt_disable_notrace(); |
| 244 | 212 | |
| 245 | 213 | /* no atomic needed, we only modify this variable by this cpu */ |
| 246 | − __this_cpu_inc(trace_active); | |
| 247 | − if (__this_cpu_read(trace_active) != 1) | |
| 214 | + __this_cpu_inc(disable_stack_tracer); | |
| 215 | + if (__this_cpu_read(disable_stack_tracer) != 1) | |
| 248 | 216 | goto out; |
| 249 | 217 | |
| 250 | 218 | ip += MCOUNT_INSN_SIZE; |
| @@ -252,7 +220,7 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip, · 1 unmodified line | ||
| 252 | 220 | check_stack(ip, &stack); |
| 253 | 221 | |
| 254 | 222 | out: |
| 255 | − __this_cpu_dec(trace_active); | |
| 223 | + __this_cpu_dec(disable_stack_tracer); | |
| 256 | 224 | /* prevent recursion in schedule */ |
| 257 | 225 | preempt_enable_notrace(); |
| 258 | 226 | } |
| @@ -294,15 +262,15 @@ stack_max_size_write(struct file *filp, const char __user *ubuf, · 35 unmodified lines | ||
| 294 | 262 | /* |
| 295 | 263 | * In case we trace inside arch_spin_lock() or after (NMI), |
| 296 | 264 | * we will cause circular lock, so we also need to increase |
| 297 | − * the percpu trace_active here. | |
| 265 | + * the percpu disable_stack_tracer here. | |
| 298 | 266 | */ |
| 299 | − __this_cpu_inc(trace_active); | |
| 267 | + __this_cpu_inc(disable_stack_tracer); | |
| 300 | 268 | |
| 301 | 269 | arch_spin_lock(&stack_trace_max_lock); |
| 302 | 270 | *ptr = val; |
| 303 | 271 | arch_spin_unlock(&stack_trace_max_lock); |
| 304 | 272 | |
| 305 | − __this_cpu_dec(trace_active); | |
| 273 | + __this_cpu_dec(disable_stack_tracer); | |
| 306 | 274 | local_irq_restore(flags); |
| 307 | 275 | |
| 308 | 276 | return count; |
| @@ -338,7 +306,7 @@ static void *t_start(struct seq_file *m, loff_t *pos) · 29 unmodified lines | ||
| 338 | 306 | { |
| 339 | 307 | local_irq_disable(); |
| 340 | 308 | |
| 341 | − __this_cpu_inc(trace_active); | |
| 309 | + __this_cpu_inc(disable_stack_tracer); | |
| 342 | 310 | |
| 343 | 311 | arch_spin_lock(&stack_trace_max_lock); |
| 344 | 312 | |
| @@ -352,7 +320,7 @@ static void t_stop(struct seq_file *m, void *p) · 7 unmodified lines | ||
| 352 | 320 | { |
| 353 | 321 | arch_spin_unlock(&stack_trace_max_lock); |
| 354 | 322 | |
| 355 | − __this_cpu_dec(trace_active); | |
| 323 | + __this_cpu_dec(disable_stack_tracer); | |
| 356 | 324 | |
| 357 | 325 | local_irq_enable(); |
| 358 | 326 | } |