Because tasks do not nest, rcu_idle_enter() and rcu_idle_exit() do not need to check for nesting. This commit therefore moves nesting checks from rcu_idle_enter_common() to rcu_irq_exit() and from rcu_idle_exit_common() to rcu_irq_enter().
Unified · Split
| @@ -350,10 +350,6 @@ static int rcu_implicit_offline_qs(struct rcu_data *rdp) · 349 unmodified lines | ||
| 350 | 350 | */ |
| 351 | 351 | static void rcu_idle_enter_common(struct rcu_dynticks *rdtp, long long oldval) |
| 352 | 352 | { |
| 353 | − if (rdtp->dynticks_nesting) { | |
| 354 | − trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting); | |
| 355 | − return; | |
| 356 | − } | |
| 357 | 353 | trace_rcu_dyntick("Start", oldval, rdtp->dynticks_nesting); |
| 358 | 354 | if (!is_idle_task(current)) { |
| 359 | 355 | struct task_struct *idle = idle_task(smp_processor_id()); |
| @@ -426,7 +422,10 @@ void rcu_irq_exit(void) · 66 unmodified lines | ||
| 426 | 422 | oldval = rdtp->dynticks_nesting; |
| 427 | 423 | rdtp->dynticks_nesting--; |
| 428 | 424 | WARN_ON_ONCE(rdtp->dynticks_nesting < 0); |
| 429 | − rcu_idle_enter_common(rdtp, oldval); | |
| 425 | + if (rdtp->dynticks_nesting) | |
| 426 | + trace_rcu_dyntick("--=", oldval, rdtp->dynticks_nesting); | |
| 427 | + else | |
| 428 | + rcu_idle_enter_common(rdtp, oldval); | |
| 430 | 429 | local_irq_restore(flags); |
| 431 | 430 | } |
| 432 | 431 | |
| @@ -439,10 +438,6 @@ void rcu_irq_exit(void) · 6 unmodified lines | ||
| 439 | 438 | */ |
| 440 | 439 | static void rcu_idle_exit_common(struct rcu_dynticks *rdtp, long long oldval) |
| 441 | 440 | { |
| 442 | − if (oldval) { | |
| 443 | − trace_rcu_dyntick("++=", oldval, rdtp->dynticks_nesting); | |
| 444 | − return; | |
| 445 | − } | |
| 446 | 441 | smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */ |
| 447 | 442 | atomic_inc(&rdtp->dynticks); |
| 448 | 443 | /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ |
| @@ -518,7 +513,10 @@ void rcu_irq_enter(void) · 69 unmodified lines | ||
| 518 | 513 | oldval = rdtp->dynticks_nesting; |
| 519 | 514 | rdtp->dynticks_nesting++; |
| 520 | 515 | WARN_ON_ONCE(rdtp->dynticks_nesting == 0); |
| 521 | − rcu_idle_exit_common(rdtp, oldval); | |
| 516 | + if (oldval) | |
| 517 | + trace_rcu_dyntick("++=", oldval, rdtp->dynticks_nesting); | |
| 518 | + else | |
| 519 | + rcu_idle_exit_common(rdtp, oldval); | |
| 522 | 520 | local_irq_restore(flags); |
| 523 | 521 | } |
| 524 | 522 | |