Now that RCU flavors have been consolidated, an RCU-preempt rcu_read_unlock() in an interrupt or softirq handler cannot possibly end the RCU read-side critical section. Consider the old vulnerability involving rcu_read_unlock() being invoked within such a handler that interrupted an __rcu_read_unlock_special(), in which a wakeup might be invoked with a scheduler lock held. Because rcu_read_unlock_special() no longer does wakeups in such situations, it is no longer necessary for __rcu_read_unlock() to set the nesting level negative. This commit therefore removes this recursion-protection code from __rcu_read_unlock(). [ paulmck: Let rcu_exp_handler() continue to call rcu_report_exp_rdp(). ] [ paulmck: Adjust other checks given no more negative nesting. ] Signed-off-by: Lai Jiangshan <laijs@linux.alibaba.com> Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Unified · Split
| @@ -639,6 +639,7 @@ static void wait_rcu_exp_gp(struct work_struct *wp) · 638 unmodified lines | |||
| 639 | */ | 639 | */ |
| 640 | static void rcu_exp_handler(void *unused) | 640 | static void rcu_exp_handler(void *unused) |
| 641 | { | 641 | { |
| 642 | int depth = rcu_preempt_depth(); | ||
| 642 | unsigned long flags; | 643 | unsigned long flags; |
| 643 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); | 644 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); |
| 644 | struct rcu_node *rnp = rdp->mynode; | 645 | struct rcu_node *rnp = rdp->mynode; |
| @@ -649,7 +650,7 @@ static void rcu_exp_handler(void *unused) · 4 unmodified lines | |||
| 649 | * critical section. If also enabled or idle, immediately | 650 | * critical section. If also enabled or idle, immediately |
| 650 | * report the quiescent state, otherwise defer. | 651 | * report the quiescent state, otherwise defer. |
| 651 | */ | 652 | */ |
| 652 | if (!rcu_preempt_depth()) { | 653 | if (!depth) { |
| 653 | if (!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)) || | 654 | if (!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)) || |
| 654 | rcu_dynticks_curr_cpu_in_eqs()) { | 655 | rcu_dynticks_curr_cpu_in_eqs()) { |
| 655 | rcu_report_exp_rdp(rdp); | 656 | rcu_report_exp_rdp(rdp); |
| @@ -673,7 +674,7 @@ static void rcu_exp_handler(void *unused) · 17 unmodified lines | |||
| 673 | * can have caused this quiescent state to already have been | 674 | * can have caused this quiescent state to already have been |
| 674 | * reported, so we really do need to check ->expmask. | 675 | * reported, so we really do need to check ->expmask. |
| 675 | */ | 676 | */ |
| 676 | if (rcu_preempt_depth() > 0) { | 677 | if (depth > 0) { |
| 677 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | 678 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
| 678 | if (rnp->expmask & rdp->grpmask) { | 679 | if (rnp->expmask & rdp->grpmask) { |
| 679 | rdp->exp_deferred_qs = true; | 680 | rdp->exp_deferred_qs = true; |
| @@ -683,30 +684,8 @@ static void rcu_exp_handler(void *unused) · 3 unmodified lines | |||
| 683 | return; | 684 | return; |
| 684 | } | 685 | } |
| 685 | 686 | ||
| 686 | /* | 687 | // Finally, negative nesting depth should not happen. |
| 687 | * The final and least likely case is where the interrupted | 688 | WARN_ON_ONCE(1); |
| 688 | * code was just about to or just finished exiting the RCU-preempt | ||
| 689 | * read-side critical section, and no, we can't tell which. | ||
| 690 | * So either way, set ->deferred_qs to flag later code that | ||
| 691 | * a quiescent state is required. | ||
| 692 | * | ||
| 693 | * If the CPU is fully enabled (or if some buggy RCU-preempt | ||
| 694 | * read-side critical section is being used from idle), just | ||
| 695 | * invoke rcu_preempt_deferred_qs() to immediately report the | ||
| 696 | * quiescent state. We cannot use rcu_read_unlock_special() | ||
| 697 | * because we are in an interrupt handler, which will cause that | ||
| 698 | * function to take an early exit without doing anything. | ||
| 699 | * | ||
| 700 | * Otherwise, force a context switch after the CPU enables everything. | ||
| 701 | */ | ||
| 702 | rdp->exp_deferred_qs = true; | ||
| 703 | if (!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)) || | ||
| 704 | WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs())) { | ||
| 705 | rcu_preempt_deferred_qs(t); | ||
| 706 | } else { | ||
| 707 | set_tsk_need_resched(t); | ||
| 708 | set_preempt_need_resched(); | ||
| 709 | } | ||
| 710 | } | 689 | } |
| 711 | 690 | ||
| 712 | /* PREEMPTION=y, so no PREEMPTION=n expedited grace period to clean up after. */ | 691 | /* PREEMPTION=y, so no PREEMPTION=n expedited grace period to clean up after. */ |
| @@ -345,9 +345,7 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) · 344 unmodified lines | |||
| 345 | return READ_ONCE(rnp->gp_tasks) != NULL; | 345 | return READ_ONCE(rnp->gp_tasks) != NULL; |
| 346 | } | 346 | } |
| 347 | 347 | ||
| 348 | /* Bias and limit values for ->rcu_read_lock_nesting. */ | 348 | /* limit value for ->rcu_read_lock_nesting. */ |
| 349 | #define RCU_NEST_BIAS INT_MAX | ||
| 350 | #define RCU_NEST_NMAX (-INT_MAX / 2) | ||
| 351 | #define RCU_NEST_PMAX (INT_MAX / 2) | 349 | #define RCU_NEST_PMAX (INT_MAX / 2) |
| 352 | 350 | ||
| 353 | static void rcu_preempt_read_enter(void) | 351 | static void rcu_preempt_read_enter(void) |
| @@ -355,9 +353,9 @@ static void rcu_preempt_read_enter(void) · 1 unmodified line | |||
| 355 | current->rcu_read_lock_nesting++; | 353 | current->rcu_read_lock_nesting++; |
| 356 | } | 354 | } |
| 357 | 355 | ||
| 358 | static void rcu_preempt_read_exit(void) | 356 | static int rcu_preempt_read_exit(void) |
| 359 | { | 357 | { |
| 360 | current->rcu_read_lock_nesting--; | 358 | return --current->rcu_read_lock_nesting; |
| 361 | } | 359 | } |
| 362 | 360 | ||
| 363 | static void rcu_preempt_depth_set(int val) | 361 | static void rcu_preempt_depth_set(int val) |
| @@ -390,21 +388,15 @@ void __rcu_read_unlock(void) · 26 unmodified lines | |||
| 390 | { | 388 | { |
| 391 | struct task_struct *t = current; | 389 | struct task_struct *t = current; |
| 392 | 390 | ||
| 393 | if (rcu_preempt_depth() != 1) { | 391 | if (rcu_preempt_read_exit() == 0) { |
| 394 | rcu_preempt_read_exit(); | ||
| 395 | } else { | ||
| 396 | barrier(); /* critical section before exit code. */ | 392 | barrier(); /* critical section before exit code. */ |
| 397 | rcu_preempt_depth_set(-RCU_NEST_BIAS); | ||
| 398 | barrier(); /* assign before ->rcu_read_unlock_special load */ | ||
| 399 | if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s))) | 393 | if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s))) |
| 400 | rcu_read_unlock_special(t); | 394 | rcu_read_unlock_special(t); |
| 401 | barrier(); /* ->rcu_read_unlock_special load before assign */ | ||
| 402 | rcu_preempt_depth_set(0); | ||
| 403 | } | 395 | } |
| 404 | if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { | 396 | if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { |
| 405 | int rrln = rcu_preempt_depth(); | 397 | int rrln = rcu_preempt_depth(); |
| 406 | 398 | ||
| 407 | WARN_ON_ONCE(rrln < 0 && rrln > RCU_NEST_NMAX); | 399 | WARN_ON_ONCE(rrln < 0 || rrln > RCU_NEST_PMAX); |
| 408 | } | 400 | } |
| 409 | } | 401 | } |
| 410 | EXPORT_SYMBOL_GPL(__rcu_read_unlock); | 402 | EXPORT_SYMBOL_GPL(__rcu_read_unlock); |
| @@ -556,7 +548,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) · 145 unmodified lines | |||
| 556 | { | 548 | { |
| 557 | return (__this_cpu_read(rcu_data.exp_deferred_qs) || | 549 | return (__this_cpu_read(rcu_data.exp_deferred_qs) || |
| 558 | READ_ONCE(t->rcu_read_unlock_special.s)) && | 550 | READ_ONCE(t->rcu_read_unlock_special.s)) && |
| 559 | rcu_preempt_depth() <= 0; | 551 | rcu_preempt_depth() == 0; |
| 560 | } | 552 | } |
| 561 | 553 | ||
| 562 | /* | 554 | /* |
| @@ -692,7 +684,7 @@ static void rcu_flavor_sched_clock_irq(int user) · 129 unmodified lines | |||
| 692 | } else if (rcu_preempt_need_deferred_qs(t)) { | 684 | } else if (rcu_preempt_need_deferred_qs(t)) { |
| 693 | rcu_preempt_deferred_qs(t); /* Report deferred QS. */ | 685 | rcu_preempt_deferred_qs(t); /* Report deferred QS. */ |
| 694 | return; | 686 | return; |
| 695 | } else if (!rcu_preempt_depth()) { | 687 | } else if (!WARN_ON_ONCE(rcu_preempt_depth())) { |
| 696 | rcu_qs(); /* Report immediate QS. */ | 688 | rcu_qs(); /* Report immediate QS. */ |
| 697 | return; | 689 | return; |
| 698 | } | 690 | } |