History

rcu: Don't use negative nesting depth in __rcu_read_unlock()

Now that RCU flavors have been consolidated, an RCU-preempt rcu_read_unlock() in an interrupt or softirq handler cannot possibly end the RCU read-side critical section. Consider the old vulnerability involving rcu_read_unlock() being invoked within such a handler that interrupted an __rcu_read_unlock_special(), in which a wakeup might be invoked with a scheduler lock held. Because rcu_read_unlock_special() no longer does wakeups in such situations, it is no longer necessary for __rcu_read_unlock() to set the nesting level negative. This commit therefore removes this recursion-protection code from __rcu_read_unlock(). [ paulmck: Let rcu_exp_handler() continue to call rcu_report_exp_rdp(). ] [ paulmck: Adjust other checks given no more negative nesting. ] Signed-off-by: Lai Jiangshan <laijs@linux.alibaba.com> Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

Lai Jiangshan committed 6 years ago

Commit 5f5fa7e1 parent f0bdf6d2 files · +12 −41Browse files

Unified · Split

2 files
  1. kernel/rcu/tree_exp.h +5 −26
  2. kernel/rcu/tree_plugin.h +7 −15
kernel/rcu/tree_exp.h+5−26
@@ -639,6 +639,7 @@ static void wait_rcu_exp_gp(struct work_struct *wp) · 638 unmodified lines
639 */639 */
640static void rcu_exp_handler(void *unused)640static void rcu_exp_handler(void *unused)
641{641{
642 int depth = rcu_preempt_depth();
642 unsigned long flags;643 unsigned long flags;
643 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);644 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
644 struct rcu_node *rnp = rdp->mynode;645 struct rcu_node *rnp = rdp->mynode;
@@ -649,7 +650,7 @@ static void rcu_exp_handler(void *unused) · 4 unmodified lines
649 * critical section. If also enabled or idle, immediately650 * critical section. If also enabled or idle, immediately
650 * report the quiescent state, otherwise defer.651 * report the quiescent state, otherwise defer.
651 */652 */
652 if (!rcu_preempt_depth()) {653 if (!depth) {
653 if (!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)) ||654 if (!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)) ||
654 rcu_dynticks_curr_cpu_in_eqs()) {655 rcu_dynticks_curr_cpu_in_eqs()) {
655 rcu_report_exp_rdp(rdp);656 rcu_report_exp_rdp(rdp);
@@ -673,7 +674,7 @@ static void rcu_exp_handler(void *unused) · 17 unmodified lines
673 * can have caused this quiescent state to already have been674 * can have caused this quiescent state to already have been
674 * reported, so we really do need to check ->expmask.675 * reported, so we really do need to check ->expmask.
675 */676 */
676 if (rcu_preempt_depth() > 0) {677 if (depth > 0) {
677 raw_spin_lock_irqsave_rcu_node(rnp, flags);678 raw_spin_lock_irqsave_rcu_node(rnp, flags);
678 if (rnp->expmask & rdp->grpmask) {679 if (rnp->expmask & rdp->grpmask) {
679 rdp->exp_deferred_qs = true;680 rdp->exp_deferred_qs = true;
@@ -683,30 +684,8 @@ static void rcu_exp_handler(void *unused) · 3 unmodified lines
683 return;684 return;
684 }685 }
685 686
686 /*687 // Finally, negative nesting depth should not happen.
687 * The final and least likely case is where the interrupted688 WARN_ON_ONCE(1);
688 * code was just about to or just finished exiting the RCU-preempt
689 * read-side critical section, and no, we can't tell which.
690 * So either way, set ->deferred_qs to flag later code that
691 * a quiescent state is required.
692 *
693 * If the CPU is fully enabled (or if some buggy RCU-preempt
694 * read-side critical section is being used from idle), just
695 * invoke rcu_preempt_deferred_qs() to immediately report the
696 * quiescent state. We cannot use rcu_read_unlock_special()
697 * because we are in an interrupt handler, which will cause that
698 * function to take an early exit without doing anything.
699 *
700 * Otherwise, force a context switch after the CPU enables everything.
701 */
702 rdp->exp_deferred_qs = true;
703 if (!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)) ||
704 WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs())) {
705 rcu_preempt_deferred_qs(t);
706 } else {
707 set_tsk_need_resched(t);
708 set_preempt_need_resched();
709 }
710}689}
711 690
712/* PREEMPTION=y, so no PREEMPTION=n expedited grace period to clean up after. */691/* PREEMPTION=y, so no PREEMPTION=n expedited grace period to clean up after. */
kernel/rcu/tree_plugin.h+7−15
@@ -345,9 +345,7 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) · 344 unmodified lines
345 return READ_ONCE(rnp->gp_tasks) != NULL;345 return READ_ONCE(rnp->gp_tasks) != NULL;
346}346}
347 347
348/* Bias and limit values for ->rcu_read_lock_nesting. */348/* limit value for ->rcu_read_lock_nesting. */
349#define RCU_NEST_BIAS INT_MAX
350#define RCU_NEST_NMAX (-INT_MAX / 2)
351#define RCU_NEST_PMAX (INT_MAX / 2)349#define RCU_NEST_PMAX (INT_MAX / 2)
352 350
353static void rcu_preempt_read_enter(void)351static void rcu_preempt_read_enter(void)
@@ -355,9 +353,9 @@ static void rcu_preempt_read_enter(void) · 1 unmodified line
355 current->rcu_read_lock_nesting++;353 current->rcu_read_lock_nesting++;
356}354}
357 355
358static void rcu_preempt_read_exit(void)356static int rcu_preempt_read_exit(void)
359{357{
360 current->rcu_read_lock_nesting--;358 return --current->rcu_read_lock_nesting;
361}359}
362 360
363static void rcu_preempt_depth_set(int val)361static void rcu_preempt_depth_set(int val)
@@ -390,21 +388,15 @@ void __rcu_read_unlock(void) · 26 unmodified lines
390{388{
391 struct task_struct *t = current;389 struct task_struct *t = current;
392 390
393 if (rcu_preempt_depth() != 1) {391 if (rcu_preempt_read_exit() == 0) {
394 rcu_preempt_read_exit();
395 } else {
396 barrier(); /* critical section before exit code. */392 barrier(); /* critical section before exit code. */
397 rcu_preempt_depth_set(-RCU_NEST_BIAS);
398 barrier(); /* assign before ->rcu_read_unlock_special load */
399 if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s)))393 if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s)))
400 rcu_read_unlock_special(t);394 rcu_read_unlock_special(t);
401 barrier(); /* ->rcu_read_unlock_special load before assign */
402 rcu_preempt_depth_set(0);
403 }395 }
404 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {396 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
405 int rrln = rcu_preempt_depth();397 int rrln = rcu_preempt_depth();
406 398
407 WARN_ON_ONCE(rrln < 0 && rrln > RCU_NEST_NMAX);399 WARN_ON_ONCE(rrln < 0 || rrln > RCU_NEST_PMAX);
408 }400 }
409}401}
410EXPORT_SYMBOL_GPL(__rcu_read_unlock);402EXPORT_SYMBOL_GPL(__rcu_read_unlock);
@@ -556,7 +548,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t) · 145 unmodified lines
556{548{
557 return (__this_cpu_read(rcu_data.exp_deferred_qs) ||549 return (__this_cpu_read(rcu_data.exp_deferred_qs) ||
558 READ_ONCE(t->rcu_read_unlock_special.s)) &&550 READ_ONCE(t->rcu_read_unlock_special.s)) &&
559 rcu_preempt_depth() <= 0;551 rcu_preempt_depth() == 0;
560}552}
561 553
562/*554/*
@@ -692,7 +684,7 @@ static void rcu_flavor_sched_clock_irq(int user) · 129 unmodified lines
692 } else if (rcu_preempt_need_deferred_qs(t)) {684 } else if (rcu_preempt_need_deferred_qs(t)) {
693 rcu_preempt_deferred_qs(t); /* Report deferred QS. */685 rcu_preempt_deferred_qs(t); /* Report deferred QS. */
694 return;686 return;
695 } else if (!rcu_preempt_depth()) {687 } else if (!WARN_ON_ONCE(rcu_preempt_depth())) {
696 rcu_qs(); /* Report immediate QS. */688 rcu_qs(); /* Report immediate QS. */
697 return;689 return;
698 }690 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t