It is illegal to have a grace period within a same-flavor RCU read-side critical section, so this commit adds lockdep-RCU checks to splat when such abuse is encountered. This commit does not detect more elaborate RCU deadlock situations. These situations might be a job for lockdep enhancements.
Unified · Split
| @@ -319,6 +319,10 @@ static void rcu_process_callbacks(struct softirq_action *unused) · 318 unmodified lines | |||
| 319 | */ | 319 | */ |
| 320 | void synchronize_sched(void) | 320 | void synchronize_sched(void) |
| 321 | { | 321 | { |
| 322 | rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && | ||
| 323 | !lock_is_held(&rcu_lock_map) && | ||
| 324 | !lock_is_held(&rcu_sched_lock_map), | ||
| 325 | "Illegal synchronize_sched() in RCU read-side critical section"); | ||
| 322 | cond_resched(); | 326 | cond_resched(); |
| 323 | } | 327 | } |
| 324 | EXPORT_SYMBOL_GPL(synchronize_sched); | 328 | EXPORT_SYMBOL_GPL(synchronize_sched); |
| @@ -706,6 +706,11 @@ EXPORT_SYMBOL_GPL(call_rcu); · 705 unmodified lines | |||
| 706 | */ | 706 | */ |
| 707 | void synchronize_rcu(void) | 707 | void synchronize_rcu(void) |
| 708 | { | 708 | { |
| 709 | rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && | ||
| 710 | !lock_is_held(&rcu_lock_map) && | ||
| 711 | !lock_is_held(&rcu_sched_lock_map), | ||
| 712 | "Illegal synchronize_rcu() in RCU read-side critical section"); | ||
| 713 | |||
| 709 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 714 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 710 | if (!rcu_scheduler_active) | 715 | if (!rcu_scheduler_active) |
| 711 | return; | 716 | return; |
| @@ -1816,6 +1816,10 @@ EXPORT_SYMBOL_GPL(call_rcu_bh); · 1815 unmodified lines | |||
| 1816 | */ | 1816 | */ |
| 1817 | void synchronize_sched(void) | 1817 | void synchronize_sched(void) |
| 1818 | { | 1818 | { |
| 1819 | rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && | ||
| 1820 | !lock_is_held(&rcu_lock_map) && | ||
| 1821 | !lock_is_held(&rcu_sched_lock_map), | ||
| 1822 | "Illegal synchronize_sched() in RCU-sched read-side critical section"); | ||
| 1819 | if (rcu_blocking_is_gp()) | 1823 | if (rcu_blocking_is_gp()) |
| 1820 | return; | 1824 | return; |
| 1821 | wait_rcu_gp(call_rcu_sched); | 1825 | wait_rcu_gp(call_rcu_sched); |
| @@ -1833,6 +1837,10 @@ EXPORT_SYMBOL_GPL(synchronize_sched); · 11 unmodified lines | |||
| 1833 | */ | 1837 | */ |
| 1834 | void synchronize_rcu_bh(void) | 1838 | void synchronize_rcu_bh(void) |
| 1835 | { | 1839 | { |
| 1840 | rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && | ||
| 1841 | !lock_is_held(&rcu_lock_map) && | ||
| 1842 | !lock_is_held(&rcu_sched_lock_map), | ||
| 1843 | "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section"); | ||
| 1836 | if (rcu_blocking_is_gp()) | 1844 | if (rcu_blocking_is_gp()) |
| 1837 | return; | 1845 | return; |
| 1838 | wait_rcu_gp(call_rcu_bh); | 1846 | wait_rcu_gp(call_rcu_bh); |
| @@ -688,6 +688,10 @@ EXPORT_SYMBOL_GPL(call_rcu); · 687 unmodified lines | |||
| 688 | */ | 688 | */ |
| 689 | void synchronize_rcu(void) | 689 | void synchronize_rcu(void) |
| 690 | { | 690 | { |
| 691 | rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && | ||
| 692 | !lock_is_held(&rcu_lock_map) && | ||
| 693 | !lock_is_held(&rcu_sched_lock_map), | ||
| 694 | "Illegal synchronize_rcu() in RCU read-side critical section"); | ||
| 691 | if (!rcu_scheduler_active) | 695 | if (!rcu_scheduler_active) |
| 692 | return; | 696 | return; |
| 693 | wait_rcu_gp(call_rcu); | 697 | wait_rcu_gp(call_rcu); |
| @@ -172,6 +172,12 @@ static void __synchronize_srcu(struct srcu_struct *sp, void (*sync_func)(void)) · 171 unmodified lines | |||
| 172 | { | 172 | { |
| 173 | int idx; | 173 | int idx; |
| 174 | 174 | ||
| 175 | rcu_lockdep_assert(!lock_is_held(&sp->dep_map) && | ||
| 176 | !lock_is_held(&rcu_bh_lock_map) && | ||
| 177 | !lock_is_held(&rcu_lock_map) && | ||
| 178 | !lock_is_held(&rcu_sched_lock_map), | ||
| 179 | "Illegal synchronize_srcu() in same-type SRCU (or RCU) read-side critical section"); | ||
| 180 | |||
| 175 | idx = sp->completed; | 181 | idx = sp->completed; |
| 176 | mutex_lock(&sp->mutex); | 182 | mutex_lock(&sp->mutex); |
| 177 | 183 | ||