This commit adds debug checks near the end of rcu_do_batch() that emit warnings if an empty rcu_segcblist structure has non-zero segment counts, or, conversely, if a non-empty structure has all-zero segment counts.
Showing the changes to kernel (3 of 3 files). Show all
Unified · Split
| @@ -94,6 +94,18 @@ static long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg) · 93 unmodified lines | ||
| 94 | 94 | return READ_ONCE(rsclp->seglen[seg]); |
| 95 | 95 | } |
| 96 | 96 | |
| 97 | +/* Return number of callbacks in segmented callback list by summing seglen. */ | |
| 98 | +long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp) | |
| 99 | +{ | |
| 100 | + long len = 0; | |
| 101 | + int i; | |
| 102 | + | |
| 103 | + for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++) | |
| 104 | + len += rcu_segcblist_get_seglen(rsclp, i); | |
| 105 | + | |
| 106 | + return len; | |
| 107 | +} | |
| 108 | + | |
| 97 | 109 | /* Set the length of a segment of the rcu_segcblist structure. */ |
| 98 | 110 | static void rcu_segcblist_set_seglen(struct rcu_segcblist *rsclp, int seg, long v) |
| 99 | 111 | { |
| @@ -15,6 +15,9 @@ static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) · 14 unmodified lines | ||
| 15 | 15 | return READ_ONCE(rclp->len); |
| 16 | 16 | } |
| 17 | 17 | |
| 18 | +/* Return number of callbacks in segmented callback list by summing seglen. */ | |
| 19 | +long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp); | |
| 20 | + | |
| 18 | 21 | void rcu_cblist_init(struct rcu_cblist *rclp); |
| 19 | 22 | void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); |
| 20 | 23 | void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, |
| @@ -2434,6 +2434,7 @@ int rcutree_dead_cpu(unsigned int cpu) · 2433 unmodified lines | ||
| 2434 | 2434 | static void rcu_do_batch(struct rcu_data *rdp) |
| 2435 | 2435 | { |
| 2436 | 2436 | int div; |
| 2437 | + bool __maybe_unused empty; | |
| 2437 | 2438 | unsigned long flags; |
| 2438 | 2439 | const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist); |
| 2439 | 2440 | struct rcu_head *rhp; |
| @@ -2548,9 +2549,12 @@ static void rcu_do_batch(struct rcu_data *rdp) · 108 unmodified lines | ||
| 2548 | 2549 | * The following usually indicates a double call_rcu(). To track |
| 2549 | 2550 | * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. |
| 2550 | 2551 | */ |
| 2551 | − WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); | |
| 2552 | + empty = rcu_segcblist_empty(&rdp->cblist); | |
| 2553 | + WARN_ON_ONCE(count == 0 && !empty); | |
| 2552 | 2554 | WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) && |
| 2553 | − count != 0 && rcu_segcblist_empty(&rdp->cblist)); | |
| 2555 | + count != 0 && empty); | |
| 2556 | + WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); | |
| 2557 | + WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); | |
| 2554 | 2558 | |
| 2555 | 2559 | rcu_nocb_unlock_irqrestore(rdp, flags); |
| 2556 | 2560 | |