This commit adds debug checks near the end of rcu_do_batch() that emit warnings if an empty rcu_segcblist structure has non-zero segment counts, or, conversely, if a non-empty structure has all-zero segment counts.
Unified · Split
| @@ -94,6 +94,18 @@ static long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg) · 93 unmodified lines | |||
| 94 | return READ_ONCE(rsclp->seglen[seg]); | 94 | return READ_ONCE(rsclp->seglen[seg]); |
| 95 | } | 95 | } |
| 96 | 96 | ||
| 97 | /* Return number of callbacks in segmented callback list by summing seglen. */ | ||
| 98 | long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp) | ||
| 99 | { | ||
| 100 | long len = 0; | ||
| 101 | int i; | ||
| 102 | |||
| 103 | for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++) | ||
| 104 | len += rcu_segcblist_get_seglen(rsclp, i); | ||
| 105 | |||
| 106 | return len; | ||
| 107 | } | ||
| 108 | |||
| 97 | /* Set the length of a segment of the rcu_segcblist structure. */ | 109 | /* Set the length of a segment of the rcu_segcblist structure. */ |
| 98 | static void rcu_segcblist_set_seglen(struct rcu_segcblist *rsclp, int seg, long v) | 110 | static void rcu_segcblist_set_seglen(struct rcu_segcblist *rsclp, int seg, long v) |
| 99 | { | 111 | { |
| @@ -15,6 +15,9 @@ static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp) · 14 unmodified lines | |||
| 15 | return READ_ONCE(rclp->len); | 15 | return READ_ONCE(rclp->len); |
| 16 | } | 16 | } |
| 17 | 17 | ||
| 18 | /* Return number of callbacks in segmented callback list by summing seglen. */ | ||
| 19 | long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp); | ||
| 20 | |||
| 18 | void rcu_cblist_init(struct rcu_cblist *rclp); | 21 | void rcu_cblist_init(struct rcu_cblist *rclp); |
| 19 | void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); | 22 | void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp); |
| 20 | void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, | 23 | void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp, |
| @@ -2434,6 +2434,7 @@ int rcutree_dead_cpu(unsigned int cpu) · 2433 unmodified lines | |||
| 2434 | static void rcu_do_batch(struct rcu_data *rdp) | 2434 | static void rcu_do_batch(struct rcu_data *rdp) |
| 2435 | { | 2435 | { |
| 2436 | int div; | 2436 | int div; |
| 2437 | bool __maybe_unused empty; | ||
| 2437 | unsigned long flags; | 2438 | unsigned long flags; |
| 2438 | const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist); | 2439 | const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist); |
| 2439 | struct rcu_head *rhp; | 2440 | struct rcu_head *rhp; |
| @@ -2548,9 +2549,12 @@ static void rcu_do_batch(struct rcu_data *rdp) · 108 unmodified lines | |||
| 2548 | * The following usually indicates a double call_rcu(). To track | 2549 | * The following usually indicates a double call_rcu(). To track |
| 2549 | * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. | 2550 | * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. |
| 2550 | */ | 2551 | */ |
| 2551 | WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist)); | 2552 | empty = rcu_segcblist_empty(&rdp->cblist); |
| 2553 | WARN_ON_ONCE(count == 0 && !empty); | ||
| 2552 | WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) && | 2554 | WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) && |
| 2553 | count != 0 && rcu_segcblist_empty(&rdp->cblist)); | 2555 | count != 0 && empty); |
| 2556 | WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); | ||
| 2557 | WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); | ||
| 2554 | 2558 | ||
| 2555 | rcu_nocb_unlock_irqrestore(rdp, flags); | 2559 | rcu_nocb_unlock_irqrestore(rdp, flags); |
| 2556 | 2560 | ||