rcu/tree: Make rcu_do_batch count how many callbacks were executed

The rcu_do_batch() function extracts the ready-to-invoke callbacks from the rcu_segcblist located in the ->cblist field of the current CPU's rcu_data structure. These callbacks are first moved to a local (unsegmented) rcu_cblist. The rcu_do_batch() function then uses this rcu_cblist's ->len field to count how many CBs it has invoked, but it does so by counting that field down from zero. Finally, this function negates the value in this ->len field (resulting in a positive number) and subtracts the result from the ->len field of the current CPU's ->cblist field. Except that it is sometimes necessary for rcu_do_batch() to stop invoking callbacks mid-stream, despite there being more ready to invoke, for example, if a high-priority task wakes up. In this case the remaining not-yet-invoked callbacks are requeued back onto the CPU's ->cblist, but remain in the ready-to-invoke segment of that list. As above, the negative of the local rcu_cblist's ->len field is still subtracted from the ->len field of the current CPU's ->cblist field. The design of counting down from 0 is confusing and error-prone, plus use of a positive count will make it easier to provide a uniform and consistent API to deal with the per-segment counts that are added later in this series. For example, rcu_segcblist_extract_done_cbs() can unconditionally populate the resulting unsegmented list's ->len field during extraction. This commit therefore explicitly counts how many callbacks were executed in rcu_do_batch() itself, counting up from zero, and then uses that to update the per-CPU segcb list's ->len field, without relying on the downcounting of rcl->len from zero.

Joel Fernandes (Google) committed 6 years ago

Commit 6bc33581 parent 5c8fe583 files · +7 −7Browse files

Unified · Split

3 files

kernel/rcu/rcu_segcblist.c +1 −1
kernel/rcu/rcu_segcblist.h +1 −0
kernel/rcu/tree.c +5 −6

kernel/rcu/rcu_segcblist.c+1−1

@@ -95,7 +95,7 @@ static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v) · 94 unmodified lines
95	* This increase is fully ordered with respect to the callers accesses	95	* This increase is fully ordered with respect to the callers accesses
96	* both before and after.	96	* both before and after.
97	*/	97	*/
98	static void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)	98	void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)
99	{	99	{
100	#ifdef CONFIG_RCU_NOCB_CPU	100	#ifdef CONFIG_RCU_NOCB_CPU
101	smp_mb__before_atomic(); /* Up to the caller! */	101	smp_mb__before_atomic(); /* Up to the caller! */

kernel/rcu/rcu_segcblist.h+1−0

@@ -76,6 +76,7 @@ static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg) · 75 unmodified lines
76	}	76	}
77		77
78	void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);	78	void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
		79	void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v);
79	void rcu_segcblist_init(struct rcu_segcblist *rsclp);	80	void rcu_segcblist_init(struct rcu_segcblist *rsclp);
80	void rcu_segcblist_disable(struct rcu_segcblist *rsclp);	81	void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
81	void rcu_segcblist_offload(struct rcu_segcblist *rsclp);	82	void rcu_segcblist_offload(struct rcu_segcblist *rsclp);

kernel/rcu/tree.c+5−6

 	const bool offloaded = rcu_segcblist_is_offloaded(&rdp->cblist);
 	struct rcu_head *rhp;
 	struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
-	long bl, count;
+	long bl, count = 0;
 	long pending, tlimit = 0;
 	/* If no callbacks are ready, just return. */
 	for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
 		rcu_callback_t f;
+		count++;
 		debug_rcu_head_unqueue(rhp);
 		rcu_lock_acquire(&rcu_callback_map);
 		/*
 		 * Stop only if limit reached and CPU has something to do.
-		 * Note: The rcl structure counts down from zero.
 		 */
-		if (-rcl.len >= bl && !offloaded &&
+		if (count >= bl && !offloaded &&
 		    (need_resched() ||
 		     (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
 			break;
 		if (unlikely(tlimit)) {
 			/* only call local_clock() every 32 callbacks */
-			if (likely((-rcl.len & 31) || local_clock() < tlimit))
+			if (likely((count & 31) || local_clock() < tlimit))
 				continue;
 			/* Exceeded the time limit, so leave. */
 			break;
 	local_irq_save(flags);
 	rcu_nocb_lock(rdp);
-	count = -rcl.len;
 	rdp->n_cbs_invoked += count;
 	trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
 			    is_idle_task(current), rcu_is_callbacks_kthread());
 	/* Update counts and requeue any remaining callbacks. */
 	rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl);
 	smp_mb(); /* List handling before counting for rcu_barrier(). */
-	rcu_segcblist_insert_count(&rdp->cblist, &rcl);
+	rcu_segcblist_add_len(&rdp->cblist, -count);
 	/* Reinstate batch limit if we have worked down the excess. */
 	count = rcu_segcblist_n_cbs(&rdp->cblist);

rcu/tree: Make rcu_do_batch count how many callbacks were executed

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code