/** * srcu_batches_completed - return batches completed. * @sp: srcu_struct on which to report batch completion. * * Report the number of batches, correlated with, but not necessarily * precisely the same as, the number of grace periods that have elapsed. */long srcu_batches_completed(struct srcu_struct *sp){	return sp->completed;}EXPORT_SYMBOL_GPL(srcu_batches_completed); #define SRCU_CALLBACK_BATCH	10#define SRCU_INTERVAL		1 /* * Move any new SRCU callbacks to the first stage of the SRCU grace * period pipeline. */static void srcu_collect_new(struct srcu_struct *sp){	if (!rcu_batch_empty(&sp->batch_queue)) {		spin_lock_irq(&sp->queue_lock);		rcu_batch_move(&sp->batch_check0, &sp->batch_queue);		spin_unlock_irq(&sp->queue_lock);	}} /* * Core SRCU state machine.  Advance callbacks from ->batch_check0 to * ->batch_check1 and then to ->batch_done as readers drain. */static void srcu_advance_batches(struct srcu_struct *sp, int trycount){	int idx = 1 ^ (sp->completed & 1); 	/*	 * Because readers might be delayed for an extended period after	 * fetching ->completed for their index, at any point in time there	 * might well be readers using both idx=0 and idx=1.  We therefore	 * need to wait for readers to clear from both index values before	 * invoking a callback.	 */ 	if (rcu_batch_empty(&sp->batch_check0) &&	    rcu_batch_empty(&sp->batch_check1))		return; /* no callbacks need to be advanced */ 	if (!try_check_zero(sp, idx, trycount))		return; /* failed to advance, will try after SRCU_INTERVAL */ 	/*	 * The callbacks in ->batch_check1 have already done with their	 * first zero check and flip back when they were enqueued on	 * ->batch_check0 in a previous invocation of srcu_advance_batches().	 * (Presumably try_check_zero() returned false during that	 * invocation, leaving the callbacks stranded on ->batch_check1.)	 * They are therefore ready to invoke, so move them to ->batch_done.	 */	rcu_batch_move(&sp->batch_done, &sp->batch_check1); 	if (rcu_batch_empty(&sp->batch_check0))		return; /* no callbacks need to be advanced */	srcu_flip(sp); 	/*	 * The callbacks in ->batch_check0 just finished their	 * first check zero and flip, so move them to ->batch_check1	 * for future checking on the other idx.	 */	rcu_batch_move(&sp->batch_check1, &sp->batch_check0); 	/*	 * SRCU read-side critical sections are normally short, so check	 * at least twice in quick succession after a flip.	 */	trycount = trycount < 2 ? 2 : trycount;	if (!try_check_zero(sp, idx^1, trycount))		return; /* failed to advance, will try after SRCU_INTERVAL */ 	/*	 * The callbacks in ->batch_check1 have now waited for all	 * pre-existing readers using both idx values.  They are therefore	 * ready to invoke, so move them to ->batch_done.	 */	rcu_batch_move(&sp->batch_done, &sp->batch_check1);} /* * Invoke a limited number of SRCU callbacks that have passed through * their grace period.  If there are more to do, SRCU will reschedule * the workqueue. */static void srcu_invoke_callbacks(struct srcu_struct *sp){	int i;	struct rcu_head *head; 	for (i = 0; i < SRCU_CALLBACK_BATCH; i++) {		head = rcu_batch_dequeue(&sp->batch_done);		if (!head)			break;		local_bh_disable();		head->func(head);		local_bh_enable();	}} /* * Finished one round of SRCU grace period.  Start another if there are * more SRCU callbacks queued, otherwise put SRCU into not-running state. */static void srcu_reschedule(struct srcu_struct *sp){	bool pending = true; 	if (rcu_batch_empty(&sp->batch_done) &&	    rcu_batch_empty(&sp->batch_check1) &&	    rcu_batch_empty(&sp->batch_check0) &&	    rcu_batch_empty(&sp->batch_queue)) {		spin_lock_irq(&sp->queue_lock);		if (rcu_batch_empty(&sp->batch_done) &&		    rcu_batch_empty(&sp->batch_check1) &&		    rcu_batch_empty(&sp->batch_check0) &&		    rcu_batch_empty(&sp->batch_queue)) {			sp->running = false;			pending = false;		}		spin_unlock_irq(&sp->queue_lock);	} 	if (pending)		queue_delayed_work(system_nrt_wq, &sp->work, SRCU_INTERVAL);} /* * This is the work-queue function that handles SRCU grace periods. */static void process_srcu(struct work_struct *work){	struct srcu_struct *sp; 	sp = container_of(work, struct srcu_struct, work.work); 	srcu_collect_new(sp);	srcu_advance_batches(sp, 1);	srcu_invoke_callbacks(sp);	srcu_reschedule(sp);}

Lines 501–650 of 650. Previous lines