/* * Sleepable Read-Copy Update mechanism for mutual exclusion. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * * Copyright (C) IBM Corporation, 2006 * Copyright (C) Fujitsu, 2012 * * Author: Paul McKenney <paulmck@us.ibm.com> *	   Lai Jiangshan <laijs@cn.fujitsu.com> * * For detailed explanation of Read-Copy Update mechanism see - * 		Documentation/RCU/ *.txt * */ #include <linux/export.h>#include <linux/mutex.h>#include <linux/percpu.h>#include <linux/preempt.h>#include <linux/rcupdate.h>#include <linux/sched.h>#include <linux/smp.h>#include <linux/delay.h>#include <linux/srcu.h> #include "rcu.h" /* * Initialize an rcu_batch structure to empty. */static inline void rcu_batch_init(struct rcu_batch *b){	b->head = NULL;	b->tail = &b->head;} /* * Enqueue a callback onto the tail of the specified rcu_batch structure. */static inline void rcu_batch_queue(struct rcu_batch *b, struct rcu_head *head){	*b->tail = head;	b->tail = &head->next;} /* * Is the specified rcu_batch structure empty? */static inline bool rcu_batch_empty(struct rcu_batch *b){	return b->tail == &b->head;} /* * Remove the callback at the head of the specified rcu_batch structure * and return a pointer to it, or return NULL if the structure is empty. */static inline struct rcu_head *rcu_batch_dequeue(struct rcu_batch *b){	struct rcu_head *head; 	if (rcu_batch_empty(b))		return NULL; 	head = b->head;	b->head = head->next;	if (b->tail == &head->next)		rcu_batch_init(b); 	return head;} /* * Move all callbacks from the rcu_batch structure specified by "from" to * the structure specified by "to". */static inline void rcu_batch_move(struct rcu_batch *to, struct rcu_batch *from){	if (!rcu_batch_empty(from)) {		*to->tail = from->head;		to->tail = from->tail;		rcu_batch_init(from);	}} static int init_srcu_struct_fields(struct srcu_struct *sp){	sp->completed = 0;	spin_lock_init(&sp->queue_lock);	sp->running = false;	rcu_batch_init(&sp->batch_queue);	rcu_batch_init(&sp->batch_check0);	rcu_batch_init(&sp->batch_check1);	rcu_batch_init(&sp->batch_done);	INIT_DELAYED_WORK(&sp->work, process_srcu);	sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array);	return sp->per_cpu_ref ? 0 : -ENOMEM;} #ifdef CONFIG_DEBUG_LOCK_ALLOC int __init_srcu_struct(struct srcu_struct *sp, const char *name,		       struct lock_class_key *key){	/* Don't re-initialize a lock while it is held. */	debug_check_no_locks_freed((void *)sp, sizeof(*sp));	lockdep_init_map(&sp->dep_map, name, key, 0);	return init_srcu_struct_fields(sp);}EXPORT_SYMBOL_GPL(__init_srcu_struct); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** * init_srcu_struct - initialize a sleep-RCU structure * @sp: structure to initialize. * * Must invoke this on a given srcu_struct before passing that srcu_struct * to any other function.  Each srcu_struct represents a separate domain * of SRCU protection. */int init_srcu_struct(struct srcu_struct *sp){	return init_srcu_struct_fields(sp);}EXPORT_SYMBOL_GPL(init_srcu_struct); #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /* * Returns approximate total of the readers' ->seq[] values for the * rank of per-CPU counters specified by idx. */static unsigned long srcu_readers_seq_idx(struct srcu_struct *sp, int idx){	int cpu;	unsigned long sum = 0;	unsigned long t; 	for_each_possible_cpu(cpu) {		t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]);		sum += t;	}	return sum;} /* * Returns approximate number of readers active on the specified rank * of the per-CPU ->c[] counters. */static unsigned long srcu_readers_active_idx(struct srcu_struct *sp, int idx){	int cpu;	unsigned long sum = 0;	unsigned long t; 	for_each_possible_cpu(cpu) {		t = READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]);		sum += t;	}	return sum;} /* * Return true if the number of pre-existing readers is determined to * be stably zero.  An example unstable zero can occur if the call * to srcu_readers_active_idx() misses an __srcu_read_lock() increment, * but due to task migration, sees the corresponding __srcu_read_unlock() * decrement.  This can happen because srcu_readers_active_idx() takes * time to sum the array, and might in fact be interrupted or preempted * partway through the summation. */static bool srcu_readers_active_idx_check(struct srcu_struct *sp, int idx){	unsigned long seq; 	seq = srcu_readers_seq_idx(sp, idx); 	/*	 * The following smp_mb() A pairs with the smp_mb() B located in	 * __srcu_read_lock().  This pairing ensures that if an	 * __srcu_read_lock() increments its counter after the summation	 * in srcu_readers_active_idx(), then the corresponding SRCU read-side	 * critical section will see any changes made prior to the start	 * of the current SRCU grace period.	 *	 * Also, if the above call to srcu_readers_seq_idx() saw the	 * increment of ->seq[], then the call to srcu_readers_active_idx()	 * must see the increment of ->c[].	 */	smp_mb(); /* A */ 	/*	 * Note that srcu_readers_active_idx() can incorrectly return	 * zero even though there is a pre-existing reader throughout.	 * To see this, suppose that task A is in a very long SRCU	 * read-side critical section that started on CPU 0, and that	 * no other reader exists, so that the sum of the counters	 * is equal to one.  Then suppose that task B starts executing	 * srcu_readers_active_idx(), summing up to CPU 1, and then that	 * task C starts reading on CPU 0, so that its increment is not	 * summed, but finishes reading on CPU 2, so that its decrement	 * -is- summed.  Then when task B completes its sum, it will	 * incorrectly get zero, despite the fact that task A has been	 * in its SRCU read-side critical section the whole time.	 *	 * We therefore do a validation step should srcu_readers_active_idx()	 * return zero.	 */	if (srcu_readers_active_idx(sp, idx) != 0)		return false; 	/*	 * The remainder of this function is the validation step.	 * The following smp_mb() D pairs with the smp_mb() C in	 * __srcu_read_unlock().  If the __srcu_read_unlock() was seen	 * by srcu_readers_active_idx() above, then any destructive	 * operation performed after the grace period will happen after	 * the corresponding SRCU read-side critical section.	 *	 * Note that there can be at most NR_CPUS worth of readers using	 * the old index, which is not enough to overflow even a 32-bit	 * integer.  (Yes, this does mean that systems having more than	 * a billion or so CPUs need to be 64-bit systems.)  Therefore,	 * the sum of the ->seq[] counters cannot possibly overflow.	 * Therefore, the only way that the return values of the two	 * calls to srcu_readers_seq_idx() can be equal is if there were	 * no increments of the corresponding rank of ->seq[] counts	 * in the interim.  But the missed-increment scenario laid out	 * above includes an increment of the ->seq[] counter by	 * the corresponding __srcu_read_lock().  Therefore, if this	 * scenario occurs, the return values from the two calls to	 * srcu_readers_seq_idx() will differ, and thus the validation	 * step below suffices.	 */	smp_mb(); /* D */ 	return srcu_readers_seq_idx(sp, idx) == seq;} /** * srcu_readers_active - returns approximate number of readers. * @sp: which srcu_struct to count active readers (holding srcu_read_lock). * * Note that this is not an atomic primitive, and can therefore suffer * severe errors when invoked on an active srcu_struct.  That said, it * can be useful as an error check at cleanup time. */static int srcu_readers_active(struct srcu_struct *sp){	int cpu;	unsigned long sum = 0; 	for_each_possible_cpu(cpu) {		sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]);		sum += READ_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]);	}	return sum;} /** * cleanup_srcu_struct - deconstruct a sleep-RCU structure * @sp: structure to clean up. * * Must invoke this after you are finished using a given srcu_struct that * was initialized via init_srcu_struct(), else you leak memory. */void cleanup_srcu_struct(struct srcu_struct *sp){	if (WARN_ON(srcu_readers_active(sp)))		return; /* Leakage unless caller handles error. */	free_percpu(sp->per_cpu_ref);	sp->per_cpu_ref = NULL;}EXPORT_SYMBOL_GPL(cleanup_srcu_struct); /* * Counts the new reader in the appropriate per-CPU element of the * srcu_struct.  Must be called from process context. * Returns an index that must be passed to the matching srcu_read_unlock(). */int __srcu_read_lock(struct srcu_struct *sp){	int idx; 	idx = READ_ONCE(sp->completed) & 0x1;	preempt_disable();	__this_cpu_inc(sp->per_cpu_ref->c[idx]);	smp_mb(); /* B */  /* Avoid leaking the critical section. */	__this_cpu_inc(sp->per_cpu_ref->seq[idx]);	preempt_enable();	return idx;}EXPORT_SYMBOL_GPL(__srcu_read_lock); /* * Removes the count for the old reader from the appropriate per-CPU * element of the srcu_struct.  Note that this may well be a different * CPU than that which was incremented by the corresponding srcu_read_lock(). * Must be called from process context. */void __srcu_read_unlock(struct srcu_struct *sp, int idx){	smp_mb(); /* C */  /* Avoid leaking the critical section. */	this_cpu_dec(sp->per_cpu_ref->c[idx]);}EXPORT_SYMBOL_GPL(__srcu_read_unlock); /* * We use an adaptive strategy for synchronize_srcu() and especially for * synchronize_srcu_expedited().  We spin for a fixed time period * (defined below) to allow SRCU readers to exit their read-side critical * sections.  If there are still some readers after 10 microseconds, * we repeatedly block for 1-millisecond time periods.  This approach * has done well in testing, so there is no need for a config parameter. */#define SRCU_RETRY_CHECK_DELAY		5#define SYNCHRONIZE_SRCU_TRYCOUNT	2#define SYNCHRONIZE_SRCU_EXP_TRYCOUNT	12 /* * @@@ Wait until all pre-existing readers complete.  Such readers * will have used the index specified by "idx". * the caller should ensures the ->completed is not changed while checking * and idx = (->completed & 1) ^ 1 */static bool try_check_zero(struct srcu_struct *sp, int idx, int trycount){	for (;;) {		if (srcu_readers_active_idx_check(sp, idx))			return true;		if (--trycount <= 0)			return false;		udelay(SRCU_RETRY_CHECK_DELAY);	}} /* * Increment the ->completed counter so that future SRCU readers will * use the other rank of the ->c[] and ->seq[] arrays.  This allows * us to wait for pre-existing readers in a starvation-free manner. */static void srcu_flip(struct srcu_struct *sp){	sp->completed++;} /* * Enqueue an SRCU callback on the specified srcu_struct structure, * initiating grace-period processing if it is not already running. * * Note that all CPUs must agree that the grace period extended beyond * all pre-existing SRCU read-side critical section.  On systems with * more than one CPU, this means that when "func()" is invoked, each CPU * is guaranteed to have executed a full memory barrier since the end of * its last corresponding SRCU read-side critical section whose beginning * preceded the call to call_rcu().  It also means that each CPU executing * an SRCU read-side critical section that continues beyond the start of * "func()" must have executed a memory barrier after the call_rcu() * but before the beginning of that SRCU read-side critical section. * Note that these guarantees include CPUs that are offline, idle, or * executing in user mode, as well as CPUs that are executing in the kernel. * * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the * resulting SRCU callback function "func()", then both CPU A and CPU * B are guaranteed to execute a full memory barrier during the time * interval between the call to call_rcu() and the invocation of "func()". * This guarantee applies even if CPU A and CPU B are the same CPU (but * again only if the system has more than one CPU). * * Of course, these guarantees apply only for invocations of call_srcu(), * srcu_read_lock(), and srcu_read_unlock() that are all passed the same * srcu_struct structure. */void call_srcu(struct srcu_struct *sp, struct rcu_head *head,		void (*func)(struct rcu_head *head)){	unsigned long flags; 	head->next = NULL;	head->func = func;	spin_lock_irqsave(&sp->queue_lock, flags);	rcu_batch_queue(&sp->batch_queue, head);	if (!sp->running) {		sp->running = true;		queue_delayed_work(system_power_efficient_wq, &sp->work, 0);	}	spin_unlock_irqrestore(&sp->queue_lock, flags);}EXPORT_SYMBOL_GPL(call_srcu); static void srcu_advance_batches(struct srcu_struct *sp, int trycount);static void srcu_reschedule(struct srcu_struct *sp); /* * Helper function for synchronize_srcu() and synchronize_srcu_expedited(). */static void __synchronize_srcu(struct srcu_struct *sp, int trycount){	struct rcu_synchronize rcu;	struct rcu_head *head = &rcu.head;	bool done = false; 	rcu_lockdep_assert(!lock_is_held(&sp->dep_map) &&			   !lock_is_held(&rcu_bh_lock_map) &&			   !lock_is_held(&rcu_lock_map) &&			   !lock_is_held(&rcu_sched_lock_map),			   "Illegal synchronize_srcu() in same-type SRCU (or RCU) read-side critical section"); 	might_sleep();	init_completion(&rcu.completion); 	head->next = NULL;	head->func = wakeme_after_rcu;	spin_lock_irq(&sp->queue_lock);	if (!sp->running) {		/* steal the processing owner */		sp->running = true;		rcu_batch_queue(&sp->batch_check0, head);		spin_unlock_irq(&sp->queue_lock); 		srcu_advance_batches(sp, trycount);		if (!rcu_batch_empty(&sp->batch_done)) {			BUG_ON(sp->batch_done.head != head);			rcu_batch_dequeue(&sp->batch_done);			done = true;		}		/* give the processing owner to work_struct */		srcu_reschedule(sp);	} else {		rcu_batch_queue(&sp->batch_queue, head);		spin_unlock_irq(&sp->queue_lock);	} 	if (!done)		wait_for_completion(&rcu.completion);} /** * synchronize_srcu - wait for prior SRCU read-side critical-section completion * @sp: srcu_struct with which to synchronize. * * Wait for the count to drain to zero of both indexes. To avoid the * possible starvation of synchronize_srcu(), it waits for the count of * the index=((->completed & 1) ^ 1) to drain to zero at first, * and then flip the completed and wait for the count of the other index. * * Can block; must be called from process context. * * Note that it is illegal to call synchronize_srcu() from the corresponding * SRCU read-side critical section; doing so will result in deadlock. * However, it is perfectly legal to call synchronize_srcu() on one * srcu_struct from some other srcu_struct's read-side critical section, * as long as the resulting graph of srcu_structs is acyclic. * * There are memory-ordering constraints implied by synchronize_srcu(). * On systems with more than one CPU, when synchronize_srcu() returns, * each CPU is guaranteed to have executed a full memory barrier since * the end of its last corresponding SRCU-sched read-side critical section * whose beginning preceded the call to synchronize_srcu().  In addition, * each CPU having an SRCU read-side critical section that extends beyond * the return from synchronize_srcu() is guaranteed to have executed a * full memory barrier after the beginning of synchronize_srcu() and before * the beginning of that SRCU read-side critical section.  Note that these * guarantees include CPUs that are offline, idle, or executing in user mode, * as well as CPUs that are executing in the kernel. * * Furthermore, if CPU A invoked synchronize_srcu(), which returned * to its caller on CPU B, then both CPU A and CPU B are guaranteed * to have executed a full memory barrier during the execution of * synchronize_srcu().  This guarantee applies even if CPU A and CPU B * are the same CPU, but again only if the system has more than one CPU. * * Of course, these memory-ordering guarantees apply only when * synchronize_srcu(), srcu_read_lock(), and srcu_read_unlock() are * passed the same srcu_struct structure. */void synchronize_srcu(struct srcu_struct *sp){	__synchronize_srcu(sp, rcu_gp_is_expedited()			   ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT			   : SYNCHRONIZE_SRCU_TRYCOUNT);}EXPORT_SYMBOL_GPL(synchronize_srcu); /** * synchronize_srcu_expedited - Brute-force SRCU grace period

Lines 1–500 of 676. Load more