History

srcu: Use "ssp" instead of "sp" for srcu_struct pointer

In RCU, the distinction between "rsp", "rnp", and "rdp" has served well for a great many years, but in SRCU, "sp" vs. "sdp" has proven confusing. This commit therefore renames SRCU's "sp" pointers to "ssp", so that there is "ssp" for srcu_struct pointer, "snp" for srcu_node pointer, and "sdp" for srcu_data pointer.

Paul E. McKenney committed 8 years ago

Commit aacb5d91 parent eb4c2385 files · +359 −359Browse files

Unified · Split

5 files
  1. include/linux/srcu.h +39 −39
  2. include/linux/srcutiny.h +12 −12
  3. include/linux/srcutree.h +4 −4
  4. kernel/rcu/srcutiny.c +60 −60
  5. kernel/rcu/srcutree.c +244 −244
include/linux/srcu.h+39−39
@@ -38,20 +38,20 @@ struct srcu_struct; · 37 unmodified lines
38 38
39#ifdef CONFIG_DEBUG_LOCK_ALLOC39#ifdef CONFIG_DEBUG_LOCK_ALLOC
40 40
41int __init_srcu_struct(struct srcu_struct *sp, const char *name,41int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
42 struct lock_class_key *key);42 struct lock_class_key *key);
43 43
44#define init_srcu_struct(sp) \44#define init_srcu_struct(ssp) \
45({ \45({ \
46 static struct lock_class_key __srcu_key; \46 static struct lock_class_key __srcu_key; \
47 \47 \
48 __init_srcu_struct((sp), #sp, &__srcu_key); \48 __init_srcu_struct((ssp), #ssp, &__srcu_key); \
49})49})
50 50
51#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },51#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name },
52#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */52#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
53 53
54int init_srcu_struct(struct srcu_struct *sp);54int init_srcu_struct(struct srcu_struct *ssp);
55 55
56#define __SRCU_DEP_MAP_INIT(srcu_name)56#define __SRCU_DEP_MAP_INIT(srcu_name)
57#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */57#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -67,28 +67,28 @@ int init_srcu_struct(struct srcu_struct *sp); · 9 unmodified lines
67struct srcu_struct { };67struct srcu_struct { };
68#endif68#endif
69 69
70void call_srcu(struct srcu_struct *sp, struct rcu_head *head,70void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
71 void (*func)(struct rcu_head *head));71 void (*func)(struct rcu_head *head));
72void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced);72void _cleanup_srcu_struct(struct srcu_struct *ssp, bool quiesced);
73int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp);73int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
74void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);74void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
75void synchronize_srcu(struct srcu_struct *sp);75void synchronize_srcu(struct srcu_struct *ssp);
76 76
77/**77/**
78 * cleanup_srcu_struct - deconstruct a sleep-RCU structure78 * cleanup_srcu_struct - deconstruct a sleep-RCU structure
79 * @sp: structure to clean up.79 * @ssp: structure to clean up.
80 *80 *
81 * Must invoke this after you are finished using a given srcu_struct that81 * Must invoke this after you are finished using a given srcu_struct that
82 * was initialized via init_srcu_struct(), else you leak memory.82 * was initialized via init_srcu_struct(), else you leak memory.
83 */83 */
84static inline void cleanup_srcu_struct(struct srcu_struct *sp)84static inline void cleanup_srcu_struct(struct srcu_struct *ssp)
85{85{
86 _cleanup_srcu_struct(sp, false);86 _cleanup_srcu_struct(ssp, false);
87}87}
88 88
89/**89/**
90 * cleanup_srcu_struct_quiesced - deconstruct a quiesced sleep-RCU structure90 * cleanup_srcu_struct_quiesced - deconstruct a quiesced sleep-RCU structure
91 * @sp: structure to clean up.91 * @ssp: structure to clean up.
92 *92 *
93 * Must invoke this after you are finished using a given srcu_struct that93 * Must invoke this after you are finished using a given srcu_struct that
94 * was initialized via init_srcu_struct(), else you leak memory. Also,94 * was initialized via init_srcu_struct(), else you leak memory. Also,
@@ -103,16 +103,16 @@ static inline void cleanup_srcu_struct(struct srcu_struct *sp) · 8 unmodified lines
103 * (with high probability, anyway), and will also cause the srcu_struct103 * (with high probability, anyway), and will also cause the srcu_struct
104 * to be leaked.104 * to be leaked.
105 */105 */
106static inline void cleanup_srcu_struct_quiesced(struct srcu_struct *sp)106static inline void cleanup_srcu_struct_quiesced(struct srcu_struct *ssp)
107{107{
108 _cleanup_srcu_struct(sp, true);108 _cleanup_srcu_struct(ssp, true);
109}109}
110 110
111#ifdef CONFIG_DEBUG_LOCK_ALLOC111#ifdef CONFIG_DEBUG_LOCK_ALLOC
112 112
113/**113/**
114 * srcu_read_lock_held - might we be in SRCU read-side critical section?114 * srcu_read_lock_held - might we be in SRCU read-side critical section?
115 * @sp: The srcu_struct structure to check115 * @ssp: The srcu_struct structure to check
116 *116 *
117 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU117 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
118 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,118 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
@@ -126,16 +126,16 @@ static inline void cleanup_srcu_struct_quiesced(struct srcu_struct *sp) · 7 unmodified lines
126 * relies on normal RCU, it can be called from the CPU which126 * relies on normal RCU, it can be called from the CPU which
127 * is in the idle loop from an RCU point of view or offline.127 * is in the idle loop from an RCU point of view or offline.
128 */128 */
129static inline int srcu_read_lock_held(const struct srcu_struct *sp)129static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
130{130{
131 if (!debug_lockdep_rcu_enabled())131 if (!debug_lockdep_rcu_enabled())
132 return 1;132 return 1;
133 return lock_is_held(&sp->dep_map);133 return lock_is_held(&ssp->dep_map);
134}134}
135 135
136#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */136#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
137 137
138static inline int srcu_read_lock_held(const struct srcu_struct *sp)138static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
139{139{
140 return 1;140 return 1;
141}141}
@@ -145,7 +145,7 @@ static inline int srcu_read_lock_held(const struct srcu_struct *sp) · 3 unmodified lines
145/**145/**
146 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing146 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
147 * @p: the pointer to fetch and protect for later dereferencing147 * @p: the pointer to fetch and protect for later dereferencing
148 * @sp: pointer to the srcu_struct, which is used to check that we148 * @ssp: pointer to the srcu_struct, which is used to check that we
149 * really are in an SRCU read-side critical section.149 * really are in an SRCU read-side critical section.
150 * @c: condition to check for update-side use150 * @c: condition to check for update-side use
151 *151 *
@@ -154,32 +154,32 @@ static inline int srcu_read_lock_held(const struct srcu_struct *sp) · 2 unmodified lines
154 * to 1. The @c argument will normally be a logical expression containing154 * to 1. The @c argument will normally be a logical expression containing
155 * lockdep_is_held() calls.155 * lockdep_is_held() calls.
156 */156 */
157#define srcu_dereference_check(p, sp, c) \157#define srcu_dereference_check(p, ssp, c) \
158 __rcu_dereference_check((p), (c) || srcu_read_lock_held(sp), __rcu)158 __rcu_dereference_check((p), (c) || srcu_read_lock_held(ssp), __rcu)
159 159
160/**160/**
161 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing161 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
162 * @p: the pointer to fetch and protect for later dereferencing162 * @p: the pointer to fetch and protect for later dereferencing
163 * @sp: pointer to the srcu_struct, which is used to check that we163 * @ssp: pointer to the srcu_struct, which is used to check that we
164 * really are in an SRCU read-side critical section.164 * really are in an SRCU read-side critical section.
165 *165 *
166 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU166 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
167 * is enabled, invoking this outside of an RCU read-side critical167 * is enabled, invoking this outside of an RCU read-side critical
168 * section will result in an RCU-lockdep splat.168 * section will result in an RCU-lockdep splat.
169 */169 */
170#define srcu_dereference(p, sp) srcu_dereference_check((p), (sp), 0)170#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)
171 171
172/**172/**
173 * srcu_dereference_notrace - no tracing and no lockdep calls from here173 * srcu_dereference_notrace - no tracing and no lockdep calls from here
174 * @p: the pointer to fetch and protect for later dereferencing174 * @p: the pointer to fetch and protect for later dereferencing
175 * @sp: pointer to the srcu_struct, which is used to check that we175 * @ssp: pointer to the srcu_struct, which is used to check that we
176 * really are in an SRCU read-side critical section.176 * really are in an SRCU read-side critical section.
177 */177 */
178#define srcu_dereference_notrace(p, sp) srcu_dereference_check((p), (sp), 1)178#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)
179 179
180/**180/**
181 * srcu_read_lock - register a new reader for an SRCU-protected structure.181 * srcu_read_lock - register a new reader for an SRCU-protected structure.
182 * @sp: srcu_struct in which to register the new reader.182 * @ssp: srcu_struct in which to register the new reader.
183 *183 *
184 * Enter an SRCU read-side critical section. Note that SRCU read-side184 * Enter an SRCU read-side critical section. Note that SRCU read-side
185 * critical sections may be nested. However, it is illegal to185 * critical sections may be nested. However, it is illegal to
@@ -194,44 +194,44 @@ static inline int srcu_read_lock_held(const struct srcu_struct *sp) · 8 unmodified lines
194 * srcu_read_unlock() in an irq handler if the matching srcu_read_lock()194 * srcu_read_unlock() in an irq handler if the matching srcu_read_lock()
195 * was invoked in process context.195 * was invoked in process context.
196 */196 */
197static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp)197static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)
198{198{
199 int retval;199 int retval;
200 200
201 retval = __srcu_read_lock(sp);201 retval = __srcu_read_lock(ssp);
202 rcu_lock_acquire(&(sp)->dep_map);202 rcu_lock_acquire(&(ssp)->dep_map);
203 return retval;203 return retval;
204}204}
205 205
206/* Used by tracing, cannot be traced and cannot invoke lockdep. */206/* Used by tracing, cannot be traced and cannot invoke lockdep. */
207static inline notrace int207static inline notrace int
208srcu_read_lock_notrace(struct srcu_struct *sp) __acquires(sp)208srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)
209{209{
210 int retval;210 int retval;
211 211
212 retval = __srcu_read_lock(sp);212 retval = __srcu_read_lock(ssp);
213 return retval;213 return retval;
214}214}
215 215
216/**216/**
217 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.217 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
218 * @sp: srcu_struct in which to unregister the old reader.218 * @ssp: srcu_struct in which to unregister the old reader.
219 * @idx: return value from corresponding srcu_read_lock().219 * @idx: return value from corresponding srcu_read_lock().
220 *220 *
221 * Exit an SRCU read-side critical section.221 * Exit an SRCU read-side critical section.
222 */222 */
223static inline void srcu_read_unlock(struct srcu_struct *sp, int idx)223static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
224 __releases(sp)224 __releases(ssp)
225{225{
226 rcu_lock_release(&(sp)->dep_map);226 rcu_lock_release(&(ssp)->dep_map);
227 __srcu_read_unlock(sp, idx);227 __srcu_read_unlock(ssp, idx);
228}228}
229 229
230/* Used by tracing, cannot be traced and cannot call lockdep. */230/* Used by tracing, cannot be traced and cannot call lockdep. */
231static inline notrace void231static inline notrace void
232srcu_read_unlock_notrace(struct srcu_struct *sp, int idx) __releases(sp)232srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp)
233{233{
234 __srcu_read_unlock(sp, idx);234 __srcu_read_unlock(ssp, idx);
235}235}
236 236
237/**237/**
include/linux/srcutiny.h+12−12
@@ -60,7 +60,7 @@ void srcu_drive_gp(struct work_struct *wp); · 59 unmodified lines
60#define DEFINE_STATIC_SRCU(name) \60#define DEFINE_STATIC_SRCU(name) \
61 static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)61 static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)
62 62
63void synchronize_srcu(struct srcu_struct *sp);63void synchronize_srcu(struct srcu_struct *ssp);
64 64
65/*65/*
66 * Counts the new reader in the appropriate per-CPU element of the66 * Counts the new reader in the appropriate per-CPU element of the
@@ -68,36 +68,36 @@ void synchronize_srcu(struct srcu_struct *sp); · 1 unmodified line
68 * __srcu_read_unlock() must be in the same handler instance. Returns an68 * __srcu_read_unlock() must be in the same handler instance. Returns an
69 * index that must be passed to the matching srcu_read_unlock().69 * index that must be passed to the matching srcu_read_unlock().
70 */70 */
71static inline int __srcu_read_lock(struct srcu_struct *sp)71static inline int __srcu_read_lock(struct srcu_struct *ssp)
72{72{
73 int idx;73 int idx;
74 74
75 idx = READ_ONCE(sp->srcu_idx);75 idx = READ_ONCE(ssp->srcu_idx);
76 WRITE_ONCE(sp->srcu_lock_nesting[idx], sp->srcu_lock_nesting[idx] + 1);76 WRITE_ONCE(ssp->srcu_lock_nesting[idx], ssp->srcu_lock_nesting[idx] + 1);
77 return idx;77 return idx;
78}78}
79 79
80static inline void synchronize_srcu_expedited(struct srcu_struct *sp)80static inline void synchronize_srcu_expedited(struct srcu_struct *ssp)
81{81{
82 synchronize_srcu(sp);82 synchronize_srcu(ssp);
83}83}
84 84
85static inline void srcu_barrier(struct srcu_struct *sp)85static inline void srcu_barrier(struct srcu_struct *ssp)
86{86{
87 synchronize_srcu(sp);87 synchronize_srcu(ssp);
88}88}
89 89
90/* Defined here to avoid size increase for non-torture kernels. */90/* Defined here to avoid size increase for non-torture kernels. */
91static inline void srcu_torture_stats_print(struct srcu_struct *sp,91static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
92 char *tt, char *tf)92 char *tt, char *tf)
93{93{
94 int idx;94 int idx;
95 95
96 idx = READ_ONCE(sp->srcu_idx) & 0x1;96 idx = READ_ONCE(ssp->srcu_idx) & 0x1;
97 pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",97 pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",
98 tt, tf, idx,98 tt, tf, idx,
99 READ_ONCE(sp->srcu_lock_nesting[!idx]),99 READ_ONCE(ssp->srcu_lock_nesting[!idx]),
100 READ_ONCE(sp->srcu_lock_nesting[idx]));100 READ_ONCE(ssp->srcu_lock_nesting[idx]));
101}101}
102 102
103#endif103#endif
include/linux/srcutree.h+4−4
@@ -51,7 +51,7 @@ struct srcu_data { · 50 unmodified lines
51 unsigned long grpmask; /* Mask for leaf srcu_node */51 unsigned long grpmask; /* Mask for leaf srcu_node */
52 /* ->srcu_data_have_cbs[]. */52 /* ->srcu_data_have_cbs[]. */
53 int cpu;53 int cpu;
54 struct srcu_struct *sp;54 struct srcu_struct *ssp;
55};55};
56 56
57/*57/*
@@ -138,8 +138,8 @@ struct srcu_struct { · 80 unmodified lines
138#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)138#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
139#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)139#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
140 140
141void synchronize_srcu_expedited(struct srcu_struct *sp);141void synchronize_srcu_expedited(struct srcu_struct *ssp);
142void srcu_barrier(struct srcu_struct *sp);142void srcu_barrier(struct srcu_struct *ssp);
143void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf);143void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf);
144 144
145#endif145#endif
kernel/rcu/srcutiny.c+60−60
@@ -37,30 +37,30 @@ int rcu_scheduler_active __read_mostly; · 36 unmodified lines
37static LIST_HEAD(srcu_boot_list);37static LIST_HEAD(srcu_boot_list);
38static bool srcu_init_done;38static bool srcu_init_done;
39 39
40static int init_srcu_struct_fields(struct srcu_struct *sp)40static int init_srcu_struct_fields(struct srcu_struct *ssp)
41{41{
42 sp->srcu_lock_nesting[0] = 0;42 ssp->srcu_lock_nesting[0] = 0;
43 sp->srcu_lock_nesting[1] = 0;43 ssp->srcu_lock_nesting[1] = 0;
44 init_swait_queue_head(&sp->srcu_wq);44 init_swait_queue_head(&ssp->srcu_wq);
45 sp->srcu_cb_head = NULL;45 ssp->srcu_cb_head = NULL;
46 sp->srcu_cb_tail = &sp->srcu_cb_head;46 ssp->srcu_cb_tail = &ssp->srcu_cb_head;
47 sp->srcu_gp_running = false;47 ssp->srcu_gp_running = false;
48 sp->srcu_gp_waiting = false;48 ssp->srcu_gp_waiting = false;
49 sp->srcu_idx = 0;49 ssp->srcu_idx = 0;
50 INIT_WORK(&sp->srcu_work, srcu_drive_gp);50 INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
51 INIT_LIST_HEAD(&sp->srcu_work.entry);51 INIT_LIST_HEAD(&ssp->srcu_work.entry);
52 return 0;52 return 0;
53}53}
54 54
55#ifdef CONFIG_DEBUG_LOCK_ALLOC55#ifdef CONFIG_DEBUG_LOCK_ALLOC
56 56
57int __init_srcu_struct(struct srcu_struct *sp, const char *name,57int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
58 struct lock_class_key *key)58 struct lock_class_key *key)
59{59{
60 /* Don't re-initialize a lock while it is held. */60 /* Don't re-initialize a lock while it is held. */
61 debug_check_no_locks_freed((void *)sp, sizeof(*sp));61 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
62 lockdep_init_map(&sp->dep_map, name, key, 0);62 lockdep_init_map(&ssp->dep_map, name, key, 0);
63 return init_srcu_struct_fields(sp);63 return init_srcu_struct_fields(ssp);
64}64}
65EXPORT_SYMBOL_GPL(__init_srcu_struct);65EXPORT_SYMBOL_GPL(__init_srcu_struct);
66 66
@@ -68,15 +68,15 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct); · 1 unmodified line
68 68
69/*69/*
70 * init_srcu_struct - initialize a sleep-RCU structure70 * init_srcu_struct - initialize a sleep-RCU structure
71 * @sp: structure to initialize.71 * @ssp: structure to initialize.
72 *72 *
73 * Must invoke this on a given srcu_struct before passing that srcu_struct73 * Must invoke this on a given srcu_struct before passing that srcu_struct
74 * to any other function. Each srcu_struct represents a separate domain74 * to any other function. Each srcu_struct represents a separate domain
75 * of SRCU protection.75 * of SRCU protection.
76 */76 */
77int init_srcu_struct(struct srcu_struct *sp)77int init_srcu_struct(struct srcu_struct *ssp)
78{78{
79 return init_srcu_struct_fields(sp);79 return init_srcu_struct_fields(ssp);
80}80}
81EXPORT_SYMBOL_GPL(init_srcu_struct);81EXPORT_SYMBOL_GPL(init_srcu_struct);
82 82
@@ -84,22 +84,22 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); · 1 unmodified line
84 84
85/*85/*
86 * cleanup_srcu_struct - deconstruct a sleep-RCU structure86 * cleanup_srcu_struct - deconstruct a sleep-RCU structure
87 * @sp: structure to clean up.87 * @ssp: structure to clean up.
88 *88 *
89 * Must invoke this after you are finished using a given srcu_struct that89 * Must invoke this after you are finished using a given srcu_struct that
90 * was initialized via init_srcu_struct(), else you leak memory.90 * was initialized via init_srcu_struct(), else you leak memory.
91 */91 */
92void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced)92void _cleanup_srcu_struct(struct srcu_struct *ssp, bool quiesced)
93{93{
94 WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]);94 WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);
95 if (quiesced)95 if (quiesced)
96 WARN_ON(work_pending(&sp->srcu_work));96 WARN_ON(work_pending(&ssp->srcu_work));
97 else97 else
98 flush_work(&sp->srcu_work);98 flush_work(&ssp->srcu_work);
99 WARN_ON(sp->srcu_gp_running);99 WARN_ON(ssp->srcu_gp_running);
100 WARN_ON(sp->srcu_gp_waiting);100 WARN_ON(ssp->srcu_gp_waiting);
101 WARN_ON(sp->srcu_cb_head);101 WARN_ON(ssp->srcu_cb_head);
102 WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail);102 WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);
103}103}
104EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);104EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);
105 105
@@ -107,13 +107,13 @@ EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); · 1 unmodified line
107 * Removes the count for the old reader from the appropriate element of107 * Removes the count for the old reader from the appropriate element of
108 * the srcu_struct.108 * the srcu_struct.
109 */109 */
110void __srcu_read_unlock(struct srcu_struct *sp, int idx)110void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
111{111{
112 int newval = sp->srcu_lock_nesting[idx] - 1;112 int newval = ssp->srcu_lock_nesting[idx] - 1;
113 113
114 WRITE_ONCE(sp->srcu_lock_nesting[idx], newval);114 WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);
115 if (!newval && READ_ONCE(sp->srcu_gp_waiting))115 if (!newval && READ_ONCE(ssp->srcu_gp_waiting))
116 swake_up_one(&sp->srcu_wq);116 swake_up_one(&ssp->srcu_wq);
117}117}
118EXPORT_SYMBOL_GPL(__srcu_read_unlock);118EXPORT_SYMBOL_GPL(__srcu_read_unlock);
119 119
@@ -127,24 +127,24 @@ void srcu_drive_gp(struct work_struct *wp) · 7 unmodified lines
127 int idx;127 int idx;
128 struct rcu_head *lh;128 struct rcu_head *lh;
129 struct rcu_head *rhp;129 struct rcu_head *rhp;
130 struct srcu_struct *sp;130 struct srcu_struct *ssp;
131 131
132 sp = container_of(wp, struct srcu_struct, srcu_work);132 ssp = container_of(wp, struct srcu_struct, srcu_work);
133 if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head))133 if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head))
134 return; /* Already running or nothing to do. */134 return; /* Already running or nothing to do. */
135 135
136 /* Remove recently arrived callbacks and wait for readers. */136 /* Remove recently arrived callbacks and wait for readers. */
137 WRITE_ONCE(sp->srcu_gp_running, true);137 WRITE_ONCE(ssp->srcu_gp_running, true);
138 local_irq_disable();138 local_irq_disable();
139 lh = sp->srcu_cb_head;139 lh = ssp->srcu_cb_head;
140 sp->srcu_cb_head = NULL;140 ssp->srcu_cb_head = NULL;
141 sp->srcu_cb_tail = &sp->srcu_cb_head;141 ssp->srcu_cb_tail = &ssp->srcu_cb_head;
142 local_irq_enable();142 local_irq_enable();
143 idx = sp->srcu_idx;143 idx = ssp->srcu_idx;
144 WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx);144 WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx);
145 WRITE_ONCE(sp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */145 WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */
146 swait_event_exclusive(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx]));146 swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));
147 WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */147 WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
148 148
149 /* Invoke the callbacks we removed above. */149 /* Invoke the callbacks we removed above. */
150 while (lh) {150 while (lh) {
@@ -161,9 +161,9 @@ void srcu_drive_gp(struct work_struct *wp) · 10 unmodified lines
161 * at interrupt level, but the ->srcu_gp_running checks will161 * at interrupt level, but the ->srcu_gp_running checks will
162 * straighten that out.162 * straighten that out.
163 */163 */
164 WRITE_ONCE(sp->srcu_gp_running, false);164 WRITE_ONCE(ssp->srcu_gp_running, false);
165 if (READ_ONCE(sp->srcu_cb_head))165 if (READ_ONCE(ssp->srcu_cb_head))
166 schedule_work(&sp->srcu_work);166 schedule_work(&ssp->srcu_work);
167}167}
168EXPORT_SYMBOL_GPL(srcu_drive_gp);168EXPORT_SYMBOL_GPL(srcu_drive_gp);
169 169
@@ -171,7 +171,7 @@ EXPORT_SYMBOL_GPL(srcu_drive_gp); · 1 unmodified line
171 * Enqueue an SRCU callback on the specified srcu_struct structure,171 * Enqueue an SRCU callback on the specified srcu_struct structure,
172 * initiating grace-period processing if it is not already running.172 * initiating grace-period processing if it is not already running.
173 */173 */
174void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,174void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
175 rcu_callback_t func)175 rcu_callback_t func)
176{176{
177 unsigned long flags;177 unsigned long flags;
@@ -179,14 +179,14 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, · 1 unmodified line
179 rhp->func = func;179 rhp->func = func;
180 rhp->next = NULL;180 rhp->next = NULL;
181 local_irq_save(flags);181 local_irq_save(flags);
182 *sp->srcu_cb_tail = rhp;182 *ssp->srcu_cb_tail = rhp;
183 sp->srcu_cb_tail = &rhp->next;183 ssp->srcu_cb_tail = &rhp->next;
184 local_irq_restore(flags);184 local_irq_restore(flags);
185 if (!READ_ONCE(sp->srcu_gp_running)) {185 if (!READ_ONCE(ssp->srcu_gp_running)) {
186 if (likely(srcu_init_done))186 if (likely(srcu_init_done))
187 schedule_work(&sp->srcu_work);187 schedule_work(&ssp->srcu_work);
188 else if (list_empty(&sp->srcu_work.entry))188 else if (list_empty(&ssp->srcu_work.entry))
189 list_add(&sp->srcu_work.entry, &srcu_boot_list);189 list_add(&ssp->srcu_work.entry, &srcu_boot_list);
190 }190 }
191}191}
192EXPORT_SYMBOL_GPL(call_srcu);192EXPORT_SYMBOL_GPL(call_srcu);
@@ -194,13 +194,13 @@ EXPORT_SYMBOL_GPL(call_srcu); · 1 unmodified line
194/*194/*
195 * synchronize_srcu - wait for prior SRCU read-side critical-section completion195 * synchronize_srcu - wait for prior SRCU read-side critical-section completion
196 */196 */
197void synchronize_srcu(struct srcu_struct *sp)197void synchronize_srcu(struct srcu_struct *ssp)
198{198{
199 struct rcu_synchronize rs;199 struct rcu_synchronize rs;
200 200
201 init_rcu_head_on_stack(&rs.head);201 init_rcu_head_on_stack(&rs.head);
202 init_completion(&rs.completion);202 init_completion(&rs.completion);
203 call_srcu(sp, &rs.head, wakeme_after_rcu);203 call_srcu(ssp, &rs.head, wakeme_after_rcu);
204 wait_for_completion(&rs.completion);204 wait_for_completion(&rs.completion);
205 destroy_rcu_head_on_stack(&rs.head);205 destroy_rcu_head_on_stack(&rs.head);
206}206}
@@ -219,13 +219,13 @@ void __init rcu_scheduler_starting(void) · 12 unmodified lines
219 */219 */
220void __init srcu_init(void)220void __init srcu_init(void)
221{221{
222 struct srcu_struct *sp;222 struct srcu_struct *ssp;
223 223
224 srcu_init_done = true;224 srcu_init_done = true;
225 while (!list_empty(&srcu_boot_list)) {225 while (!list_empty(&srcu_boot_list)) {
226 sp = list_first_entry(&srcu_boot_list,226 ssp = list_first_entry(&srcu_boot_list,
227 struct srcu_struct, srcu_work.entry);227 struct srcu_struct, srcu_work.entry);
228 list_del_init(&sp->srcu_work.entry);228 list_del_init(&ssp->srcu_work.entry);
229 schedule_work(&sp->srcu_work);229 schedule_work(&ssp->srcu_work);
230 }230 }
231}231}
kernel/rcu/srcutree.c+244−244
@@ -56,7 +56,7 @@ static LIST_HEAD(srcu_boot_list); · 55 unmodified lines
56static bool __read_mostly srcu_init_done;56static bool __read_mostly srcu_init_done;
57 57
58static void srcu_invoke_callbacks(struct work_struct *work);58static void srcu_invoke_callbacks(struct work_struct *work);
59static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay);59static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
60static void process_srcu(struct work_struct *work);60static void process_srcu(struct work_struct *work);
61 61
62/* Wrappers for lock acquisition and release, see raw_spin_lock_rcu_node(). */62/* Wrappers for lock acquisition and release, see raw_spin_lock_rcu_node(). */
@@ -92,7 +92,7 @@ do { \ · 29 unmodified lines
92 * srcu_read_unlock() running against them. So if the is_static parameter92 * srcu_read_unlock() running against them. So if the is_static parameter
93 * is set, don't initialize ->srcu_lock_count[] and ->srcu_unlock_count[].93 * is set, don't initialize ->srcu_lock_count[] and ->srcu_unlock_count[].
94 */94 */
95static void init_srcu_struct_nodes(struct srcu_struct *sp, bool is_static)95static void init_srcu_struct_nodes(struct srcu_struct *ssp, bool is_static)
96{96{
97 int cpu;97 int cpu;
98 int i;98 int i;
@@ -103,13 +103,13 @@ static void init_srcu_struct_nodes(struct srcu_struct *sp, bool is_static) · 4 unmodified lines
103 struct srcu_node *snp_first;103 struct srcu_node *snp_first;
104 104
105 /* Work out the overall tree geometry. */105 /* Work out the overall tree geometry. */
106 sp->level[0] = &sp->node[0];106 ssp->level[0] = &ssp->node[0];
107 for (i = 1; i < rcu_num_lvls; i++)107 for (i = 1; i < rcu_num_lvls; i++)
108 sp->level[i] = sp->level[i - 1] + num_rcu_lvl[i - 1];108 ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1];
109 rcu_init_levelspread(levelspread, num_rcu_lvl);109 rcu_init_levelspread(levelspread, num_rcu_lvl);
110 110
111 /* Each pass through this loop initializes one srcu_node structure. */111 /* Each pass through this loop initializes one srcu_node structure. */
112 srcu_for_each_node_breadth_first(sp, snp) {112 srcu_for_each_node_breadth_first(ssp, snp) {
113 spin_lock_init(&ACCESS_PRIVATE(snp, lock));113 spin_lock_init(&ACCESS_PRIVATE(snp, lock));
114 WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=114 WARN_ON_ONCE(ARRAY_SIZE(snp->srcu_have_cbs) !=
115 ARRAY_SIZE(snp->srcu_data_have_cbs));115 ARRAY_SIZE(snp->srcu_data_have_cbs));
@@ -120,17 +120,17 @@ static void init_srcu_struct_nodes(struct srcu_struct *sp, bool is_static) · 4 unmodified lines
120 snp->srcu_gp_seq_needed_exp = 0;120 snp->srcu_gp_seq_needed_exp = 0;
121 snp->grplo = -1;121 snp->grplo = -1;
122 snp->grphi = -1;122 snp->grphi = -1;
123 if (snp == &sp->node[0]) {123 if (snp == &ssp->node[0]) {
124 /* Root node, special case. */124 /* Root node, special case. */
125 snp->srcu_parent = NULL;125 snp->srcu_parent = NULL;
126 continue;126 continue;
127 }127 }
128 128
129 /* Non-root node. */129 /* Non-root node. */
130 if (snp == sp->level[level + 1])130 if (snp == ssp->level[level + 1])
131 level++;131 level++;
132 snp->srcu_parent = sp->level[level - 1] +132 snp->srcu_parent = ssp->level[level - 1] +
133 (snp - sp->level[level]) /133 (snp - ssp->level[level]) /
134 levelspread[level - 1];134 levelspread[level - 1];
135 }135 }
136 136
@@ -141,14 +141,14 @@ static void init_srcu_struct_nodes(struct srcu_struct *sp, bool is_static) · 4 unmodified lines
141 WARN_ON_ONCE(ARRAY_SIZE(sdp->srcu_lock_count) !=141 WARN_ON_ONCE(ARRAY_SIZE(sdp->srcu_lock_count) !=
142 ARRAY_SIZE(sdp->srcu_unlock_count));142 ARRAY_SIZE(sdp->srcu_unlock_count));
143 level = rcu_num_lvls - 1;143 level = rcu_num_lvls - 1;
144 snp_first = sp->level[level];144 snp_first = ssp->level[level];
145 for_each_possible_cpu(cpu) {145 for_each_possible_cpu(cpu) {
146 sdp = per_cpu_ptr(sp->sda, cpu);146 sdp = per_cpu_ptr(ssp->sda, cpu);
147 spin_lock_init(&ACCESS_PRIVATE(sdp, lock));147 spin_lock_init(&ACCESS_PRIVATE(sdp, lock));
148 rcu_segcblist_init(&sdp->srcu_cblist);148 rcu_segcblist_init(&sdp->srcu_cblist);
149 sdp->srcu_cblist_invoking = false;149 sdp->srcu_cblist_invoking = false;
150 sdp->srcu_gp_seq_needed = sp->srcu_gp_seq;150 sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq;
151 sdp->srcu_gp_seq_needed_exp = sp->srcu_gp_seq;151 sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq;
152 sdp->mynode = &snp_first[cpu / levelspread[level]];152 sdp->mynode = &snp_first[cpu / levelspread[level]];
153 for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {153 for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {
154 if (snp->grplo < 0)154 if (snp->grplo < 0)
@@ -157,7 +157,7 @@ static void init_srcu_struct_nodes(struct srcu_struct *sp, bool is_static) · 2 unmodified lines
157 }157 }
158 sdp->cpu = cpu;158 sdp->cpu = cpu;
159 INIT_DELAYED_WORK(&sdp->work, srcu_invoke_callbacks);159 INIT_DELAYED_WORK(&sdp->work, srcu_invoke_callbacks);
160 sdp->sp = sp;160 sdp->ssp = ssp;
161 sdp->grpmask = 1 << (cpu - sdp->mynode->grplo);161 sdp->grpmask = 1 << (cpu - sdp->mynode->grplo);
162 if (is_static)162 if (is_static)
163 continue;163 continue;
@@ -176,35 +176,35 @@ static void init_srcu_struct_nodes(struct srcu_struct *sp, bool is_static) · 12 unmodified lines
176 * parameter is passed through to init_srcu_struct_nodes(), and176 * parameter is passed through to init_srcu_struct_nodes(), and
177 * also tells us that ->sda has already been wired up to srcu_data.177 * also tells us that ->sda has already been wired up to srcu_data.
178 */178 */
179static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static)179static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
180{180{
181 mutex_init(&sp->srcu_cb_mutex);181 mutex_init(&ssp->srcu_cb_mutex);
182 mutex_init(&sp->srcu_gp_mutex);182 mutex_init(&ssp->srcu_gp_mutex);
183 sp->srcu_idx = 0;183 ssp->srcu_idx = 0;
184 sp->srcu_gp_seq = 0;184 ssp->srcu_gp_seq = 0;
185 sp->srcu_barrier_seq = 0;185 ssp->srcu_barrier_seq = 0;
186 mutex_init(&sp->srcu_barrier_mutex);186 mutex_init(&ssp->srcu_barrier_mutex);
187 atomic_set(&sp->srcu_barrier_cpu_cnt, 0);187 atomic_set(&ssp->srcu_barrier_cpu_cnt, 0);
188 INIT_DELAYED_WORK(&sp->work, process_srcu);188 INIT_DELAYED_WORK(&ssp->work, process_srcu);
189 if (!is_static)189 if (!is_static)
190 sp->sda = alloc_percpu(struct srcu_data);190 ssp->sda = alloc_percpu(struct srcu_data);
191 init_srcu_struct_nodes(sp, is_static);191 init_srcu_struct_nodes(ssp, is_static);
192 sp->srcu_gp_seq_needed_exp = 0;192 ssp->srcu_gp_seq_needed_exp = 0;
193 sp->srcu_last_gp_end = ktime_get_mono_fast_ns();193 ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();
194 smp_store_release(&sp->srcu_gp_seq_needed, 0); /* Init done. */194 smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */
195 return sp->sda ? 0 : -ENOMEM;195 return ssp->sda ? 0 : -ENOMEM;
196}196}
197 197
198#ifdef CONFIG_DEBUG_LOCK_ALLOC198#ifdef CONFIG_DEBUG_LOCK_ALLOC
199 199
200int __init_srcu_struct(struct srcu_struct *sp, const char *name,200int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
201 struct lock_class_key *key)201 struct lock_class_key *key)
202{202{
203 /* Don't re-initialize a lock while it is held. */203 /* Don't re-initialize a lock while it is held. */
204 debug_check_no_locks_freed((void *)sp, sizeof(*sp));204 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
205 lockdep_init_map(&sp->dep_map, name, key, 0);205 lockdep_init_map(&ssp->dep_map, name, key, 0);
206 spin_lock_init(&ACCESS_PRIVATE(sp, lock));206 spin_lock_init(&ACCESS_PRIVATE(ssp, lock));
207 return init_srcu_struct_fields(sp, false);207 return init_srcu_struct_fields(ssp, false);
208}208}
209EXPORT_SYMBOL_GPL(__init_srcu_struct);209EXPORT_SYMBOL_GPL(__init_srcu_struct);
210 210
@@ -212,16 +212,16 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct); · 1 unmodified line
212 212
213/**213/**
214 * init_srcu_struct - initialize a sleep-RCU structure214 * init_srcu_struct - initialize a sleep-RCU structure
215 * @sp: structure to initialize.215 * @ssp: structure to initialize.
216 *216 *
217 * Must invoke this on a given srcu_struct before passing that srcu_struct217 * Must invoke this on a given srcu_struct before passing that srcu_struct
218 * to any other function. Each srcu_struct represents a separate domain218 * to any other function. Each srcu_struct represents a separate domain
219 * of SRCU protection.219 * of SRCU protection.
220 */220 */
221int init_srcu_struct(struct srcu_struct *sp)221int init_srcu_struct(struct srcu_struct *ssp)
222{222{
223 spin_lock_init(&ACCESS_PRIVATE(sp, lock));223 spin_lock_init(&ACCESS_PRIVATE(ssp, lock));
224 return init_srcu_struct_fields(sp, false);224 return init_srcu_struct_fields(ssp, false);
225}225}
226EXPORT_SYMBOL_GPL(init_srcu_struct);226EXPORT_SYMBOL_GPL(init_srcu_struct);
227 227
@@ -231,37 +231,37 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); · 3 unmodified lines
231 * First-use initialization of statically allocated srcu_struct231 * First-use initialization of statically allocated srcu_struct
232 * structure. Wiring up the combining tree is more than can be232 * structure. Wiring up the combining tree is more than can be
233 * done with compile-time initialization, so this check is added233 * done with compile-time initialization, so this check is added
234 * to each update-side SRCU primitive. Use sp->lock, which -is-234 * to each update-side SRCU primitive. Use ssp->lock, which -is-
235 * compile-time initialized, to resolve races involving multiple235 * compile-time initialized, to resolve races involving multiple
236 * CPUs trying to garner first-use privileges.236 * CPUs trying to garner first-use privileges.
237 */237 */
238static void check_init_srcu_struct(struct srcu_struct *sp)238static void check_init_srcu_struct(struct srcu_struct *ssp)
239{239{
240 unsigned long flags;240 unsigned long flags;
241 241
242 /* The smp_load_acquire() pairs with the smp_store_release(). */242 /* The smp_load_acquire() pairs with the smp_store_release(). */
243 if (!rcu_seq_state(smp_load_acquire(&sp->srcu_gp_seq_needed))) /*^^^*/243 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq_needed))) /*^^^*/
244 return; /* Already initialized. */244 return; /* Already initialized. */
245 spin_lock_irqsave_rcu_node(sp, flags);245 spin_lock_irqsave_rcu_node(ssp, flags);
246 if (!rcu_seq_state(sp->srcu_gp_seq_needed)) {246 if (!rcu_seq_state(ssp->srcu_gp_seq_needed)) {
247 spin_unlock_irqrestore_rcu_node(sp, flags);247 spin_unlock_irqrestore_rcu_node(ssp, flags);
248 return;248 return;
249 }249 }
250 init_srcu_struct_fields(sp, true);250 init_srcu_struct_fields(ssp, true);
251 spin_unlock_irqrestore_rcu_node(sp, flags);251 spin_unlock_irqrestore_rcu_node(ssp, flags);
252}252}
253 253
254/*254/*
255 * Returns approximate total of the readers' ->srcu_lock_count[] values255 * Returns approximate total of the readers' ->srcu_lock_count[] values
256 * for the rank of per-CPU counters specified by idx.256 * for the rank of per-CPU counters specified by idx.
257 */257 */
258static unsigned long srcu_readers_lock_idx(struct srcu_struct *sp, int idx)258static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)
259{259{
260 int cpu;260 int cpu;
261 unsigned long sum = 0;261 unsigned long sum = 0;
262 262
263 for_each_possible_cpu(cpu) {263 for_each_possible_cpu(cpu) {
264 struct srcu_data *cpuc = per_cpu_ptr(sp->sda, cpu);264 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);
265 265
266 sum += READ_ONCE(cpuc->srcu_lock_count[idx]);266 sum += READ_ONCE(cpuc->srcu_lock_count[idx]);
267 }267 }
@@ -272,13 +272,13 @@ static unsigned long srcu_readers_lock_idx(struct srcu_struct *sp, int idx) · 4 unmodified lines
272 * Returns approximate total of the readers' ->srcu_unlock_count[] values272 * Returns approximate total of the readers' ->srcu_unlock_count[] values
273 * for the rank of per-CPU counters specified by idx.273 * for the rank of per-CPU counters specified by idx.
274 */274 */
275static unsigned long srcu_readers_unlock_idx(struct srcu_struct *sp, int idx)275static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)
276{276{
277 int cpu;277 int cpu;
278 unsigned long sum = 0;278 unsigned long sum = 0;
279 279
280 for_each_possible_cpu(cpu) {280 for_each_possible_cpu(cpu) {
281 struct srcu_data *cpuc = per_cpu_ptr(sp->sda, cpu);281 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);
282 282
283 sum += READ_ONCE(cpuc->srcu_unlock_count[idx]);283 sum += READ_ONCE(cpuc->srcu_unlock_count[idx]);
284 }284 }
@@ -289,11 +289,11 @@ static unsigned long srcu_readers_unlock_idx(struct srcu_struct *sp, int idx) · 4 unmodified lines
289 * Return true if the number of pre-existing readers is determined to289 * Return true if the number of pre-existing readers is determined to
290 * be zero.290 * be zero.
291 */291 */
292static bool srcu_readers_active_idx_check(struct srcu_struct *sp, int idx)292static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)
293{293{
294 unsigned long unlocks;294 unsigned long unlocks;
295 295
296 unlocks = srcu_readers_unlock_idx(sp, idx);296 unlocks = srcu_readers_unlock_idx(ssp, idx);
297 297
298 /*298 /*
299 * Make sure that a lock is always counted if the corresponding299 * Make sure that a lock is always counted if the corresponding
@@ -329,25 +329,25 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *sp, int idx) · 29 unmodified lines
329 * of floor(ULONG_MAX/NR_CPUS/2), which should be sufficient,329 * of floor(ULONG_MAX/NR_CPUS/2), which should be sufficient,
330 * especially on 64-bit systems.330 * especially on 64-bit systems.
331 */331 */
332 return srcu_readers_lock_idx(sp, idx) == unlocks;332 return srcu_readers_lock_idx(ssp, idx) == unlocks;
333}333}
334 334
335/**335/**
336 * srcu_readers_active - returns true if there are readers. and false336 * srcu_readers_active - returns true if there are readers. and false
337 * otherwise337 * otherwise
338 * @sp: which srcu_struct to count active readers (holding srcu_read_lock).338 * @ssp: which srcu_struct to count active readers (holding srcu_read_lock).
339 *339 *
340 * Note that this is not an atomic primitive, and can therefore suffer340 * Note that this is not an atomic primitive, and can therefore suffer
341 * severe errors when invoked on an active srcu_struct. That said, it341 * severe errors when invoked on an active srcu_struct. That said, it
342 * can be useful as an error check at cleanup time.342 * can be useful as an error check at cleanup time.
343 */343 */
344static bool srcu_readers_active(struct srcu_struct *sp)344static bool srcu_readers_active(struct srcu_struct *ssp)
345{345{
346 int cpu;346 int cpu;
347 unsigned long sum = 0;347 unsigned long sum = 0;
348 348
349 for_each_possible_cpu(cpu) {349 for_each_possible_cpu(cpu) {
350 struct srcu_data *cpuc = per_cpu_ptr(sp->sda, cpu);350 struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);
351 351
352 sum += READ_ONCE(cpuc->srcu_lock_count[0]);352 sum += READ_ONCE(cpuc->srcu_lock_count[0]);
353 sum += READ_ONCE(cpuc->srcu_lock_count[1]);353 sum += READ_ONCE(cpuc->srcu_lock_count[1]);
@@ -363,44 +363,44 @@ static bool srcu_readers_active(struct srcu_struct *sp) · 9 unmodified lines
363 * Return grace-period delay, zero if there are expedited grace363 * Return grace-period delay, zero if there are expedited grace
364 * periods pending, SRCU_INTERVAL otherwise.364 * periods pending, SRCU_INTERVAL otherwise.
365 */365 */
366static unsigned long srcu_get_delay(struct srcu_struct *sp)366static unsigned long srcu_get_delay(struct srcu_struct *ssp)
367{367{
368 if (ULONG_CMP_LT(READ_ONCE(sp->srcu_gp_seq),368 if (ULONG_CMP_LT(READ_ONCE(ssp->srcu_gp_seq),
369 READ_ONCE(sp->srcu_gp_seq_needed_exp)))369 READ_ONCE(ssp->srcu_gp_seq_needed_exp)))
370 return 0;370 return 0;
371 return SRCU_INTERVAL;371 return SRCU_INTERVAL;
372}372}
373 373
374/* Helper for cleanup_srcu_struct() and cleanup_srcu_struct_quiesced(). */374/* Helper for cleanup_srcu_struct() and cleanup_srcu_struct_quiesced(). */
375void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced)375void _cleanup_srcu_struct(struct srcu_struct *ssp, bool quiesced)
376{376{
377 int cpu;377 int cpu;
378 378
379 if (WARN_ON(!srcu_get_delay(sp)))379 if (WARN_ON(!srcu_get_delay(ssp)))
380 return; /* Just leak it! */380 return; /* Just leak it! */
381 if (WARN_ON(srcu_readers_active(sp)))381 if (WARN_ON(srcu_readers_active(ssp)))
382 return; /* Just leak it! */382 return; /* Just leak it! */
383 if (quiesced) {383 if (quiesced) {
384 if (WARN_ON(delayed_work_pending(&sp->work)))384 if (WARN_ON(delayed_work_pending(&ssp->work)))
385 return; /* Just leak it! */385 return; /* Just leak it! */
386 } else {386 } else {
387 flush_delayed_work(&sp->work);387 flush_delayed_work(&ssp->work);
388 }388 }
389 for_each_possible_cpu(cpu)389 for_each_possible_cpu(cpu)
390 if (quiesced) {390 if (quiesced) {
391 if (WARN_ON(delayed_work_pending(&per_cpu_ptr(sp->sda, cpu)->work)))391 if (WARN_ON(delayed_work_pending(&per_cpu_ptr(ssp->sda, cpu)->work)))
392 return; /* Just leak it! */392 return; /* Just leak it! */
393 } else {393 } else {
394 flush_delayed_work(&per_cpu_ptr(sp->sda, cpu)->work);394 flush_delayed_work(&per_cpu_ptr(ssp->sda, cpu)->work);
395 }395 }
396 if (WARN_ON(rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)) != SRCU_STATE_IDLE) ||396 if (WARN_ON(rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) != SRCU_STATE_IDLE) ||
397 WARN_ON(srcu_readers_active(sp))) {397 WARN_ON(srcu_readers_active(ssp))) {
398 pr_info("%s: Active srcu_struct %p state: %d\n",398 pr_info("%s: Active srcu_struct %p state: %d\n",
399 __func__, sp, rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)));399 __func__, ssp, rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)));
400 return; /* Caller forgot to stop doing call_srcu()? */400 return; /* Caller forgot to stop doing call_srcu()? */
401 }401 }
402 free_percpu(sp->sda);402 free_percpu(ssp->sda);
403 sp->sda = NULL;403 ssp->sda = NULL;
404}404}
405EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);405EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);
406 406
@@ -409,12 +409,12 @@ EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); · 2 unmodified lines
409 * srcu_struct.409 * srcu_struct.
410 * Returns an index that must be passed to the matching srcu_read_unlock().410 * Returns an index that must be passed to the matching srcu_read_unlock().
411 */411 */
412int __srcu_read_lock(struct srcu_struct *sp)412int __srcu_read_lock(struct srcu_struct *ssp)
413{413{
414 int idx;414 int idx;
415 415
416 idx = READ_ONCE(sp->srcu_idx) & 0x1;416 idx = READ_ONCE(ssp->srcu_idx) & 0x1;
417 this_cpu_inc(sp->sda->srcu_lock_count[idx]);417 this_cpu_inc(ssp->sda->srcu_lock_count[idx]);
418 smp_mb(); /* B */ /* Avoid leaking the critical section. */418 smp_mb(); /* B */ /* Avoid leaking the critical section. */
419 return idx;419 return idx;
420}420}
@@ -425,10 +425,10 @@ EXPORT_SYMBOL_GPL(__srcu_read_lock); · 4 unmodified lines
425 * element of the srcu_struct. Note that this may well be a different425 * element of the srcu_struct. Note that this may well be a different
426 * CPU than that which was incremented by the corresponding srcu_read_lock().426 * CPU than that which was incremented by the corresponding srcu_read_lock().
427 */427 */
428void __srcu_read_unlock(struct srcu_struct *sp, int idx)428void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
429{429{
430 smp_mb(); /* C */ /* Avoid leaking the critical section. */430 smp_mb(); /* C */ /* Avoid leaking the critical section. */
431 this_cpu_inc(sp->sda->srcu_unlock_count[idx]);431 this_cpu_inc(ssp->sda->srcu_unlock_count[idx]);
432}432}
433EXPORT_SYMBOL_GPL(__srcu_read_unlock);433EXPORT_SYMBOL_GPL(__srcu_read_unlock);
434 434
@@ -444,22 +444,22 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock); · 9 unmodified lines
444/*444/*
445 * Start an SRCU grace period.445 * Start an SRCU grace period.
446 */446 */
447static void srcu_gp_start(struct srcu_struct *sp)447static void srcu_gp_start(struct srcu_struct *ssp)
448{448{
449 struct srcu_data *sdp = this_cpu_ptr(sp->sda);449 struct srcu_data *sdp = this_cpu_ptr(ssp->sda);
450 int state;450 int state;
451 451
452 lockdep_assert_held(&ACCESS_PRIVATE(sp, lock));452 lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock));
453 WARN_ON_ONCE(ULONG_CMP_GE(sp->srcu_gp_seq, sp->srcu_gp_seq_needed));453 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed));
454 spin_lock_rcu_node(sdp); /* Interrupts already disabled. */454 spin_lock_rcu_node(sdp); /* Interrupts already disabled. */
455 rcu_segcblist_advance(&sdp->srcu_cblist,455 rcu_segcblist_advance(&sdp->srcu_cblist,
456 rcu_seq_current(&sp->srcu_gp_seq));456 rcu_seq_current(&ssp->srcu_gp_seq));
457 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist,457 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist,
458 rcu_seq_snap(&sp->srcu_gp_seq));458 rcu_seq_snap(&ssp->srcu_gp_seq));
459 spin_unlock_rcu_node(sdp); /* Interrupts remain disabled. */459 spin_unlock_rcu_node(sdp); /* Interrupts remain disabled. */
460 smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */460 smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */
461 rcu_seq_start(&sp->srcu_gp_seq);461 rcu_seq_start(&ssp->srcu_gp_seq);
462 state = rcu_seq_state(READ_ONCE(sp->srcu_gp_seq));462 state = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));
463 WARN_ON_ONCE(state != SRCU_STATE_SCAN1);463 WARN_ON_ONCE(state != SRCU_STATE_SCAN1);
464}464}
465 465
@@ -513,7 +513,7 @@ static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay) · 47 unmodified lines
513 * just-completed grace period, the one corresponding to idx. If possible,513 * just-completed grace period, the one corresponding to idx. If possible,
514 * schedule this invocation on the corresponding CPUs.514 * schedule this invocation on the corresponding CPUs.
515 */515 */
516static void srcu_schedule_cbs_snp(struct srcu_struct *sp, struct srcu_node *snp,516static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,
517 unsigned long mask, unsigned long delay)517 unsigned long mask, unsigned long delay)
518{518{
519 int cpu;519 int cpu;
@@ -521,7 +521,7 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *sp, struct srcu_node *snp, · 1 unmodified line
521 for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {521 for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {
522 if (!(mask & (1 << (cpu - snp->grplo))))522 if (!(mask & (1 << (cpu - snp->grplo))))
523 continue;523 continue;
524 srcu_schedule_cbs_sdp(per_cpu_ptr(sp->sda, cpu), delay);524 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);
525 }525 }
526}526}
527 527
@@ -534,7 +534,7 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *sp, struct srcu_node *snp, · 6 unmodified lines
534 * are initiating callback invocation. This allows the ->srcu_have_cbs[]534 * are initiating callback invocation. This allows the ->srcu_have_cbs[]
535 * array to have a finite number of elements.535 * array to have a finite number of elements.
536 */536 */
537static void srcu_gp_end(struct srcu_struct *sp)537static void srcu_gp_end(struct srcu_struct *ssp)
538{538{
539 unsigned long cbdelay;539 unsigned long cbdelay;
540 bool cbs;540 bool cbs;
@@ -548,28 +548,28 @@ static void srcu_gp_end(struct srcu_struct *sp) · 7 unmodified lines
548 struct srcu_node *snp;548 struct srcu_node *snp;
549 549
550 /* Prevent more than one additional grace period. */550 /* Prevent more than one additional grace period. */
551 mutex_lock(&sp->srcu_cb_mutex);551 mutex_lock(&ssp->srcu_cb_mutex);
552 552
553 /* End the current grace period. */553 /* End the current grace period. */
554 spin_lock_irq_rcu_node(sp);554 spin_lock_irq_rcu_node(ssp);
555 idx = rcu_seq_state(sp->srcu_gp_seq);555 idx = rcu_seq_state(ssp->srcu_gp_seq);
556 WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);556 WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);
557 cbdelay = srcu_get_delay(sp);557 cbdelay = srcu_get_delay(ssp);
558 sp->srcu_last_gp_end = ktime_get_mono_fast_ns();558 ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();
559 rcu_seq_end(&sp->srcu_gp_seq);559 rcu_seq_end(&ssp->srcu_gp_seq);
560 gpseq = rcu_seq_current(&sp->srcu_gp_seq);560 gpseq = rcu_seq_current(&ssp->srcu_gp_seq);
561 if (ULONG_CMP_LT(sp->srcu_gp_seq_needed_exp, gpseq))561 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq))
562 sp->srcu_gp_seq_needed_exp = gpseq;562 ssp->srcu_gp_seq_needed_exp = gpseq;
563 spin_unlock_irq_rcu_node(sp);563 spin_unlock_irq_rcu_node(ssp);
564 mutex_unlock(&sp->srcu_gp_mutex);564 mutex_unlock(&ssp->srcu_gp_mutex);
565 /* A new grace period can start at this point. But only one. */565 /* A new grace period can start at this point. But only one. */
566 566
567 /* Initiate callback invocation as needed. */567 /* Initiate callback invocation as needed. */
568 idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);568 idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);
569 srcu_for_each_node_breadth_first(sp, snp) {569 srcu_for_each_node_breadth_first(ssp, snp) {
570 spin_lock_irq_rcu_node(snp);570 spin_lock_irq_rcu_node(snp);
571 cbs = false;571 cbs = false;
572 last_lvl = snp >= sp->level[rcu_num_lvls - 1];572 last_lvl = snp >= ssp->level[rcu_num_lvls - 1];
573 if (last_lvl)573 if (last_lvl)
574 cbs = snp->srcu_have_cbs[idx] == gpseq;574 cbs = snp->srcu_have_cbs[idx] == gpseq;
575 snp->srcu_have_cbs[idx] = gpseq;575 snp->srcu_have_cbs[idx] = gpseq;
@@ -580,12 +580,12 @@ static void srcu_gp_end(struct srcu_struct *sp) · 4 unmodified lines
580 snp->srcu_data_have_cbs[idx] = 0;580 snp->srcu_data_have_cbs[idx] = 0;
581 spin_unlock_irq_rcu_node(snp);581 spin_unlock_irq_rcu_node(snp);
582 if (cbs)582 if (cbs)
583 srcu_schedule_cbs_snp(sp, snp, mask, cbdelay);583 srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);
584 584
585 /* Occasionally prevent srcu_data counter wrap. */585 /* Occasionally prevent srcu_data counter wrap. */
586 if (!(gpseq & counter_wrap_check) && last_lvl)586 if (!(gpseq & counter_wrap_check) && last_lvl)
587 for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {587 for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {
588 sdp = per_cpu_ptr(sp->sda, cpu);588 sdp = per_cpu_ptr(ssp->sda, cpu);
589 spin_lock_irqsave_rcu_node(sdp, flags);589 spin_lock_irqsave_rcu_node(sdp, flags);
590 if (ULONG_CMP_GE(gpseq,590 if (ULONG_CMP_GE(gpseq,
591 sdp->srcu_gp_seq_needed + 100))591 sdp->srcu_gp_seq_needed + 100))
@@ -598,18 +598,18 @@ static void srcu_gp_end(struct srcu_struct *sp) · 6 unmodified lines
598 }598 }
599 599
600 /* Callback initiation done, allow grace periods after next. */600 /* Callback initiation done, allow grace periods after next. */
601 mutex_unlock(&sp->srcu_cb_mutex);601 mutex_unlock(&ssp->srcu_cb_mutex);
602 602
603 /* Start a new grace period if needed. */603 /* Start a new grace period if needed. */
604 spin_lock_irq_rcu_node(sp);604 spin_lock_irq_rcu_node(ssp);
605 gpseq = rcu_seq_current(&sp->srcu_gp_seq);605 gpseq = rcu_seq_current(&ssp->srcu_gp_seq);
606 if (!rcu_seq_state(gpseq) &&606 if (!rcu_seq_state(gpseq) &&
607 ULONG_CMP_LT(gpseq, sp->srcu_gp_seq_needed)) {607 ULONG_CMP_LT(gpseq, ssp->srcu_gp_seq_needed)) {
608 srcu_gp_start(sp);608 srcu_gp_start(ssp);
609 spin_unlock_irq_rcu_node(sp);609 spin_unlock_irq_rcu_node(ssp);
610 srcu_reschedule(sp, 0);610 srcu_reschedule(ssp, 0);
611 } else {611 } else {
612 spin_unlock_irq_rcu_node(sp);612 spin_unlock_irq_rcu_node(ssp);
613 }613 }
614}614}
615 615
@@ -620,13 +620,13 @@ static void srcu_gp_end(struct srcu_struct *sp) · 4 unmodified lines
620 * but without expediting. To start a completely new grace period,620 * but without expediting. To start a completely new grace period,
621 * whether expedited or not, use srcu_funnel_gp_start() instead.621 * whether expedited or not, use srcu_funnel_gp_start() instead.
622 */622 */
623static void srcu_funnel_exp_start(struct srcu_struct *sp, struct srcu_node *snp,623static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,
624 unsigned long s)624 unsigned long s)
625{625{
626 unsigned long flags;626 unsigned long flags;
627 627
628 for (; snp != NULL; snp = snp->srcu_parent) {628 for (; snp != NULL; snp = snp->srcu_parent) {
629 if (rcu_seq_done(&sp->srcu_gp_seq, s) ||629 if (rcu_seq_done(&ssp->srcu_gp_seq, s) ||
630 ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))630 ULONG_CMP_GE(READ_ONCE(snp->srcu_gp_seq_needed_exp), s))
631 return;631 return;
632 spin_lock_irqsave_rcu_node(snp, flags);632 spin_lock_irqsave_rcu_node(snp, flags);
@@ -637,10 +637,10 @@ static void srcu_funnel_exp_start(struct srcu_struct *sp, struct srcu_node *snp, · 4 unmodified lines
637 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);637 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
638 spin_unlock_irqrestore_rcu_node(snp, flags);638 spin_unlock_irqrestore_rcu_node(snp, flags);
639 }639 }
640 spin_lock_irqsave_rcu_node(sp, flags);640 spin_lock_irqsave_rcu_node(ssp, flags);
641 if (ULONG_CMP_LT(sp->srcu_gp_seq_needed_exp, s))641 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
642 sp->srcu_gp_seq_needed_exp = s;642 ssp->srcu_gp_seq_needed_exp = s;
643 spin_unlock_irqrestore_rcu_node(sp, flags);643 spin_unlock_irqrestore_rcu_node(ssp, flags);
644}644}
645 645
646/*646/*
@@ -653,7 +653,7 @@ static void srcu_funnel_exp_start(struct srcu_struct *sp, struct srcu_node *snp, · 6 unmodified lines
653 * Note that this function also does the work of srcu_funnel_exp_start(),653 * Note that this function also does the work of srcu_funnel_exp_start(),
654 * in some cases by directly invoking it.654 * in some cases by directly invoking it.
655 */655 */
656static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp,656static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
657 unsigned long s, bool do_norm)657 unsigned long s, bool do_norm)
658{658{
659 unsigned long flags;659 unsigned long flags;
@@ -663,7 +663,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, · 3 unmodified lines
663 663
664 /* Each pass through the loop does one level of the srcu_node tree. */664 /* Each pass through the loop does one level of the srcu_node tree. */
665 for (; snp != NULL; snp = snp->srcu_parent) {665 for (; snp != NULL; snp = snp->srcu_parent) {
666 if (rcu_seq_done(&sp->srcu_gp_seq, s) && snp != sdp->mynode)666 if (rcu_seq_done(&ssp->srcu_gp_seq, s) && snp != sdp->mynode)
667 return; /* GP already done and CBs recorded. */667 return; /* GP already done and CBs recorded. */
668 spin_lock_irqsave_rcu_node(snp, flags);668 spin_lock_irqsave_rcu_node(snp, flags);
669 if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {669 if (ULONG_CMP_GE(snp->srcu_have_cbs[idx], s)) {
@@ -678,7 +678,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, · 8 unmodified lines
678 return;678 return;
679 }679 }
680 if (!do_norm)680 if (!do_norm)
681 srcu_funnel_exp_start(sp, snp, s);681 srcu_funnel_exp_start(ssp, snp, s);
682 return;682 return;
683 }683 }
684 snp->srcu_have_cbs[idx] = s;684 snp->srcu_have_cbs[idx] = s;
@@ -690,29 +690,29 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, · 5 unmodified lines
690 }690 }
691 691
692 /* Top of tree, must ensure the grace period will be started. */692 /* Top of tree, must ensure the grace period will be started. */
693 spin_lock_irqsave_rcu_node(sp, flags);693 spin_lock_irqsave_rcu_node(ssp, flags);
694 if (ULONG_CMP_LT(sp->srcu_gp_seq_needed, s)) {694 if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed, s)) {
695 /*695 /*
696 * Record need for grace period s. Pair with load696 * Record need for grace period s. Pair with load
697 * acquire setting up for initialization.697 * acquire setting up for initialization.
698 */698 */
699 smp_store_release(&sp->srcu_gp_seq_needed, s); /*^^^*/699 smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/
700 }700 }
701 if (!do_norm && ULONG_CMP_LT(sp->srcu_gp_seq_needed_exp, s))701 if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
702 sp->srcu_gp_seq_needed_exp = s;702 ssp->srcu_gp_seq_needed_exp = s;
703 703
704 /* If grace period not already done and none in progress, start it. */704 /* If grace period not already done and none in progress, start it. */
705 if (!rcu_seq_done(&sp->srcu_gp_seq, s) &&705 if (!rcu_seq_done(&ssp->srcu_gp_seq, s) &&
706 rcu_seq_state(sp->srcu_gp_seq) == SRCU_STATE_IDLE) {706 rcu_seq_state(ssp->srcu_gp_seq) == SRCU_STATE_IDLE) {
707 WARN_ON_ONCE(ULONG_CMP_GE(sp->srcu_gp_seq, sp->srcu_gp_seq_needed));707 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed));
708 srcu_gp_start(sp);708 srcu_gp_start(ssp);
709 if (likely(srcu_init_done))709 if (likely(srcu_init_done))
710 queue_delayed_work(rcu_gp_wq, &sp->work,710 queue_delayed_work(rcu_gp_wq, &ssp->work,
711 srcu_get_delay(sp));711 srcu_get_delay(ssp));
712 else if (list_empty(&sp->work.work.entry))712 else if (list_empty(&ssp->work.work.entry))
713 list_add(&sp->work.work.entry, &srcu_boot_list);713 list_add(&ssp->work.work.entry, &srcu_boot_list);
714 }714 }
715 spin_unlock_irqrestore_rcu_node(sp, flags);715 spin_unlock_irqrestore_rcu_node(ssp, flags);
716}716}
717 717
718/*718/*
@@ -720,12 +720,12 @@ static void srcu_funnel_gp_start(struct srcu_struct *sp, struct srcu_data *sdp, · 1 unmodified line
720 * loop an additional time if there is an expedited grace period pending.720 * loop an additional time if there is an expedited grace period pending.
721 * The caller must ensure that ->srcu_idx is not changed while checking.721 * The caller must ensure that ->srcu_idx is not changed while checking.
722 */722 */
723static bool try_check_zero(struct srcu_struct *sp, int idx, int trycount)723static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)
724{724{
725 for (;;) {725 for (;;) {
726 if (srcu_readers_active_idx_check(sp, idx))726 if (srcu_readers_active_idx_check(ssp, idx))
727 return true;727 return true;
728 if (--trycount + !srcu_get_delay(sp) <= 0)728 if (--trycount + !srcu_get_delay(ssp) <= 0)
729 return false;729 return false;
730 udelay(SRCU_RETRY_CHECK_DELAY);730 udelay(SRCU_RETRY_CHECK_DELAY);
731 }731 }
@@ -736,7 +736,7 @@ static bool try_check_zero(struct srcu_struct *sp, int idx, int trycount) · 4 unmodified lines
736 * use the other rank of the ->srcu_(un)lock_count[] arrays. This allows736 * use the other rank of the ->srcu_(un)lock_count[] arrays. This allows
737 * us to wait for pre-existing readers in a starvation-free manner.737 * us to wait for pre-existing readers in a starvation-free manner.
738 */738 */
739static void srcu_flip(struct srcu_struct *sp)739static void srcu_flip(struct srcu_struct *ssp)
740{740{
741 /*741 /*
742 * Ensure that if this updater saw a given reader's increment742 * Ensure that if this updater saw a given reader's increment
@@ -748,7 +748,7 @@ static void srcu_flip(struct srcu_struct *sp) · 5 unmodified lines
748 */748 */
749 smp_mb(); /* E */ /* Pairs with B and C. */749 smp_mb(); /* E */ /* Pairs with B and C. */
750 750
751 WRITE_ONCE(sp->srcu_idx, sp->srcu_idx + 1);751 WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
752 752
753 /*753 /*
754 * Ensure that if the updater misses an __srcu_read_unlock()754 * Ensure that if the updater misses an __srcu_read_unlock()
@@ -781,7 +781,7 @@ static void srcu_flip(struct srcu_struct *sp) · 26 unmodified lines
781 * negligible when amoritized over that time period, and the extra latency781 * negligible when amoritized over that time period, and the extra latency
782 * of a needlessly non-expedited grace period is similarly negligible.782 * of a needlessly non-expedited grace period is similarly negligible.
783 */783 */
784static bool srcu_might_be_idle(struct srcu_struct *sp)784static bool srcu_might_be_idle(struct srcu_struct *ssp)
785{785{
786 unsigned long curseq;786 unsigned long curseq;
787 unsigned long flags;787 unsigned long flags;
@@ -790,7 +790,7 @@ static bool srcu_might_be_idle(struct srcu_struct *sp) · 2 unmodified lines
790 790
791 /* If the local srcu_data structure has callbacks, not idle. */791 /* If the local srcu_data structure has callbacks, not idle. */
792 local_irq_save(flags);792 local_irq_save(flags);
793 sdp = this_cpu_ptr(sp->sda);793 sdp = this_cpu_ptr(ssp->sda);
794 if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {794 if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) {
795 local_irq_restore(flags);795 local_irq_restore(flags);
796 return false; /* Callbacks already present, so not idle. */796 return false; /* Callbacks already present, so not idle. */
@@ -806,17 +806,17 @@ static bool srcu_might_be_idle(struct srcu_struct *sp) · 9 unmodified lines
806 /* First, see if enough time has passed since the last GP. */806 /* First, see if enough time has passed since the last GP. */
807 t = ktime_get_mono_fast_ns();807 t = ktime_get_mono_fast_ns();
808 if (exp_holdoff == 0 ||808 if (exp_holdoff == 0 ||
809 time_in_range_open(t, sp->srcu_last_gp_end,809 time_in_range_open(t, ssp->srcu_last_gp_end,
810 sp->srcu_last_gp_end + exp_holdoff))810 ssp->srcu_last_gp_end + exp_holdoff))
811 return false; /* Too soon after last GP. */811 return false; /* Too soon after last GP. */
812 812
813 /* Next, check for probable idleness. */813 /* Next, check for probable idleness. */
814 curseq = rcu_seq_current(&sp->srcu_gp_seq);814 curseq = rcu_seq_current(&ssp->srcu_gp_seq);
815 smp_mb(); /* Order ->srcu_gp_seq with ->srcu_gp_seq_needed. */815 smp_mb(); /* Order ->srcu_gp_seq with ->srcu_gp_seq_needed. */
816 if (ULONG_CMP_LT(curseq, READ_ONCE(sp->srcu_gp_seq_needed)))816 if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_gp_seq_needed)))
817 return false; /* Grace period in progress, so not idle. */817 return false; /* Grace period in progress, so not idle. */
818 smp_mb(); /* Order ->srcu_gp_seq with prior access. */818 smp_mb(); /* Order ->srcu_gp_seq with prior access. */
819 if (curseq != rcu_seq_current(&sp->srcu_gp_seq))819 if (curseq != rcu_seq_current(&ssp->srcu_gp_seq))
820 return false; /* GP # changed, so not idle. */820 return false; /* GP # changed, so not idle. */
821 return true; /* With reasonable probability, idle! */821 return true; /* With reasonable probability, idle! */
822}822}
@@ -856,7 +856,7 @@ static void srcu_leak_callback(struct rcu_head *rhp) · 33 unmodified lines
856 * srcu_read_lock(), and srcu_read_unlock() that are all passed the same856 * srcu_read_lock(), and srcu_read_unlock() that are all passed the same
857 * srcu_struct structure.857 * srcu_struct structure.
858 */858 */
859void __call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,859void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
860 rcu_callback_t func, bool do_norm)860 rcu_callback_t func, bool do_norm)
861{861{
862 unsigned long flags;862 unsigned long flags;
@@ -866,7 +866,7 @@ void __call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, · 3 unmodified lines
866 unsigned long s;866 unsigned long s;
867 struct srcu_data *sdp;867 struct srcu_data *sdp;
868 868
869 check_init_srcu_struct(sp);869 check_init_srcu_struct(ssp);
870 if (debug_rcu_head_queue(rhp)) {870 if (debug_rcu_head_queue(rhp)) {
871 /* Probable double call_srcu(), so leak the callback. */871 /* Probable double call_srcu(), so leak the callback. */
872 WRITE_ONCE(rhp->func, srcu_leak_callback);872 WRITE_ONCE(rhp->func, srcu_leak_callback);
@@ -874,14 +874,14 @@ void __call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, · 1 unmodified line
874 return;874 return;
875 }875 }
876 rhp->func = func;876 rhp->func = func;
877 idx = srcu_read_lock(sp);877 idx = srcu_read_lock(ssp);
878 local_irq_save(flags);878 local_irq_save(flags);
879 sdp = this_cpu_ptr(sp->sda);879 sdp = this_cpu_ptr(ssp->sda);
880 spin_lock_rcu_node(sdp);880 spin_lock_rcu_node(sdp);
881 rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp, false);881 rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp, false);
882 rcu_segcblist_advance(&sdp->srcu_cblist,882 rcu_segcblist_advance(&sdp->srcu_cblist,
883 rcu_seq_current(&sp->srcu_gp_seq));883 rcu_seq_current(&ssp->srcu_gp_seq));
884 s = rcu_seq_snap(&sp->srcu_gp_seq);884 s = rcu_seq_snap(&ssp->srcu_gp_seq);
885 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s);885 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s);
886 if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {886 if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) {
887 sdp->srcu_gp_seq_needed = s;887 sdp->srcu_gp_seq_needed = s;
@@ -893,15 +893,15 @@ void __call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, · 5 unmodified lines
893 }893 }
894 spin_unlock_irqrestore_rcu_node(sdp, flags);894 spin_unlock_irqrestore_rcu_node(sdp, flags);
895 if (needgp)895 if (needgp)
896 srcu_funnel_gp_start(sp, sdp, s, do_norm);896 srcu_funnel_gp_start(ssp, sdp, s, do_norm);
897 else if (needexp)897 else if (needexp)
898 srcu_funnel_exp_start(sp, sdp->mynode, s);898 srcu_funnel_exp_start(ssp, sdp->mynode, s);
899 srcu_read_unlock(sp, idx);899 srcu_read_unlock(ssp, idx);
900}900}
901 901
902/**902/**
903 * call_srcu() - Queue a callback for invocation after an SRCU grace period903 * call_srcu() - Queue a callback for invocation after an SRCU grace period
904 * @sp: srcu_struct in queue the callback904 * @ssp: srcu_struct in queue the callback
905 * @rhp: structure to be used for queueing the SRCU callback.905 * @rhp: structure to be used for queueing the SRCU callback.
906 * @func: function to be invoked after the SRCU grace period906 * @func: function to be invoked after the SRCU grace period
907 *907 *
@@ -916,21 +916,21 @@ void __call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, · 8 unmodified lines
916 * The callback will be invoked from process context, but must nevertheless916 * The callback will be invoked from process context, but must nevertheless
917 * be fast and must not block.917 * be fast and must not block.
918 */918 */
919void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,919void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
920 rcu_callback_t func)920 rcu_callback_t func)
921{921{
922 __call_srcu(sp, rhp, func, true);922 __call_srcu(ssp, rhp, func, true);
923}923}
924EXPORT_SYMBOL_GPL(call_srcu);924EXPORT_SYMBOL_GPL(call_srcu);
925 925
926/*926/*
927 * Helper function for synchronize_srcu() and synchronize_srcu_expedited().927 * Helper function for synchronize_srcu() and synchronize_srcu_expedited().
928 */928 */
929static void __synchronize_srcu(struct srcu_struct *sp, bool do_norm)929static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm)
930{930{
931 struct rcu_synchronize rcu;931 struct rcu_synchronize rcu;
932 932
933 RCU_LOCKDEP_WARN(lock_is_held(&sp->dep_map) ||933 RCU_LOCKDEP_WARN(lock_is_held(&ssp->dep_map) ||
934 lock_is_held(&rcu_bh_lock_map) ||934 lock_is_held(&rcu_bh_lock_map) ||
935 lock_is_held(&rcu_lock_map) ||935 lock_is_held(&rcu_lock_map) ||
936 lock_is_held(&rcu_sched_lock_map),936 lock_is_held(&rcu_sched_lock_map),
@@ -939,10 +939,10 @@ static void __synchronize_srcu(struct srcu_struct *sp, bool do_norm) · 2 unmodified lines
939 if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)939 if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
940 return;940 return;
941 might_sleep();941 might_sleep();
942 check_init_srcu_struct(sp);942 check_init_srcu_struct(ssp);
943 init_completion(&rcu.completion);943 init_completion(&rcu.completion);
944 init_rcu_head_on_stack(&rcu.head);944 init_rcu_head_on_stack(&rcu.head);
945 __call_srcu(sp, &rcu.head, wakeme_after_rcu, do_norm);945 __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm);
946 wait_for_completion(&rcu.completion);946 wait_for_completion(&rcu.completion);
947 destroy_rcu_head_on_stack(&rcu.head);947 destroy_rcu_head_on_stack(&rcu.head);
948 948
@@ -958,7 +958,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, bool do_norm) · 9 unmodified lines
958 958
959/**959/**
960 * synchronize_srcu_expedited - Brute-force SRCU grace period960 * synchronize_srcu_expedited - Brute-force SRCU grace period
961 * @sp: srcu_struct with which to synchronize.961 * @ssp: srcu_struct with which to synchronize.
962 *962 *
963 * Wait for an SRCU grace period to elapse, but be more aggressive about963 * Wait for an SRCU grace period to elapse, but be more aggressive about
964 * spinning rather than blocking when waiting.964 * spinning rather than blocking when waiting.
@@ -966,15 +966,15 @@ static void __synchronize_srcu(struct srcu_struct *sp, bool do_norm) · 1 unmodified line
966 * Note that synchronize_srcu_expedited() has the same deadlock and966 * Note that synchronize_srcu_expedited() has the same deadlock and
967 * memory-ordering properties as does synchronize_srcu().967 * memory-ordering properties as does synchronize_srcu().
968 */968 */
969void synchronize_srcu_expedited(struct srcu_struct *sp)969void synchronize_srcu_expedited(struct srcu_struct *ssp)
970{970{
971 __synchronize_srcu(sp, rcu_gp_is_normal());971 __synchronize_srcu(ssp, rcu_gp_is_normal());
972}972}
973EXPORT_SYMBOL_GPL(synchronize_srcu_expedited);973EXPORT_SYMBOL_GPL(synchronize_srcu_expedited);
974 974
975/**975/**
976 * synchronize_srcu - wait for prior SRCU read-side critical-section completion976 * synchronize_srcu - wait for prior SRCU read-side critical-section completion
977 * @sp: srcu_struct with which to synchronize.977 * @ssp: srcu_struct with which to synchronize.
978 *978 *
979 * Wait for the count to drain to zero of both indexes. To avoid the979 * Wait for the count to drain to zero of both indexes. To avoid the
980 * possible starvation of synchronize_srcu(), it waits for the count of980 * possible starvation of synchronize_srcu(), it waits for the count of
@@ -1016,12 +1016,12 @@ EXPORT_SYMBOL_GPL(synchronize_srcu_expedited); · 35 unmodified lines
1016 * SRCU must also provide it. Note that detecting idleness is heuristic1016 * SRCU must also provide it. Note that detecting idleness is heuristic
1017 * and subject to both false positives and negatives.1017 * and subject to both false positives and negatives.
1018 */1018 */
1019void synchronize_srcu(struct srcu_struct *sp)1019void synchronize_srcu(struct srcu_struct *ssp)
1020{1020{
1021 if (srcu_might_be_idle(sp) || rcu_gp_is_expedited())1021 if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited())
1022 synchronize_srcu_expedited(sp);1022 synchronize_srcu_expedited(ssp);
1023 else1023 else
1024 __synchronize_srcu(sp, true);1024 __synchronize_srcu(ssp, true);
1025}1025}
1026EXPORT_SYMBOL_GPL(synchronize_srcu);1026EXPORT_SYMBOL_GPL(synchronize_srcu);
1027 1027
@@ -1031,36 +1031,36 @@ EXPORT_SYMBOL_GPL(synchronize_srcu); · 3 unmodified lines
1031static void srcu_barrier_cb(struct rcu_head *rhp)1031static void srcu_barrier_cb(struct rcu_head *rhp)
1032{1032{
1033 struct srcu_data *sdp;1033 struct srcu_data *sdp;
1034 struct srcu_struct *sp;1034 struct srcu_struct *ssp;
1035 1035
1036 sdp = container_of(rhp, struct srcu_data, srcu_barrier_head);1036 sdp = container_of(rhp, struct srcu_data, srcu_barrier_head);
1037 sp = sdp->sp;1037 ssp = sdp->ssp;
1038 if (atomic_dec_and_test(&sp->srcu_barrier_cpu_cnt))1038 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt))
1039 complete(&sp->srcu_barrier_completion);1039 complete(&ssp->srcu_barrier_completion);
1040}1040}
1041 1041
1042/**1042/**
1043 * srcu_barrier - Wait until all in-flight call_srcu() callbacks complete.1043 * srcu_barrier - Wait until all in-flight call_srcu() callbacks complete.
1044 * @sp: srcu_struct on which to wait for in-flight callbacks.1044 * @ssp: srcu_struct on which to wait for in-flight callbacks.
1045 */1045 */
1046void srcu_barrier(struct srcu_struct *sp)1046void srcu_barrier(struct srcu_struct *ssp)
1047{1047{
1048 int cpu;1048 int cpu;
1049 struct srcu_data *sdp;1049 struct srcu_data *sdp;
1050 unsigned long s = rcu_seq_snap(&sp->srcu_barrier_seq);1050 unsigned long s = rcu_seq_snap(&ssp->srcu_barrier_seq);
1051 1051
1052 check_init_srcu_struct(sp);1052 check_init_srcu_struct(ssp);
1053 mutex_lock(&sp->srcu_barrier_mutex);1053 mutex_lock(&ssp->srcu_barrier_mutex);
1054 if (rcu_seq_done(&sp->srcu_barrier_seq, s)) {1054 if (rcu_seq_done(&ssp->srcu_barrier_seq, s)) {
1055 smp_mb(); /* Force ordering following return. */1055 smp_mb(); /* Force ordering following return. */
1056 mutex_unlock(&sp->srcu_barrier_mutex);1056 mutex_unlock(&ssp->srcu_barrier_mutex);
1057 return; /* Someone else did our work for us. */1057 return; /* Someone else did our work for us. */
1058 }1058 }
1059 rcu_seq_start(&sp->srcu_barrier_seq);1059 rcu_seq_start(&ssp->srcu_barrier_seq);
1060 init_completion(&sp->srcu_barrier_completion);1060 init_completion(&ssp->srcu_barrier_completion);
1061 1061
1062 /* Initial count prevents reaching zero until all CBs are posted. */1062 /* Initial count prevents reaching zero until all CBs are posted. */
1063 atomic_set(&sp->srcu_barrier_cpu_cnt, 1);1063 atomic_set(&ssp->srcu_barrier_cpu_cnt, 1);
1064 1064
1065 /*1065 /*
1066 * Each pass through this loop enqueues a callback, but only1066 * Each pass through this loop enqueues a callback, but only
@@ -1071,39 +1071,39 @@ void srcu_barrier(struct srcu_struct *sp) · 4 unmodified lines
1071 * grace period as the last callback already in the queue.1071 * grace period as the last callback already in the queue.
1072 */1072 */
1073 for_each_possible_cpu(cpu) {1073 for_each_possible_cpu(cpu) {
1074 sdp = per_cpu_ptr(sp->sda, cpu);1074 sdp = per_cpu_ptr(ssp->sda, cpu);
1075 spin_lock_irq_rcu_node(sdp);1075 spin_lock_irq_rcu_node(sdp);
1076 atomic_inc(&sp->srcu_barrier_cpu_cnt);1076 atomic_inc(&ssp->srcu_barrier_cpu_cnt);
1077 sdp->srcu_barrier_head.func = srcu_barrier_cb;1077 sdp->srcu_barrier_head.func = srcu_barrier_cb;
1078 debug_rcu_head_queue(&sdp->srcu_barrier_head);1078 debug_rcu_head_queue(&sdp->srcu_barrier_head);
1079 if (!rcu_segcblist_entrain(&sdp->srcu_cblist,1079 if (!rcu_segcblist_entrain(&sdp->srcu_cblist,
1080 &sdp->srcu_barrier_head, 0)) {1080 &sdp->srcu_barrier_head, 0)) {
1081 debug_rcu_head_unqueue(&sdp->srcu_barrier_head);1081 debug_rcu_head_unqueue(&sdp->srcu_barrier_head);
1082 atomic_dec(&sp->srcu_barrier_cpu_cnt);1082 atomic_dec(&ssp->srcu_barrier_cpu_cnt);
1083 }1083 }
1084 spin_unlock_irq_rcu_node(sdp);1084 spin_unlock_irq_rcu_node(sdp);
1085 }1085 }
1086 1086
1087 /* Remove the initial count, at which point reaching zero can happen. */1087 /* Remove the initial count, at which point reaching zero can happen. */
1088 if (atomic_dec_and_test(&sp->srcu_barrier_cpu_cnt))1088 if (atomic_dec_and_test(&ssp->srcu_barrier_cpu_cnt))
1089 complete(&sp->srcu_barrier_completion);1089 complete(&ssp->srcu_barrier_completion);
1090 wait_for_completion(&sp->srcu_barrier_completion);1090 wait_for_completion(&ssp->srcu_barrier_completion);
1091 1091
1092 rcu_seq_end(&sp->srcu_barrier_seq);1092 rcu_seq_end(&ssp->srcu_barrier_seq);
1093 mutex_unlock(&sp->srcu_barrier_mutex);1093 mutex_unlock(&ssp->srcu_barrier_mutex);
1094}1094}
1095EXPORT_SYMBOL_GPL(srcu_barrier);1095EXPORT_SYMBOL_GPL(srcu_barrier);
1096 1096
1097/**1097/**
1098 * srcu_batches_completed - return batches completed.1098 * srcu_batches_completed - return batches completed.
1099 * @sp: srcu_struct on which to report batch completion.1099 * @ssp: srcu_struct on which to report batch completion.
1100 *1100 *
1101 * Report the number of batches, correlated with, but not necessarily1101 * Report the number of batches, correlated with, but not necessarily
1102 * precisely the same as, the number of grace periods that have elapsed.1102 * precisely the same as, the number of grace periods that have elapsed.
1103 */1103 */
1104unsigned long srcu_batches_completed(struct srcu_struct *sp)1104unsigned long srcu_batches_completed(struct srcu_struct *ssp)
1105{1105{
1106 return sp->srcu_idx;1106 return ssp->srcu_idx;
1107}1107}
1108EXPORT_SYMBOL_GPL(srcu_batches_completed);1108EXPORT_SYMBOL_GPL(srcu_batches_completed);
1109 1109
@@ -1112,11 +1112,11 @@ EXPORT_SYMBOL_GPL(srcu_batches_completed); · 2 unmodified lines
1112 * to SRCU_STATE_SCAN2, and invoke srcu_gp_end() when scan has1112 * to SRCU_STATE_SCAN2, and invoke srcu_gp_end() when scan has
1113 * completed in that state.1113 * completed in that state.
1114 */1114 */
1115static void srcu_advance_state(struct srcu_struct *sp)1115static void srcu_advance_state(struct srcu_struct *ssp)
1116{1116{
1117 int idx;1117 int idx;
1118 1118
1119 mutex_lock(&sp->srcu_gp_mutex);1119 mutex_lock(&ssp->srcu_gp_mutex);
1120 1120
1121 /*1121 /*
1122 * Because readers might be delayed for an extended period after1122 * Because readers might be delayed for an extended period after
@@ -1128,47 +1128,47 @@ static void srcu_advance_state(struct srcu_struct *sp) · 5 unmodified lines
1128 * The load-acquire ensures that we see the accesses performed1128 * The load-acquire ensures that we see the accesses performed
1129 * by the prior grace period.1129 * by the prior grace period.
1130 */1130 */
1131 idx = rcu_seq_state(smp_load_acquire(&sp->srcu_gp_seq)); /* ^^^ */1131 idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_gp_seq)); /* ^^^ */
1132 if (idx == SRCU_STATE_IDLE) {1132 if (idx == SRCU_STATE_IDLE) {
1133 spin_lock_irq_rcu_node(sp);1133 spin_lock_irq_rcu_node(ssp);
1134 if (ULONG_CMP_GE(sp->srcu_gp_seq, sp->srcu_gp_seq_needed)) {1134 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) {
1135 WARN_ON_ONCE(rcu_seq_state(sp->srcu_gp_seq));1135 WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq));
1136 spin_unlock_irq_rcu_node(sp);1136 spin_unlock_irq_rcu_node(ssp);
1137 mutex_unlock(&sp->srcu_gp_mutex);1137 mutex_unlock(&ssp->srcu_gp_mutex);
1138 return;1138 return;
1139 }1139 }
1140 idx = rcu_seq_state(READ_ONCE(sp->srcu_gp_seq));1140 idx = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));
1141 if (idx == SRCU_STATE_IDLE)1141 if (idx == SRCU_STATE_IDLE)
1142 srcu_gp_start(sp);1142 srcu_gp_start(ssp);
1143 spin_unlock_irq_rcu_node(sp);1143 spin_unlock_irq_rcu_node(ssp);
1144 if (idx != SRCU_STATE_IDLE) {1144 if (idx != SRCU_STATE_IDLE) {
1145 mutex_unlock(&sp->srcu_gp_mutex);1145 mutex_unlock(&ssp->srcu_gp_mutex);
1146 return; /* Someone else started the grace period. */1146 return; /* Someone else started the grace period. */
1147 }1147 }
1148 }1148 }
1149 1149
1150 if (rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)) == SRCU_STATE_SCAN1) {1150 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN1) {
1151 idx = 1 ^ (sp->srcu_idx & 1);1151 idx = 1 ^ (ssp->srcu_idx & 1);
1152 if (!try_check_zero(sp, idx, 1)) {1152 if (!try_check_zero(ssp, idx, 1)) {
1153 mutex_unlock(&sp->srcu_gp_mutex);1153 mutex_unlock(&ssp->srcu_gp_mutex);
1154 return; /* readers present, retry later. */1154 return; /* readers present, retry later. */
1155 }1155 }
1156 srcu_flip(sp);1156 srcu_flip(ssp);
1157 rcu_seq_set_state(&sp->srcu_gp_seq, SRCU_STATE_SCAN2);1157 rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2);
1158 }1158 }
1159 1159
1160 if (rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)) == SRCU_STATE_SCAN2) {1160 if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) {
1161 1161
1162 /*1162 /*
1163 * SRCU read-side critical sections are normally short,1163 * SRCU read-side critical sections are normally short,
1164 * so check at least twice in quick succession after a flip.1164 * so check at least twice in quick succession after a flip.
1165 */1165 */
1166 idx = 1 ^ (sp->srcu_idx & 1);1166 idx = 1 ^ (ssp->srcu_idx & 1);
1167 if (!try_check_zero(sp, idx, 2)) {1167 if (!try_check_zero(ssp, idx, 2)) {
1168 mutex_unlock(&sp->srcu_gp_mutex);1168 mutex_unlock(&ssp->srcu_gp_mutex);
1169 return; /* readers present, retry later. */1169 return; /* readers present, retry later. */
1170 }1170 }
1171 srcu_gp_end(sp); /* Releases ->srcu_gp_mutex. */1171 srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */
1172 }1172 }
1173}1173}
1174 1174
@@ -1184,14 +1184,14 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 9 unmodified lines
1184 struct rcu_cblist ready_cbs;1184 struct rcu_cblist ready_cbs;
1185 struct rcu_head *rhp;1185 struct rcu_head *rhp;
1186 struct srcu_data *sdp;1186 struct srcu_data *sdp;
1187 struct srcu_struct *sp;1187 struct srcu_struct *ssp;
1188 1188
1189 sdp = container_of(work, struct srcu_data, work.work);1189 sdp = container_of(work, struct srcu_data, work.work);
1190 sp = sdp->sp;1190 ssp = sdp->ssp;
1191 rcu_cblist_init(&ready_cbs);1191 rcu_cblist_init(&ready_cbs);
1192 spin_lock_irq_rcu_node(sdp);1192 spin_lock_irq_rcu_node(sdp);
1193 rcu_segcblist_advance(&sdp->srcu_cblist,1193 rcu_segcblist_advance(&sdp->srcu_cblist,
1194 rcu_seq_current(&sp->srcu_gp_seq));1194 rcu_seq_current(&ssp->srcu_gp_seq));
1195 if (sdp->srcu_cblist_invoking ||1195 if (sdp->srcu_cblist_invoking ||
1196 !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {1196 !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) {
1197 spin_unlock_irq_rcu_node(sdp);1197 spin_unlock_irq_rcu_node(sdp);
@@ -1217,7 +1217,7 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 19 unmodified lines
1217 spin_lock_irq_rcu_node(sdp);1217 spin_lock_irq_rcu_node(sdp);
1218 rcu_segcblist_insert_count(&sdp->srcu_cblist, &ready_cbs);1218 rcu_segcblist_insert_count(&sdp->srcu_cblist, &ready_cbs);
1219 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist,1219 (void)rcu_segcblist_accelerate(&sdp->srcu_cblist,
1220 rcu_seq_snap(&sp->srcu_gp_seq));1220 rcu_seq_snap(&ssp->srcu_gp_seq));
1221 sdp->srcu_cblist_invoking = false;1221 sdp->srcu_cblist_invoking = false;
1222 more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);1222 more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist);
1223 spin_unlock_irq_rcu_node(sdp);1223 spin_unlock_irq_rcu_node(sdp);
@@ -1229,24 +1229,24 @@ static void srcu_invoke_callbacks(struct work_struct *work) · 5 unmodified lines
1229 * Finished one round of SRCU grace period. Start another if there are1229 * Finished one round of SRCU grace period. Start another if there are
1230 * more SRCU callbacks queued, otherwise put SRCU into not-running state.1230 * more SRCU callbacks queued, otherwise put SRCU into not-running state.
1231 */1231 */
1232static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay)1232static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)
1233{1233{
1234 bool pushgp = true;1234 bool pushgp = true;
1235 1235
1236 spin_lock_irq_rcu_node(sp);1236 spin_lock_irq_rcu_node(ssp);
1237 if (ULONG_CMP_GE(sp->srcu_gp_seq, sp->srcu_gp_seq_needed)) {1237 if (ULONG_CMP_GE(ssp->srcu_gp_seq, ssp->srcu_gp_seq_needed)) {
1238 if (!WARN_ON_ONCE(rcu_seq_state(sp->srcu_gp_seq))) {1238 if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_gp_seq))) {
1239 /* All requests fulfilled, time to go idle. */1239 /* All requests fulfilled, time to go idle. */
1240 pushgp = false;1240 pushgp = false;
1241 }1241 }
1242 } else if (!rcu_seq_state(sp->srcu_gp_seq)) {1242 } else if (!rcu_seq_state(ssp->srcu_gp_seq)) {
1243 /* Outstanding request and no GP. Start one. */1243 /* Outstanding request and no GP. Start one. */
1244 srcu_gp_start(sp);1244 srcu_gp_start(ssp);
1245 }1245 }
1246 spin_unlock_irq_rcu_node(sp);1246 spin_unlock_irq_rcu_node(ssp);
1247 1247
1248 if (pushgp)1248 if (pushgp)
1249 queue_delayed_work(rcu_gp_wq, &sp->work, delay);1249 queue_delayed_work(rcu_gp_wq, &ssp->work, delay);
1250}1250}
1251 1251
1252/*1252/*
@@ -1254,41 +1254,41 @@ static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay) · 1 unmodified line
1254 */1254 */
1255static void process_srcu(struct work_struct *work)1255static void process_srcu(struct work_struct *work)
1256{1256{
1257 struct srcu_struct *sp;1257 struct srcu_struct *ssp;
1258 1258
1259 sp = container_of(work, struct srcu_struct, work.work);1259 ssp = container_of(work, struct srcu_struct, work.work);
1260 1260
1261 srcu_advance_state(sp);1261 srcu_advance_state(ssp);
1262 srcu_reschedule(sp, srcu_get_delay(sp));1262 srcu_reschedule(ssp, srcu_get_delay(ssp));
1263}1263}
1264 1264
1265void srcutorture_get_gp_data(enum rcutorture_type test_type,1265void srcutorture_get_gp_data(enum rcutorture_type test_type,
1266 struct srcu_struct *sp, int *flags,1266 struct srcu_struct *ssp, int *flags,
1267 unsigned long *gp_seq)1267 unsigned long *gp_seq)
1268{1268{
1269 if (test_type != SRCU_FLAVOR)1269 if (test_type != SRCU_FLAVOR)
1270 return;1270 return;
1271 *flags = 0;1271 *flags = 0;
1272 *gp_seq = rcu_seq_current(&sp->srcu_gp_seq);1272 *gp_seq = rcu_seq_current(&ssp->srcu_gp_seq);
1273}1273}
1274EXPORT_SYMBOL_GPL(srcutorture_get_gp_data);1274EXPORT_SYMBOL_GPL(srcutorture_get_gp_data);
1275 1275
1276void srcu_torture_stats_print(struct srcu_struct *sp, char *tt, char *tf)1276void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)
1277{1277{
1278 int cpu;1278 int cpu;
1279 int idx;1279 int idx;
1280 unsigned long s0 = 0, s1 = 0;1280 unsigned long s0 = 0, s1 = 0;
1281 1281
1282 idx = sp->srcu_idx & 0x1;1282 idx = ssp->srcu_idx & 0x1;
1283 pr_alert("%s%s Tree SRCU g%ld per-CPU(idx=%d):",1283 pr_alert("%s%s Tree SRCU g%ld per-CPU(idx=%d):",
1284 tt, tf, rcu_seq_current(&sp->srcu_gp_seq), idx);1284 tt, tf, rcu_seq_current(&ssp->srcu_gp_seq), idx);
1285 for_each_possible_cpu(cpu) {1285 for_each_possible_cpu(cpu) {
1286 unsigned long l0, l1;1286 unsigned long l0, l1;
1287 unsigned long u0, u1;1287 unsigned long u0, u1;
1288 long c0, c1;1288 long c0, c1;
1289 struct srcu_data *sdp;1289 struct srcu_data *sdp;
1290 1290
1291 sdp = per_cpu_ptr(sp->sda, cpu);1291 sdp = per_cpu_ptr(ssp->sda, cpu);
1292 u0 = sdp->srcu_unlock_count[!idx];1292 u0 = sdp->srcu_unlock_count[!idx];
1293 u1 = sdp->srcu_unlock_count[idx];1293 u1 = sdp->srcu_unlock_count[idx];
1294 1294
@@ -1323,14 +1323,14 @@ early_initcall(srcu_bootup_announce); · 28 unmodified lines
1323 1323
1324void __init srcu_init(void)1324void __init srcu_init(void)
1325{1325{
1326 struct srcu_struct *sp;1326 struct srcu_struct *ssp;
1327 1327
1328 srcu_init_done = true;1328 srcu_init_done = true;
1329 while (!list_empty(&srcu_boot_list)) {1329 while (!list_empty(&srcu_boot_list)) {
1330 sp = list_first_entry(&srcu_boot_list, struct srcu_struct,1330 ssp = list_first_entry(&srcu_boot_list, struct srcu_struct,
1331 work.work.entry);1331 work.work.entry);
1332 check_init_srcu_struct(sp);1332 check_init_srcu_struct(ssp);
1333 list_del_init(&sp->work.work.entry);1333 list_del_init(&ssp->work.work.entry);
1334 queue_work(rcu_gp_wq, &sp->work.work);1334 queue_work(rcu_gp_wq, &ssp->work.work);
1335 }1335 }
1336}1336}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t