srcu: Provide polling interfaces for Tiny SRCU grace periods

There is a need for a polling interface for SRCU grace periods, so this commit supplies get_state_synchronize_srcu(), start_poll_synchronize_srcu(), and poll_state_synchronize_srcu() for this purpose. The first can be used if future grace periods are inevitable (perhaps due to a later call_srcu() invocation), the second if future grace periods might not otherwise happen, and the third to check if a grace period has elapsed since the corresponding call to either of the first two. As with get_state_synchronize_rcu() and cond_synchronize_rcu(), the return value from either get_state_synchronize_srcu() or start_poll_synchronize_srcu() must be passed in to a later call to poll_state_synchronize_srcu().

Paul E. McKenney committed 6 years ago

Commit 8b5bd671 parent 29d2bb94 files · +59 −2Browse files

Unified · Split

4 files

include/linux/rcupdate.h +2 −0
include/linux/srcu.h +3 −0
include/linux/srcutiny.h +1 −0
kernel/rcu/srcutiny.c +53 −2

include/linux/rcupdate.h+2−0

@@ -33,6 +33,8 @@ · 32 unmodified lines
33	33	#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
34	34	#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
35	35	#define ulong2long(a) ((long )(&(a)))
	36	+#define USHORT_CMP_GE(a, b) (USHRT_MAX / 2 >= (unsigned short)((a) - (b)))
	37	+#define USHORT_CMP_LT(a, b) (USHRT_MAX / 2 < (unsigned short)((a) - (b)))
36	38
37	39	/* Exported common interfaces */
38	40	void call_rcu(struct rcu_head *head, rcu_callback_t func);

include/linux/srcu.h+3−0

@@ -60,6 +60,9 @@ void cleanup_srcu_struct(struct srcu_struct *ssp); · 59 unmodified lines
60	60	int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
61	61	void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
62	62	void synchronize_srcu(struct srcu_struct *ssp);
	63	+unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
	64	+unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
	65	+bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
63	66
64	67	#ifdef CONFIG_DEBUG_LOCK_ALLOC
65	68

include/linux/srcutiny.h+1−0

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	struct srcu_struct {
17	17	short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */
18	18	unsigned short srcu_idx; /* Current reader array element in bit 0x2. */
	19	+ unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */
19	20	u8 srcu_gp_running; /* GP workqueue running? */
20	21	u8 srcu_gp_waiting; /* GP waiting for readers? */
21	22	struct swait_queue_head srcu_wq;

kernel/rcu/srcutiny.c+53−2

@@ -34,6 +34,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) · 33 unmodified lines
34	34	ssp->srcu_gp_running = false;
35	35	ssp->srcu_gp_waiting = false;
36	36	ssp->srcu_idx = 0;
	37	+ ssp->srcu_idx_max = 0;
37	38	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
38	39	INIT_LIST_HEAD(&ssp->srcu_work.entry);
39	40	return 0;
@@ -84,6 +85,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) · 44 unmodified lines
84	85	WARN_ON(ssp->srcu_gp_waiting);
85	86	WARN_ON(ssp->srcu_cb_head);
86	87	WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);
	88	+ WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);
	89	+ WARN_ON(ssp->srcu_idx & 0x1);
87	90	}
88	91	EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
89	92
@@ -114,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) · 24 unmodified lines
114	117	struct srcu_struct *ssp;
115	118
116	119	ssp = container_of(wp, struct srcu_struct, srcu_work);
117		− if (ssp->srcu_gp_running \|\| !READ_ONCE(ssp->srcu_cb_head))
	120	+ if (ssp->srcu_gp_running \|\| USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
118	121	return; /* Already running or nothing to do. */
119	122
120	123	/* Remove recently arrived callbacks and wait for readers. */
@@ -147,13 +150,19 @@ void srcu_drive_gp(struct work_struct *wp) · 26 unmodified lines
147	150	* straighten that out.
148	151	*/
149	152	WRITE_ONCE(ssp->srcu_gp_running, false);
150		− if (READ_ONCE(ssp->srcu_cb_head))
	153	+ if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
151	154	schedule_work(&ssp->srcu_work);
152	155	}
153	156	EXPORT_SYMBOL_GPL(srcu_drive_gp);
154	157
155	158	static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
156	159	{
	160	+ unsigned short cookie;
	161	+
	162	+ cookie = get_state_synchronize_srcu(ssp);
	163	+ if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
	164	+ return;
	165	+ WRITE_ONCE(ssp->srcu_idx_max, cookie);
157	166	if (!READ_ONCE(ssp->srcu_gp_running)) {
158	167	if (likely(srcu_init_done))
159	168	schedule_work(&ssp->srcu_work);
@@ -196,6 +205,48 @@ void synchronize_srcu(struct srcu_struct *ssp) · 36 unmodified lines
196	205	}
197	206	EXPORT_SYMBOL_GPL(synchronize_srcu);
198	207
	208	+/*
	209	+ * get_state_synchronize_srcu - Provide an end-of-grace-period cookie
	210	+ */
	211	+unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
	212	+{
	213	+ unsigned long ret;
	214	+
	215	+ barrier();
	216	+ ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
	217	+ barrier();
	218	+ return ret & USHRT_MAX;
	219	+}
	220	+EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
	221	+
	222	+/*
	223	+ * start_poll_synchronize_srcu - Provide cookie and start grace period
	224	+ *
	225	+ * The difference between this and get_state_synchronize_srcu() is that
	226	+ * this function ensures that the poll_state_synchronize_srcu() will
	227	+ * eventually return the value true.
	228	+ */
	229	+unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)
	230	+{
	231	+ unsigned long ret = get_state_synchronize_srcu(ssp);
	232	+
	233	+ srcu_gp_start_if_needed(ssp);
	234	+ return ret;
	235	+}
	236	+EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
	237	+
	238	+/*
	239	+ * poll_state_synchronize_srcu - Has cookie's grace period ended?
	240	+ */
	241	+bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
	242	+{
	243	+ bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie);
	244	+
	245	+ barrier();
	246	+ return ret;
	247	+}
	248	+EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);
	249	+
199	250	/* Lockdep diagnostics. */
200	251	void __init rcu_scheduler_starting(void)
201	252	{

srcu: Provide polling interfaces for Tiny SRCU grace periods

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code