There is a need for a polling interface for SRCU grace periods, so this commit supplies get_state_synchronize_srcu(), start_poll_synchronize_srcu(), and poll_state_synchronize_srcu() for this purpose. The first can be used if future grace periods are inevitable (perhaps due to a later call_srcu() invocation), the second if future grace periods might not otherwise happen, and the third to check if a grace period has elapsed since the corresponding call to either of the first two. As with get_state_synchronize_rcu() and cond_synchronize_rcu(), the return value from either get_state_synchronize_srcu() or start_poll_synchronize_srcu() must be passed in to a later call to poll_state_synchronize_srcu().
Showing the changes to kernel (1 of 4 files). Show all
Unified · Split
| @@ -34,6 +34,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) · 33 unmodified lines | |||
| 34 | ssp->srcu_gp_running = false; | 34 | ssp->srcu_gp_running = false; |
| 35 | ssp->srcu_gp_waiting = false; | 35 | ssp->srcu_gp_waiting = false; |
| 36 | ssp->srcu_idx = 0; | 36 | ssp->srcu_idx = 0; |
| 37 | ssp->srcu_idx_max = 0; | ||
| 37 | INIT_WORK(&ssp->srcu_work, srcu_drive_gp); | 38 | INIT_WORK(&ssp->srcu_work, srcu_drive_gp); |
| 38 | INIT_LIST_HEAD(&ssp->srcu_work.entry); | 39 | INIT_LIST_HEAD(&ssp->srcu_work.entry); |
| 39 | return 0; | 40 | return 0; |
| @@ -84,6 +85,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) · 44 unmodified lines | |||
| 84 | WARN_ON(ssp->srcu_gp_waiting); | 85 | WARN_ON(ssp->srcu_gp_waiting); |
| 85 | WARN_ON(ssp->srcu_cb_head); | 86 | WARN_ON(ssp->srcu_cb_head); |
| 86 | WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail); | 87 | WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail); |
| 88 | WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max); | ||
| 89 | WARN_ON(ssp->srcu_idx & 0x1); | ||
| 87 | } | 90 | } |
| 88 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | 91 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); |
| 89 | 92 | ||
| @@ -114,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) · 24 unmodified lines | |||
| 114 | struct srcu_struct *ssp; | 117 | struct srcu_struct *ssp; |
| 115 | 118 | ||
| 116 | ssp = container_of(wp, struct srcu_struct, srcu_work); | 119 | ssp = container_of(wp, struct srcu_struct, srcu_work); |
| 117 | if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head)) | 120 | if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) |
| 118 | return; /* Already running or nothing to do. */ | 121 | return; /* Already running or nothing to do. */ |
| 119 | 122 | ||
| 120 | /* Remove recently arrived callbacks and wait for readers. */ | 123 | /* Remove recently arrived callbacks and wait for readers. */ |
| @@ -147,13 +150,19 @@ void srcu_drive_gp(struct work_struct *wp) · 26 unmodified lines | |||
| 147 | * straighten that out. | 150 | * straighten that out. |
| 148 | */ | 151 | */ |
| 149 | WRITE_ONCE(ssp->srcu_gp_running, false); | 152 | WRITE_ONCE(ssp->srcu_gp_running, false); |
| 150 | if (READ_ONCE(ssp->srcu_cb_head)) | 153 | if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) |
| 151 | schedule_work(&ssp->srcu_work); | 154 | schedule_work(&ssp->srcu_work); |
| 152 | } | 155 | } |
| 153 | EXPORT_SYMBOL_GPL(srcu_drive_gp); | 156 | EXPORT_SYMBOL_GPL(srcu_drive_gp); |
| 154 | 157 | ||
| 155 | static void srcu_gp_start_if_needed(struct srcu_struct *ssp) | 158 | static void srcu_gp_start_if_needed(struct srcu_struct *ssp) |
| 156 | { | 159 | { |
| 160 | unsigned short cookie; | ||
| 161 | |||
| 162 | cookie = get_state_synchronize_srcu(ssp); | ||
| 163 | if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) | ||
| 164 | return; | ||
| 165 | WRITE_ONCE(ssp->srcu_idx_max, cookie); | ||
| 157 | if (!READ_ONCE(ssp->srcu_gp_running)) { | 166 | if (!READ_ONCE(ssp->srcu_gp_running)) { |
| 158 | if (likely(srcu_init_done)) | 167 | if (likely(srcu_init_done)) |
| 159 | schedule_work(&ssp->srcu_work); | 168 | schedule_work(&ssp->srcu_work); |
| @@ -196,6 +205,48 @@ void synchronize_srcu(struct srcu_struct *ssp) · 36 unmodified lines | |||
| 196 | } | 205 | } |
| 197 | EXPORT_SYMBOL_GPL(synchronize_srcu); | 206 | EXPORT_SYMBOL_GPL(synchronize_srcu); |
| 198 | 207 | ||
| 208 | /* | ||
| 209 | * get_state_synchronize_srcu - Provide an end-of-grace-period cookie | ||
| 210 | */ | ||
| 211 | unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) | ||
| 212 | { | ||
| 213 | unsigned long ret; | ||
| 214 | |||
| 215 | barrier(); | ||
| 216 | ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1; | ||
| 217 | barrier(); | ||
| 218 | return ret & USHRT_MAX; | ||
| 219 | } | ||
| 220 | EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); | ||
| 221 | |||
| 222 | /* | ||
| 223 | * start_poll_synchronize_srcu - Provide cookie and start grace period | ||
| 224 | * | ||
| 225 | * The difference between this and get_state_synchronize_srcu() is that | ||
| 226 | * this function ensures that the poll_state_synchronize_srcu() will | ||
| 227 | * eventually return the value true. | ||
| 228 | */ | ||
| 229 | unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) | ||
| 230 | { | ||
| 231 | unsigned long ret = get_state_synchronize_srcu(ssp); | ||
| 232 | |||
| 233 | srcu_gp_start_if_needed(ssp); | ||
| 234 | return ret; | ||
| 235 | } | ||
| 236 | EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); | ||
| 237 | |||
| 238 | /* | ||
| 239 | * poll_state_synchronize_srcu - Has cookie's grace period ended? | ||
| 240 | */ | ||
| 241 | bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) | ||
| 242 | { | ||
| 243 | bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie); | ||
| 244 | |||
| 245 | barrier(); | ||
| 246 | return ret; | ||
| 247 | } | ||
| 248 | EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); | ||
| 249 | |||
| 199 | /* Lockdep diagnostics. */ | 250 | /* Lockdep diagnostics. */ |
| 200 | void __init rcu_scheduler_starting(void) | 251 | void __init rcu_scheduler_starting(void) |
| 201 | { | 252 | { |