seqlock.h is now included by kernel's RST documentation, but a small number of the the exported seqlock.h functions are kernel-doc annotated. Add kernel-doc for all seqlock.h exported APIs.
Showing the changes to include/linux (1 of 1 file). Show all
Unified · Split
| @@ -75,6 +75,10 @@ static inline void __seqcount_init(seqcount_t *s, const char *name, · 74 unmodified lines | ||
| 75 | 75 | # define SEQCOUNT_DEP_MAP_INIT(lockname) \ |
| 76 | 76 | .dep_map = { .name = #lockname } \ |
| 77 | 77 | |
| 78 | +/** | |
| 79 | + * seqcount_init() - runtime initializer for seqcount_t | |
| 80 | + * @s: Pointer to the seqcount_t instance | |
| 81 | + */ | |
| 78 | 82 | # define seqcount_init(s) \ |
| 79 | 83 | do { \ |
| 80 | 84 | static struct lock_class_key __key; \ |
| @@ -98,13 +102,15 @@ static inline void seqcount_lockdep_reader_access(const seqcount_t *s) · 17 unmodified lines | ||
| 98 | 102 | # define seqcount_lockdep_reader_access(x) |
| 99 | 103 | #endif |
| 100 | 104 | |
| 101 | −#define SEQCNT_ZERO(lockname) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(lockname)} | |
| 102 | − | |
| 105 | +/** | |
| 106 | + * SEQCNT_ZERO() - static initializer for seqcount_t | |
| 107 | + * @name: Name of the seqcount_t instance | |
| 108 | + */ | |
| 109 | +#define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) } | |
| 103 | 110 | |
| 104 | 111 | /** |
| 105 | − * __read_seqcount_begin - begin a seq-read critical section (without barrier) | |
| 106 | − * @s: pointer to seqcount_t | |
| 107 | − * Returns: count to be passed to read_seqcount_retry | |
| 112 | + * __read_seqcount_begin() - begin a seqcount_t read section w/o barrier | |
| 113 | + * @s: Pointer to seqcount_t | |
| 108 | 114 | * |
| 109 | 115 | * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb() |
| 110 | 116 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is |
| @@ -113,6 +119,8 @@ static inline void seqcount_lockdep_reader_access(const seqcount_t *s) · 2 unmodified lines | ||
| 113 | 119 | * |
| 114 | 120 | * Use carefully, only in critical code, and comment how the barrier is |
| 115 | 121 | * provided. |
| 122 | + * | |
| 123 | + * Return: count to be passed to read_seqcount_retry() | |
| 116 | 124 | */ |
| 117 | 125 | static inline unsigned __read_seqcount_begin(const seqcount_t *s) |
| 118 | 126 | { |
| @@ -129,13 +137,10 @@ repeat: · 10 unmodified lines | ||
| 129 | 137 | } |
| 130 | 138 | |
| 131 | 139 | /** |
| 132 | − * raw_read_seqcount_begin - start seq-read critical section w/o lockdep | |
| 133 | − * @s: pointer to seqcount_t | |
| 134 | − * Returns: count to be passed to read_seqcount_retry | |
| 140 | + * raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep | |
| 141 | + * @s: Pointer to seqcount_t | |
| 135 | 142 | * |
| 136 | − * raw_read_seqcount_begin opens a read critical section of the given | |
| 137 | − * seqcount, but without any lockdep checking. Validity of the critical | |
| 138 | − * section is tested by checking read_seqcount_retry function. | |
| 143 | + * Return: count to be passed to read_seqcount_retry() | |
| 139 | 144 | */ |
| 140 | 145 | static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) |
| 141 | 146 | { |
| @@ -145,13 +150,10 @@ static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) · 3 unmodified lines | ||
| 145 | 150 | } |
| 146 | 151 | |
| 147 | 152 | /** |
| 148 | − * read_seqcount_begin - begin a seq-read critical section | |
| 149 | − * @s: pointer to seqcount_t | |
| 150 | − * Returns: count to be passed to read_seqcount_retry | |
| 153 | + * read_seqcount_begin() - begin a seqcount_t read critical section | |
| 154 | + * @s: Pointer to seqcount_t | |
| 151 | 155 | * |
| 152 | − * read_seqcount_begin opens a read critical section of the given seqcount. | |
| 153 | − * Validity of the critical section is tested by checking read_seqcount_retry | |
| 154 | − * function. | |
| 156 | + * Return: count to be passed to read_seqcount_retry() | |
| 155 | 157 | */ |
| 156 | 158 | static inline unsigned read_seqcount_begin(const seqcount_t *s) |
| 157 | 159 | { |
| @@ -160,13 +162,15 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s) · 2 unmodified lines | ||
| 160 | 162 | } |
| 161 | 163 | |
| 162 | 164 | /** |
| 163 | − * raw_read_seqcount - Read the raw seqcount | |
| 164 | − * @s: pointer to seqcount_t | |
| 165 | − * Returns: count to be passed to read_seqcount_retry | |
| 165 | + * raw_read_seqcount() - read the raw seqcount_t counter value | |
| 166 | + * @s: Pointer to seqcount_t | |
| 166 | 167 | * |
| 167 | 168 | * raw_read_seqcount opens a read critical section of the given |
| 168 | − * seqcount without any lockdep checking and without checking or | |
| 169 | − * masking the LSB. Calling code is responsible for handling that. | |
| 169 | + * seqcount_t, without any lockdep checking, and without checking or | |
| 170 | + * masking the sequence counter LSB. Calling code is responsible for | |
| 171 | + * handling that. | |
| 172 | + * | |
| 173 | + * Return: count to be passed to read_seqcount_retry() | |
| 170 | 174 | */ |
| 171 | 175 | static inline unsigned raw_read_seqcount(const seqcount_t *s) |
| 172 | 176 | { |
| @@ -177,18 +181,21 @@ static inline unsigned raw_read_seqcount(const seqcount_t *s) · 4 unmodified lines | ||
| 177 | 181 | } |
| 178 | 182 | |
| 179 | 183 | /** |
| 180 | − * raw_seqcount_begin - begin a seq-read critical section | |
| 181 | − * @s: pointer to seqcount_t | |
| 182 | − * Returns: count to be passed to read_seqcount_retry | |
| 184 | + * raw_seqcount_begin() - begin a seqcount_t read critical section w/o | |
| 185 | + * lockdep and w/o counter stabilization | |
| 186 | + * @s: Pointer to seqcount_t | |
| 183 | 187 | * |
| 184 | − * raw_seqcount_begin opens a read critical section of the given seqcount. | |
| 185 | − * Validity of the critical section is tested by checking read_seqcount_retry | |
| 186 | − * function. | |
| 188 | + * raw_seqcount_begin opens a read critical section of the given | |
| 189 | + * seqcount_t. Unlike read_seqcount_begin(), this function will not wait | |
| 190 | + * for the count to stabilize. If a writer is active when it begins, it | |
| 191 | + * will fail the read_seqcount_retry() at the end of the read critical | |
| 192 | + * section instead of stabilizing at the beginning of it. | |
| 187 | 193 | * |
| 188 | − * Unlike read_seqcount_begin(), this function will not wait for the count | |
| 189 | − * to stabilize. If a writer is active when we begin, we will fail the | |
| 190 | − * read_seqcount_retry() instead of stabilizing at the beginning of the | |
| 191 | − * critical section. | |
| 194 | + * Use this only in special kernel hot paths where the read section is | |
| 195 | + * small and has a high probability of success through other external | |
| 196 | + * means. It will save a single branching instruction. | |
| 197 | + * | |
| 198 | + * Return: count to be passed to read_seqcount_retry() | |
| 192 | 199 | */ |
| 193 | 200 | static inline unsigned raw_seqcount_begin(const seqcount_t *s) |
| 194 | 201 | { |
| @@ -199,10 +206,9 @@ static inline unsigned raw_seqcount_begin(const seqcount_t *s) · 4 unmodified lines | ||
| 199 | 206 | } |
| 200 | 207 | |
| 201 | 208 | /** |
| 202 | − * __read_seqcount_retry - end a seq-read critical section (without barrier) | |
| 203 | − * @s: pointer to seqcount_t | |
| 204 | − * @start: count, from read_seqcount_begin | |
| 205 | − * Returns: 1 if retry is required, else 0 | |
| 209 | + * __read_seqcount_retry() - end a seqcount_t read section w/o barrier | |
| 210 | + * @s: Pointer to seqcount_t | |
| 211 | + * @start: count, from read_seqcount_begin() | |
| 206 | 212 | * |
| 207 | 213 | * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() |
| 208 | 214 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is |
| @@ -211,6 +217,8 @@ static inline unsigned raw_seqcount_begin(const seqcount_t *s) · 2 unmodified lines | ||
| 211 | 217 | * |
| 212 | 218 | * Use carefully, only in critical code, and comment how the barrier is |
| 213 | 219 | * provided. |
| 220 | + * | |
| 221 | + * Return: true if a read section retry is required, else false | |
| 214 | 222 | */ |
| 215 | 223 | static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) |
| 216 | 224 | { |
| @@ -219,14 +227,15 @@ static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) · 2 unmodified lines | ||
| 219 | 227 | } |
| 220 | 228 | |
| 221 | 229 | /** |
| 222 | − * read_seqcount_retry - end a seq-read critical section | |
| 223 | − * @s: pointer to seqcount_t | |
| 224 | − * @start: count, from read_seqcount_begin | |
| 225 | − * Returns: 1 if retry is required, else 0 | |
| 230 | + * read_seqcount_retry() - end a seqcount_t read critical section | |
| 231 | + * @s: Pointer to seqcount_t | |
| 232 | + * @start: count, from read_seqcount_begin() | |
| 226 | 233 | * |
| 227 | − * read_seqcount_retry closes a read critical section of the given seqcount. | |
| 228 | − * If the critical section was invalid, it must be ignored (and typically | |
| 229 | − * retried). | |
| 234 | + * read_seqcount_retry closes the read critical section of given | |
| 235 | + * seqcount_t. If the critical section was invalid, it must be ignored | |
| 236 | + * (and typically retried). | |
| 237 | + * | |
| 238 | + * Return: true if a read section retry is required, else false | |
| 230 | 239 | */ |
| 231 | 240 | static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) |
| 232 | 241 | { |
| @@ -234,6 +243,10 @@ static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) · 1 unmodified line | ||
| 234 | 243 | return __read_seqcount_retry(s, start); |
| 235 | 244 | } |
| 236 | 245 | |
| 246 | +/** | |
| 247 | + * raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep | |
| 248 | + * @s: Pointer to seqcount_t | |
| 249 | + */ | |
| 237 | 250 | static inline void raw_write_seqcount_begin(seqcount_t *s) |
| 238 | 251 | { |
| 239 | 252 | kcsan_nestable_atomic_begin(); |
| @@ -241,6 +254,10 @@ static inline void raw_write_seqcount_begin(seqcount_t *s) · 1 unmodified line | ||
| 241 | 254 | smp_wmb(); |
| 242 | 255 | } |
| 243 | 256 | |
| 257 | +/** | |
| 258 | + * raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep | |
| 259 | + * @s: Pointer to seqcount_t | |
| 260 | + */ | |
| 244 | 261 | static inline void raw_write_seqcount_end(seqcount_t *s) |
| 245 | 262 | { |
| 246 | 263 | smp_wmb(); |
| @@ -248,17 +265,42 @@ static inline void raw_write_seqcount_end(seqcount_t *s) · 1 unmodified line | ||
| 248 | 265 | kcsan_nestable_atomic_end(); |
| 249 | 266 | } |
| 250 | 267 | |
| 268 | +/** | |
| 269 | + * write_seqcount_begin_nested() - start a seqcount_t write section with | |
| 270 | + * custom lockdep nesting level | |
| 271 | + * @s: Pointer to seqcount_t | |
| 272 | + * @subclass: lockdep nesting level | |
| 273 | + * | |
| 274 | + * See Documentation/locking/lockdep-design.rst | |
| 275 | + */ | |
| 251 | 276 | static inline void write_seqcount_begin_nested(seqcount_t *s, int subclass) |
| 252 | 277 | { |
| 253 | 278 | raw_write_seqcount_begin(s); |
| 254 | 279 | seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_); |
| 255 | 280 | } |
| 256 | 281 | |
| 282 | +/** | |
| 283 | + * write_seqcount_begin() - start a seqcount_t write side critical section | |
| 284 | + * @s: Pointer to seqcount_t | |
| 285 | + * | |
| 286 | + * write_seqcount_begin opens a write side critical section of the given | |
| 287 | + * seqcount_t. | |
| 288 | + * | |
| 289 | + * Context: seqcount_t write side critical sections must be serialized and | |
| 290 | + * non-preemptible. If readers can be invoked from hardirq or softirq | |
| 291 | + * context, interrupts or bottom halves must be respectively disabled. | |
| 292 | + */ | |
| 257 | 293 | static inline void write_seqcount_begin(seqcount_t *s) |
| 258 | 294 | { |
| 259 | 295 | write_seqcount_begin_nested(s, 0); |
| 260 | 296 | } |
| 261 | 297 | |
| 298 | +/** | |
| 299 | + * write_seqcount_end() - end a seqcount_t write side critical section | |
| 300 | + * @s: Pointer to seqcount_t | |
| 301 | + * | |
| 302 | + * The write section must've been opened with write_seqcount_begin(). | |
| 303 | + */ | |
| 262 | 304 | static inline void write_seqcount_end(seqcount_t *s) |
| 263 | 305 | { |
| 264 | 306 | seqcount_release(&s->dep_map, _RET_IP_); |
| @@ -266,12 +308,12 @@ static inline void write_seqcount_end(seqcount_t *s) · 1 unmodified line | ||
| 266 | 308 | } |
| 267 | 309 | |
| 268 | 310 | /** |
| 269 | − * raw_write_seqcount_barrier - do a seq write barrier | |
| 270 | − * @s: pointer to seqcount_t | |
| 311 | + * raw_write_seqcount_barrier() - do a seqcount_t write barrier | |
| 312 | + * @s: Pointer to seqcount_t | |
| 271 | 313 | * |
| 272 | − * This can be used to provide an ordering guarantee instead of the | |
| 273 | − * usual consistency guarantee. It is one wmb cheaper, because we can | |
| 274 | − * collapse the two back-to-back wmb()s. | |
| 314 | + * This can be used to provide an ordering guarantee instead of the usual | |
| 315 | + * consistency guarantee. It is one wmb cheaper, because it can collapse | |
| 316 | + * the two back-to-back wmb()s. | |
| 275 | 317 | * |
| 276 | 318 | * Note that writes surrounding the barrier should be declared atomic (e.g. |
| 277 | 319 | * via WRITE_ONCE): a) to ensure the writes become visible to other threads |
| @@ -316,11 +358,12 @@ static inline void raw_write_seqcount_barrier(seqcount_t *s) · 38 unmodified lines | ||
| 316 | 358 | } |
| 317 | 359 | |
| 318 | 360 | /** |
| 319 | − * write_seqcount_invalidate - invalidate in-progress read-side seq operations | |
| 320 | − * @s: pointer to seqcount_t | |
| 361 | + * write_seqcount_invalidate() - invalidate in-progress seqcount_t read | |
| 362 | + * side operations | |
| 363 | + * @s: Pointer to seqcount_t | |
| 321 | 364 | * |
| 322 | − * After write_seqcount_invalidate, no read-side seq operations will complete | |
| 323 | − * successfully and see data older than this. | |
| 365 | + * After write_seqcount_invalidate, no seqcount_t read side operations | |
| 366 | + * will complete successfully and see data older than this. | |
| 324 | 367 | */ |
| 325 | 368 | static inline void write_seqcount_invalidate(seqcount_t *s) |
| 326 | 369 | { |
| @@ -330,6 +373,21 @@ static inline void write_seqcount_invalidate(seqcount_t *s) · 3 unmodified lines | ||
| 330 | 373 | kcsan_nestable_atomic_end(); |
| 331 | 374 | } |
| 332 | 375 | |
| 376 | +/** | |
| 377 | + * raw_read_seqcount_latch() - pick even/odd seqcount_t latch data copy | |
| 378 | + * @s: Pointer to seqcount_t | |
| 379 | + * | |
| 380 | + * Use seqcount_t latching to switch between two storage places protected | |
| 381 | + * by a sequence counter. Doing so allows having interruptible, preemptible, | |
| 382 | + * seqcount_t write side critical sections. | |
| 383 | + * | |
| 384 | + * Check raw_write_seqcount_latch() for more details and a full reader and | |
| 385 | + * writer usage example. | |
| 386 | + * | |
| 387 | + * Return: sequence counter raw value. Use the lowest bit as an index for | |
| 388 | + * picking which data copy to read. The full counter value must then be | |
| 389 | + * checked with read_seqcount_retry(). | |
| 390 | + */ | |
| 333 | 391 | static inline int raw_read_seqcount_latch(seqcount_t *s) |
| 334 | 392 | { |
| 335 | 393 | /* Pairs with the first smp_wmb() in raw_write_seqcount_latch() */ |
| @@ -338,8 +396,8 @@ static inline int raw_read_seqcount_latch(seqcount_t *s) · 2 unmodified lines | ||
| 338 | 396 | } |
| 339 | 397 | |
| 340 | 398 | /** |
| 341 | − * raw_write_seqcount_latch - redirect readers to even/odd copy | |
| 342 | − * @s: pointer to seqcount_t | |
| 399 | + * raw_write_seqcount_latch() - redirect readers to even/odd copy | |
| 400 | + * @s: Pointer to seqcount_t | |
| 343 | 401 | * |
| 344 | 402 | * The latch technique is a multiversion concurrency control method that allows |
| 345 | 403 | * queries during non-atomic modifications. If you can guarantee queries never |
| @@ -446,17 +504,28 @@ typedef struct { · 100 unmodified lines | ||
| 446 | 504 | .lock = __SPIN_LOCK_UNLOCKED(lockname) \ |
| 447 | 505 | } |
| 448 | 506 | |
| 449 | −#define seqlock_init(x) \ | |
| 507 | +/** | |
| 508 | + * seqlock_init() - dynamic initializer for seqlock_t | |
| 509 | + * @sl: Pointer to the seqlock_t instance | |
| 510 | + */ | |
| 511 | +#define seqlock_init(sl) \ | |
| 450 | 512 | do { \ |
| 451 | − seqcount_init(&(x)->seqcount); \ | |
| 452 | − spin_lock_init(&(x)->lock); \ | |
| 513 | + seqcount_init(&(sl)->seqcount); \ | |
| 514 | + spin_lock_init(&(sl)->lock); \ | |
| 453 | 515 | } while (0) |
| 454 | 516 | |
| 455 | −#define DEFINE_SEQLOCK(x) \ | |
| 456 | − seqlock_t x = __SEQLOCK_UNLOCKED(x) | |
| 517 | +/** | |
| 518 | + * DEFINE_SEQLOCK() - Define a statically allocated seqlock_t | |
| 519 | + * @sl: Name of the seqlock_t instance | |
| 520 | + */ | |
| 521 | +#define DEFINE_SEQLOCK(sl) \ | |
| 522 | + seqlock_t sl = __SEQLOCK_UNLOCKED(sl) | |
| 457 | 523 | |
| 458 | −/* | |
| 459 | − * Read side functions for starting and finalizing a read side section. | |
| 524 | +/** | |
| 525 | + * read_seqbegin() - start a seqlock_t read side critical section | |
| 526 | + * @sl: Pointer to seqlock_t | |
| 527 | + * | |
| 528 | + * Return: count, to be passed to read_seqretry() | |
| 460 | 529 | */ |
| 461 | 530 | static inline unsigned read_seqbegin(const seqlock_t *sl) |
| 462 | 531 | { |
| @@ -467,6 +536,17 @@ static inline unsigned read_seqbegin(const seqlock_t *sl) · 4 unmodified lines | ||
| 467 | 536 | return ret; |
| 468 | 537 | } |
| 469 | 538 | |
| 539 | +/** | |
| 540 | + * read_seqretry() - end a seqlock_t read side section | |
| 541 | + * @sl: Pointer to seqlock_t | |
| 542 | + * @start: count, from read_seqbegin() | |
| 543 | + * | |
| 544 | + * read_seqretry closes the read side critical section of given seqlock_t. | |
| 545 | + * If the critical section was invalid, it must be ignored (and typically | |
| 546 | + * retried). | |
| 547 | + * | |
| 548 | + * Return: true if a read section retry is required, else false | |
| 549 | + */ | |
| 470 | 550 | static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) |
| 471 | 551 | { |
| 472 | 552 | /* |
| @@ -478,10 +558,18 @@ static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) · 5 unmodified lines | ||
| 478 | 558 | return read_seqcount_retry(&sl->seqcount, start); |
| 479 | 559 | } |
| 480 | 560 | |
| 481 | −/* | |
| 482 | − * Lock out other writers and update the count. | |
| 483 | − * Acts like a normal spin_lock/unlock. | |
| 484 | − * Don't need preempt_disable() because that is in the spin_lock already. | |
| 561 | +/** | |
| 562 | + * write_seqlock() - start a seqlock_t write side critical section | |
| 563 | + * @sl: Pointer to seqlock_t | |
| 564 | + * | |
| 565 | + * write_seqlock opens a write side critical section for the given | |
| 566 | + * seqlock_t. It also implicitly acquires the spinlock_t embedded inside | |
| 567 | + * that sequential lock. All seqlock_t write side sections are thus | |
| 568 | + * automatically serialized and non-preemptible. | |
| 569 | + * | |
| 570 | + * Context: if the seqlock_t read section, or other write side critical | |
| 571 | + * sections, can be invoked from hardirq or softirq contexts, use the | |
| 572 | + * _irqsave or _bh variants of this function instead. | |
| 485 | 573 | */ |
| 486 | 574 | static inline void write_seqlock(seqlock_t *sl) |
| 487 | 575 | { |
| @@ -489,30 +577,66 @@ static inline void write_seqlock(seqlock_t *sl) · 1 unmodified line | ||
| 489 | 577 | write_seqcount_begin(&sl->seqcount); |
| 490 | 578 | } |
| 491 | 579 | |
| 580 | +/** | |
| 581 | + * write_sequnlock() - end a seqlock_t write side critical section | |
| 582 | + * @sl: Pointer to seqlock_t | |
| 583 | + * | |
| 584 | + * write_sequnlock closes the (serialized and non-preemptible) write side | |
| 585 | + * critical section of given seqlock_t. | |
| 586 | + */ | |
| 492 | 587 | static inline void write_sequnlock(seqlock_t *sl) |
| 493 | 588 | { |
| 494 | 589 | write_seqcount_end(&sl->seqcount); |
| 495 | 590 | spin_unlock(&sl->lock); |
| 496 | 591 | } |
| 497 | 592 | |
| 593 | +/** | |
| 594 | + * write_seqlock_bh() - start a softirqs-disabled seqlock_t write section | |
| 595 | + * @sl: Pointer to seqlock_t | |
| 596 | + * | |
| 597 | + * _bh variant of write_seqlock(). Use only if the read side section, or | |
| 598 | + * other write side sections, can be invoked from softirq contexts. | |
| 599 | + */ | |
| 498 | 600 | static inline void write_seqlock_bh(seqlock_t *sl) |
| 499 | 601 | { |
| 500 | 602 | spin_lock_bh(&sl->lock); |
| 501 | 603 | write_seqcount_begin(&sl->seqcount); |
| 502 | 604 | } |
| 503 | 605 | |
| 606 | +/** | |
| 607 | + * write_sequnlock_bh() - end a softirqs-disabled seqlock_t write section | |
| 608 | + * @sl: Pointer to seqlock_t | |
| 609 | + * | |
| 610 | + * write_sequnlock_bh closes the serialized, non-preemptible, and | |
| 611 | + * softirqs-disabled, seqlock_t write side critical section opened with | |
| 612 | + * write_seqlock_bh(). | |
| 613 | + */ | |
| 504 | 614 | static inline void write_sequnlock_bh(seqlock_t *sl) |
| 505 | 615 | { |
| 506 | 616 | write_seqcount_end(&sl->seqcount); |
| 507 | 617 | spin_unlock_bh(&sl->lock); |
| 508 | 618 | } |
| 509 | 619 | |
| 620 | +/** | |
| 621 | + * write_seqlock_irq() - start a non-interruptible seqlock_t write section | |
| 622 | + * @sl: Pointer to seqlock_t | |
| 623 | + * | |
| 624 | + * _irq variant of write_seqlock(). Use only if the read side section, or | |
| 625 | + * other write sections, can be invoked from hardirq contexts. | |
| 626 | + */ | |
| 510 | 627 | static inline void write_seqlock_irq(seqlock_t *sl) |
| 511 | 628 | { |
| 512 | 629 | spin_lock_irq(&sl->lock); |
| 513 | 630 | write_seqcount_begin(&sl->seqcount); |
| 514 | 631 | } |
| 515 | 632 | |
| 633 | +/** | |
| 634 | + * write_sequnlock_irq() - end a non-interruptible seqlock_t write section | |
| 635 | + * @sl: Pointer to seqlock_t | |
| 636 | + * | |
| 637 | + * write_sequnlock_irq closes the serialized and non-interruptible | |
| 638 | + * seqlock_t write side section opened with write_seqlock_irq(). | |
| 639 | + */ | |
| 516 | 640 | static inline void write_sequnlock_irq(seqlock_t *sl) |
| 517 | 641 | { |
| 518 | 642 | write_seqcount_end(&sl->seqcount); |
| @@ -528,9 +652,28 @@ static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl) · 9 unmodified lines | ||
| 528 | 652 | return flags; |
| 529 | 653 | } |
| 530 | 654 | |
| 655 | +/** | |
| 656 | + * write_seqlock_irqsave() - start a non-interruptible seqlock_t write | |
| 657 | + * section | |
| 658 | + * @lock: Pointer to seqlock_t | |
| 659 | + * @flags: Stack-allocated storage for saving caller's local interrupt | |
| 660 | + * state, to be passed to write_sequnlock_irqrestore(). | |
| 661 | + * | |
| 662 | + * _irqsave variant of write_seqlock(). Use it only if the read side | |
| 663 | + * section, or other write sections, can be invoked from hardirq context. | |
| 664 | + */ | |
| 531 | 665 | #define write_seqlock_irqsave(lock, flags) \ |
| 532 | 666 | do { flags = __write_seqlock_irqsave(lock); } while (0) |
| 533 | 667 | |
| 668 | +/** | |
| 669 | + * write_sequnlock_irqrestore() - end non-interruptible seqlock_t write | |
| 670 | + * section | |
| 671 | + * @sl: Pointer to seqlock_t | |
| 672 | + * @flags: Caller's saved interrupt state, from write_seqlock_irqsave() | |
| 673 | + * | |
| 674 | + * write_sequnlock_irqrestore closes the serialized and non-interruptible | |
| 675 | + * seqlock_t write section previously opened with write_seqlock_irqsave(). | |
| 676 | + */ | |
| 534 | 677 | static inline void |
| 535 | 678 | write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) |
| 536 | 679 | { |
| @@ -538,36 +681,79 @@ write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) · 1 unmodified line | ||
| 538 | 681 | spin_unlock_irqrestore(&sl->lock, flags); |
| 539 | 682 | } |
| 540 | 683 | |
| 541 | −/* | |
| 542 | − * A locking reader exclusively locks out other writers and locking readers, | |
| 543 | − * but doesn't update the sequence number. Acts like a normal spin_lock/unlock. | |
| 544 | − * Don't need preempt_disable() because that is in the spin_lock already. | |
| 684 | +/** | |
| 685 | + * read_seqlock_excl() - begin a seqlock_t locking reader section | |
| 686 | + * @sl: Pointer to seqlock_t | |
| 687 | + * | |
| 688 | + * read_seqlock_excl opens a seqlock_t locking reader critical section. A | |
| 689 | + * locking reader exclusively locks out *both* other writers *and* other | |
| 690 | + * locking readers, but it does not update the embedded sequence number. | |
| 691 | + * | |
| 692 | + * Locking readers act like a normal spin_lock()/spin_unlock(). | |
| 693 | + * | |
| 694 | + * Context: if the seqlock_t write section, *or other read sections*, can | |
| 695 | + * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | |
| 696 | + * variant of this function instead. | |
| 697 | + * | |
| 698 | + * The opened read section must be closed with read_sequnlock_excl(). | |
| 545 | 699 | */ |
| 546 | 700 | static inline void read_seqlock_excl(seqlock_t *sl) |
| 547 | 701 | { |
| 548 | 702 | spin_lock(&sl->lock); |
| 549 | 703 | } |
| 550 | 704 | |
| 705 | +/** | |
| 706 | + * read_sequnlock_excl() - end a seqlock_t locking reader critical section | |
| 707 | + * @sl: Pointer to seqlock_t | |
| 708 | + */ | |
| 551 | 709 | static inline void read_sequnlock_excl(seqlock_t *sl) |
| 552 | 710 | { |
| 553 | 711 | spin_unlock(&sl->lock); |
| 554 | 712 | } |
| 555 | 713 | |
| 714 | +/** | |
| 715 | + * read_seqlock_excl_bh() - start a seqlock_t locking reader section with | |
| 716 | + * softirqs disabled | |
| 717 | + * @sl: Pointer to seqlock_t | |
| 718 | + * | |
| 719 | + * _bh variant of read_seqlock_excl(). Use this variant only if the | |
| 720 | + * seqlock_t write side section, *or other read sections*, can be invoked | |
| 721 | + * from softirq contexts. | |
| 722 | + */ | |
| 556 | 723 | static inline void read_seqlock_excl_bh(seqlock_t *sl) |
| 557 | 724 | { |
| 558 | 725 | spin_lock_bh(&sl->lock); |
| 559 | 726 | } |
| 560 | 727 | |
| 728 | +/** | |
| 729 | + * read_sequnlock_excl_bh() - stop a seqlock_t softirq-disabled locking | |
| 730 | + * reader section | |
| 731 | + * @sl: Pointer to seqlock_t | |
| 732 | + */ | |
| 561 | 733 | static inline void read_sequnlock_excl_bh(seqlock_t *sl) |
| 562 | 734 | { |
| 563 | 735 | spin_unlock_bh(&sl->lock); |
| 564 | 736 | } |
| 565 | 737 | |
| 738 | +/** | |
| 739 | + * read_seqlock_excl_irq() - start a non-interruptible seqlock_t locking | |
| 740 | + * reader section | |
| 741 | + * @sl: Pointer to seqlock_t | |
| 742 | + * | |
| 743 | + * _irq variant of read_seqlock_excl(). Use this only if the seqlock_t | |
| 744 | + * write side section, *or other read sections*, can be invoked from a | |
| 745 | + * hardirq context. | |
| 746 | + */ | |
| 566 | 747 | static inline void read_seqlock_excl_irq(seqlock_t *sl) |
| 567 | 748 | { |
| 568 | 749 | spin_lock_irq(&sl->lock); |
| 569 | 750 | } |
| 570 | 751 | |
| 752 | +/** | |
| 753 | + * read_sequnlock_excl_irq() - end an interrupts-disabled seqlock_t | |
| 754 | + * locking reader section | |
| 755 | + * @sl: Pointer to seqlock_t | |
| 756 | + */ | |
| 571 | 757 | static inline void read_sequnlock_excl_irq(seqlock_t *sl) |
| 572 | 758 | { |
| 573 | 759 | spin_unlock_irq(&sl->lock); |
| @@ -581,9 +767,26 @@ static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl) · 7 unmodified lines | ||
| 581 | 767 | return flags; |
| 582 | 768 | } |
| 583 | 769 | |
| 770 | +/** | |
| 771 | + * read_seqlock_excl_irqsave() - start a non-interruptible seqlock_t | |
| 772 | + * locking reader section | |
| 773 | + * @lock: Pointer to seqlock_t | |
| 774 | + * @flags: Stack-allocated storage for saving caller's local interrupt | |
| 775 | + * state, to be passed to read_sequnlock_excl_irqrestore(). | |
| 776 | + * | |
| 777 | + * _irqsave variant of read_seqlock_excl(). Use this only if the seqlock_t | |
| 778 | + * write side section, *or other read sections*, can be invoked from a | |
| 779 | + * hardirq context. | |
| 780 | + */ | |
| 584 | 781 | #define read_seqlock_excl_irqsave(lock, flags) \ |
| 585 | 782 | do { flags = __read_seqlock_excl_irqsave(lock); } while (0) |
| 586 | 783 | |
| 784 | +/** | |
| 785 | + * read_sequnlock_excl_irqrestore() - end non-interruptible seqlock_t | |
| 786 | + * locking reader section | |
| 787 | + * @sl: Pointer to seqlock_t | |
| 788 | + * @flags: Caller saved interrupt state, from read_seqlock_excl_irqsave() | |
| 789 | + */ | |
| 587 | 790 | static inline void |
| 588 | 791 | read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) |
| 589 | 792 | { |
| @@ -591,14 +794,35 @@ read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) · 1 unmodified line | ||
| 591 | 794 | } |
| 592 | 795 | |
| 593 | 796 | /** |
| 594 | − * read_seqbegin_or_lock - begin a sequence number check or locking block | |
| 595 | − * @lock: sequence lock | |
| 596 | − * @seq : sequence number to be checked | |
| 597 | − * | |
| 598 | − * First try it once optimistically without taking the lock. If that fails, | |
| 599 | − * take the lock. The sequence number is also used as a marker for deciding | |
| 600 | − * whether to be a reader (even) or writer (odd). | |
| 601 | − * N.B. seq must be initialized to an even number to begin with. | |
| 797 | + * read_seqbegin_or_lock() - begin a seqlock_t lockless or locking reader | |
| 798 | + * @lock: Pointer to seqlock_t | |
| 799 | + * @seq : Marker and return parameter. If the passed value is even, the | |
| 800 | + * reader will become a *lockless* seqlock_t reader as in read_seqbegin(). | |
| 801 | + * If the passed value is odd, the reader will become a *locking* reader | |
| 802 | + * as in read_seqlock_excl(). In the first call to this function, the | |
| 803 | + * caller *must* initialize and pass an even value to @seq; this way, a | |
| 804 | + * lockless read can be optimistically tried first. | |
| 805 | + * | |
| 806 | + * read_seqbegin_or_lock is an API designed to optimistically try a normal | |
| 807 | + * lockless seqlock_t read section first. If an odd counter is found, the | |
| 808 | + * lockless read trial has failed, and the next read iteration transforms | |
| 809 | + * itself into a full seqlock_t locking reader. | |
| 810 | + * | |
| 811 | + * This is typically used to avoid seqlock_t lockless readers starvation | |
| 812 | + * (too much retry loops) in the case of a sharp spike in write side | |
| 813 | + * activity. | |
| 814 | + * | |
| 815 | + * Context: if the seqlock_t write section, *or other read sections*, can | |
| 816 | + * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | |
| 817 | + * variant of this function instead. | |
| 818 | + * | |
| 819 | + * Check Documentation/locking/seqlock.rst for template example code. | |
| 820 | + * | |
| 821 | + * Return: the encountered sequence counter value, through the @seq | |
| 822 | + * parameter, which is overloaded as a return parameter. This returned | |
| 823 | + * value must be checked with need_seqretry(). If the read section need to | |
| 824 | + * be retried, this returned value must also be passed as the @seq | |
| 825 | + * parameter of the next read_seqbegin_or_lock() iteration. | |
| 602 | 826 | */ |
| 603 | 827 | static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) |
| 604 | 828 | { |
| @@ -608,17 +832,52 @@ static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) · 3 unmodified lines | ||
| 608 | 832 | read_seqlock_excl(lock); |
| 609 | 833 | } |
| 610 | 834 | |
| 835 | +/** | |
| 836 | + * need_seqretry() - validate seqlock_t "locking or lockless" read section | |
| 837 | + * @lock: Pointer to seqlock_t | |
| 838 | + * @seq: sequence count, from read_seqbegin_or_lock() | |
| 839 | + * | |
| 840 | + * Return: true if a read section retry is required, false otherwise | |
| 841 | + */ | |
| 611 | 842 | static inline int need_seqretry(seqlock_t *lock, int seq) |
| 612 | 843 | { |
| 613 | 844 | return !(seq & 1) && read_seqretry(lock, seq); |
| 614 | 845 | } |
| 615 | 846 | |
| 847 | +/** | |
| 848 | + * done_seqretry() - end seqlock_t "locking or lockless" reader section | |
| 849 | + * @lock: Pointer to seqlock_t | |
| 850 | + * @seq: count, from read_seqbegin_or_lock() | |
| 851 | + * | |
| 852 | + * done_seqretry finishes the seqlock_t read side critical section started | |
| 853 | + * with read_seqbegin_or_lock() and validated by need_seqretry(). | |
| 854 | + */ | |
| 616 | 855 | static inline void done_seqretry(seqlock_t *lock, int seq) |
| 617 | 856 | { |
| 618 | 857 | if (seq & 1) |
| 619 | 858 | read_sequnlock_excl(lock); |
| 620 | 859 | } |
| 621 | 860 | |
| 861 | +/** | |
| 862 | + * read_seqbegin_or_lock_irqsave() - begin a seqlock_t lockless reader, or | |
| 863 | + * a non-interruptible locking reader | |
| 864 | + * @lock: Pointer to seqlock_t | |
| 865 | + * @seq: Marker and return parameter. Check read_seqbegin_or_lock(). | |
| 866 | + * | |
| 867 | + * This is the _irqsave variant of read_seqbegin_or_lock(). Use it only if | |
| 868 | + * the seqlock_t write section, *or other read sections*, can be invoked | |
| 869 | + * from hardirq context. | |
| 870 | + * | |
| 871 | + * Note: Interrupts will be disabled only for "locking reader" mode. | |
| 872 | + * | |
| 873 | + * Return: | |
| 874 | + * | |
| 875 | + * 1. The saved local interrupts state in case of a locking reader, to | |
| 876 | + * be passed to done_seqretry_irqrestore(). | |
| 877 | + * | |
| 878 | + * 2. The encountered sequence counter value, returned through @seq | |
| 879 | + * overloaded as a return parameter. Check read_seqbegin_or_lock(). | |
| 880 | + */ | |
| 622 | 881 | static inline unsigned long |
| 623 | 882 | read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) |
| 624 | 883 | { |
| @@ -632,6 +891,18 @@ read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) · 7 unmodified lines | ||
| 632 | 891 | return flags; |
| 633 | 892 | } |
| 634 | 893 | |
| 894 | +/** | |
| 895 | + * done_seqretry_irqrestore() - end a seqlock_t lockless reader, or a | |
| 896 | + * non-interruptible locking reader section | |
| 897 | + * @lock: Pointer to seqlock_t | |
| 898 | + * @seq: Count, from read_seqbegin_or_lock_irqsave() | |
| 899 | + * @flags: Caller's saved local interrupt state in case of a locking | |
| 900 | + * reader, also from read_seqbegin_or_lock_irqsave() | |
| 901 | + * | |
| 902 | + * This is the _irqrestore variant of done_seqretry(). The read section | |
| 903 | + * must've been opened with read_seqbegin_or_lock_irqsave(), and validated | |
| 904 | + * by need_seqretry(). | |
| 905 | + */ | |
| 635 | 906 | static inline void |
| 636 | 907 | done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags) |
| 637 | 908 | { |