seqlock.h is now included by kernel's RST documentation, but a small number of the the exported seqlock.h functions are kernel-doc annotated. Add kernel-doc for all seqlock.h exported APIs.
Showing the changes to include/linux (1 of 1 file). Show all
Unified · Split
| @@ -75,6 +75,10 @@ static inline void __seqcount_init(seqcount_t *s, const char *name, · 74 unmodified lines | |||
| 75 | # define SEQCOUNT_DEP_MAP_INIT(lockname) \ | 75 | # define SEQCOUNT_DEP_MAP_INIT(lockname) \ |
| 76 | .dep_map = { .name = #lockname } \ | 76 | .dep_map = { .name = #lockname } \ |
| 77 | 77 | ||
| 78 | /** | ||
| 79 | * seqcount_init() - runtime initializer for seqcount_t | ||
| 80 | * @s: Pointer to the seqcount_t instance | ||
| 81 | */ | ||
| 78 | # define seqcount_init(s) \ | 82 | # define seqcount_init(s) \ |
| 79 | do { \ | 83 | do { \ |
| 80 | static struct lock_class_key __key; \ | 84 | static struct lock_class_key __key; \ |
| @@ -98,13 +102,15 @@ static inline void seqcount_lockdep_reader_access(const seqcount_t *s) · 17 unmodified lines | |||
| 98 | # define seqcount_lockdep_reader_access(x) | 102 | # define seqcount_lockdep_reader_access(x) |
| 99 | #endif | 103 | #endif |
| 100 | 104 | ||
| 101 | #define SEQCNT_ZERO(lockname) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(lockname)} | 105 | /** |
| 102 | 106 | * SEQCNT_ZERO() - static initializer for seqcount_t | |
| 107 | * @name: Name of the seqcount_t instance | ||
| 108 | */ | ||
| 109 | #define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) } | ||
| 103 | 110 | ||
| 104 | /** | 111 | /** |
| 105 | * __read_seqcount_begin - begin a seq-read critical section (without barrier) | 112 | * __read_seqcount_begin() - begin a seqcount_t read section w/o barrier |
| 106 | * @s: pointer to seqcount_t | 113 | * @s: Pointer to seqcount_t |
| 107 | * Returns: count to be passed to read_seqcount_retry | ||
| 108 | * | 114 | * |
| 109 | * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb() | 115 | * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb() |
| 110 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | 116 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is |
| @@ -113,6 +119,8 @@ static inline void seqcount_lockdep_reader_access(const seqcount_t *s) · 2 unmodified lines | |||
| 113 | * | 119 | * |
| 114 | * Use carefully, only in critical code, and comment how the barrier is | 120 | * Use carefully, only in critical code, and comment how the barrier is |
| 115 | * provided. | 121 | * provided. |
| 122 | * | ||
| 123 | * Return: count to be passed to read_seqcount_retry() | ||
| 116 | */ | 124 | */ |
| 117 | static inline unsigned __read_seqcount_begin(const seqcount_t *s) | 125 | static inline unsigned __read_seqcount_begin(const seqcount_t *s) |
| 118 | { | 126 | { |
| @@ -129,13 +137,10 @@ repeat: · 10 unmodified lines | |||
| 129 | } | 137 | } |
| 130 | 138 | ||
| 131 | /** | 139 | /** |
| 132 | * raw_read_seqcount_begin - start seq-read critical section w/o lockdep | 140 | * raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep |
| 133 | * @s: pointer to seqcount_t | 141 | * @s: Pointer to seqcount_t |
| 134 | * Returns: count to be passed to read_seqcount_retry | ||
| 135 | * | 142 | * |
| 136 | * raw_read_seqcount_begin opens a read critical section of the given | 143 | * Return: count to be passed to read_seqcount_retry() |
| 137 | * seqcount, but without any lockdep checking. Validity of the critical | ||
| 138 | * section is tested by checking read_seqcount_retry function. | ||
| 139 | */ | 144 | */ |
| 140 | static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) | 145 | static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) |
| 141 | { | 146 | { |
| @@ -145,13 +150,10 @@ static inline unsigned raw_read_seqcount_begin(const seqcount_t *s) · 3 unmodified lines | |||
| 145 | } | 150 | } |
| 146 | 151 | ||
| 147 | /** | 152 | /** |
| 148 | * read_seqcount_begin - begin a seq-read critical section | 153 | * read_seqcount_begin() - begin a seqcount_t read critical section |
| 149 | * @s: pointer to seqcount_t | 154 | * @s: Pointer to seqcount_t |
| 150 | * Returns: count to be passed to read_seqcount_retry | ||
| 151 | * | 155 | * |
| 152 | * read_seqcount_begin opens a read critical section of the given seqcount. | 156 | * Return: count to be passed to read_seqcount_retry() |
| 153 | * Validity of the critical section is tested by checking read_seqcount_retry | ||
| 154 | * function. | ||
| 155 | */ | 157 | */ |
| 156 | static inline unsigned read_seqcount_begin(const seqcount_t *s) | 158 | static inline unsigned read_seqcount_begin(const seqcount_t *s) |
| 157 | { | 159 | { |
| @@ -160,13 +162,15 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s) · 2 unmodified lines | |||
| 160 | } | 162 | } |
| 161 | 163 | ||
| 162 | /** | 164 | /** |
| 163 | * raw_read_seqcount - Read the raw seqcount | 165 | * raw_read_seqcount() - read the raw seqcount_t counter value |
| 164 | * @s: pointer to seqcount_t | 166 | * @s: Pointer to seqcount_t |
| 165 | * Returns: count to be passed to read_seqcount_retry | ||
| 166 | * | 167 | * |
| 167 | * raw_read_seqcount opens a read critical section of the given | 168 | * raw_read_seqcount opens a read critical section of the given |
| 168 | * seqcount without any lockdep checking and without checking or | 169 | * seqcount_t, without any lockdep checking, and without checking or |
| 169 | * masking the LSB. Calling code is responsible for handling that. | 170 | * masking the sequence counter LSB. Calling code is responsible for |
| 171 | * handling that. | ||
| 172 | * | ||
| 173 | * Return: count to be passed to read_seqcount_retry() | ||
| 170 | */ | 174 | */ |
| 171 | static inline unsigned raw_read_seqcount(const seqcount_t *s) | 175 | static inline unsigned raw_read_seqcount(const seqcount_t *s) |
| 172 | { | 176 | { |
| @@ -177,18 +181,21 @@ static inline unsigned raw_read_seqcount(const seqcount_t *s) · 4 unmodified lines | |||
| 177 | } | 181 | } |
| 178 | 182 | ||
| 179 | /** | 183 | /** |
| 180 | * raw_seqcount_begin - begin a seq-read critical section | 184 | * raw_seqcount_begin() - begin a seqcount_t read critical section w/o |
| 181 | * @s: pointer to seqcount_t | 185 | * lockdep and w/o counter stabilization |
| 182 | * Returns: count to be passed to read_seqcount_retry | 186 | * @s: Pointer to seqcount_t |
| 183 | * | 187 | * |
| 184 | * raw_seqcount_begin opens a read critical section of the given seqcount. | 188 | * raw_seqcount_begin opens a read critical section of the given |
| 185 | * Validity of the critical section is tested by checking read_seqcount_retry | 189 | * seqcount_t. Unlike read_seqcount_begin(), this function will not wait |
| 186 | * function. | 190 | * for the count to stabilize. If a writer is active when it begins, it |
| 191 | * will fail the read_seqcount_retry() at the end of the read critical | ||
| 192 | * section instead of stabilizing at the beginning of it. | ||
| 187 | * | 193 | * |
| 188 | * Unlike read_seqcount_begin(), this function will not wait for the count | 194 | * Use this only in special kernel hot paths where the read section is |
| 189 | * to stabilize. If a writer is active when we begin, we will fail the | 195 | * small and has a high probability of success through other external |
| 190 | * read_seqcount_retry() instead of stabilizing at the beginning of the | 196 | * means. It will save a single branching instruction. |
| 191 | * critical section. | 197 | * |
| 198 | * Return: count to be passed to read_seqcount_retry() | ||
| 192 | */ | 199 | */ |
| 193 | static inline unsigned raw_seqcount_begin(const seqcount_t *s) | 200 | static inline unsigned raw_seqcount_begin(const seqcount_t *s) |
| 194 | { | 201 | { |
| @@ -199,10 +206,9 @@ static inline unsigned raw_seqcount_begin(const seqcount_t *s) · 4 unmodified lines | |||
| 199 | } | 206 | } |
| 200 | 207 | ||
| 201 | /** | 208 | /** |
| 202 | * __read_seqcount_retry - end a seq-read critical section (without barrier) | 209 | * __read_seqcount_retry() - end a seqcount_t read section w/o barrier |
| 203 | * @s: pointer to seqcount_t | 210 | * @s: Pointer to seqcount_t |
| 204 | * @start: count, from read_seqcount_begin | 211 | * @start: count, from read_seqcount_begin() |
| 205 | * Returns: 1 if retry is required, else 0 | ||
| 206 | * | 212 | * |
| 207 | * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() | 213 | * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb() |
| 208 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is | 214 | * barrier. Callers should ensure that smp_rmb() or equivalent ordering is |
| @@ -211,6 +217,8 @@ static inline unsigned raw_seqcount_begin(const seqcount_t *s) · 2 unmodified lines | |||
| 211 | * | 217 | * |
| 212 | * Use carefully, only in critical code, and comment how the barrier is | 218 | * Use carefully, only in critical code, and comment how the barrier is |
| 213 | * provided. | 219 | * provided. |
| 220 | * | ||
| 221 | * Return: true if a read section retry is required, else false | ||
| 214 | */ | 222 | */ |
| 215 | static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) | 223 | static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) |
| 216 | { | 224 | { |
| @@ -219,14 +227,15 @@ static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) · 2 unmodified lines | |||
| 219 | } | 227 | } |
| 220 | 228 | ||
| 221 | /** | 229 | /** |
| 222 | * read_seqcount_retry - end a seq-read critical section | 230 | * read_seqcount_retry() - end a seqcount_t read critical section |
| 223 | * @s: pointer to seqcount_t | 231 | * @s: Pointer to seqcount_t |
| 224 | * @start: count, from read_seqcount_begin | 232 | * @start: count, from read_seqcount_begin() |
| 225 | * Returns: 1 if retry is required, else 0 | ||
| 226 | * | 233 | * |
| 227 | * read_seqcount_retry closes a read critical section of the given seqcount. | 234 | * read_seqcount_retry closes the read critical section of given |
| 228 | * If the critical section was invalid, it must be ignored (and typically | 235 | * seqcount_t. If the critical section was invalid, it must be ignored |
| 229 | * retried). | 236 | * (and typically retried). |
| 237 | * | ||
| 238 | * Return: true if a read section retry is required, else false | ||
| 230 | */ | 239 | */ |
| 231 | static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) | 240 | static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) |
| 232 | { | 241 | { |
| @@ -234,6 +243,10 @@ static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) · 1 unmodified line | |||
| 234 | return __read_seqcount_retry(s, start); | 243 | return __read_seqcount_retry(s, start); |
| 235 | } | 244 | } |
| 236 | 245 | ||
| 246 | /** | ||
| 247 | * raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep | ||
| 248 | * @s: Pointer to seqcount_t | ||
| 249 | */ | ||
| 237 | static inline void raw_write_seqcount_begin(seqcount_t *s) | 250 | static inline void raw_write_seqcount_begin(seqcount_t *s) |
| 238 | { | 251 | { |
| 239 | kcsan_nestable_atomic_begin(); | 252 | kcsan_nestable_atomic_begin(); |
| @@ -241,6 +254,10 @@ static inline void raw_write_seqcount_begin(seqcount_t *s) · 1 unmodified line | |||
| 241 | smp_wmb(); | 254 | smp_wmb(); |
| 242 | } | 255 | } |
| 243 | 256 | ||
| 257 | /** | ||
| 258 | * raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep | ||
| 259 | * @s: Pointer to seqcount_t | ||
| 260 | */ | ||
| 244 | static inline void raw_write_seqcount_end(seqcount_t *s) | 261 | static inline void raw_write_seqcount_end(seqcount_t *s) |
| 245 | { | 262 | { |
| 246 | smp_wmb(); | 263 | smp_wmb(); |
| @@ -248,17 +265,42 @@ static inline void raw_write_seqcount_end(seqcount_t *s) · 1 unmodified line | |||
| 248 | kcsan_nestable_atomic_end(); | 265 | kcsan_nestable_atomic_end(); |
| 249 | } | 266 | } |
| 250 | 267 | ||
| 268 | /** | ||
| 269 | * write_seqcount_begin_nested() - start a seqcount_t write section with | ||
| 270 | * custom lockdep nesting level | ||
| 271 | * @s: Pointer to seqcount_t | ||
| 272 | * @subclass: lockdep nesting level | ||
| 273 | * | ||
| 274 | * See Documentation/locking/lockdep-design.rst | ||
| 275 | */ | ||
| 251 | static inline void write_seqcount_begin_nested(seqcount_t *s, int subclass) | 276 | static inline void write_seqcount_begin_nested(seqcount_t *s, int subclass) |
| 252 | { | 277 | { |
| 253 | raw_write_seqcount_begin(s); | 278 | raw_write_seqcount_begin(s); |
| 254 | seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_); | 279 | seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_); |
| 255 | } | 280 | } |
| 256 | 281 | ||
| 282 | /** | ||
| 283 | * write_seqcount_begin() - start a seqcount_t write side critical section | ||
| 284 | * @s: Pointer to seqcount_t | ||
| 285 | * | ||
| 286 | * write_seqcount_begin opens a write side critical section of the given | ||
| 287 | * seqcount_t. | ||
| 288 | * | ||
| 289 | * Context: seqcount_t write side critical sections must be serialized and | ||
| 290 | * non-preemptible. If readers can be invoked from hardirq or softirq | ||
| 291 | * context, interrupts or bottom halves must be respectively disabled. | ||
| 292 | */ | ||
| 257 | static inline void write_seqcount_begin(seqcount_t *s) | 293 | static inline void write_seqcount_begin(seqcount_t *s) |
| 258 | { | 294 | { |
| 259 | write_seqcount_begin_nested(s, 0); | 295 | write_seqcount_begin_nested(s, 0); |
| 260 | } | 296 | } |
| 261 | 297 | ||
| 298 | /** | ||
| 299 | * write_seqcount_end() - end a seqcount_t write side critical section | ||
| 300 | * @s: Pointer to seqcount_t | ||
| 301 | * | ||
| 302 | * The write section must've been opened with write_seqcount_begin(). | ||
| 303 | */ | ||
| 262 | static inline void write_seqcount_end(seqcount_t *s) | 304 | static inline void write_seqcount_end(seqcount_t *s) |
| 263 | { | 305 | { |
| 264 | seqcount_release(&s->dep_map, _RET_IP_); | 306 | seqcount_release(&s->dep_map, _RET_IP_); |
| @@ -266,12 +308,12 @@ static inline void write_seqcount_end(seqcount_t *s) · 1 unmodified line | |||
| 266 | } | 308 | } |
| 267 | 309 | ||
| 268 | /** | 310 | /** |
| 269 | * raw_write_seqcount_barrier - do a seq write barrier | 311 | * raw_write_seqcount_barrier() - do a seqcount_t write barrier |
| 270 | * @s: pointer to seqcount_t | 312 | * @s: Pointer to seqcount_t |
| 271 | * | 313 | * |
| 272 | * This can be used to provide an ordering guarantee instead of the | 314 | * This can be used to provide an ordering guarantee instead of the usual |
| 273 | * usual consistency guarantee. It is one wmb cheaper, because we can | 315 | * consistency guarantee. It is one wmb cheaper, because it can collapse |
| 274 | * collapse the two back-to-back wmb()s. | 316 | * the two back-to-back wmb()s. |
| 275 | * | 317 | * |
| 276 | * Note that writes surrounding the barrier should be declared atomic (e.g. | 318 | * Note that writes surrounding the barrier should be declared atomic (e.g. |
| 277 | * via WRITE_ONCE): a) to ensure the writes become visible to other threads | 319 | * via WRITE_ONCE): a) to ensure the writes become visible to other threads |
| @@ -316,11 +358,12 @@ static inline void raw_write_seqcount_barrier(seqcount_t *s) · 38 unmodified lines | |||
| 316 | } | 358 | } |
| 317 | 359 | ||
| 318 | /** | 360 | /** |
| 319 | * write_seqcount_invalidate - invalidate in-progress read-side seq operations | 361 | * write_seqcount_invalidate() - invalidate in-progress seqcount_t read |
| 320 | * @s: pointer to seqcount_t | 362 | * side operations |
| 363 | * @s: Pointer to seqcount_t | ||
| 321 | * | 364 | * |
| 322 | * After write_seqcount_invalidate, no read-side seq operations will complete | 365 | * After write_seqcount_invalidate, no seqcount_t read side operations |
| 323 | * successfully and see data older than this. | 366 | * will complete successfully and see data older than this. |
| 324 | */ | 367 | */ |
| 325 | static inline void write_seqcount_invalidate(seqcount_t *s) | 368 | static inline void write_seqcount_invalidate(seqcount_t *s) |
| 326 | { | 369 | { |
| @@ -330,6 +373,21 @@ static inline void write_seqcount_invalidate(seqcount_t *s) · 3 unmodified lines | |||
| 330 | kcsan_nestable_atomic_end(); | 373 | kcsan_nestable_atomic_end(); |
| 331 | } | 374 | } |
| 332 | 375 | ||
| 376 | /** | ||
| 377 | * raw_read_seqcount_latch() - pick even/odd seqcount_t latch data copy | ||
| 378 | * @s: Pointer to seqcount_t | ||
| 379 | * | ||
| 380 | * Use seqcount_t latching to switch between two storage places protected | ||
| 381 | * by a sequence counter. Doing so allows having interruptible, preemptible, | ||
| 382 | * seqcount_t write side critical sections. | ||
| 383 | * | ||
| 384 | * Check raw_write_seqcount_latch() for more details and a full reader and | ||
| 385 | * writer usage example. | ||
| 386 | * | ||
| 387 | * Return: sequence counter raw value. Use the lowest bit as an index for | ||
| 388 | * picking which data copy to read. The full counter value must then be | ||
| 389 | * checked with read_seqcount_retry(). | ||
| 390 | */ | ||
| 333 | static inline int raw_read_seqcount_latch(seqcount_t *s) | 391 | static inline int raw_read_seqcount_latch(seqcount_t *s) |
| 334 | { | 392 | { |
| 335 | /* Pairs with the first smp_wmb() in raw_write_seqcount_latch() */ | 393 | /* Pairs with the first smp_wmb() in raw_write_seqcount_latch() */ |
| @@ -338,8 +396,8 @@ static inline int raw_read_seqcount_latch(seqcount_t *s) · 2 unmodified lines | |||
| 338 | } | 396 | } |
| 339 | 397 | ||
| 340 | /** | 398 | /** |
| 341 | * raw_write_seqcount_latch - redirect readers to even/odd copy | 399 | * raw_write_seqcount_latch() - redirect readers to even/odd copy |
| 342 | * @s: pointer to seqcount_t | 400 | * @s: Pointer to seqcount_t |
| 343 | * | 401 | * |
| 344 | * The latch technique is a multiversion concurrency control method that allows | 402 | * The latch technique is a multiversion concurrency control method that allows |
| 345 | * queries during non-atomic modifications. If you can guarantee queries never | 403 | * queries during non-atomic modifications. If you can guarantee queries never |
| @@ -446,17 +504,28 @@ typedef struct { · 100 unmodified lines | |||
| 446 | .lock = __SPIN_LOCK_UNLOCKED(lockname) \ | 504 | .lock = __SPIN_LOCK_UNLOCKED(lockname) \ |
| 447 | } | 505 | } |
| 448 | 506 | ||
| 449 | #define seqlock_init(x) \ | 507 | /** |
| 508 | * seqlock_init() - dynamic initializer for seqlock_t | ||
| 509 | * @sl: Pointer to the seqlock_t instance | ||
| 510 | */ | ||
| 511 | #define seqlock_init(sl) \ | ||
| 450 | do { \ | 512 | do { \ |
| 451 | seqcount_init(&(x)->seqcount); \ | 513 | seqcount_init(&(sl)->seqcount); \ |
| 452 | spin_lock_init(&(x)->lock); \ | 514 | spin_lock_init(&(sl)->lock); \ |
| 453 | } while (0) | 515 | } while (0) |
| 454 | 516 | ||
| 455 | #define DEFINE_SEQLOCK(x) \ | 517 | /** |
| 456 | seqlock_t x = __SEQLOCK_UNLOCKED(x) | 518 | * DEFINE_SEQLOCK() - Define a statically allocated seqlock_t |
| 519 | * @sl: Name of the seqlock_t instance | ||
| 520 | */ | ||
| 521 | #define DEFINE_SEQLOCK(sl) \ | ||
| 522 | seqlock_t sl = __SEQLOCK_UNLOCKED(sl) | ||
| 457 | 523 | ||
| 458 | /* | 524 | /** |
| 459 | * Read side functions for starting and finalizing a read side section. | 525 | * read_seqbegin() - start a seqlock_t read side critical section |
| 526 | * @sl: Pointer to seqlock_t | ||
| 527 | * | ||
| 528 | * Return: count, to be passed to read_seqretry() | ||
| 460 | */ | 529 | */ |
| 461 | static inline unsigned read_seqbegin(const seqlock_t *sl) | 530 | static inline unsigned read_seqbegin(const seqlock_t *sl) |
| 462 | { | 531 | { |
| @@ -467,6 +536,17 @@ static inline unsigned read_seqbegin(const seqlock_t *sl) · 4 unmodified lines | |||
| 467 | return ret; | 536 | return ret; |
| 468 | } | 537 | } |
| 469 | 538 | ||
| 539 | /** | ||
| 540 | * read_seqretry() - end a seqlock_t read side section | ||
| 541 | * @sl: Pointer to seqlock_t | ||
| 542 | * @start: count, from read_seqbegin() | ||
| 543 | * | ||
| 544 | * read_seqretry closes the read side critical section of given seqlock_t. | ||
| 545 | * If the critical section was invalid, it must be ignored (and typically | ||
| 546 | * retried). | ||
| 547 | * | ||
| 548 | * Return: true if a read section retry is required, else false | ||
| 549 | */ | ||
| 470 | static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) | 550 | static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) |
| 471 | { | 551 | { |
| 472 | /* | 552 | /* |
| @@ -478,10 +558,18 @@ static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) · 5 unmodified lines | |||
| 478 | return read_seqcount_retry(&sl->seqcount, start); | 558 | return read_seqcount_retry(&sl->seqcount, start); |
| 479 | } | 559 | } |
| 480 | 560 | ||
| 481 | /* | 561 | /** |
| 482 | * Lock out other writers and update the count. | 562 | * write_seqlock() - start a seqlock_t write side critical section |
| 483 | * Acts like a normal spin_lock/unlock. | 563 | * @sl: Pointer to seqlock_t |
| 484 | * Don't need preempt_disable() because that is in the spin_lock already. | 564 | * |
| 565 | * write_seqlock opens a write side critical section for the given | ||
| 566 | * seqlock_t. It also implicitly acquires the spinlock_t embedded inside | ||
| 567 | * that sequential lock. All seqlock_t write side sections are thus | ||
| 568 | * automatically serialized and non-preemptible. | ||
| 569 | * | ||
| 570 | * Context: if the seqlock_t read section, or other write side critical | ||
| 571 | * sections, can be invoked from hardirq or softirq contexts, use the | ||
| 572 | * _irqsave or _bh variants of this function instead. | ||
| 485 | */ | 573 | */ |
| 486 | static inline void write_seqlock(seqlock_t *sl) | 574 | static inline void write_seqlock(seqlock_t *sl) |
| 487 | { | 575 | { |
| @@ -489,30 +577,66 @@ static inline void write_seqlock(seqlock_t *sl) · 1 unmodified line | |||
| 489 | write_seqcount_begin(&sl->seqcount); | 577 | write_seqcount_begin(&sl->seqcount); |
| 490 | } | 578 | } |
| 491 | 579 | ||
| 580 | /** | ||
| 581 | * write_sequnlock() - end a seqlock_t write side critical section | ||
| 582 | * @sl: Pointer to seqlock_t | ||
| 583 | * | ||
| 584 | * write_sequnlock closes the (serialized and non-preemptible) write side | ||
| 585 | * critical section of given seqlock_t. | ||
| 586 | */ | ||
| 492 | static inline void write_sequnlock(seqlock_t *sl) | 587 | static inline void write_sequnlock(seqlock_t *sl) |
| 493 | { | 588 | { |
| 494 | write_seqcount_end(&sl->seqcount); | 589 | write_seqcount_end(&sl->seqcount); |
| 495 | spin_unlock(&sl->lock); | 590 | spin_unlock(&sl->lock); |
| 496 | } | 591 | } |
| 497 | 592 | ||
| 593 | /** | ||
| 594 | * write_seqlock_bh() - start a softirqs-disabled seqlock_t write section | ||
| 595 | * @sl: Pointer to seqlock_t | ||
| 596 | * | ||
| 597 | * _bh variant of write_seqlock(). Use only if the read side section, or | ||
| 598 | * other write side sections, can be invoked from softirq contexts. | ||
| 599 | */ | ||
| 498 | static inline void write_seqlock_bh(seqlock_t *sl) | 600 | static inline void write_seqlock_bh(seqlock_t *sl) |
| 499 | { | 601 | { |
| 500 | spin_lock_bh(&sl->lock); | 602 | spin_lock_bh(&sl->lock); |
| 501 | write_seqcount_begin(&sl->seqcount); | 603 | write_seqcount_begin(&sl->seqcount); |
| 502 | } | 604 | } |
| 503 | 605 | ||
| 606 | /** | ||
| 607 | * write_sequnlock_bh() - end a softirqs-disabled seqlock_t write section | ||
| 608 | * @sl: Pointer to seqlock_t | ||
| 609 | * | ||
| 610 | * write_sequnlock_bh closes the serialized, non-preemptible, and | ||
| 611 | * softirqs-disabled, seqlock_t write side critical section opened with | ||
| 612 | * write_seqlock_bh(). | ||
| 613 | */ | ||
| 504 | static inline void write_sequnlock_bh(seqlock_t *sl) | 614 | static inline void write_sequnlock_bh(seqlock_t *sl) |
| 505 | { | 615 | { |
| 506 | write_seqcount_end(&sl->seqcount); | 616 | write_seqcount_end(&sl->seqcount); |
| 507 | spin_unlock_bh(&sl->lock); | 617 | spin_unlock_bh(&sl->lock); |
| 508 | } | 618 | } |
| 509 | 619 | ||
| 620 | /** | ||
| 621 | * write_seqlock_irq() - start a non-interruptible seqlock_t write section | ||
| 622 | * @sl: Pointer to seqlock_t | ||
| 623 | * | ||
| 624 | * _irq variant of write_seqlock(). Use only if the read side section, or | ||
| 625 | * other write sections, can be invoked from hardirq contexts. | ||
| 626 | */ | ||
| 510 | static inline void write_seqlock_irq(seqlock_t *sl) | 627 | static inline void write_seqlock_irq(seqlock_t *sl) |
| 511 | { | 628 | { |
| 512 | spin_lock_irq(&sl->lock); | 629 | spin_lock_irq(&sl->lock); |
| 513 | write_seqcount_begin(&sl->seqcount); | 630 | write_seqcount_begin(&sl->seqcount); |
| 514 | } | 631 | } |
| 515 | 632 | ||
| 633 | /** | ||
| 634 | * write_sequnlock_irq() - end a non-interruptible seqlock_t write section | ||
| 635 | * @sl: Pointer to seqlock_t | ||
| 636 | * | ||
| 637 | * write_sequnlock_irq closes the serialized and non-interruptible | ||
| 638 | * seqlock_t write side section opened with write_seqlock_irq(). | ||
| 639 | */ | ||
| 516 | static inline void write_sequnlock_irq(seqlock_t *sl) | 640 | static inline void write_sequnlock_irq(seqlock_t *sl) |
| 517 | { | 641 | { |
| 518 | write_seqcount_end(&sl->seqcount); | 642 | write_seqcount_end(&sl->seqcount); |
| @@ -528,9 +652,28 @@ static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl) · 9 unmodified lines | |||
| 528 | return flags; | 652 | return flags; |
| 529 | } | 653 | } |
| 530 | 654 | ||
| 655 | /** | ||
| 656 | * write_seqlock_irqsave() - start a non-interruptible seqlock_t write | ||
| 657 | * section | ||
| 658 | * @lock: Pointer to seqlock_t | ||
| 659 | * @flags: Stack-allocated storage for saving caller's local interrupt | ||
| 660 | * state, to be passed to write_sequnlock_irqrestore(). | ||
| 661 | * | ||
| 662 | * _irqsave variant of write_seqlock(). Use it only if the read side | ||
| 663 | * section, or other write sections, can be invoked from hardirq context. | ||
| 664 | */ | ||
| 531 | #define write_seqlock_irqsave(lock, flags) \ | 665 | #define write_seqlock_irqsave(lock, flags) \ |
| 532 | do { flags = __write_seqlock_irqsave(lock); } while (0) | 666 | do { flags = __write_seqlock_irqsave(lock); } while (0) |
| 533 | 667 | ||
| 668 | /** | ||
| 669 | * write_sequnlock_irqrestore() - end non-interruptible seqlock_t write | ||
| 670 | * section | ||
| 671 | * @sl: Pointer to seqlock_t | ||
| 672 | * @flags: Caller's saved interrupt state, from write_seqlock_irqsave() | ||
| 673 | * | ||
| 674 | * write_sequnlock_irqrestore closes the serialized and non-interruptible | ||
| 675 | * seqlock_t write section previously opened with write_seqlock_irqsave(). | ||
| 676 | */ | ||
| 534 | static inline void | 677 | static inline void |
| 535 | write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) | 678 | write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) |
| 536 | { | 679 | { |
| @@ -538,36 +681,79 @@ write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) · 1 unmodified line | |||
| 538 | spin_unlock_irqrestore(&sl->lock, flags); | 681 | spin_unlock_irqrestore(&sl->lock, flags); |
| 539 | } | 682 | } |
| 540 | 683 | ||
| 541 | /* | 684 | /** |
| 542 | * A locking reader exclusively locks out other writers and locking readers, | 685 | * read_seqlock_excl() - begin a seqlock_t locking reader section |
| 543 | * but doesn't update the sequence number. Acts like a normal spin_lock/unlock. | 686 | * @sl: Pointer to seqlock_t |
| 544 | * Don't need preempt_disable() because that is in the spin_lock already. | 687 | * |
| 688 | * read_seqlock_excl opens a seqlock_t locking reader critical section. A | ||
| 689 | * locking reader exclusively locks out *both* other writers *and* other | ||
| 690 | * locking readers, but it does not update the embedded sequence number. | ||
| 691 | * | ||
| 692 | * Locking readers act like a normal spin_lock()/spin_unlock(). | ||
| 693 | * | ||
| 694 | * Context: if the seqlock_t write section, *or other read sections*, can | ||
| 695 | * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | ||
| 696 | * variant of this function instead. | ||
| 697 | * | ||
| 698 | * The opened read section must be closed with read_sequnlock_excl(). | ||
| 545 | */ | 699 | */ |
| 546 | static inline void read_seqlock_excl(seqlock_t *sl) | 700 | static inline void read_seqlock_excl(seqlock_t *sl) |
| 547 | { | 701 | { |
| 548 | spin_lock(&sl->lock); | 702 | spin_lock(&sl->lock); |
| 549 | } | 703 | } |
| 550 | 704 | ||
| 705 | /** | ||
| 706 | * read_sequnlock_excl() - end a seqlock_t locking reader critical section | ||
| 707 | * @sl: Pointer to seqlock_t | ||
| 708 | */ | ||
| 551 | static inline void read_sequnlock_excl(seqlock_t *sl) | 709 | static inline void read_sequnlock_excl(seqlock_t *sl) |
| 552 | { | 710 | { |
| 553 | spin_unlock(&sl->lock); | 711 | spin_unlock(&sl->lock); |
| 554 | } | 712 | } |
| 555 | 713 | ||
| 714 | /** | ||
| 715 | * read_seqlock_excl_bh() - start a seqlock_t locking reader section with | ||
| 716 | * softirqs disabled | ||
| 717 | * @sl: Pointer to seqlock_t | ||
| 718 | * | ||
| 719 | * _bh variant of read_seqlock_excl(). Use this variant only if the | ||
| 720 | * seqlock_t write side section, *or other read sections*, can be invoked | ||
| 721 | * from softirq contexts. | ||
| 722 | */ | ||
| 556 | static inline void read_seqlock_excl_bh(seqlock_t *sl) | 723 | static inline void read_seqlock_excl_bh(seqlock_t *sl) |
| 557 | { | 724 | { |
| 558 | spin_lock_bh(&sl->lock); | 725 | spin_lock_bh(&sl->lock); |
| 559 | } | 726 | } |
| 560 | 727 | ||
| 728 | /** | ||
| 729 | * read_sequnlock_excl_bh() - stop a seqlock_t softirq-disabled locking | ||
| 730 | * reader section | ||
| 731 | * @sl: Pointer to seqlock_t | ||
| 732 | */ | ||
| 561 | static inline void read_sequnlock_excl_bh(seqlock_t *sl) | 733 | static inline void read_sequnlock_excl_bh(seqlock_t *sl) |
| 562 | { | 734 | { |
| 563 | spin_unlock_bh(&sl->lock); | 735 | spin_unlock_bh(&sl->lock); |
| 564 | } | 736 | } |
| 565 | 737 | ||
| 738 | /** | ||
| 739 | * read_seqlock_excl_irq() - start a non-interruptible seqlock_t locking | ||
| 740 | * reader section | ||
| 741 | * @sl: Pointer to seqlock_t | ||
| 742 | * | ||
| 743 | * _irq variant of read_seqlock_excl(). Use this only if the seqlock_t | ||
| 744 | * write side section, *or other read sections*, can be invoked from a | ||
| 745 | * hardirq context. | ||
| 746 | */ | ||
| 566 | static inline void read_seqlock_excl_irq(seqlock_t *sl) | 747 | static inline void read_seqlock_excl_irq(seqlock_t *sl) |
| 567 | { | 748 | { |
| 568 | spin_lock_irq(&sl->lock); | 749 | spin_lock_irq(&sl->lock); |
| 569 | } | 750 | } |
| 570 | 751 | ||
| 752 | /** | ||
| 753 | * read_sequnlock_excl_irq() - end an interrupts-disabled seqlock_t | ||
| 754 | * locking reader section | ||
| 755 | * @sl: Pointer to seqlock_t | ||
| 756 | */ | ||
| 571 | static inline void read_sequnlock_excl_irq(seqlock_t *sl) | 757 | static inline void read_sequnlock_excl_irq(seqlock_t *sl) |
| 572 | { | 758 | { |
| 573 | spin_unlock_irq(&sl->lock); | 759 | spin_unlock_irq(&sl->lock); |
| @@ -581,9 +767,26 @@ static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl) · 7 unmodified lines | |||
| 581 | return flags; | 767 | return flags; |
| 582 | } | 768 | } |
| 583 | 769 | ||
| 770 | /** | ||
| 771 | * read_seqlock_excl_irqsave() - start a non-interruptible seqlock_t | ||
| 772 | * locking reader section | ||
| 773 | * @lock: Pointer to seqlock_t | ||
| 774 | * @flags: Stack-allocated storage for saving caller's local interrupt | ||
| 775 | * state, to be passed to read_sequnlock_excl_irqrestore(). | ||
| 776 | * | ||
| 777 | * _irqsave variant of read_seqlock_excl(). Use this only if the seqlock_t | ||
| 778 | * write side section, *or other read sections*, can be invoked from a | ||
| 779 | * hardirq context. | ||
| 780 | */ | ||
| 584 | #define read_seqlock_excl_irqsave(lock, flags) \ | 781 | #define read_seqlock_excl_irqsave(lock, flags) \ |
| 585 | do { flags = __read_seqlock_excl_irqsave(lock); } while (0) | 782 | do { flags = __read_seqlock_excl_irqsave(lock); } while (0) |
| 586 | 783 | ||
| 784 | /** | ||
| 785 | * read_sequnlock_excl_irqrestore() - end non-interruptible seqlock_t | ||
| 786 | * locking reader section | ||
| 787 | * @sl: Pointer to seqlock_t | ||
| 788 | * @flags: Caller saved interrupt state, from read_seqlock_excl_irqsave() | ||
| 789 | */ | ||
| 587 | static inline void | 790 | static inline void |
| 588 | read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) | 791 | read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) |
| 589 | { | 792 | { |
| @@ -591,14 +794,35 @@ read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) · 1 unmodified line | |||
| 591 | } | 794 | } |
| 592 | 795 | ||
| 593 | /** | 796 | /** |
| 594 | * read_seqbegin_or_lock - begin a sequence number check or locking block | 797 | * read_seqbegin_or_lock() - begin a seqlock_t lockless or locking reader |
| 595 | * @lock: sequence lock | 798 | * @lock: Pointer to seqlock_t |
| 596 | * @seq : sequence number to be checked | 799 | * @seq : Marker and return parameter. If the passed value is even, the |
| 597 | * | 800 | * reader will become a *lockless* seqlock_t reader as in read_seqbegin(). |
| 598 | * First try it once optimistically without taking the lock. If that fails, | 801 | * If the passed value is odd, the reader will become a *locking* reader |
| 599 | * take the lock. The sequence number is also used as a marker for deciding | 802 | * as in read_seqlock_excl(). In the first call to this function, the |
| 600 | * whether to be a reader (even) or writer (odd). | 803 | * caller *must* initialize and pass an even value to @seq; this way, a |
| 601 | * N.B. seq must be initialized to an even number to begin with. | 804 | * lockless read can be optimistically tried first. |
| 805 | * | ||
| 806 | * read_seqbegin_or_lock is an API designed to optimistically try a normal | ||
| 807 | * lockless seqlock_t read section first. If an odd counter is found, the | ||
| 808 | * lockless read trial has failed, and the next read iteration transforms | ||
| 809 | * itself into a full seqlock_t locking reader. | ||
| 810 | * | ||
| 811 | * This is typically used to avoid seqlock_t lockless readers starvation | ||
| 812 | * (too much retry loops) in the case of a sharp spike in write side | ||
| 813 | * activity. | ||
| 814 | * | ||
| 815 | * Context: if the seqlock_t write section, *or other read sections*, can | ||
| 816 | * be invoked from hardirq or softirq contexts, use the _irqsave or _bh | ||
| 817 | * variant of this function instead. | ||
| 818 | * | ||
| 819 | * Check Documentation/locking/seqlock.rst for template example code. | ||
| 820 | * | ||
| 821 | * Return: the encountered sequence counter value, through the @seq | ||
| 822 | * parameter, which is overloaded as a return parameter. This returned | ||
| 823 | * value must be checked with need_seqretry(). If the read section need to | ||
| 824 | * be retried, this returned value must also be passed as the @seq | ||
| 825 | * parameter of the next read_seqbegin_or_lock() iteration. | ||
| 602 | */ | 826 | */ |
| 603 | static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) | 827 | static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) |
| 604 | { | 828 | { |
| @@ -608,17 +832,52 @@ static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) · 3 unmodified lines | |||
| 608 | read_seqlock_excl(lock); | 832 | read_seqlock_excl(lock); |
| 609 | } | 833 | } |
| 610 | 834 | ||
| 835 | /** | ||
| 836 | * need_seqretry() - validate seqlock_t "locking or lockless" read section | ||
| 837 | * @lock: Pointer to seqlock_t | ||
| 838 | * @seq: sequence count, from read_seqbegin_or_lock() | ||
| 839 | * | ||
| 840 | * Return: true if a read section retry is required, false otherwise | ||
| 841 | */ | ||
| 611 | static inline int need_seqretry(seqlock_t *lock, int seq) | 842 | static inline int need_seqretry(seqlock_t *lock, int seq) |
| 612 | { | 843 | { |
| 613 | return !(seq & 1) && read_seqretry(lock, seq); | 844 | return !(seq & 1) && read_seqretry(lock, seq); |
| 614 | } | 845 | } |
| 615 | 846 | ||
| 847 | /** | ||
| 848 | * done_seqretry() - end seqlock_t "locking or lockless" reader section | ||
| 849 | * @lock: Pointer to seqlock_t | ||
| 850 | * @seq: count, from read_seqbegin_or_lock() | ||
| 851 | * | ||
| 852 | * done_seqretry finishes the seqlock_t read side critical section started | ||
| 853 | * with read_seqbegin_or_lock() and validated by need_seqretry(). | ||
| 854 | */ | ||
| 616 | static inline void done_seqretry(seqlock_t *lock, int seq) | 855 | static inline void done_seqretry(seqlock_t *lock, int seq) |
| 617 | { | 856 | { |
| 618 | if (seq & 1) | 857 | if (seq & 1) |
| 619 | read_sequnlock_excl(lock); | 858 | read_sequnlock_excl(lock); |
| 620 | } | 859 | } |
| 621 | 860 | ||
| 861 | /** | ||
| 862 | * read_seqbegin_or_lock_irqsave() - begin a seqlock_t lockless reader, or | ||
| 863 | * a non-interruptible locking reader | ||
| 864 | * @lock: Pointer to seqlock_t | ||
| 865 | * @seq: Marker and return parameter. Check read_seqbegin_or_lock(). | ||
| 866 | * | ||
| 867 | * This is the _irqsave variant of read_seqbegin_or_lock(). Use it only if | ||
| 868 | * the seqlock_t write section, *or other read sections*, can be invoked | ||
| 869 | * from hardirq context. | ||
| 870 | * | ||
| 871 | * Note: Interrupts will be disabled only for "locking reader" mode. | ||
| 872 | * | ||
| 873 | * Return: | ||
| 874 | * | ||
| 875 | * 1. The saved local interrupts state in case of a locking reader, to | ||
| 876 | * be passed to done_seqretry_irqrestore(). | ||
| 877 | * | ||
| 878 | * 2. The encountered sequence counter value, returned through @seq | ||
| 879 | * overloaded as a return parameter. Check read_seqbegin_or_lock(). | ||
| 880 | */ | ||
| 622 | static inline unsigned long | 881 | static inline unsigned long |
| 623 | read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) | 882 | read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) |
| 624 | { | 883 | { |
| @@ -632,6 +891,18 @@ read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) · 7 unmodified lines | |||
| 632 | return flags; | 891 | return flags; |
| 633 | } | 892 | } |
| 634 | 893 | ||
| 894 | /** | ||
| 895 | * done_seqretry_irqrestore() - end a seqlock_t lockless reader, or a | ||
| 896 | * non-interruptible locking reader section | ||
| 897 | * @lock: Pointer to seqlock_t | ||
| 898 | * @seq: Count, from read_seqbegin_or_lock_irqsave() | ||
| 899 | * @flags: Caller's saved local interrupt state in case of a locking | ||
| 900 | * reader, also from read_seqbegin_or_lock_irqsave() | ||
| 901 | * | ||
| 902 | * This is the _irqrestore variant of done_seqretry(). The read section | ||
| 903 | * must've been opened with read_seqbegin_or_lock_irqsave(), and validated | ||
| 904 | * by need_seqretry(). | ||
| 905 | */ | ||
| 635 | static inline void | 906 | static inline void |
| 636 | done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags) | 907 | done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags) |
| 637 | { | 908 | { |