The code in rwbase_write_lock() is a little non-obvious vs the read+set 'trylock', extract the sequence into a helper function to clarify the code. This also provides a single site to fix fast-path ordering.
Showing the changes to kernel/locking (1 of 1 file). Show all
Unified · Split
| @@ -196,6 +196,19 @@ static inline void rwbase_write_downgrade(struct rwbase_rt *rwb) · 195 unmodified lines | |||
| 196 | __rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags); | 196 | __rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags); |
| 197 | } | 197 | } |
| 198 | 198 | ||
| 199 | static inline bool __rwbase_write_trylock(struct rwbase_rt *rwb) | ||
| 200 | { | ||
| 201 | /* Can do without CAS because we're serialized by wait_lock. */ | ||
| 202 | lockdep_assert_held(&rwb->rtmutex.wait_lock); | ||
| 203 | |||
| 204 | if (!atomic_read(&rwb->readers)) { | ||
| 205 | atomic_set(&rwb->readers, WRITER_BIAS); | ||
| 206 | return 1; | ||
| 207 | } | ||
| 208 | |||
| 209 | return 0; | ||
| 210 | } | ||
| 211 | |||
| 199 | static int __sched rwbase_write_lock(struct rwbase_rt *rwb, | 212 | static int __sched rwbase_write_lock(struct rwbase_rt *rwb, |
| 200 | unsigned int state) | 213 | unsigned int state) |
| 201 | { | 214 | { |
| @@ -210,34 +223,30 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb, · 8 unmodified lines | |||
| 210 | atomic_sub(READER_BIAS, &rwb->readers); | 223 | atomic_sub(READER_BIAS, &rwb->readers); |
| 211 | 224 | ||
| 212 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); | 225 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); |
| 213 | /* | 226 | if (__rwbase_write_trylock(rwb)) |
| 214 | * set_current_state() for rw_semaphore | 227 | goto out_unlock; |
| 215 | * current_save_and_set_rtlock_wait_state() for rwlock | ||
| 216 | */ | ||
| 217 | rwbase_set_and_save_current_state(state); | ||
| 218 | 228 | ||
| 219 | /* Block until all readers have left the critical section. */ | 229 | rwbase_set_and_save_current_state(state); |
| 220 | for (; atomic_read(&rwb->readers);) { | 230 | for (;;) { |
| 221 | /* Optimized out for rwlocks */ | 231 | /* Optimized out for rwlocks */ |
| 222 | if (rwbase_signal_pending_state(state, current)) { | 232 | if (rwbase_signal_pending_state(state, current)) { |
| 223 | rwbase_restore_current_state(); | 233 | rwbase_restore_current_state(); |
| 224 | __rwbase_write_unlock(rwb, 0, flags); | 234 | __rwbase_write_unlock(rwb, 0, flags); |
| 225 | return -EINTR; | 235 | return -EINTR; |
| 226 | } | 236 | } |
| 237 | |||
| 238 | if (__rwbase_write_trylock(rwb)) | ||
| 239 | break; | ||
| 240 | |||
| 227 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); | 241 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); |
| 242 | rwbase_schedule(); | ||
| 243 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); | ||
| 228 | 244 | ||
| 229 | /* | ||
| 230 | * Schedule and wait for the readers to leave the critical | ||
| 231 | * section. The last reader leaving it wakes the waiter. | ||
| 232 | */ | ||
| 233 | if (atomic_read(&rwb->readers) != 0) | ||
| 234 | rwbase_schedule(); | ||
| 235 | set_current_state(state); | 245 | set_current_state(state); |
| 236 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); | ||
| 237 | } | 246 | } |
| 238 | |||
| 239 | atomic_set(&rwb->readers, WRITER_BIAS); | ||
| 240 | rwbase_restore_current_state(); | 247 | rwbase_restore_current_state(); |
| 248 | |||
| 249 | out_unlock: | ||
| 241 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); | 250 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); |
| 242 | return 0; | 251 | return 0; |
| 243 | } | 252 | } |
| @@ -253,8 +262,7 @@ static inline int rwbase_write_trylock(struct rwbase_rt *rwb) · 9 unmodified lines | |||
| 253 | atomic_sub(READER_BIAS, &rwb->readers); | 262 | atomic_sub(READER_BIAS, &rwb->readers); |
| 254 | 263 | ||
| 255 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); | 264 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); |
| 256 | if (!atomic_read(&rwb->readers)) { | 265 | if (__rwbase_write_trylock(rwb)) { |
| 257 | atomic_set(&rwb->readers, WRITER_BIAS); | ||
| 258 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); | 266 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); |
| 259 | return 1; | 267 | return 1; |
| 260 | } | 268 | } |