The code in rwbase_write_lock() is a little non-obvious vs the read+set 'trylock', extract the sequence into a helper function to clarify the code. This also provides a single site to fix fast-path ordering.
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -196,6 +196,19 @@ static inline void rwbase_write_downgrade(struct rwbase_rt *rwb) · 195 unmodified lines | ||
| 196 | 196 | __rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags); |
| 197 | 197 | } |
| 198 | 198 | |
| 199 | +static inline bool __rwbase_write_trylock(struct rwbase_rt *rwb) | |
| 200 | +{ | |
| 201 | + /* Can do without CAS because we're serialized by wait_lock. */ | |
| 202 | + lockdep_assert_held(&rwb->rtmutex.wait_lock); | |
| 203 | + | |
| 204 | + if (!atomic_read(&rwb->readers)) { | |
| 205 | + atomic_set(&rwb->readers, WRITER_BIAS); | |
| 206 | + return 1; | |
| 207 | + } | |
| 208 | + | |
| 209 | + return 0; | |
| 210 | +} | |
| 211 | + | |
| 199 | 212 | static int __sched rwbase_write_lock(struct rwbase_rt *rwb, |
| 200 | 213 | unsigned int state) |
| 201 | 214 | { |
| @@ -210,34 +223,30 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb, · 8 unmodified lines | ||
| 210 | 223 | atomic_sub(READER_BIAS, &rwb->readers); |
| 211 | 224 | |
| 212 | 225 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); |
| 213 | − /* | |
| 214 | − * set_current_state() for rw_semaphore | |
| 215 | − * current_save_and_set_rtlock_wait_state() for rwlock | |
| 216 | − */ | |
| 217 | − rwbase_set_and_save_current_state(state); | |
| 226 | + if (__rwbase_write_trylock(rwb)) | |
| 227 | + goto out_unlock; | |
| 218 | 228 | |
| 219 | − /* Block until all readers have left the critical section. */ | |
| 220 | − for (; atomic_read(&rwb->readers);) { | |
| 229 | + rwbase_set_and_save_current_state(state); | |
| 230 | + for (;;) { | |
| 221 | 231 | /* Optimized out for rwlocks */ |
| 222 | 232 | if (rwbase_signal_pending_state(state, current)) { |
| 223 | 233 | rwbase_restore_current_state(); |
| 224 | 234 | __rwbase_write_unlock(rwb, 0, flags); |
| 225 | 235 | return -EINTR; |
| 226 | 236 | } |
| 237 | + | |
| 238 | + if (__rwbase_write_trylock(rwb)) | |
| 239 | + break; | |
| 240 | + | |
| 227 | 241 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); |
| 242 | + rwbase_schedule(); | |
| 243 | + raw_spin_lock_irqsave(&rtm->wait_lock, flags); | |
| 228 | 244 | |
| 229 | − /* | |
| 230 | − * Schedule and wait for the readers to leave the critical | |
| 231 | − * section. The last reader leaving it wakes the waiter. | |
| 232 | − */ | |
| 233 | − if (atomic_read(&rwb->readers) != 0) | |
| 234 | − rwbase_schedule(); | |
| 235 | 245 | set_current_state(state); |
| 236 | − raw_spin_lock_irqsave(&rtm->wait_lock, flags); | |
| 237 | 246 | } |
| 238 | − | |
| 239 | − atomic_set(&rwb->readers, WRITER_BIAS); | |
| 240 | 247 | rwbase_restore_current_state(); |
| 248 | + | |
| 249 | +out_unlock: | |
| 241 | 250 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); |
| 242 | 251 | return 0; |
| 243 | 252 | } |
| @@ -253,8 +262,7 @@ static inline int rwbase_write_trylock(struct rwbase_rt *rwb) · 9 unmodified lines | ||
| 253 | 262 | atomic_sub(READER_BIAS, &rwb->readers); |
| 254 | 263 | |
| 255 | 264 | raw_spin_lock_irqsave(&rtm->wait_lock, flags); |
| 256 | − if (!atomic_read(&rwb->readers)) { | |
| 257 | − atomic_set(&rwb->readers, WRITER_BIAS); | |
| 265 | + if (__rwbase_write_trylock(rwb)) { | |
| 258 | 266 | raw_spin_unlock_irqrestore(&rtm->wait_lock, flags); |
| 259 | 267 | return 1; |
| 260 | 268 | } |