History

locking/rwsem: Support optimistic spinning

We have reached the point where our mutexes are quite fine tuned for a number of situations. This includes the use of heuristics and optimistic spinning, based on MCS locking techniques. Exclusive ownership of read-write semaphores are, conceptually, just about the same as mutexes, making them close cousins. To this end we need to make them both perform similarly, and right now, rwsems are simply not up to it. This was discovered by both reverting commit 4fc3f1d6 (mm/rmap, migration: Make rmap_walk_anon() and try_to_unmap_anon() more scalable) and similarly, converting some other mutexes (ie: i_mmap_mutex) to rwsems. This creates a situation where users have to choose between a rwsem and mutex taking into account this important performance difference. Specifically, biggest difference between both locks is when we fail to acquire a mutex in the fastpath, optimistic spinning comes in to play and we can avoid a large amount of unnecessary sleeping and overhead of moving tasks in and out of wait queue. Rwsems do not have such logic. This patch, based on the work from Tim Chen and I, adds support for write-side optimistic spinning when the lock is contended. It also includes support for the recently added cancelable MCS locking for adaptive spinning. Note that is is only applicable to the xadd method, and the spinlock rwsem variant remains intact. Allowing optimistic spinning before putting the writer on the wait queue reduces wait queue contention and provided greater chance for the rwsem to get acquired. With these changes, rwsem is on par with mutex. The performance benefits can be seen on a number of workloads. For instance, on a 8 socket, 80 core 64bit Westmere box, aim7 shows the following improvements in throughput: +--------------+---------------------+-----------------+ | Workload | throughput-increase | number of users | +--------------+---------------------+-----------------+ | alltests | 20% | >1000 | | custom | 27%, 60% | 10-100, >1000 | | high_systime | 36%, 30% | >100, >1000 | | shared | 58%, 29% | 10-100, >1000 | +--------------+---------------------+-----------------+ There was also improvement on smaller systems, such as a quad-core x86-64 laptop running a 30Gb PostgreSQL (pgbench) workload for up to +60% in throughput for over 50 clients. Additionally, benefits were also noticed in exim (mail server) workloads. Furthermore, no performance regression have been seen at all.

Davidlohr Bueso committed 12 years ago

Commit 4fc828e1 parent 3cf2f343 files · +248 −33Browse files

Unified · Split

3 files
  1. include/linux/rwsem.h +22 −3
  2. kernel/locking/rwsem-xadd.c +196 −29
  3. kernel/locking/rwsem.c +30 −1
include/linux/rwsem.h+22−3
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616
1717 #include <linux/atomic.h>
1818
19+struct optimistic_spin_queue;
1920 struct rw_semaphore;
2021
2122 #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
@@ -23,9 +24,17 @@ struct rw_semaphore; · 1 unmodified line
2324 #else
2425 /* All arch specific implementations share the same struct */
2526 struct rw_semaphore {
26− long count;
27− raw_spinlock_t wait_lock;
28− struct list_head wait_list;
27+ long count;
28+ raw_spinlock_t wait_lock;
29+ struct list_head wait_list;
30+#ifdef CONFIG_SMP
31+ /*
32+ * Write owner. Used as a speculative check to see
33+ * if the owner is running on the cpu.
34+ */
35+ struct task_struct *owner;
36+ struct optimistic_spin_queue *osq; /* spinner MCS lock */
37+#endif
2938 #ifdef CONFIG_DEBUG_LOCK_ALLOC
3039 struct lockdep_map dep_map;
3140 #endif
@@ -55,11 +64,21 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) · 23 unmodified lines
5564 # define __RWSEM_DEP_MAP_INIT(lockname)
5665 #endif
5766
67+#ifdef CONFIG_SMP
68+#define __RWSEM_INITIALIZER(name) \
69+ { RWSEM_UNLOCKED_VALUE, \
70+ __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \
71+ LIST_HEAD_INIT((name).wait_list), \
72+ NULL, /* owner */ \
73+ NULL /* mcs lock */ \
74+ __RWSEM_DEP_MAP_INIT(name) }
75+#else
5876 #define __RWSEM_INITIALIZER(name) \
5977 { RWSEM_UNLOCKED_VALUE, \
6078 __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \
6179 LIST_HEAD_INIT((name).wait_list) \
6280 __RWSEM_DEP_MAP_INIT(name) }
81+#endif
6382
6483 #define DECLARE_RWSEM(name) \
6584 struct rw_semaphore name = __RWSEM_INITIALIZER(name)
kernel/locking/rwsem-xadd.c+196−29
@@ -5,11 +5,17 @@ · 4 unmodified lines
55 *
66 * Writer lock-stealing by Alex Shi <alex.shi@intel.com>
77 * and Michel Lespinasse <walken@google.com>
8+ *
9+ * Optimistic spinning by Tim Chen <tim.c.chen@intel.com>
10+ * and Davidlohr Bueso <davidlohr@hp.com>. Based on mutexes.
811 */
912 #include <linux/rwsem.h>
1013 #include <linux/sched.h>
1114 #include <linux/init.h>
1215 #include <linux/export.h>
16+#include <linux/sched/rt.h>
17+
18+#include "mcs_spinlock.h"
1319
1420 /*
1521 * Guide to the rw_semaphore's count field for common values.
@@ -76,6 +82,10 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name, · 60 unmodified lines
7682 sem->count = RWSEM_UNLOCKED_VALUE;
7783 raw_spin_lock_init(&sem->wait_lock);
7884 INIT_LIST_HEAD(&sem->wait_list);
85+#ifdef CONFIG_SMP
86+ sem->owner = NULL;
87+ sem->osq = NULL;
88+#endif
7989 }
8090
8191 EXPORT_SYMBOL(__init_rwsem);
@@ -190,7 +200,7 @@ __rwsem_do_wake(struct rw_semaphore *sem, enum rwsem_wake_type wake_type) · 108 unmodified lines
190200 }
191201
192202 /*
193− * wait for the read lock to be granted
203+ * Wait for the read lock to be granted
194204 */
195205 __visible
196206 struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
@@ -237,64 +247,221 @@ struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem) · 40 unmodified lines
237247 return sem;
238248 }
239249
250+static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)
251+{
252+ if (!(count & RWSEM_ACTIVE_MASK)) {
253+ /* try acquiring the write lock */
254+ if (sem->count == RWSEM_WAITING_BIAS &&
255+ cmpxchg(&sem->count, RWSEM_WAITING_BIAS,
256+ RWSEM_ACTIVE_WRITE_BIAS) == RWSEM_WAITING_BIAS) {
257+ if (!list_is_singular(&sem->wait_list))
258+ rwsem_atomic_update(RWSEM_WAITING_BIAS, sem);
259+ return true;
260+ }
261+ }
262+ return false;
263+}
264+
265+#ifdef CONFIG_SMP
240266 /*
241− * wait until we successfully acquire the write lock
267+ * Try to acquire write lock before the writer has been put on wait queue.
268+ */
269+static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)
270+{
271+ long old, count = ACCESS_ONCE(sem->count);
272+
273+ while (true) {
274+ if (!(count == 0 || count == RWSEM_WAITING_BIAS))
275+ return false;
276+
277+ old = cmpxchg(&sem->count, count, count + RWSEM_ACTIVE_WRITE_BIAS);
278+ if (old == count)
279+ return true;
280+
281+ count = old;
282+ }
283+}
284+
285+static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
286+{
287+ struct task_struct *owner;
288+ bool on_cpu = true;
289+
290+ if (need_resched())
291+ return 0;
292+
293+ rcu_read_lock();
294+ owner = ACCESS_ONCE(sem->owner);
295+ if (owner)
296+ on_cpu = owner->on_cpu;
297+ rcu_read_unlock();
298+
299+ /*
300+ * If sem->owner is not set, the rwsem owner may have
301+ * just acquired it and not set the owner yet or the rwsem
302+ * has been released.
303+ */
304+ return on_cpu;
305+}
306+
307+static inline bool owner_running(struct rw_semaphore *sem,
308+ struct task_struct *owner)
309+{
310+ if (sem->owner != owner)
311+ return false;
312+
313+ /*
314+ * Ensure we emit the owner->on_cpu, dereference _after_ checking
315+ * sem->owner still matches owner, if that fails, owner might
316+ * point to free()d memory, if it still matches, the rcu_read_lock()
317+ * ensures the memory stays valid.
318+ */
319+ barrier();
320+
321+ return owner->on_cpu;
322+}
323+
324+static noinline
325+bool rwsem_spin_on_owner(struct rw_semaphore *sem, struct task_struct *owner)
326+{
327+ rcu_read_lock();
328+ while (owner_running(sem, owner)) {
329+ if (need_resched())
330+ break;
331+
332+ arch_mutex_cpu_relax();
333+ }
334+ rcu_read_unlock();
335+
336+ /*
337+ * We break out the loop above on need_resched() or when the
338+ * owner changed, which is a sign for heavy contention. Return
339+ * success only when sem->owner is NULL.
340+ */
341+ return sem->owner == NULL;
342+}
343+
344+static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
345+{
346+ struct task_struct *owner;
347+ bool taken = false;
348+
349+ preempt_disable();
350+
351+ /* sem->wait_lock should not be held when doing optimistic spinning */
352+ if (!rwsem_can_spin_on_owner(sem))
353+ goto done;
354+
355+ if (!osq_lock(&sem->osq))
356+ goto done;
357+
358+ while (true) {
359+ owner = ACCESS_ONCE(sem->owner);
360+ if (owner && !rwsem_spin_on_owner(sem, owner))
361+ break;
362+
363+ /* wait_lock will be acquired if write_lock is obtained */
364+ if (rwsem_try_write_lock_unqueued(sem)) {
365+ taken = true;
366+ break;
367+ }
368+
369+ /*
370+ * When there's no owner, we might have preempted between the
371+ * owner acquiring the lock and setting the owner field. If
372+ * we're an RT task that will live-lock because we won't let
373+ * the owner complete.
374+ */
375+ if (!owner && (need_resched() || rt_task(current)))
376+ break;
377+
378+ /*
379+ * The cpu_relax() call is a compiler barrier which forces
380+ * everything in this loop to be re-loaded. We don't need
381+ * memory barriers as we'll eventually observe the right
382+ * values at the cost of a few extra spins.
383+ */
384+ arch_mutex_cpu_relax();
385+ }
386+ osq_unlock(&sem->osq);
387+done:
388+ preempt_enable();
389+ return taken;
390+}
391+
392+#else
393+static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
394+{
395+ return false;
396+}
397+#endif
398+
399+/*
400+ * Wait until we successfully acquire the write lock
242401 */
243402 __visible
244403 struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
245404 {
246− long count, adjustment = -RWSEM_ACTIVE_WRITE_BIAS;
405+ long count;
406+ bool waiting = true; /* any queued threads before us */
247407 struct rwsem_waiter waiter;
248− struct task_struct *tsk = current;
249408
250− /* set up my own style of waitqueue */
251− waiter.task = tsk;
409+ /* undo write bias from down_write operation, stop active locking */
410+ count = rwsem_atomic_update(-RWSEM_ACTIVE_WRITE_BIAS, sem);
411+
412+ /* do optimistic spinning and steal lock if possible */
413+ if (rwsem_optimistic_spin(sem))
414+ return sem;
415+
416+ /*
417+ * Optimistic spinning failed, proceed to the slowpath
418+ * and block until we can acquire the sem.
419+ */
420+ waiter.task = current;
252421 waiter.type = RWSEM_WAITING_FOR_WRITE;
253422
254423 raw_spin_lock_irq(&sem->wait_lock);
424+
425+ /* account for this before adding a new element to the list */
255426 if (list_empty(&sem->wait_list))
256− adjustment += RWSEM_WAITING_BIAS;
427+ waiting = false;
428+
257429 list_add_tail(&waiter.list, &sem->wait_list);
258430
259431 /* we're now waiting on the lock, but no longer actively locking */
260− count = rwsem_atomic_update(adjustment, sem);
432+ if (waiting) {
433+ count = ACCESS_ONCE(sem->count);
261434
262− /* If there were already threads queued before us and there are no
263− * active writers, the lock must be read owned; so we try to wake
264− * any read locks that were queued ahead of us. */
265− if (count > RWSEM_WAITING_BIAS &&
266− adjustment == -RWSEM_ACTIVE_WRITE_BIAS)
267− sem = __rwsem_do_wake(sem, RWSEM_WAKE_READERS);
435+ /*
436+ * If there were already threads queued before us and there are no
437+ * active writers, the lock must be read owned; so we try to wake
438+ * any read locks that were queued ahead of us.
439+ */
440+ if (count > RWSEM_WAITING_BIAS)
441+ sem = __rwsem_do_wake(sem, RWSEM_WAKE_READERS);
442+
443+ } else
444+ count = rwsem_atomic_update(RWSEM_WAITING_BIAS, sem);
268445
269446 /* wait until we successfully acquire the lock */
270− set_task_state(tsk, TASK_UNINTERRUPTIBLE);
447+ set_current_state(TASK_UNINTERRUPTIBLE);
271448 while (true) {
272− if (!(count & RWSEM_ACTIVE_MASK)) {
273− /* Try acquiring the write lock. */
274− count = RWSEM_ACTIVE_WRITE_BIAS;
275− if (!list_is_singular(&sem->wait_list))
276− count += RWSEM_WAITING_BIAS;
277−
278− if (sem->count == RWSEM_WAITING_BIAS &&
279− cmpxchg(&sem->count, RWSEM_WAITING_BIAS, count) ==
280− RWSEM_WAITING_BIAS)
281− break;
282− }
283−
449+ if (rwsem_try_write_lock(count, sem))
450+ break;
284451 raw_spin_unlock_irq(&sem->wait_lock);
285452
286453 /* Block until there are no active lockers. */
287454 do {
288455 schedule();
289− set_task_state(tsk, TASK_UNINTERRUPTIBLE);
456+ set_current_state(TASK_UNINTERRUPTIBLE);
290457 } while ((count = sem->count) & RWSEM_ACTIVE_MASK);
291458
292459 raw_spin_lock_irq(&sem->wait_lock);
293460 }
461+ __set_current_state(TASK_RUNNING);
294462
295463 list_del(&waiter.list);
296464 raw_spin_unlock_irq(&sem->wait_lock);
297− tsk->state = TASK_RUNNING;
298465
299466 return sem;
300467 }
kernel/locking/rwsem.c+30−1
@@ -12,6 +12,27 @@ · 11 unmodified lines
1212
1313 #include <linux/atomic.h>
1414
15+#if defined(CONFIG_SMP) && defined(CONFIG_RWSEM_XCHGADD_ALGORITHM)
16+static inline void rwsem_set_owner(struct rw_semaphore *sem)
17+{
18+ sem->owner = current;
19+}
20+
21+static inline void rwsem_clear_owner(struct rw_semaphore *sem)
22+{
23+ sem->owner = NULL;
24+}
25+
26+#else
27+static inline void rwsem_set_owner(struct rw_semaphore *sem)
28+{
29+}
30+
31+static inline void rwsem_clear_owner(struct rw_semaphore *sem)
32+{
33+}
34+#endif
35+
1536 /*
1637 * lock for reading
1738 */
@@ -48,6 +69,7 @@ void __sched down_write(struct rw_semaphore *sem) · 30 unmodified lines
4869 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
4970
5071 LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
72+ rwsem_set_owner(sem);
5173 }
5274
5375 EXPORT_SYMBOL(down_write);
@@ -59,8 +81,11 @@ int down_write_trylock(struct rw_semaphore *sem) · 5 unmodified lines
5981 {
6082 int ret = __down_write_trylock(sem);
6183
62− if (ret == 1)
84+ if (ret == 1) {
6385 rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
86+ rwsem_set_owner(sem);
87+ }
88+
6489 return ret;
6590 }
6691
@@ -85,6 +110,7 @@ void up_write(struct rw_semaphore *sem) · 18 unmodified lines
85110 {
86111 rwsem_release(&sem->dep_map, 1, _RET_IP_);
87112
113+ rwsem_clear_owner(sem);
88114 __up_write(sem);
89115 }
90116
@@ -99,6 +125,7 @@ void downgrade_write(struct rw_semaphore *sem) · 8 unmodified lines
99125 * lockdep: a downgraded write will live on as a write
100126 * dependency.
101127 */
128+ rwsem_clear_owner(sem);
102129 __downgrade_write(sem);
103130 }
104131
@@ -122,6 +149,7 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest) · 17 unmodified lines
122149 rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
123150
124151 LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
152+ rwsem_set_owner(sem);
125153 }
126154
127155 EXPORT_SYMBOL(_down_write_nest_lock);
@@ -141,6 +169,7 @@ void down_write_nested(struct rw_semaphore *sem, int subclass) · 13 unmodified lines
141169 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
142170
143171 LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
172+ rwsem_set_owner(sem);
144173 }
145174
146175 EXPORT_SYMBOL(down_write_nested);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t