History

locking: Move the mutex code to kernel/locking/

Peter Zijlstra committed 13 years ago

Commit 01768b41 parent c90423d6 files · +11 −4Browse files

Showing the changes to kernel/locking (5 of 6 files). Show all

Unified · Split

5 files
  1. kernel/locking/Makefile +9 −0
  2. kernel/locking/mutex-debug.c +110 −0
  3. kernel/locking/mutex-debug.h +55 −0
  4. kernel/locking/mutex.c +960 −0
  5. kernel/locking/mutex.h +48 −0
kernel/locking/Makefile+9−0
@@ -0,0 +1,9 @@
1
2obj-y += mutex.o
3
4ifdef CONFIG_FUNCTION_TRACER
5CFLAGS_REMOVE_mutex-debug.o = -pg
6CFLAGS_REMOVE_rtmutex-debug.o = -pg
7endif
8
9obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o
kernel/locking/mutex-debug.c+110−0
@@ -0,0 +1,110 @@
1/*
2 * kernel/mutex-debug.c
3 *
4 * Debugging code for mutexes
5 *
6 * Started by Ingo Molnar:
7 *
8 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
9 *
10 * lock debugging, locking tree, deadlock detection started by:
11 *
12 * Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey
13 * Released under the General Public License (GPL).
14 */
15#include <linux/mutex.h>
16#include <linux/delay.h>
17#include <linux/export.h>
18#include <linux/poison.h>
19#include <linux/sched.h>
20#include <linux/spinlock.h>
21#include <linux/kallsyms.h>
22#include <linux/interrupt.h>
23#include <linux/debug_locks.h>
24
25#include "mutex-debug.h"
26
27/*
28 * Must be called with lock->wait_lock held.
29 */
30void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)
31{
32 memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter));
33 waiter->magic = waiter;
34 INIT_LIST_HEAD(&waiter->list);
35}
36
37void debug_mutex_wake_waiter(struct mutex *lock, struct mutex_waiter *waiter)
38{
39 SMP_DEBUG_LOCKS_WARN_ON(!spin_is_locked(&lock->wait_lock));
40 DEBUG_LOCKS_WARN_ON(list_empty(&lock->wait_list));
41 DEBUG_LOCKS_WARN_ON(waiter->magic != waiter);
42 DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
43}
44
45void debug_mutex_free_waiter(struct mutex_waiter *waiter)
46{
47 DEBUG_LOCKS_WARN_ON(!list_empty(&waiter->list));
48 memset(waiter, MUTEX_DEBUG_FREE, sizeof(*waiter));
49}
50
51void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
52 struct thread_info *ti)
53{
54 SMP_DEBUG_LOCKS_WARN_ON(!spin_is_locked(&lock->wait_lock));
55
56 /* Mark the current thread as blocked on the lock: */
57 ti->task->blocked_on = waiter;
58}
59
60void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
61 struct thread_info *ti)
62{
63 DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
64 DEBUG_LOCKS_WARN_ON(waiter->task != ti->task);
65 DEBUG_LOCKS_WARN_ON(ti->task->blocked_on != waiter);
66 ti->task->blocked_on = NULL;
67
68 list_del_init(&waiter->list);
69 waiter->task = NULL;
70}
71
72void debug_mutex_unlock(struct mutex *lock)
73{
74 if (unlikely(!debug_locks))
75 return;
76
77 DEBUG_LOCKS_WARN_ON(lock->magic != lock);
78 DEBUG_LOCKS_WARN_ON(lock->owner != current);
79 DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
80 mutex_clear_owner(lock);
81}
82
83void debug_mutex_init(struct mutex *lock, const char *name,
84 struct lock_class_key *key)
85{
86#ifdef CONFIG_DEBUG_LOCK_ALLOC
87 /*
88 * Make sure we are not reinitializing a held lock:
89 */
90 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
91 lockdep_init_map(&lock->dep_map, name, key, 0);
92#endif
93 lock->magic = lock;
94}
95
96/***
97 * mutex_destroy - mark a mutex unusable
98 * @lock: the mutex to be destroyed
99 *
100 * This function marks the mutex uninitialized, and any subsequent
101 * use of the mutex is forbidden. The mutex must not be locked when
102 * this function is called.
103 */
104void mutex_destroy(struct mutex *lock)
105{
106 DEBUG_LOCKS_WARN_ON(mutex_is_locked(lock));
107 lock->magic = NULL;
108}
109
110EXPORT_SYMBOL_GPL(mutex_destroy);
kernel/locking/mutex-debug.h+55−0
@@ -0,0 +1,55 @@
1/*
2 * Mutexes: blocking mutual exclusion locks
3 *
4 * started by Ingo Molnar:
5 *
6 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7 *
8 * This file contains mutex debugging related internal declarations,
9 * prototypes and inline functions, for the CONFIG_DEBUG_MUTEXES case.
10 * More details are in kernel/mutex-debug.c.
11 */
12
13/*
14 * This must be called with lock->wait_lock held.
15 */
16extern void debug_mutex_lock_common(struct mutex *lock,
17 struct mutex_waiter *waiter);
18extern void debug_mutex_wake_waiter(struct mutex *lock,
19 struct mutex_waiter *waiter);
20extern void debug_mutex_free_waiter(struct mutex_waiter *waiter);
21extern void debug_mutex_add_waiter(struct mutex *lock,
22 struct mutex_waiter *waiter,
23 struct thread_info *ti);
24extern void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
25 struct thread_info *ti);
26extern void debug_mutex_unlock(struct mutex *lock);
27extern void debug_mutex_init(struct mutex *lock, const char *name,
28 struct lock_class_key *key);
29
30static inline void mutex_set_owner(struct mutex *lock)
31{
32 lock->owner = current;
33}
34
35static inline void mutex_clear_owner(struct mutex *lock)
36{
37 lock->owner = NULL;
38}
39
40#define spin_lock_mutex(lock, flags) \
41 do { \
42 struct mutex *l = container_of(lock, struct mutex, wait_lock); \
43 \
44 DEBUG_LOCKS_WARN_ON(in_interrupt()); \
45 local_irq_save(flags); \
46 arch_spin_lock(&(lock)->rlock.raw_lock);\
47 DEBUG_LOCKS_WARN_ON(l->magic != l); \
48 } while (0)
49
50#define spin_unlock_mutex(lock, flags) \
51 do { \
52 arch_spin_unlock(&(lock)->rlock.raw_lock); \
53 local_irq_restore(flags); \
54 preempt_check_resched(); \
55 } while (0)
kernel/locking/mutex.c+960−0
@@ -0,0 +1,960 @@
1/*
2 * kernel/mutex.c
3 *
4 * Mutexes: blocking mutual exclusion locks
5 *
6 * Started by Ingo Molnar:
7 *
8 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
9 *
10 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
11 * David Howells for suggestions and improvements.
12 *
13 * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
14 * from the -rt tree, where it was originally implemented for rtmutexes
15 * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
16 * and Sven Dietrich.
17 *
18 * Also see Documentation/mutex-design.txt.
19 */
20#include <linux/mutex.h>
21#include <linux/ww_mutex.h>
22#include <linux/sched.h>
23#include <linux/sched/rt.h>
24#include <linux/export.h>
25#include <linux/spinlock.h>
26#include <linux/interrupt.h>
27#include <linux/debug_locks.h>
28
29/*
30 * In the DEBUG case we are using the "NULL fastpath" for mutexes,
31 * which forces all calls into the slowpath:
32 */
33#ifdef CONFIG_DEBUG_MUTEXES
34# include "mutex-debug.h"
35# include <asm-generic/mutex-null.h>
36#else
37# include "mutex.h"
38# include <asm/mutex.h>
39#endif
40
41/*
42 * A negative mutex count indicates that waiters are sleeping waiting for the
43 * mutex.
44 */
45#define MUTEX_SHOW_NO_WAITER(mutex) (atomic_read(&(mutex)->count) >= 0)
46
47void
48__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
49{
50 atomic_set(&lock->count, 1);
51 spin_lock_init(&lock->wait_lock);
52 INIT_LIST_HEAD(&lock->wait_list);
53 mutex_clear_owner(lock);
54#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
55 lock->spin_mlock = NULL;
56#endif
57
58 debug_mutex_init(lock, name, key);
59}
60
61EXPORT_SYMBOL(__mutex_init);
62
63#ifndef CONFIG_DEBUG_LOCK_ALLOC
64/*
65 * We split the mutex lock/unlock logic into separate fastpath and
66 * slowpath functions, to reduce the register pressure on the fastpath.
67 * We also put the fastpath first in the kernel image, to make sure the
68 * branch is predicted by the CPU as default-untaken.
69 */
70static __used noinline void __sched
71__mutex_lock_slowpath(atomic_t *lock_count);
72
73/**
74 * mutex_lock - acquire the mutex
75 * @lock: the mutex to be acquired
76 *
77 * Lock the mutex exclusively for this task. If the mutex is not
78 * available right now, it will sleep until it can get it.
79 *
80 * The mutex must later on be released by the same task that
81 * acquired it. Recursive locking is not allowed. The task
82 * may not exit without first unlocking the mutex. Also, kernel
83 * memory where the mutex resides mutex must not be freed with
84 * the mutex still locked. The mutex must first be initialized
85 * (or statically defined) before it can be locked. memset()-ing
86 * the mutex to 0 is not allowed.
87 *
88 * ( The CONFIG_DEBUG_MUTEXES .config option turns on debugging
89 * checks that will enforce the restrictions and will also do
90 * deadlock debugging. )
91 *
92 * This function is similar to (but not equivalent to) down().
93 */
94void __sched mutex_lock(struct mutex *lock)
95{
96 might_sleep();
97 /*
98 * The locking fastpath is the 1->0 transition from
99 * 'unlocked' into 'locked' state.
100 */
101 __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
102 mutex_set_owner(lock);
103}
104
105EXPORT_SYMBOL(mutex_lock);
106#endif
107
108#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
109/*
110 * In order to avoid a stampede of mutex spinners from acquiring the mutex
111 * more or less simultaneously, the spinners need to acquire a MCS lock
112 * first before spinning on the owner field.
113 *
114 * We don't inline mspin_lock() so that perf can correctly account for the
115 * time spent in this lock function.
116 */
117struct mspin_node {
118 struct mspin_node *next ;
119 int locked; /* 1 if lock acquired */
120};
121#define MLOCK(mutex) ((struct mspin_node **)&((mutex)->spin_mlock))
122
123static noinline
124void mspin_lock(struct mspin_node **lock, struct mspin_node *node)
125{
126 struct mspin_node *prev;
127
128 /* Init node */
129 node->locked = 0;
130 node->next = NULL;
131
132 prev = xchg(lock, node);
133 if (likely(prev == NULL)) {
134 /* Lock acquired */
135 node->locked = 1;
136 return;
137 }
138 ACCESS_ONCE(prev->next) = node;
139 smp_wmb();
140 /* Wait until the lock holder passes the lock down */
141 while (!ACCESS_ONCE(node->locked))
142 arch_mutex_cpu_relax();
143}
144
145static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node)
146{
147 struct mspin_node *next = ACCESS_ONCE(node->next);
148
149 if (likely(!next)) {
150 /*
151 * Release the lock by setting it to NULL
152 */
153 if (cmpxchg(lock, node, NULL) == node)
154 return;
155 /* Wait until the next pointer is set */
156 while (!(next = ACCESS_ONCE(node->next)))
157 arch_mutex_cpu_relax();
158 }
159 ACCESS_ONCE(next->locked) = 1;
160 smp_wmb();
161}
162
163/*
164 * Mutex spinning code migrated from kernel/sched/core.c
165 */
166
167static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
168{
169 if (lock->owner != owner)
170 return false;
171
172 /*
173 * Ensure we emit the owner->on_cpu, dereference _after_ checking
174 * lock->owner still matches owner, if that fails, owner might
175 * point to free()d memory, if it still matches, the rcu_read_lock()
176 * ensures the memory stays valid.
177 */
178 barrier();
179
180 return owner->on_cpu;
181}
182
183/*
184 * Look out! "owner" is an entirely speculative pointer
185 * access and not reliable.
186 */
187static noinline
188int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
189{
190 rcu_read_lock();
191 while (owner_running(lock, owner)) {
192 if (need_resched())
193 break;
194
195 arch_mutex_cpu_relax();
196 }
197 rcu_read_unlock();
198
199 /*
200 * We break out the loop above on need_resched() and when the
201 * owner changed, which is a sign for heavy contention. Return
202 * success only when lock->owner is NULL.
203 */
204 return lock->owner == NULL;
205}
206
207/*
208 * Initial check for entering the mutex spinning loop
209 */
210static inline int mutex_can_spin_on_owner(struct mutex *lock)
211{
212 struct task_struct *owner;
213 int retval = 1;
214
215 rcu_read_lock();
216 owner = ACCESS_ONCE(lock->owner);
217 if (owner)
218 retval = owner->on_cpu;
219 rcu_read_unlock();
220 /*
221 * if lock->owner is not set, the mutex owner may have just acquired
222 * it and not set the owner yet or the mutex has been released.
223 */
224 return retval;
225}
226#endif
227
228static __used noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count);
229
230/**
231 * mutex_unlock - release the mutex
232 * @lock: the mutex to be released
233 *
234 * Unlock a mutex that has been locked by this task previously.
235 *
236 * This function must not be used in interrupt context. Unlocking
237 * of a not locked mutex is not allowed.
238 *
239 * This function is similar to (but not equivalent to) up().
240 */
241void __sched mutex_unlock(struct mutex *lock)
242{
243 /*
244 * The unlocking fastpath is the 0->1 transition from 'locked'
245 * into 'unlocked' state:
246 */
247#ifndef CONFIG_DEBUG_MUTEXES
248 /*
249 * When debugging is enabled we must not clear the owner before time,
250 * the slow path will always be taken, and that clears the owner field
251 * after verifying that it was indeed current.
252 */
253 mutex_clear_owner(lock);
254#endif
255 __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
256}
257
258EXPORT_SYMBOL(mutex_unlock);
259
260/**
261 * ww_mutex_unlock - release the w/w mutex
262 * @lock: the mutex to be released
263 *
264 * Unlock a mutex that has been locked by this task previously with any of the
265 * ww_mutex_lock* functions (with or without an acquire context). It is
266 * forbidden to release the locks after releasing the acquire context.
267 *
268 * This function must not be used in interrupt context. Unlocking
269 * of a unlocked mutex is not allowed.
270 */
271void __sched ww_mutex_unlock(struct ww_mutex *lock)
272{
273 /*
274 * The unlocking fastpath is the 0->1 transition from 'locked'
275 * into 'unlocked' state:
276 */
277 if (lock->ctx) {
278#ifdef CONFIG_DEBUG_MUTEXES
279 DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
280#endif
281 if (lock->ctx->acquired > 0)
282 lock->ctx->acquired--;
283 lock->ctx = NULL;
284 }
285
286#ifndef CONFIG_DEBUG_MUTEXES
287 /*
288 * When debugging is enabled we must not clear the owner before time,
289 * the slow path will always be taken, and that clears the owner field
290 * after verifying that it was indeed current.
291 */
292 mutex_clear_owner(&lock->base);
293#endif
294 __mutex_fastpath_unlock(&lock->base.count, __mutex_unlock_slowpath);
295}
296EXPORT_SYMBOL(ww_mutex_unlock);
297
298static inline int __sched
299__mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
300{
301 struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
302 struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
303
304 if (!hold_ctx)
305 return 0;
306
307 if (unlikely(ctx == hold_ctx))
308 return -EALREADY;
309
310 if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
311 (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
312#ifdef CONFIG_DEBUG_MUTEXES
313 DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
314 ctx->contending_lock = ww;
315#endif
316 return -EDEADLK;
317 }
318
319 return 0;
320}
321
322static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
323 struct ww_acquire_ctx *ww_ctx)
324{
325#ifdef CONFIG_DEBUG_MUTEXES
326 /*
327 * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
328 * but released with a normal mutex_unlock in this call.
329 *
330 * This should never happen, always use ww_mutex_unlock.
331 */
332 DEBUG_LOCKS_WARN_ON(ww->ctx);
333
334 /*
335 * Not quite done after calling ww_acquire_done() ?
336 */
337 DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
338
339 if (ww_ctx->contending_lock) {
340 /*
341 * After -EDEADLK you tried to
342 * acquire a different ww_mutex? Bad!
343 */
344 DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
345
346 /*
347 * You called ww_mutex_lock after receiving -EDEADLK,
348 * but 'forgot' to unlock everything else first?
349 */
350 DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
351 ww_ctx->contending_lock = NULL;
352 }
353
354 /*
355 * Naughty, using a different class will lead to undefined behavior!
356 */
357 DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
358#endif
359 ww_ctx->acquired++;
360}
361
362/*
363 * after acquiring lock with fastpath or when we lost out in contested
364 * slowpath, set ctx and wake up any waiters so they can recheck.
365 *
366 * This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
367 * as the fastpath and opportunistic spinning are disabled in that case.
368 */
369static __always_inline void
370ww_mutex_set_context_fastpath(struct ww_mutex *lock,
371 struct ww_acquire_ctx *ctx)
372{
373 unsigned long flags;
374 struct mutex_waiter *cur;
375
376 ww_mutex_lock_acquired(lock, ctx);
377
378 lock->ctx = ctx;
379
380 /*
381 * The lock->ctx update should be visible on all cores before
382 * the atomic read is done, otherwise contended waiters might be
383 * missed. The contended waiters will either see ww_ctx == NULL
384 * and keep spinning, or it will acquire wait_lock, add itself
385 * to waiter list and sleep.
386 */
387 smp_mb(); /* ^^^ */
388
389 /*
390 * Check if lock is contended, if not there is nobody to wake up
391 */
392 if (likely(atomic_read(&lock->base.count) == 0))
393 return;
394
395 /*
396 * Uh oh, we raced in fastpath, wake up everyone in this case,
397 * so they can see the new lock->ctx.
398 */
399 spin_lock_mutex(&lock->base.wait_lock, flags);
400 list_for_each_entry(cur, &lock->base.wait_list, list) {
401 debug_mutex_wake_waiter(&lock->base, cur);
402 wake_up_process(cur->task);
403 }
404 spin_unlock_mutex(&lock->base.wait_lock, flags);
405}
406
407/*
408 * Lock a mutex (possibly interruptible), slowpath:
409 */
410static __always_inline int __sched
411__mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
412 struct lockdep_map *nest_lock, unsigned long ip,
413 struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
414{
415 struct task_struct *task = current;
416 struct mutex_waiter waiter;
417 unsigned long flags;
418 int ret;
419
420 preempt_disable();
421 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
422
423#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
424 /*
425 * Optimistic spinning.
426 *
427 * We try to spin for acquisition when we find that there are no
428 * pending waiters and the lock owner is currently running on a
429 * (different) CPU.
430 *
431 * The rationale is that if the lock owner is running, it is likely to
432 * release the lock soon.
433 *
434 * Since this needs the lock owner, and this mutex implementation
435 * doesn't track the owner atomically in the lock field, we need to
436 * track it non-atomically.
437 *
438 * We can't do this for DEBUG_MUTEXES because that relies on wait_lock
439 * to serialize everything.
440 *
441 * The mutex spinners are queued up using MCS lock so that only one
442 * spinner can compete for the mutex. However, if mutex spinning isn't
443 * going to happen, there is no point in going through the lock/unlock
444 * overhead.
445 */
446 if (!mutex_can_spin_on_owner(lock))
447 goto slowpath;
448
449 for (;;) {
450 struct task_struct *owner;
451 struct mspin_node node;
452
453 if (use_ww_ctx && ww_ctx->acquired > 0) {
454 struct ww_mutex *ww;
455
456 ww = container_of(lock, struct ww_mutex, base);
457 /*
458 * If ww->ctx is set the contents are undefined, only
459 * by acquiring wait_lock there is a guarantee that
460 * they are not invalid when reading.
461 *
462 * As such, when deadlock detection needs to be
463 * performed the optimistic spinning cannot be done.
464 */
465 if (ACCESS_ONCE(ww->ctx))
466 goto slowpath;
467 }
468
469 /*
470 * If there's an owner, wait for it to either
471 * release the lock or go to sleep.
472 */
473 mspin_lock(MLOCK(lock), &node);
474 owner = ACCESS_ONCE(lock->owner);
475 if (owner && !mutex_spin_on_owner(lock, owner)) {
476 mspin_unlock(MLOCK(lock), &node);
477 goto slowpath;
478 }
479
480 if ((atomic_read(&lock->count) == 1) &&
481 (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
482 lock_acquired(&lock->dep_map, ip);
483 if (use_ww_ctx) {
484 struct ww_mutex *ww;
485 ww = container_of(lock, struct ww_mutex, base);
486
487 ww_mutex_set_context_fastpath(ww, ww_ctx);
488 }
489
490 mutex_set_owner(lock);
491 mspin_unlock(MLOCK(lock), &node);
492 preempt_enable();
493 return 0;
494 }
495 mspin_unlock(MLOCK(lock), &node);
496
497 /*
498 * When there's no owner, we might have preempted between the
499 * owner acquiring the lock and setting the owner field. If
500 * we're an RT task that will live-lock because we won't let
501 * the owner complete.
502 */
503 if (!owner && (need_resched() || rt_task(task)))
504 goto slowpath;
505
506 /*
507 * The cpu_relax() call is a compiler barrier which forces
508 * everything in this loop to be re-loaded. We don't need
509 * memory barriers as we'll eventually observe the right
510 * values at the cost of a few extra spins.
511 */
512 arch_mutex_cpu_relax();
513 }
514slowpath:
515#endif
516 spin_lock_mutex(&lock->wait_lock, flags);
517
518 /* once more, can we acquire the lock? */
519 if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
520 goto skip_wait;
521
522 debug_mutex_lock_common(lock, &waiter);
523 debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
524
525 /* add waiting tasks to the end of the waitqueue (FIFO): */
526 list_add_tail(&waiter.list, &lock->wait_list);
527 waiter.task = task;
528
529 lock_contended(&lock->dep_map, ip);
530
531 for (;;) {
532 /*
533 * Lets try to take the lock again - this is needed even if
534 * we get here for the first time (shortly after failing to
535 * acquire the lock), to make sure that we get a wakeup once
536 * it's unlocked. Later on, if we sleep, this is the
537 * operation that gives us the lock. We xchg it to -1, so
538 * that when we release the lock, we properly wake up the
539 * other waiters:
540 */
541 if (MUTEX_SHOW_NO_WAITER(lock) &&
542 (atomic_xchg(&lock->count, -1) == 1))
543 break;
544
545 /*
546 * got a signal? (This code gets eliminated in the
547 * TASK_UNINTERRUPTIBLE case.)
548 */
549 if (unlikely(signal_pending_state(state, task))) {
550 ret = -EINTR;
551 goto err;
552 }
553
554 if (use_ww_ctx && ww_ctx->acquired > 0) {
555 ret = __mutex_lock_check_stamp(lock, ww_ctx);
556 if (ret)
557 goto err;
558 }
559
560 __set_task_state(task, state);
561
562 /* didn't get the lock, go to sleep: */
563 spin_unlock_mutex(&lock->wait_lock, flags);
564 schedule_preempt_disabled();
565 spin_lock_mutex(&lock->wait_lock, flags);
566 }
567 mutex_remove_waiter(lock, &waiter, current_thread_info());
568 /* set it to 0 if there are no waiters left: */
569 if (likely(list_empty(&lock->wait_list)))
570 atomic_set(&lock->count, 0);
571 debug_mutex_free_waiter(&waiter);
572
573skip_wait:
574 /* got the lock - cleanup and rejoice! */
575 lock_acquired(&lock->dep_map, ip);
576 mutex_set_owner(lock);
577
578 if (use_ww_ctx) {
579 struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
580 struct mutex_waiter *cur;
581
582 /*
583 * This branch gets optimized out for the common case,
584 * and is only important for ww_mutex_lock.
585 */
586 ww_mutex_lock_acquired(ww, ww_ctx);
587 ww->ctx = ww_ctx;
588
589 /*
590 * Give any possible sleeping processes the chance to wake up,
591 * so they can recheck if they have to back off.
592 */
593 list_for_each_entry(cur, &lock->wait_list, list) {
594 debug_mutex_wake_waiter(lock, cur);
595 wake_up_process(cur->task);
596 }
597 }
598
599 spin_unlock_mutex(&lock->wait_lock, flags);
600 preempt_enable();
601 return 0;
602
603err:
604 mutex_remove_waiter(lock, &waiter, task_thread_info(task));
605 spin_unlock_mutex(&lock->wait_lock, flags);
606 debug_mutex_free_waiter(&waiter);
607 mutex_release(&lock->dep_map, 1, ip);
608 preempt_enable();
609 return ret;
610}
611
612#ifdef CONFIG_DEBUG_LOCK_ALLOC
613void __sched
614mutex_lock_nested(struct mutex *lock, unsigned int subclass)
615{
616 might_sleep();
617 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
618 subclass, NULL, _RET_IP_, NULL, 0);
619}
620
621EXPORT_SYMBOL_GPL(mutex_lock_nested);
622
623void __sched
624_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
625{
626 might_sleep();
627 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
628 0, nest, _RET_IP_, NULL, 0);
629}
630
631EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
632
633int __sched
634mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
635{
636 might_sleep();
637 return __mutex_lock_common(lock, TASK_KILLABLE,
638 subclass, NULL, _RET_IP_, NULL, 0);
639}
640EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
641
642int __sched
643mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
644{
645 might_sleep();
646 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
647 subclass, NULL, _RET_IP_, NULL, 0);
648}
649
650EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
651
652static inline int
653ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
654{
655#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
656 unsigned tmp;
657
658 if (ctx->deadlock_inject_countdown-- == 0) {
659 tmp = ctx->deadlock_inject_interval;
660 if (tmp > UINT_MAX/4)
661 tmp = UINT_MAX;
662 else
663 tmp = tmp*2 + tmp + tmp/2;
664
665 ctx->deadlock_inject_interval = tmp;
666 ctx->deadlock_inject_countdown = tmp;
667 ctx->contending_lock = lock;
668
669 ww_mutex_unlock(lock);
670
671 return -EDEADLK;
672 }
673#endif
674
675 return 0;
676}
677
678int __sched
679__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
680{
681 int ret;
682
683 might_sleep();
684 ret = __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
685 0, &ctx->dep_map, _RET_IP_, ctx, 1);
686 if (!ret && ctx->acquired > 1)
687 return ww_mutex_deadlock_injection(lock, ctx);
688
689 return ret;
690}
691EXPORT_SYMBOL_GPL(__ww_mutex_lock);
692
693int __sched
694__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
695{
696 int ret;
697
698 might_sleep();
699 ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
700 0, &ctx->dep_map, _RET_IP_, ctx, 1);
701
702 if (!ret && ctx->acquired > 1)
703 return ww_mutex_deadlock_injection(lock, ctx);
704
705 return ret;
706}
707EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
708
709#endif
710
711/*
712 * Release the lock, slowpath:
713 */
714static inline void
715__mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
716{
717 struct mutex *lock = container_of(lock_count, struct mutex, count);
718 unsigned long flags;
719
720 spin_lock_mutex(&lock->wait_lock, flags);
721 mutex_release(&lock->dep_map, nested, _RET_IP_);
722 debug_mutex_unlock(lock);
723
724 /*
725 * some architectures leave the lock unlocked in the fastpath failure
726 * case, others need to leave it locked. In the later case we have to
727 * unlock it here
728 */
729 if (__mutex_slowpath_needs_to_unlock())
730 atomic_set(&lock->count, 1);
731
732 if (!list_empty(&lock->wait_list)) {
733 /* get the first entry from the wait-list: */
734 struct mutex_waiter *waiter =
735 list_entry(lock->wait_list.next,
736 struct mutex_waiter, list);
737
738 debug_mutex_wake_waiter(lock, waiter);
739
740 wake_up_process(waiter->task);
741 }
742
743 spin_unlock_mutex(&lock->wait_lock, flags);
744}
745
746/*
747 * Release the lock, slowpath:
748 */
749static __used noinline void
750__mutex_unlock_slowpath(atomic_t *lock_count)
751{
752 __mutex_unlock_common_slowpath(lock_count, 1);
753}
754
755#ifndef CONFIG_DEBUG_LOCK_ALLOC
756/*
757 * Here come the less common (and hence less performance-critical) APIs:
758 * mutex_lock_interruptible() and mutex_trylock().
759 */
760static noinline int __sched
761__mutex_lock_killable_slowpath(struct mutex *lock);
762
763static noinline int __sched
764__mutex_lock_interruptible_slowpath(struct mutex *lock);
765
766/**
767 * mutex_lock_interruptible - acquire the mutex, interruptible
768 * @lock: the mutex to be acquired
769 *
770 * Lock the mutex like mutex_lock(), and return 0 if the mutex has
771 * been acquired or sleep until the mutex becomes available. If a
772 * signal arrives while waiting for the lock then this function
773 * returns -EINTR.
774 *
775 * This function is similar to (but not equivalent to) down_interruptible().
776 */
777int __sched mutex_lock_interruptible(struct mutex *lock)
778{
779 int ret;
780
781 might_sleep();
782 ret = __mutex_fastpath_lock_retval(&lock->count);
783 if (likely(!ret)) {
784 mutex_set_owner(lock);
785 return 0;
786 } else
787 return __mutex_lock_interruptible_slowpath(lock);
788}
789
790EXPORT_SYMBOL(mutex_lock_interruptible);
791
792int __sched mutex_lock_killable(struct mutex *lock)
793{
794 int ret;
795
796 might_sleep();
797 ret = __mutex_fastpath_lock_retval(&lock->count);
798 if (likely(!ret)) {
799 mutex_set_owner(lock);
800 return 0;
801 } else
802 return __mutex_lock_killable_slowpath(lock);
803}
804EXPORT_SYMBOL(mutex_lock_killable);
805
806static __used noinline void __sched
807__mutex_lock_slowpath(atomic_t *lock_count)
808{
809 struct mutex *lock = container_of(lock_count, struct mutex, count);
810
811 __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0,
812 NULL, _RET_IP_, NULL, 0);
813}
814
815static noinline int __sched
816__mutex_lock_killable_slowpath(struct mutex *lock)
817{
818 return __mutex_lock_common(lock, TASK_KILLABLE, 0,
819 NULL, _RET_IP_, NULL, 0);
820}
821
822static noinline int __sched
823__mutex_lock_interruptible_slowpath(struct mutex *lock)
824{
825 return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0,
826 NULL, _RET_IP_, NULL, 0);
827}
828
829static noinline int __sched
830__ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
831{
832 return __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, 0,
833 NULL, _RET_IP_, ctx, 1);
834}
835
836static noinline int __sched
837__ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
838 struct ww_acquire_ctx *ctx)
839{
840 return __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, 0,
841 NULL, _RET_IP_, ctx, 1);
842}
843
844#endif
845
846/*
847 * Spinlock based trylock, we take the spinlock and check whether we
848 * can get the lock:
849 */
850static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
851{
852 struct mutex *lock = container_of(lock_count, struct mutex, count);
853 unsigned long flags;
854 int prev;
855
856 spin_lock_mutex(&lock->wait_lock, flags);
857
858 prev = atomic_xchg(&lock->count, -1);
859 if (likely(prev == 1)) {
860 mutex_set_owner(lock);
861 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
862 }
863
864 /* Set it back to 0 if there are no waiters: */
865 if (likely(list_empty(&lock->wait_list)))
866 atomic_set(&lock->count, 0);
867
868 spin_unlock_mutex(&lock->wait_lock, flags);
869
870 return prev == 1;
871}
872
873/**
874 * mutex_trylock - try to acquire the mutex, without waiting
875 * @lock: the mutex to be acquired
876 *
877 * Try to acquire the mutex atomically. Returns 1 if the mutex
878 * has been acquired successfully, and 0 on contention.
879 *
880 * NOTE: this function follows the spin_trylock() convention, so
881 * it is negated from the down_trylock() return values! Be careful
882 * about this when converting semaphore users to mutexes.
883 *
884 * This function must not be used in interrupt context. The
885 * mutex must be released by the same task that acquired it.
886 */
887int __sched mutex_trylock(struct mutex *lock)
888{
889 int ret;
890
891 ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);
892 if (ret)
893 mutex_set_owner(lock);
894
895 return ret;
896}
897EXPORT_SYMBOL(mutex_trylock);
898
899#ifndef CONFIG_DEBUG_LOCK_ALLOC
900int __sched
901__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
902{
903 int ret;
904
905 might_sleep();
906
907 ret = __mutex_fastpath_lock_retval(&lock->base.count);
908
909 if (likely(!ret)) {
910 ww_mutex_set_context_fastpath(lock, ctx);
911 mutex_set_owner(&lock->base);
912 } else
913 ret = __ww_mutex_lock_slowpath(lock, ctx);
914 return ret;
915}
916EXPORT_SYMBOL(__ww_mutex_lock);
917
918int __sched
919__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
920{
921 int ret;
922
923 might_sleep();
924
925 ret = __mutex_fastpath_lock_retval(&lock->base.count);
926
927 if (likely(!ret)) {
928 ww_mutex_set_context_fastpath(lock, ctx);
929 mutex_set_owner(&lock->base);
930 } else
931 ret = __ww_mutex_lock_interruptible_slowpath(lock, ctx);
932 return ret;
933}
934EXPORT_SYMBOL(__ww_mutex_lock_interruptible);
935
936#endif
937
938/**
939 * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
940 * @cnt: the atomic which we are to dec
941 * @lock: the mutex to return holding if we dec to 0
942 *
943 * return true and hold lock if we dec to 0, return false otherwise
944 */
945int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
946{
947 /* dec if we can't possibly hit 0 */
948 if (atomic_add_unless(cnt, -1, 1))
949 return 0;
950 /* we might hit 0, so take the lock */
951 mutex_lock(lock);
952 if (!atomic_dec_and_test(cnt)) {
953 /* when we actually did the dec, we didn't hit 0 */
954 mutex_unlock(lock);
955 return 0;
956 }
957 /* we hit 0, and we hold the lock */
958 return 1;
959}
960EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
kernel/locking/mutex.h+48−0
@@ -0,0 +1,48 @@
1/*
2 * Mutexes: blocking mutual exclusion locks
3 *
4 * started by Ingo Molnar:
5 *
6 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7 *
8 * This file contains mutex debugging related internal prototypes, for the
9 * !CONFIG_DEBUG_MUTEXES case. Most of them are NOPs:
10 */
11
12#define spin_lock_mutex(lock, flags) \
13 do { spin_lock(lock); (void)(flags); } while (0)
14#define spin_unlock_mutex(lock, flags) \
15 do { spin_unlock(lock); (void)(flags); } while (0)
16#define mutex_remove_waiter(lock, waiter, ti) \
17 __list_del((waiter)->list.prev, (waiter)->list.next)
18
19#ifdef CONFIG_SMP
20static inline void mutex_set_owner(struct mutex *lock)
21{
22 lock->owner = current;
23}
24
25static inline void mutex_clear_owner(struct mutex *lock)
26{
27 lock->owner = NULL;
28}
29#else
30static inline void mutex_set_owner(struct mutex *lock)
31{
32}
33
34static inline void mutex_clear_owner(struct mutex *lock)
35{
36}
37#endif
38
39#define debug_mutex_wake_waiter(lock, waiter) do { } while (0)
40#define debug_mutex_free_waiter(waiter) do { } while (0)
41#define debug_mutex_add_waiter(lock, waiter, ti) do { } while (0)
42#define debug_mutex_unlock(lock) do { } while (0)
43#define debug_mutex_init(lock, name, key) do { } while (0)
44
45static inline void
46debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter)
47{
48}