History

Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar: - improve rwsem scalability - add uninitialized rwsem debugging check - reduce lockdep's stacktrace memory usage and add diagnostics - misc cleanups, code consolidation and constification * 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: mutex: Fix up mutex_waiter usage locking/mutex: Use mutex flags macro instead of hard code locking/mutex: Make __mutex_owner static to mutex.c locking/qspinlock,x86: Clarify virt_spin_lock_key locking/rwsem: Check for operations on an uninitialized rwsem locking/rwsem: Make handoff writer optimistically spin on owner locking/lockdep: Report more stack trace statistics locking/lockdep: Reduce space occupied by stack traces stacktrace: Constify 'entries' arguments locking/lockdep: Make it clear that what lock_class::key points at is not modified

Linus Torvalds committed 7 years ago

Commit c7eba512 parents cc9b499 e57d14311 files · +241 −84Browse files

Unified · Split

11 files
  1. arch/x86/include/asm/qspinlock.h +15 −0
  2. include/linux/lockdep.h +4 −7
  3. include/linux/mutex.h +3 −22
  4. include/linux/rwsem.h +10 −0
  5. include/linux/stacktrace.h +2 −2
  6. kernel/locking/lockdep.c +120 −39
  7. kernel/locking/lockdep_internals.h +8 −1
  8. kernel/locking/lockdep_proc.c +7 −1
  9. kernel/locking/mutex.c +26 −0
  10. kernel/locking/rwsem.c +44 −10
  11. kernel/stacktrace.c +2 −2
arch/x86/include/asm/qspinlock.h+15−0
@@ -63,10 +63,25 @@ static inline bool vcpu_is_preempted(long cpu) · 62 unmodified lines
63#endif63#endif
64 64
65#ifdef CONFIG_PARAVIRT65#ifdef CONFIG_PARAVIRT
66/*
67 * virt_spin_lock_key - enables (by default) the virt_spin_lock() hijack.
68 *
69 * Native (and PV wanting native due to vCPU pinning) should disable this key.
70 * It is done in this backwards fashion to only have a single direction change,
71 * which removes ordering between native_pv_spin_init() and HV setup.
72 */
66DECLARE_STATIC_KEY_TRUE(virt_spin_lock_key);73DECLARE_STATIC_KEY_TRUE(virt_spin_lock_key);
67 74
68void native_pv_lock_init(void) __init;75void native_pv_lock_init(void) __init;
69 76
77/*
78 * Shortcut for the queued_spin_lock_slowpath() function that allows
79 * virt to hijack it.
80 *
81 * Returns:
82 * true - lock has been negotiated, all done;
83 * false - queued_spin_lock_slowpath() will do its thing.
84 */
70#define virt_spin_lock virt_spin_lock85#define virt_spin_lock virt_spin_lock
71static inline bool virt_spin_lock(struct qspinlock *lock)86static inline bool virt_spin_lock(struct qspinlock *lock)
72{87{
include/linux/lockdep.h+4−7
@@ -66,10 +66,7 @@ struct lock_class_key { · 65 unmodified lines
66 66
67extern struct lock_class_key __lockdep_no_validate__;67extern struct lock_class_key __lockdep_no_validate__;
68 68
69struct lock_trace {69struct lock_trace;
70 unsigned int nr_entries;
71 unsigned int offset;
72};
73 70
74#define LOCKSTAT_POINTS 471#define LOCKSTAT_POINTS 4
75 72
@@ -97,7 +94,7 @@ struct lock_class { · 21 unmodified lines
97 */94 */
98 struct list_head locks_after, locks_before;95 struct list_head locks_after, locks_before;
99 96
100 struct lockdep_subclass_key *key;97 const struct lockdep_subclass_key *key;
101 unsigned int subclass;98 unsigned int subclass;
102 unsigned int dep_gen_id;99 unsigned int dep_gen_id;
103 100
@@ -105,7 +102,7 @@ struct lock_class { · 1 unmodified line
105 * IRQ/softirq usage tracking bits:102 * IRQ/softirq usage tracking bits:
106 */103 */
107 unsigned long usage_mask;104 unsigned long usage_mask;
108 struct lock_trace usage_traces[XXX_LOCK_USAGE_STATES];105 const struct lock_trace *usage_traces[XXX_LOCK_USAGE_STATES];
109 106
110 /*107 /*
111 * Generation counter, when doing certain classes of graph walking,108 * Generation counter, when doing certain classes of graph walking,
@@ -193,7 +190,7 @@ struct lock_list { · 81 unmodified lines
193 struct list_head entry;190 struct list_head entry;
194 struct lock_class *class;191 struct lock_class *class;
195 struct lock_class *links_to;192 struct lock_class *links_to;
196 struct lock_trace trace;193 const struct lock_trace *trace;
197 int distance;194 int distance;
198 195
199 /*196 /*
include/linux/mutex.h+3−22
@@ -65,16 +65,6 @@ struct mutex { · 64 unmodified lines
65#endif65#endif
66};66};
67 67
68/*
69 * Internal helper function; C doesn't allow us to hide it :/
70 *
71 * DO NOT USE (outside of mutex code).
72 */
73static inline struct task_struct *__mutex_owner(struct mutex *lock)
74{
75 return (struct task_struct *)(atomic_long_read(&lock->owner) & ~0x07);
76}
77
78/*68/*
79 * This is the control structure for tasks blocked on mutex,69 * This is the control structure for tasks blocked on mutex,
80 * which resides on the blocked task's kernel stack:70 * which resides on the blocked task's kernel stack:
@@ -144,10 +134,7 @@ extern void __mutex_init(struct mutex *lock, const char *name, · 63 unmodified lines
144 *134 *
145 * Returns true if the mutex is locked, false if unlocked.135 * Returns true if the mutex is locked, false if unlocked.
146 */136 */
147static inline bool mutex_is_locked(struct mutex *lock)137extern bool mutex_is_locked(struct mutex *lock);
148{
149 return __mutex_owner(lock) != NULL;
150}
151 138
152/*139/*
153 * See kernel/locking/mutex.c for detailed documentation of these APIs.140 * See kernel/locking/mutex.c for detailed documentation of these APIs.
@@ -220,13 +207,7 @@ enum mutex_trylock_recursive_enum { · 66 unmodified lines
220 * - MUTEX_TRYLOCK_SUCCESS - lock acquired,207 * - MUTEX_TRYLOCK_SUCCESS - lock acquired,
221 * - MUTEX_TRYLOCK_RECURSIVE - we already owned the lock.208 * - MUTEX_TRYLOCK_RECURSIVE - we already owned the lock.
222 */209 */
223static inline /* __deprecated */ __must_check enum mutex_trylock_recursive_enum210extern /* __deprecated */ __must_check enum mutex_trylock_recursive_enum
224mutex_trylock_recursive(struct mutex *lock)211mutex_trylock_recursive(struct mutex *lock);
225{
226 if (unlikely(__mutex_owner(lock) == current))
227 return MUTEX_TRYLOCK_RECURSIVE;
228
229 return mutex_trylock(lock);
230}
231 212
232#endif /* __LINUX_MUTEX_H */213#endif /* __LINUX_MUTEX_H */
include/linux/rwsem.h+10−0
@@ -45,6 +45,9 @@ struct rw_semaphore { · 44 unmodified lines
45#endif45#endif
46 raw_spinlock_t wait_lock;46 raw_spinlock_t wait_lock;
47 struct list_head wait_list;47 struct list_head wait_list;
48#ifdef CONFIG_DEBUG_RWSEMS
49 void *magic;
50#endif
48#ifdef CONFIG_DEBUG_LOCK_ALLOC51#ifdef CONFIG_DEBUG_LOCK_ALLOC
49 struct lockdep_map dep_map;52 struct lockdep_map dep_map;
50#endif53#endif
@@ -73,6 +76,12 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) · 22 unmodified lines
73# define __RWSEM_DEP_MAP_INIT(lockname)76# define __RWSEM_DEP_MAP_INIT(lockname)
74#endif77#endif
75 78
79#ifdef CONFIG_DEBUG_RWSEMS
80# define __DEBUG_RWSEM_INITIALIZER(lockname) , .magic = &lockname
81#else
82# define __DEBUG_RWSEM_INITIALIZER(lockname)
83#endif
84
76#ifdef CONFIG_RWSEM_SPIN_ON_OWNER85#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
77#define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED86#define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED
78#else87#else
@@ -85,6 +94,7 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) · 6 unmodified lines
85 .wait_list = LIST_HEAD_INIT((name).wait_list), \94 .wait_list = LIST_HEAD_INIT((name).wait_list), \
86 .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock) \95 .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock) \
87 __RWSEM_OPT_INIT(name) \96 __RWSEM_OPT_INIT(name) \
97 __DEBUG_RWSEM_INITIALIZER(name) \
88 __RWSEM_DEP_MAP_INIT(name) }98 __RWSEM_DEP_MAP_INIT(name) }
89 99
90#define DECLARE_RWSEM(name) \100#define DECLARE_RWSEM(name) \
include/linux/stacktrace.h+2−2
@@ -9,9 +9,9 @@ struct task_struct; · 8 unmodified lines
9struct pt_regs;9struct pt_regs;
10 10
11#ifdef CONFIG_STACKTRACE11#ifdef CONFIG_STACKTRACE
12void stack_trace_print(unsigned long *trace, unsigned int nr_entries,12void stack_trace_print(const unsigned long *trace, unsigned int nr_entries,
13 int spaces);13 int spaces);
14int stack_trace_snprint(char *buf, size_t size, unsigned long *entries,14int stack_trace_snprint(char *buf, size_t size, const unsigned long *entries,
15 unsigned int nr_entries, int spaces);15 unsigned int nr_entries, int spaces);
16unsigned int stack_trace_save(unsigned long *store, unsigned int size,16unsigned int stack_trace_save(unsigned long *store, unsigned int size,
17 unsigned int skipnr);17 unsigned int skipnr);
kernel/locking/lockdep.c+120−39
@@ -449,33 +449,101 @@ static void print_lockdep_off(const char *bug_msg) · 448 unmodified lines
449unsigned long nr_stack_trace_entries;449unsigned long nr_stack_trace_entries;
450 450
451#ifdef CONFIG_PROVE_LOCKING451#ifdef CONFIG_PROVE_LOCKING
452/**
453 * struct lock_trace - single stack backtrace
454 * @hash_entry: Entry in a stack_trace_hash[] list.
455 * @hash: jhash() of @entries.
456 * @nr_entries: Number of entries in @entries.
457 * @entries: Actual stack backtrace.
458 */
459struct lock_trace {
460 struct hlist_node hash_entry;
461 u32 hash;
462 u32 nr_entries;
463 unsigned long entries[0] __aligned(sizeof(unsigned long));
464};
465#define LOCK_TRACE_SIZE_IN_LONGS \
466 (sizeof(struct lock_trace) / sizeof(unsigned long))
452/*467/*
453 * Stack-trace: tightly packed array of stack backtrace468 * Stack-trace: sequence of lock_trace structures. Protected by the graph_lock.
454 * addresses. Protected by the graph_lock.
455 */469 */
456static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];470static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];
471static struct hlist_head stack_trace_hash[STACK_TRACE_HASH_SIZE];
472
473static bool traces_identical(struct lock_trace *t1, struct lock_trace *t2)
474{
475 return t1->hash == t2->hash && t1->nr_entries == t2->nr_entries &&
476 memcmp(t1->entries, t2->entries,
477 t1->nr_entries * sizeof(t1->entries[0])) == 0;
478}
457 479
458static int save_trace(struct lock_trace *trace)480static struct lock_trace *save_trace(void)
459{481{
460 unsigned long *entries = stack_trace + nr_stack_trace_entries;482 struct lock_trace *trace, *t2;
483 struct hlist_head *hash_head;
484 u32 hash;
461 unsigned int max_entries;485 unsigned int max_entries;
462 486
463 trace->offset = nr_stack_trace_entries;487 BUILD_BUG_ON_NOT_POWER_OF_2(STACK_TRACE_HASH_SIZE);
464 max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;488 BUILD_BUG_ON(LOCK_TRACE_SIZE_IN_LONGS >= MAX_STACK_TRACE_ENTRIES);
465 trace->nr_entries = stack_trace_save(entries, max_entries, 3);489
466 nr_stack_trace_entries += trace->nr_entries;490 trace = (struct lock_trace *)(stack_trace + nr_stack_trace_entries);
491 max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries -
492 LOCK_TRACE_SIZE_IN_LONGS;
493 trace->nr_entries = stack_trace_save(trace->entries, max_entries, 3);
467 494
468 if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {495 if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES -
496 LOCK_TRACE_SIZE_IN_LONGS - 1) {
469 if (!debug_locks_off_graph_unlock())497 if (!debug_locks_off_graph_unlock())
470 return 0;498 return NULL;
471 499
472 print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");500 print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");
473 dump_stack();501 dump_stack();
474 502
475 return 0;503 return NULL;
476 }504 }
477 505
478 return 1;506 hash = jhash(trace->entries, trace->nr_entries *
507 sizeof(trace->entries[0]), 0);
508 trace->hash = hash;
509 hash_head = stack_trace_hash + (hash & (STACK_TRACE_HASH_SIZE - 1));
510 hlist_for_each_entry(t2, hash_head, hash_entry) {
511 if (traces_identical(trace, t2))
512 return t2;
513 }
514 nr_stack_trace_entries += LOCK_TRACE_SIZE_IN_LONGS + trace->nr_entries;
515 hlist_add_head(&trace->hash_entry, hash_head);
516
517 return trace;
518}
519
520/* Return the number of stack traces in the stack_trace[] array. */
521u64 lockdep_stack_trace_count(void)
522{
523 struct lock_trace *trace;
524 u64 c = 0;
525 int i;
526
527 for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++) {
528 hlist_for_each_entry(trace, &stack_trace_hash[i], hash_entry) {
529 c++;
530 }
531 }
532
533 return c;
534}
535
536/* Return the number of stack hash chains that have at least one stack trace. */
537u64 lockdep_stack_hash_count(void)
538{
539 u64 c = 0;
540 int i;
541
542 for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++)
543 if (!hlist_empty(&stack_trace_hash[i]))
544 c++;
545
546 return c;
479}547}
480#endif548#endif
481 549
@@ -511,7 +579,7 @@ static const char *usage_str[] = · 29 unmodified lines
511};579};
512#endif580#endif
513 581
514const char * __get_key_name(struct lockdep_subclass_key *key, char *str)582const char *__get_key_name(const struct lockdep_subclass_key *key, char *str)
515{583{
516 return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);584 return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);
517}585}
@@ -1235,7 +1303,7 @@ static struct lock_list *alloc_list_entry(void) · 717 unmodified lines
1235static int add_lock_to_list(struct lock_class *this,1303static int add_lock_to_list(struct lock_class *this,
1236 struct lock_class *links_to, struct list_head *head,1304 struct lock_class *links_to, struct list_head *head,
1237 unsigned long ip, int distance,1305 unsigned long ip, int distance,
1238 struct lock_trace *trace)1306 const struct lock_trace *trace)
1239{1307{
1240 struct lock_list *entry;1308 struct lock_list *entry;
1241 /*1309 /*
@@ -1249,7 +1317,7 @@ static int add_lock_to_list(struct lock_class *this, · 7 unmodified lines
1249 entry->class = this;1317 entry->class = this;
1250 entry->links_to = links_to;1318 entry->links_to = links_to;
1251 entry->distance = distance;1319 entry->distance = distance;
1252 entry->trace = *trace;1320 entry->trace = trace;
1253 /*1321 /*
1254 * Both allocation and removal are done under the graph lock; but1322 * Both allocation and removal are done under the graph lock; but
1255 * iteration is under RCU-sched; see look_up_lock_class() and1323 * iteration is under RCU-sched; see look_up_lock_class() and
@@ -1470,11 +1538,10 @@ static inline int __bfs_backwards(struct lock_list *src_entry, · 214 unmodified lines
1470 1538
1471}1539}
1472 1540
1473static void print_lock_trace(struct lock_trace *trace, unsigned int spaces)1541static void print_lock_trace(const struct lock_trace *trace,
1542 unsigned int spaces)
1474{1543{
1475 unsigned long *entries = stack_trace + trace->offset;1544 stack_trace_print(trace->entries, trace->nr_entries, spaces);
1476
1477 stack_trace_print(entries, trace->nr_entries, spaces);
1478}1545}
1479 1546
1480/*1547/*
@@ -1489,7 +1556,7 @@ print_circular_bug_entry(struct lock_list *target, int depth) · 8 unmodified lines
1489 printk("\n-> #%u", depth);1556 printk("\n-> #%u", depth);
1490 print_lock_name(target->class);1557 print_lock_name(target->class);
1491 printk(KERN_CONT ":\n");1558 printk(KERN_CONT ":\n");
1492 print_lock_trace(&target->trace, 6);1559 print_lock_trace(target->trace, 6);
1493}1560}
1494 1561
1495static void1562static void
@@ -1592,7 +1659,8 @@ static noinline void print_circular_bug(struct lock_list *this, · 96 unmodified lines
1592 if (!debug_locks_off_graph_unlock() || debug_locks_silent)1659 if (!debug_locks_off_graph_unlock() || debug_locks_silent)
1593 return;1660 return;
1594 1661
1595 if (!save_trace(&this->trace))1662 this->trace = save_trace();
1663 if (!this->trace)
1596 return;1664 return;
1597 1665
1598 depth = get_lock_depth(target);1666 depth = get_lock_depth(target);
@@ -1715,7 +1783,7 @@ check_path(struct lock_class *target, struct lock_list *src_entry, · 116 unmodified lines
1715 */1783 */
1716static noinline int1784static noinline int
1717check_noncircular(struct held_lock *src, struct held_lock *target,1785check_noncircular(struct held_lock *src, struct held_lock *target,
1718 struct lock_trace *trace)1786 struct lock_trace **const trace)
1719{1787{
1720 int ret;1788 int ret;
1721 struct lock_list *uninitialized_var(target_entry);1789 struct lock_list *uninitialized_var(target_entry);
@@ -1729,13 +1797,13 @@ check_noncircular(struct held_lock *src, struct held_lock *target, · 7 unmodified lines
1729 ret = check_path(hlock_class(target), &src_entry, &target_entry);1797 ret = check_path(hlock_class(target), &src_entry, &target_entry);
1730 1798
1731 if (unlikely(!ret)) {1799 if (unlikely(!ret)) {
1732 if (!trace->nr_entries) {1800 if (!*trace) {
1733 /*1801 /*
1734 * If save_trace fails here, the printing might1802 * If save_trace fails here, the printing might
1735 * trigger a WARN but because of the !nr_entries it1803 * trigger a WARN but because of the !nr_entries it
1736 * should not do bad things.1804 * should not do bad things.
1737 */1805 */
1738 save_trace(trace);1806 *trace = save_trace();
1739 }1807 }
1740 1808
1741 print_circular_bug(&src_entry, target_entry, src, target);1809 print_circular_bug(&src_entry, target_entry, src, target);
@@ -1859,7 +1927,7 @@ static void print_lock_class_header(struct lock_class *class, int depth) · 117 unmodified lines
1859 1927
1860 len += printk("%*s %s", depth, "", usage_str[bit]);1928 len += printk("%*s %s", depth, "", usage_str[bit]);
1861 len += printk(KERN_CONT " at:\n");1929 len += printk(KERN_CONT " at:\n");
1862 print_lock_trace(class->usage_traces + bit, len);1930 print_lock_trace(class->usage_traces[bit], len);
1863 }1931 }
1864 }1932 }
1865 printk("%*s }\n", depth, "");1933 printk("%*s }\n", depth, "");
@@ -1884,7 +1952,7 @@ print_shortest_lock_dependencies(struct lock_list *leaf, · 18 unmodified lines
1884 do {1952 do {
1885 print_lock_class_header(entry->class, depth);1953 print_lock_class_header(entry->class, depth);
1886 printk("%*s ... acquired at:\n", depth, "");1954 printk("%*s ... acquired at:\n", depth, "");
1887 print_lock_trace(&entry->trace, 2);1955 print_lock_trace(entry->trace, 2);
1888 printk("\n");1956 printk("\n");
1889 1957
1890 if (depth == 0 && (entry != root)) {1958 if (depth == 0 && (entry != root)) {
@@ -1995,14 +2063,14 @@ print_bad_irq_dependency(struct task_struct *curr, · 104 unmodified lines
1995 print_lock_name(backwards_entry->class);2063 print_lock_name(backwards_entry->class);
1996 pr_warn("\n... which became %s-irq-safe at:\n", irqclass);2064 pr_warn("\n... which became %s-irq-safe at:\n", irqclass);
1997 2065
1998 print_lock_trace(backwards_entry->class->usage_traces + bit1, 1);2066 print_lock_trace(backwards_entry->class->usage_traces[bit1], 1);
1999 2067
2000 pr_warn("\nto a %s-irq-unsafe lock:\n", irqclass);2068 pr_warn("\nto a %s-irq-unsafe lock:\n", irqclass);
2001 print_lock_name(forwards_entry->class);2069 print_lock_name(forwards_entry->class);
2002 pr_warn("\n... which became %s-irq-unsafe at:\n", irqclass);2070 pr_warn("\n... which became %s-irq-unsafe at:\n", irqclass);
2003 pr_warn("...");2071 pr_warn("...");
2004 2072
2005 print_lock_trace(forwards_entry->class->usage_traces + bit2, 1);2073 print_lock_trace(forwards_entry->class->usage_traces[bit2], 1);
2006 2074
2007 pr_warn("\nother info that might help us debug this:\n\n");2075 pr_warn("\nother info that might help us debug this:\n\n");
2008 print_irq_lock_scenario(backwards_entry, forwards_entry,2076 print_irq_lock_scenario(backwards_entry, forwards_entry,
@@ -2011,13 +2079,15 @@ print_bad_irq_dependency(struct task_struct *curr, · 2 unmodified lines
2011 lockdep_print_held_locks(curr);2079 lockdep_print_held_locks(curr);
2012 2080
2013 pr_warn("\nthe dependencies between %s-irq-safe lock and the holding lock:\n", irqclass);2081 pr_warn("\nthe dependencies between %s-irq-safe lock and the holding lock:\n", irqclass);
2014 if (!save_trace(&prev_root->trace))2082 prev_root->trace = save_trace();
2083 if (!prev_root->trace)
2015 return;2084 return;
2016 print_shortest_lock_dependencies(backwards_entry, prev_root);2085 print_shortest_lock_dependencies(backwards_entry, prev_root);
2017 2086
2018 pr_warn("\nthe dependencies between the lock to be acquired");2087 pr_warn("\nthe dependencies between the lock to be acquired");
2019 pr_warn(" and %s-irq-unsafe lock:\n", irqclass);2088 pr_warn(" and %s-irq-unsafe lock:\n", irqclass);
2020 if (!save_trace(&next_root->trace))2089 next_root->trace = save_trace();
2090 if (!next_root->trace)
2021 return;2091 return;
2022 print_shortest_lock_dependencies(forwards_entry, next_root);2092 print_shortest_lock_dependencies(forwards_entry, next_root);
2023 2093
@@ -2369,7 +2439,8 @@ check_deadlock(struct task_struct *curr, struct held_lock *next) · 345 unmodified lines
2369 */2439 */
2370static int2440static int
2371check_prev_add(struct task_struct *curr, struct held_lock *prev,2441check_prev_add(struct task_struct *curr, struct held_lock *prev,
2372 struct held_lock *next, int distance, struct lock_trace *trace)2442 struct held_lock *next, int distance,
2443 struct lock_trace **const trace)
2373{2444{
2374 struct lock_list *entry;2445 struct lock_list *entry;
2375 int ret;2446 int ret;
@@ -2444,8 +2515,11 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev, · 68 unmodified lines
2444 return ret;2515 return ret;
2445#endif2516#endif
2446 2517
2447 if (!trace->nr_entries && !save_trace(trace))2518 if (!*trace) {
2448 return 0;2519 *trace = save_trace();
2520 if (!*trace)
2521 return 0;
2522 }
2449 2523
2450 /*2524 /*
2451 * Ok, all validations passed, add the new lock2525 * Ok, all validations passed, add the new lock
@@ -2453,14 +2527,14 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev, · 1 unmodified line
2453 */2527 */
2454 ret = add_lock_to_list(hlock_class(next), hlock_class(prev),2528 ret = add_lock_to_list(hlock_class(next), hlock_class(prev),
2455 &hlock_class(prev)->locks_after,2529 &hlock_class(prev)->locks_after,
2456 next->acquire_ip, distance, trace);2530 next->acquire_ip, distance, *trace);
2457 2531
2458 if (!ret)2532 if (!ret)
2459 return 0;2533 return 0;
2460 2534
2461 ret = add_lock_to_list(hlock_class(prev), hlock_class(next),2535 ret = add_lock_to_list(hlock_class(prev), hlock_class(next),
2462 &hlock_class(next)->locks_before,2536 &hlock_class(next)->locks_before,
2463 next->acquire_ip, distance, trace);2537 next->acquire_ip, distance, *trace);
2464 if (!ret)2538 if (!ret)
2465 return 0;2539 return 0;
2466 2540
@@ -2476,7 +2550,7 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev, · 9 unmodified lines
2476static int2550static int
2477check_prevs_add(struct task_struct *curr, struct held_lock *next)2551check_prevs_add(struct task_struct *curr, struct held_lock *next)
2478{2552{
2479 struct lock_trace trace = { .nr_entries = 0 };2553 struct lock_trace *trace = NULL;
2480 int depth = curr->lockdep_depth;2554 int depth = curr->lockdep_depth;
2481 struct held_lock *hlock;2555 struct held_lock *hlock;
2482 2556
@@ -3015,7 +3089,7 @@ print_usage_bug(struct task_struct *curr, struct held_lock *this, · 532 unmodified lines
3015 print_lock(this);3089 print_lock(this);
3016 3090
3017 pr_warn("{%s} state was registered at:\n", usage_str[prev_bit]);3091 pr_warn("{%s} state was registered at:\n", usage_str[prev_bit]);
3018 print_lock_trace(hlock_class(this)->usage_traces + prev_bit, 1);3092 print_lock_trace(hlock_class(this)->usage_traces[prev_bit], 1);
3019 3093
3020 print_irqtrace_events(curr);3094 print_irqtrace_events(curr);
3021 pr_warn("\nother info that might help us debug this:\n");3095 pr_warn("\nother info that might help us debug this:\n");
@@ -3096,7 +3170,8 @@ print_irq_inversion_bug(struct task_struct *curr, · 74 unmodified lines
3096 lockdep_print_held_locks(curr);3170 lockdep_print_held_locks(curr);
3097 3171
3098 pr_warn("\nthe shortest dependencies between 2nd lock and 1st lock:\n");3172 pr_warn("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
3099 if (!save_trace(&root->trace))3173 root->trace = save_trace();
3174 if (!root->trace)
3100 return;3175 return;
3101 print_shortest_lock_dependencies(other, root);3176 print_shortest_lock_dependencies(other, root);
3102 3177
@@ -3580,7 +3655,7 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this, · 477 unmodified lines
3580 3655
3581 hlock_class(this)->usage_mask |= new_mask;3656 hlock_class(this)->usage_mask |= new_mask;
3582 3657
3583 if (!save_trace(hlock_class(this)->usage_traces + new_bit))3658 if (!(hlock_class(this)->usage_traces[new_bit] = save_trace()))
3584 return 0;3659 return 0;
3585 3660
3586 switch (new_bit) {3661 switch (new_bit) {
@@ -5157,6 +5232,12 @@ void __init lockdep_init(void) · 1570 unmodified lines
5157 ) / 10245232 ) / 1024
5158 );5233 );
5159 5234
5235#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
5236 printk(" memory used for stack traces: %zu kB\n",
5237 (sizeof(stack_trace) + sizeof(stack_trace_hash)) / 1024
5238 );
5239#endif
5240
5160 printk(" per task-struct memory footprint: %zu bytes\n",5241 printk(" per task-struct memory footprint: %zu bytes\n",
5161 sizeof(((struct task_struct *)NULL)->held_locks));5242 sizeof(((struct task_struct *)NULL)->held_locks));
5162}5243}
kernel/locking/lockdep_internals.h+8−1
@@ -92,6 +92,7 @@ static const unsigned long LOCKF_USED_IN_IRQ_READ = · 91 unmodified lines
92#define MAX_LOCKDEP_ENTRIES 16384UL92#define MAX_LOCKDEP_ENTRIES 16384UL
93#define MAX_LOCKDEP_CHAINS_BITS 1593#define MAX_LOCKDEP_CHAINS_BITS 15
94#define MAX_STACK_TRACE_ENTRIES 262144UL94#define MAX_STACK_TRACE_ENTRIES 262144UL
95#define STACK_TRACE_HASH_SIZE 8192
95#else96#else
96#define MAX_LOCKDEP_ENTRIES 32768UL97#define MAX_LOCKDEP_ENTRIES 32768UL
97 98
@@ -102,6 +103,7 @@ static const unsigned long LOCKF_USED_IN_IRQ_READ = · 4 unmodified lines
102 * addresses. Protected by the hash_lock.103 * addresses. Protected by the hash_lock.
103 */104 */
104#define MAX_STACK_TRACE_ENTRIES 524288UL105#define MAX_STACK_TRACE_ENTRIES 524288UL
106#define STACK_TRACE_HASH_SIZE 16384
105#endif107#endif
106 108
107#define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS)109#define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS)
@@ -116,7 +118,8 @@ extern struct lock_chain lock_chains[]; · 8 unmodified lines
116extern void get_usage_chars(struct lock_class *class,118extern void get_usage_chars(struct lock_class *class,
117 char usage[LOCK_USAGE_CHARS]);119 char usage[LOCK_USAGE_CHARS]);
118 120
119extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str);121extern const char *__get_key_name(const struct lockdep_subclass_key *key,
122 char *str);
120 123
121struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i);124struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i);
122 125
@@ -137,6 +140,10 @@ extern unsigned int max_bfs_queue_depth; · 14 unmodified lines
137#ifdef CONFIG_PROVE_LOCKING140#ifdef CONFIG_PROVE_LOCKING
138extern unsigned long lockdep_count_forward_deps(struct lock_class *);141extern unsigned long lockdep_count_forward_deps(struct lock_class *);
139extern unsigned long lockdep_count_backward_deps(struct lock_class *);142extern unsigned long lockdep_count_backward_deps(struct lock_class *);
143#ifdef CONFIG_TRACE_IRQFLAGS
144u64 lockdep_stack_trace_count(void);
145u64 lockdep_stack_hash_count(void);
146#endif
140#else147#else
141static inline unsigned long148static inline unsigned long
142lockdep_count_forward_deps(struct lock_class *class)149lockdep_count_forward_deps(struct lock_class *class)
kernel/locking/lockdep_proc.c+7−1
@@ -285,6 +285,12 @@ static int lockdep_stats_show(struct seq_file *m, void *v) · 284 unmodified lines
285 nr_process_chains);285 nr_process_chains);
286 seq_printf(m, " stack-trace entries: %11lu [max: %lu]\n",286 seq_printf(m, " stack-trace entries: %11lu [max: %lu]\n",
287 nr_stack_trace_entries, MAX_STACK_TRACE_ENTRIES);287 nr_stack_trace_entries, MAX_STACK_TRACE_ENTRIES);
288#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
289 seq_printf(m, " number of stack traces: %llu\n",
290 lockdep_stack_trace_count());
291 seq_printf(m, " number of stack hash chains: %llu\n",
292 lockdep_stack_hash_count());
293#endif
288 seq_printf(m, " combined max dependencies: %11u\n",294 seq_printf(m, " combined max dependencies: %11u\n",
289 (nr_hardirq_chains + 1) *295 (nr_hardirq_chains + 1) *
290 (nr_softirq_chains + 1) *296 (nr_softirq_chains + 1) *
@@ -399,7 +405,7 @@ static void seq_lock_time(struct seq_file *m, struct lock_time *lt) · 108 unmodified lines
399 405
400static void seq_stats(struct seq_file *m, struct lock_stat_data *data)406static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
401{407{
402 struct lockdep_subclass_key *ckey;408 const struct lockdep_subclass_key *ckey;
403 struct lock_class_stats *stats;409 struct lock_class_stats *stats;
404 struct lock_class *class;410 struct lock_class *class;
405 const char *cname;411 const char *cname;
kernel/locking/mutex.c+26−0
@@ -65,11 +65,37 @@ EXPORT_SYMBOL(__mutex_init); · 64 unmodified lines
65 65
66#define MUTEX_FLAGS 0x0766#define MUTEX_FLAGS 0x07
67 67
68/*
69 * Internal helper function; C doesn't allow us to hide it :/
70 *
71 * DO NOT USE (outside of mutex code).
72 */
73static inline struct task_struct *__mutex_owner(struct mutex *lock)
74{
75 return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);
76}
77
68static inline struct task_struct *__owner_task(unsigned long owner)78static inline struct task_struct *__owner_task(unsigned long owner)
69{79{
70 return (struct task_struct *)(owner & ~MUTEX_FLAGS);80 return (struct task_struct *)(owner & ~MUTEX_FLAGS);
71}81}
72 82
83bool mutex_is_locked(struct mutex *lock)
84{
85 return __mutex_owner(lock) != NULL;
86}
87EXPORT_SYMBOL(mutex_is_locked);
88
89__must_check enum mutex_trylock_recursive_enum
90mutex_trylock_recursive(struct mutex *lock)
91{
92 if (unlikely(__mutex_owner(lock) == current))
93 return MUTEX_TRYLOCK_RECURSIVE;
94
95 return mutex_trylock(lock);
96}
97EXPORT_SYMBOL(mutex_trylock_recursive);
98
73static inline unsigned long __owner_flags(unsigned long owner)99static inline unsigned long __owner_flags(unsigned long owner)
74{100{
75 return owner & MUTEX_FLAGS;101 return owner & MUTEX_FLAGS;
kernel/locking/rwsem.c+44−10
@@ -105,8 +105,9 @@ · 104 unmodified lines
105#ifdef CONFIG_DEBUG_RWSEMS105#ifdef CONFIG_DEBUG_RWSEMS
106# define DEBUG_RWSEMS_WARN_ON(c, sem) do { \106# define DEBUG_RWSEMS_WARN_ON(c, sem) do { \
107 if (!debug_locks_silent && \107 if (!debug_locks_silent && \
108 WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\108 WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, magic = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\
109 #c, atomic_long_read(&(sem)->count), \109 #c, atomic_long_read(&(sem)->count), \
110 (unsigned long) sem->magic, \
110 atomic_long_read(&(sem)->owner), (long)current, \111 atomic_long_read(&(sem)->owner), (long)current, \
111 list_empty(&(sem)->wait_list) ? "" : "not ")) \112 list_empty(&(sem)->wait_list) ? "" : "not ")) \
112 debug_locks_off(); \113 debug_locks_off(); \
@@ -329,6 +330,9 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name, · 216 unmodified lines
329 */330 */
330 debug_check_no_locks_freed((void *)sem, sizeof(*sem));331 debug_check_no_locks_freed((void *)sem, sizeof(*sem));
331 lockdep_init_map(&sem->dep_map, name, key, 0);332 lockdep_init_map(&sem->dep_map, name, key, 0);
333#endif
334#ifdef CONFIG_DEBUG_RWSEMS
335 sem->magic = sem;
332#endif336#endif
333 atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);337 atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
334 raw_spin_lock_init(&sem->wait_lock);338 raw_spin_lock_init(&sem->wait_lock);
@@ -724,11 +728,12 @@ rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable) · 389 unmodified lines
724 728
725 rcu_read_lock();729 rcu_read_lock();
726 for (;;) {730 for (;;) {
727 if (atomic_long_read(&sem->count) & RWSEM_FLAG_HANDOFF) {731 /*
728 state = OWNER_NONSPINNABLE;732 * When a waiting writer set the handoff flag, it may spin
729 break;733 * on the owner as well. Once that writer acquires the lock,
730 }734 * we can spin on it. So we don't need to quit even when the
731 735 * handoff bit is set.
736 */
732 new = rwsem_owner_flags(sem, &new_flags);737 new = rwsem_owner_flags(sem, &new_flags);
733 if ((new != owner) || (new_flags != flags)) {738 if ((new != owner) || (new_flags != flags)) {
734 state = rwsem_owner_state(new, new_flags, nonspinnable);739 state = rwsem_owner_state(new, new_flags, nonspinnable);
@@ -974,6 +979,13 @@ static inline bool rwsem_reader_phase_trylock(struct rw_semaphore *sem, · 239 unmodified lines
974{979{
975 return false;980 return false;
976}981}
982
983static inline int
984rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable)
985{
986 return 0;
987}
988#define OWNER_NULL 1
977#endif989#endif
978 990
979/*991/*
@@ -1206,6 +1218,18 @@ wait: · 226 unmodified lines
1206 1218
1207 raw_spin_unlock_irq(&sem->wait_lock);1219 raw_spin_unlock_irq(&sem->wait_lock);
1208 1220
1221 /*
1222 * After setting the handoff bit and failing to acquire
1223 * the lock, attempt to spin on owner to accelerate lock
1224 * transfer. If the previous owner is a on-cpu writer and it
1225 * has just released the lock, OWNER_NULL will be returned.
1226 * In this case, we attempt to acquire the lock again
1227 * without sleeping.
1228 */
1229 if ((wstate == WRITER_HANDOFF) &&
1230 (rwsem_spin_on_owner(sem, 0) == OWNER_NULL))
1231 goto trylock_again;
1232
1209 /* Block until there are no active lockers. */1233 /* Block until there are no active lockers. */
1210 for (;;) {1234 for (;;) {
1211 if (signal_pending_state(state, current))1235 if (signal_pending_state(state, current))
@@ -1240,7 +1264,7 @@ wait: · 28 unmodified lines
1240 break;1264 break;
1241 }1265 }
1242 }1266 }
1243 1267trylock_again:
1244 raw_spin_lock_irq(&sem->wait_lock);1268 raw_spin_lock_irq(&sem->wait_lock);
1245 }1269 }
1246 __set_current_state(TASK_RUNNING);1270 __set_current_state(TASK_RUNNING);
@@ -1338,11 +1362,14 @@ static inline int __down_read_killable(struct rw_semaphore *sem) · 91 unmodified lines
1338 1362
1339static inline int __down_read_trylock(struct rw_semaphore *sem)1363static inline int __down_read_trylock(struct rw_semaphore *sem)
1340{1364{
1365 long tmp;
1366
1367 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
1368
1341 /*1369 /*
1342 * Optimize for the case when the rwsem is not locked at all.1370 * Optimize for the case when the rwsem is not locked at all.
1343 */1371 */
1344 long tmp = RWSEM_UNLOCKED_VALUE;1372 tmp = RWSEM_UNLOCKED_VALUE;
1345
1346 do {1373 do {
1347 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,1374 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,
1348 tmp + RWSEM_READER_BIAS)) {1375 tmp + RWSEM_READER_BIAS)) {
@@ -1383,8 +1410,11 @@ static inline int __down_write_killable(struct rw_semaphore *sem) · 34 unmodified lines
1383 1410
1384static inline int __down_write_trylock(struct rw_semaphore *sem)1411static inline int __down_write_trylock(struct rw_semaphore *sem)
1385{1412{
1386 long tmp = RWSEM_UNLOCKED_VALUE;1413 long tmp;
1387 1414
1415 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
1416
1417 tmp = RWSEM_UNLOCKED_VALUE;
1388 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,1418 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,
1389 RWSEM_WRITER_LOCKED)) {1419 RWSEM_WRITER_LOCKED)) {
1390 rwsem_set_owner(sem);1420 rwsem_set_owner(sem);
@@ -1400,7 +1430,9 @@ inline void __up_read(struct rw_semaphore *sem) · 9 unmodified lines
1400{1430{
1401 long tmp;1431 long tmp;
1402 1432
1433 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
1403 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);1434 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
1435
1404 rwsem_clear_reader_owned(sem);1436 rwsem_clear_reader_owned(sem);
1405 tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);1437 tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);
1406 DEBUG_RWSEMS_WARN_ON(tmp < 0, sem);1438 DEBUG_RWSEMS_WARN_ON(tmp < 0, sem);
@@ -1418,12 +1450,14 @@ static inline void __up_write(struct rw_semaphore *sem) · 11 unmodified lines
1418{1450{
1419 long tmp;1451 long tmp;
1420 1452
1453 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
1421 /*1454 /*
1422 * sem->owner may differ from current if the ownership is transferred1455 * sem->owner may differ from current if the ownership is transferred
1423 * to an anonymous writer by setting the RWSEM_NONSPINNABLE bits.1456 * to an anonymous writer by setting the RWSEM_NONSPINNABLE bits.
1424 */1457 */
1425 DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) &&1458 DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) &&
1426 !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);1459 !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);
1460
1427 rwsem_clear_owner(sem);1461 rwsem_clear_owner(sem);
1428 tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);1462 tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
1429 if (unlikely(tmp & RWSEM_FLAG_WAITERS))1463 if (unlikely(tmp & RWSEM_FLAG_WAITERS))
kernel/stacktrace.c+2−2
@@ -20,7 +20,7 @@ · 19 unmodified lines
20 * @nr_entries: Number of entries in the storage array20 * @nr_entries: Number of entries in the storage array
21 * @spaces: Number of leading spaces to print21 * @spaces: Number of leading spaces to print
22 */22 */
23void stack_trace_print(unsigned long *entries, unsigned int nr_entries,23void stack_trace_print(const unsigned long *entries, unsigned int nr_entries,
24 int spaces)24 int spaces)
25{25{
26 unsigned int i;26 unsigned int i;
@@ -43,7 +43,7 @@ EXPORT_SYMBOL_GPL(stack_trace_print); · 16 unmodified lines
43 *43 *
44 * Return: Number of bytes printed.44 * Return: Number of bytes printed.
45 */45 */
46int stack_trace_snprint(char *buf, size_t size, unsigned long *entries,46int stack_trace_snprint(char *buf, size_t size, const unsigned long *entries,
47 unsigned int nr_entries, int spaces)47 unsigned int nr_entries, int spaces)
48{48{
49 unsigned int generated, i, total = 0;49 unsigned int generated, i, total = 0;