Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Ingo Molnar: - improve rwsem scalability - add uninitialized rwsem debugging check - reduce lockdep's stacktrace memory usage and add diagnostics - misc cleanups, code consolidation and constification * 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: mutex: Fix up mutex_waiter usage locking/mutex: Use mutex flags macro instead of hard code locking/mutex: Make __mutex_owner static to mutex.c locking/qspinlock,x86: Clarify virt_spin_lock_key locking/rwsem: Check for operations on an uninitialized rwsem locking/rwsem: Make handoff writer optimistically spin on owner locking/lockdep: Report more stack trace statistics locking/lockdep: Reduce space occupied by stack traces stacktrace: Constify 'entries' arguments locking/lockdep: Make it clear that what lock_class::key points at is not modified

Linus Torvalds committed 7 years ago

Commit c7eba512 parents cc9b499 e57d14311 files · +241 −84Browse files

Unified · Split

11 files

arch/x86/include/asm/qspinlock.h +15 −0
include/linux/lockdep.h +4 −7
include/linux/mutex.h +3 −22
include/linux/rwsem.h +10 −0
include/linux/stacktrace.h +2 −2
kernel/locking/lockdep.c +120 −39
kernel/locking/lockdep_internals.h +8 −1
kernel/locking/lockdep_proc.c +7 −1
kernel/locking/mutex.c +26 −0
kernel/locking/rwsem.c +44 −10
kernel/stacktrace.c +2 −2

arch/x86/include/asm/qspinlock.h+15−0

@@ -63,10 +63,25 @@ static inline bool vcpu_is_preempted(long cpu) · 62 unmodified lines
63	63	#endif
64	64
65	65	#ifdef CONFIG_PARAVIRT
	66	+/*
	67	+ * virt_spin_lock_key - enables (by default) the virt_spin_lock() hijack.
	68	+ *
	69	+ * Native (and PV wanting native due to vCPU pinning) should disable this key.
	70	+ * It is done in this backwards fashion to only have a single direction change,
	71	+ * which removes ordering between native_pv_spin_init() and HV setup.
	72	+ */
66	73	DECLARE_STATIC_KEY_TRUE(virt_spin_lock_key);
67	74
68	75	void native_pv_lock_init(void) __init;
69	76
	77	+/*
	78	+ * Shortcut for the queued_spin_lock_slowpath() function that allows
	79	+ * virt to hijack it.
	80	+ *
	81	+ * Returns:
	82	+ * true - lock has been negotiated, all done;
	83	+ * false - queued_spin_lock_slowpath() will do its thing.
	84	+ */
70	85	#define virt_spin_lock virt_spin_lock
71	86	static inline bool virt_spin_lock(struct qspinlock *lock)
72	87	{

include/linux/lockdep.h+4−7

@@ -66,10 +66,7 @@ struct lock_class_key { · 65 unmodified lines
66	66
67	67	extern struct lock_class_key __lockdep_no_validate__;
68	68
69		−struct lock_trace {
70		− unsigned int nr_entries;
71		− unsigned int offset;
72		−};
	69	+struct lock_trace;
73	70
74	71	#define LOCKSTAT_POINTS 4
75	72
@@ -97,7 +94,7 @@ struct lock_class { · 21 unmodified lines
97	94	*/
98	95	struct list_head locks_after, locks_before;
99	96
100		− struct lockdep_subclass_key *key;
	97	+ const struct lockdep_subclass_key *key;
101	98	unsigned int subclass;
102	99	unsigned int dep_gen_id;
103	100
@@ -105,7 +102,7 @@ struct lock_class { · 1 unmodified line
105	102	* IRQ/softirq usage tracking bits:
106	103	*/
107	104	unsigned long usage_mask;
108		− struct lock_trace usage_traces[XXX_LOCK_USAGE_STATES];
	105	+ const struct lock_trace *usage_traces[XXX_LOCK_USAGE_STATES];
109	106
110	107	/*
111	108	* Generation counter, when doing certain classes of graph walking,
@@ -193,7 +190,7 @@ struct lock_list { · 81 unmodified lines
193	190	struct list_head entry;
194	191	struct lock_class *class;
195	192	struct lock_class *links_to;
196		− struct lock_trace trace;
	193	+ const struct lock_trace *trace;
197	194	int distance;
198	195
199	196	/*

include/linux/mutex.h+3−22

@@ -65,16 +65,6 @@ struct mutex { · 64 unmodified lines
65	65	#endif
66	66	};
67	67
68		−/*
69		− * Internal helper function; C doesn't allow us to hide it :/
70		− *
71		− * DO NOT USE (outside of mutex code).
72		− */
73		−static inline struct task_struct __mutex_owner(struct mutex lock)
74		−{
75		− return (struct task_struct *)(atomic_long_read(&lock->owner) & ~0x07);
76		−}
77		−
78	68	/*
79	69	* This is the control structure for tasks blocked on mutex,
80	70	* which resides on the blocked task's kernel stack:
@@ -144,10 +134,7 @@ extern void __mutex_init(struct mutex lock, const char name, · 63 unmodified lines
144	134	*
145	135	* Returns true if the mutex is locked, false if unlocked.
146	136	*/
147		−static inline bool mutex_is_locked(struct mutex *lock)
148		−{
149		− return __mutex_owner(lock) != NULL;
150		−}
	137	+extern bool mutex_is_locked(struct mutex *lock);
151	138
152	139	/*
153	140	* See kernel/locking/mutex.c for detailed documentation of these APIs.
@@ -220,13 +207,7 @@ enum mutex_trylock_recursive_enum { · 66 unmodified lines
220	207	* - MUTEX_TRYLOCK_SUCCESS - lock acquired,
221	208	* - MUTEX_TRYLOCK_RECURSIVE - we already owned the lock.
222	209	*/
223		−static inline /* __deprecated */ __must_check enum mutex_trylock_recursive_enum
224		−mutex_trylock_recursive(struct mutex *lock)
225		−{
226		− if (unlikely(__mutex_owner(lock) == current))
227		− return MUTEX_TRYLOCK_RECURSIVE;
228		−
229		− return mutex_trylock(lock);
230		−}
	210	+extern /* __deprecated */ __must_check enum mutex_trylock_recursive_enum
	211	+mutex_trylock_recursive(struct mutex *lock);
231	212
232	213	#endif /* __LINUX_MUTEX_H */

include/linux/rwsem.h+10−0

@@ -45,6 +45,9 @@ struct rw_semaphore { · 44 unmodified lines
45	45	#endif
46	46	raw_spinlock_t wait_lock;
47	47	struct list_head wait_list;
	48	+#ifdef CONFIG_DEBUG_RWSEMS
	49	+ void *magic;
	50	+#endif
48	51	#ifdef CONFIG_DEBUG_LOCK_ALLOC
49	52	struct lockdep_map dep_map;
50	53	#endif
@@ -73,6 +76,12 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) · 22 unmodified lines
73	76	# define __RWSEM_DEP_MAP_INIT(lockname)
74	77	#endif
75	78
	79	+#ifdef CONFIG_DEBUG_RWSEMS
	80	+# define __DEBUG_RWSEM_INITIALIZER(lockname) , .magic = &lockname
	81	+#else
	82	+# define __DEBUG_RWSEM_INITIALIZER(lockname)
	83	+#endif
	84	+
76	85	#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
77	86	#define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED
78	87	#else
@@ -85,6 +94,7 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) · 6 unmodified lines
85	94	.wait_list = LIST_HEAD_INIT((name).wait_list), \
86	95	.wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock) \
87	96	__RWSEM_OPT_INIT(name) \
	97	+ __DEBUG_RWSEM_INITIALIZER(name) \
88	98	__RWSEM_DEP_MAP_INIT(name) }
89	99
90	100	#define DECLARE_RWSEM(name) \

include/linux/stacktrace.h+2−2

@@ -9,9 +9,9 @@ struct task_struct; · 8 unmodified lines
9	9	struct pt_regs;
10	10
11	11	#ifdef CONFIG_STACKTRACE
12		−void stack_trace_print(unsigned long *trace, unsigned int nr_entries,
	12	+void stack_trace_print(const unsigned long *trace, unsigned int nr_entries,
13	13	int spaces);
14		−int stack_trace_snprint(char buf, size_t size, unsigned long entries,
	14	+int stack_trace_snprint(char buf, size_t size, const unsigned long entries,
15	15	unsigned int nr_entries, int spaces);
16	16	unsigned int stack_trace_save(unsigned long *store, unsigned int size,
17	17	unsigned int skipnr);

kernel/locking/lockdep.c+120−39

@@ -449,33 +449,101 @@ static void print_lockdep_off(const char *bug_msg) · 448 unmodified lines
449	449	unsigned long nr_stack_trace_entries;
450	450
451	451	#ifdef CONFIG_PROVE_LOCKING
	452	+/**
	453	+ * struct lock_trace - single stack backtrace
	454	+ * @hash_entry: Entry in a stack_trace_hash[] list.
	455	+ * @hash: jhash() of @entries.
	456	+ * @nr_entries: Number of entries in @entries.
	457	+ * @entries: Actual stack backtrace.
	458	+ */
	459	+struct lock_trace {
	460	+ struct hlist_node hash_entry;
	461	+ u32 hash;
	462	+ u32 nr_entries;
	463	+ unsigned long entries[0] __aligned(sizeof(unsigned long));
	464	+};
	465	+#define LOCK_TRACE_SIZE_IN_LONGS \
	466	+ (sizeof(struct lock_trace) / sizeof(unsigned long))
452	467	/*
453		− * Stack-trace: tightly packed array of stack backtrace
454		− * addresses. Protected by the graph_lock.
	468	+ * Stack-trace: sequence of lock_trace structures. Protected by the graph_lock.
455	469	*/
456	470	static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];
	471	+static struct hlist_head stack_trace_hash[STACK_TRACE_HASH_SIZE];
	472	+
	473	+static bool traces_identical(struct lock_trace t1, struct lock_trace t2)
	474	+{
	475	+ return t1->hash == t2->hash && t1->nr_entries == t2->nr_entries &&
	476	+ memcmp(t1->entries, t2->entries,
	477	+ t1->nr_entries * sizeof(t1->entries[0])) == 0;
	478	+}
457	479
458		−static int save_trace(struct lock_trace *trace)
	480	+static struct lock_trace *save_trace(void)
459	481	{
460		− unsigned long *entries = stack_trace + nr_stack_trace_entries;
	482	+ struct lock_trace trace, t2;
	483	+ struct hlist_head *hash_head;
	484	+ u32 hash;
461	485	unsigned int max_entries;
462	486
463		− trace->offset = nr_stack_trace_entries;
464		− max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;
465		− trace->nr_entries = stack_trace_save(entries, max_entries, 3);
466		− nr_stack_trace_entries += trace->nr_entries;
	487	+ BUILD_BUG_ON_NOT_POWER_OF_2(STACK_TRACE_HASH_SIZE);
	488	+ BUILD_BUG_ON(LOCK_TRACE_SIZE_IN_LONGS >= MAX_STACK_TRACE_ENTRIES);
	489	+
	490	+ trace = (struct lock_trace *)(stack_trace + nr_stack_trace_entries);
	491	+ max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries -
	492	+ LOCK_TRACE_SIZE_IN_LONGS;
	493	+ trace->nr_entries = stack_trace_save(trace->entries, max_entries, 3);
467	494
468		− if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {
	495	+ if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES -
	496	+ LOCK_TRACE_SIZE_IN_LONGS - 1) {
469	497	if (!debug_locks_off_graph_unlock())
470		− return 0;
	498	+ return NULL;
471	499
472	500	print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");
473	501	dump_stack();
474	502
475		− return 0;
	503	+ return NULL;
476	504	}
477	505
478		− return 1;
	506	+ hash = jhash(trace->entries, trace->nr_entries *
	507	+ sizeof(trace->entries[0]), 0);
	508	+ trace->hash = hash;
	509	+ hash_head = stack_trace_hash + (hash & (STACK_TRACE_HASH_SIZE - 1));
	510	+ hlist_for_each_entry(t2, hash_head, hash_entry) {
	511	+ if (traces_identical(trace, t2))
	512	+ return t2;
	513	+ }
	514	+ nr_stack_trace_entries += LOCK_TRACE_SIZE_IN_LONGS + trace->nr_entries;
	515	+ hlist_add_head(&trace->hash_entry, hash_head);
	516	+
	517	+ return trace;
	518	+}
	519	+
	520	+/* Return the number of stack traces in the stack_trace[] array. */
	521	+u64 lockdep_stack_trace_count(void)
	522	+{
	523	+ struct lock_trace *trace;
	524	+ u64 c = 0;
	525	+ int i;
	526	+
	527	+ for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++) {
	528	+ hlist_for_each_entry(trace, &stack_trace_hash[i], hash_entry) {
	529	+ c++;
	530	+ }
	531	+ }
	532	+
	533	+ return c;
	534	+}
	535	+
	536	+/* Return the number of stack hash chains that have at least one stack trace. */
	537	+u64 lockdep_stack_hash_count(void)
	538	+{
	539	+ u64 c = 0;
	540	+ int i;
	541	+
	542	+ for (i = 0; i < ARRAY_SIZE(stack_trace_hash); i++)
	543	+ if (!hlist_empty(&stack_trace_hash[i]))
	544	+ c++;
	545	+
	546	+ return c;
479	547	}
480	548	#endif
481	549
@@ -511,7 +579,7 @@ static const char *usage_str[] = · 29 unmodified lines
511	579	};
512	580	#endif
513	581
514		−const char * __get_key_name(struct lockdep_subclass_key key, char str)
	582	+const char __get_key_name(const struct lockdep_subclass_key key, char *str)
515	583	{
516	584	return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);
517	585	}
@@ -1235,7 +1303,7 @@ static struct lock_list *alloc_list_entry(void) · 717 unmodified lines
1235	1303	static int add_lock_to_list(struct lock_class *this,
1236	1304	struct lock_class links_to, struct list_head head,
1237	1305	unsigned long ip, int distance,
1238		− struct lock_trace *trace)
	1306	+ const struct lock_trace *trace)
1239	1307	{
1240	1308	struct lock_list *entry;
1241	1309	/*
@@ -1249,7 +1317,7 @@ static int add_lock_to_list(struct lock_class *this, · 7 unmodified lines
1249	1317	entry->class = this;
1250	1318	entry->links_to = links_to;
1251	1319	entry->distance = distance;
1252		− entry->trace = *trace;
	1320	+ entry->trace = trace;
1253	1321	/*
1254	1322	* Both allocation and removal are done under the graph lock; but
1255	1323	* iteration is under RCU-sched; see look_up_lock_class() and
@@ -1470,11 +1538,10 @@ static inline int __bfs_backwards(struct lock_list *src_entry, · 214 unmodified lines
1470	1538
1471	1539	}
1472	1540
1473		−static void print_lock_trace(struct lock_trace *trace, unsigned int spaces)
	1541	+static void print_lock_trace(const struct lock_trace *trace,
	1542	+ unsigned int spaces)
1474	1543	{
1475		− unsigned long *entries = stack_trace + trace->offset;
1476		−
1477		− stack_trace_print(entries, trace->nr_entries, spaces);
	1544	+ stack_trace_print(trace->entries, trace->nr_entries, spaces);
1478	1545	}
1479	1546
1480	1547	/*
@@ -1489,7 +1556,7 @@ print_circular_bug_entry(struct lock_list *target, int depth) · 8 unmodified lines
1489	1556	printk("\n-> #%u", depth);
1490	1557	print_lock_name(target->class);
1491	1558	printk(KERN_CONT ":\n");
1492		− print_lock_trace(&target->trace, 6);
	1559	+ print_lock_trace(target->trace, 6);
1493	1560	}
1494	1561
1495	1562	static void
@@ -1592,7 +1659,8 @@ static noinline void print_circular_bug(struct lock_list *this, · 96 unmodified lines
1592	1659	if (!debug_locks_off_graph_unlock() \|\| debug_locks_silent)
1593	1660	return;
1594	1661
1595		− if (!save_trace(&this->trace))
	1662	+ this->trace = save_trace();
	1663	+ if (!this->trace)
1596	1664	return;
1597	1665
1598	1666	depth = get_lock_depth(target);
@@ -1715,7 +1783,7 @@ check_path(struct lock_class target, struct lock_list src_entry, · 116 unmodified lines
1715	1783	*/
1716	1784	static noinline int
1717	1785	check_noncircular(struct held_lock src, struct held_lock target,
1718		− struct lock_trace *trace)
	1786	+ struct lock_trace **const trace)
1719	1787	{
1720	1788	int ret;
1721	1789	struct lock_list *uninitialized_var(target_entry);
@@ -1729,13 +1797,13 @@ check_noncircular(struct held_lock src, struct held_lock target, · 7 unmodified lines
1729	1797	ret = check_path(hlock_class(target), &src_entry, &target_entry);
1730	1798
1731	1799	if (unlikely(!ret)) {
1732		− if (!trace->nr_entries) {
	1800	+ if (!*trace) {
1733	1801	/*
1734	1802	* If save_trace fails here, the printing might
1735	1803	* trigger a WARN but because of the !nr_entries it
1736	1804	* should not do bad things.
1737	1805	*/
1738		− save_trace(trace);
	1806	+ *trace = save_trace();
1739	1807	}
1740	1808
1741	1809	print_circular_bug(&src_entry, target_entry, src, target);
@@ -1859,7 +1927,7 @@ static void print_lock_class_header(struct lock_class *class, int depth) · 117 unmodified lines
1859	1927
1860	1928	len += printk("%*s %s", depth, "", usage_str[bit]);
1861	1929	len += printk(KERN_CONT " at:\n");
1862		− print_lock_trace(class->usage_traces + bit, len);
	1930	+ print_lock_trace(class->usage_traces[bit], len);
1863	1931	}
1864	1932	}
1865	1933	printk("%*s }\n", depth, "");
@@ -1884,7 +1952,7 @@ print_shortest_lock_dependencies(struct lock_list *leaf, · 18 unmodified lines
1884	1952	do {
1885	1953	print_lock_class_header(entry->class, depth);
1886	1954	printk("%*s ... acquired at:\n", depth, "");
1887		− print_lock_trace(&entry->trace, 2);
	1955	+ print_lock_trace(entry->trace, 2);
1888	1956	printk("\n");
1889	1957
1890	1958	if (depth == 0 && (entry != root)) {
@@ -1995,14 +2063,14 @@ print_bad_irq_dependency(struct task_struct *curr, · 104 unmodified lines
1995	2063	print_lock_name(backwards_entry->class);
1996	2064	pr_warn("\n... which became %s-irq-safe at:\n", irqclass);
1997	2065
1998		− print_lock_trace(backwards_entry->class->usage_traces + bit1, 1);
	2066	+ print_lock_trace(backwards_entry->class->usage_traces[bit1], 1);
1999	2067
2000	2068	pr_warn("\nto a %s-irq-unsafe lock:\n", irqclass);
2001	2069	print_lock_name(forwards_entry->class);
2002	2070	pr_warn("\n... which became %s-irq-unsafe at:\n", irqclass);
2003	2071	pr_warn("...");
2004	2072
2005		− print_lock_trace(forwards_entry->class->usage_traces + bit2, 1);
	2073	+ print_lock_trace(forwards_entry->class->usage_traces[bit2], 1);
2006	2074
2007	2075	pr_warn("\nother info that might help us debug this:\n\n");
2008	2076	print_irq_lock_scenario(backwards_entry, forwards_entry,
@@ -2011,13 +2079,15 @@ print_bad_irq_dependency(struct task_struct *curr, · 2 unmodified lines
2011	2079	lockdep_print_held_locks(curr);
2012	2080
2013	2081	pr_warn("\nthe dependencies between %s-irq-safe lock and the holding lock:\n", irqclass);
2014		− if (!save_trace(&prev_root->trace))
	2082	+ prev_root->trace = save_trace();
	2083	+ if (!prev_root->trace)
2015	2084	return;
2016	2085	print_shortest_lock_dependencies(backwards_entry, prev_root);
2017	2086
2018	2087	pr_warn("\nthe dependencies between the lock to be acquired");
2019	2088	pr_warn(" and %s-irq-unsafe lock:\n", irqclass);
2020		− if (!save_trace(&next_root->trace))
	2089	+ next_root->trace = save_trace();
	2090	+ if (!next_root->trace)
2021	2091	return;
2022	2092	print_shortest_lock_dependencies(forwards_entry, next_root);
2023	2093
@@ -2369,7 +2439,8 @@ check_deadlock(struct task_struct curr, struct held_lock next) · 345 unmodified lines
2369	2439	*/
2370	2440	static int
2371	2441	check_prev_add(struct task_struct curr, struct held_lock prev,
2372		− struct held_lock next, int distance, struct lock_trace trace)
	2442	+ struct held_lock *next, int distance,
	2443	+ struct lock_trace **const trace)
2373	2444	{
2374	2445	struct lock_list *entry;
2375	2446	int ret;
@@ -2444,8 +2515,11 @@ check_prev_add(struct task_struct curr, struct held_lock prev, · 68 unmodified lines
2444	2515	return ret;
2445	2516	#endif
2446	2517
2447		− if (!trace->nr_entries && !save_trace(trace))
2448		− return 0;
	2518	+ if (!*trace) {
	2519	+ *trace = save_trace();
	2520	+ if (!*trace)
	2521	+ return 0;
	2522	+ }
2449	2523
2450	2524	/*
2451	2525	* Ok, all validations passed, add the new lock
@@ -2453,14 +2527,14 @@ check_prev_add(struct task_struct curr, struct held_lock prev, · 1 unmodified line
2453	2527	*/
2454	2528	ret = add_lock_to_list(hlock_class(next), hlock_class(prev),
2455	2529	&hlock_class(prev)->locks_after,
2456		− next->acquire_ip, distance, trace);
	2530	+ next->acquire_ip, distance, *trace);
2457	2531
2458	2532	if (!ret)
2459	2533	return 0;
2460	2534
2461	2535	ret = add_lock_to_list(hlock_class(prev), hlock_class(next),
2462	2536	&hlock_class(next)->locks_before,
2463		− next->acquire_ip, distance, trace);
	2537	+ next->acquire_ip, distance, *trace);
2464	2538	if (!ret)
2465	2539	return 0;
2466	2540
@@ -2476,7 +2550,7 @@ check_prev_add(struct task_struct curr, struct held_lock prev, · 9 unmodified lines
2476	2550	static int
2477	2551	check_prevs_add(struct task_struct curr, struct held_lock next)
2478	2552	{
2479		− struct lock_trace trace = { .nr_entries = 0 };
	2553	+ struct lock_trace *trace = NULL;
2480	2554	int depth = curr->lockdep_depth;
2481	2555	struct held_lock *hlock;
2482	2556
@@ -3015,7 +3089,7 @@ print_usage_bug(struct task_struct curr, struct held_lock this, · 532 unmodified lines
3015	3089	print_lock(this);
3016	3090
3017	3091	pr_warn("{%s} state was registered at:\n", usage_str[prev_bit]);
3018		− print_lock_trace(hlock_class(this)->usage_traces + prev_bit, 1);
	3092	+ print_lock_trace(hlock_class(this)->usage_traces[prev_bit], 1);
3019	3093
3020	3094	print_irqtrace_events(curr);
3021	3095	pr_warn("\nother info that might help us debug this:\n");
@@ -3096,7 +3170,8 @@ print_irq_inversion_bug(struct task_struct *curr, · 74 unmodified lines
3096	3170	lockdep_print_held_locks(curr);
3097	3171
3098	3172	pr_warn("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
3099		− if (!save_trace(&root->trace))
	3173	+ root->trace = save_trace();
	3174	+ if (!root->trace)
3100	3175	return;
3101	3176	print_shortest_lock_dependencies(other, root);
3102	3177
@@ -3580,7 +3655,7 @@ static int mark_lock(struct task_struct curr, struct held_lock this, · 477 unmodified lines
3580	3655
3581	3656	hlock_class(this)->usage_mask \|= new_mask;
3582	3657
3583		− if (!save_trace(hlock_class(this)->usage_traces + new_bit))
	3658	+ if (!(hlock_class(this)->usage_traces[new_bit] = save_trace()))
3584	3659	return 0;
3585	3660
3586	3661	switch (new_bit) {
@@ -5157,6 +5232,12 @@ void __init lockdep_init(void) · 1570 unmodified lines
5157	5232	) / 1024
5158	5233	);
5159	5234
	5235	+#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
	5236	+ printk(" memory used for stack traces: %zu kB\n",
	5237	+ (sizeof(stack_trace) + sizeof(stack_trace_hash)) / 1024
	5238	+ );
	5239	+#endif
	5240	+
5160	5241	printk(" per task-struct memory footprint: %zu bytes\n",
5161	5242	sizeof(((struct task_struct *)NULL)->held_locks));
5162	5243	}

kernel/locking/lockdep_internals.h+8−1

@@ -92,6 +92,7 @@ static const unsigned long LOCKF_USED_IN_IRQ_READ = · 91 unmodified lines
92	92	#define MAX_LOCKDEP_ENTRIES 16384UL
93	93	#define MAX_LOCKDEP_CHAINS_BITS 15
94	94	#define MAX_STACK_TRACE_ENTRIES 262144UL
	95	+#define STACK_TRACE_HASH_SIZE 8192
95	96	#else
96	97	#define MAX_LOCKDEP_ENTRIES 32768UL
97	98
@@ -102,6 +103,7 @@ static const unsigned long LOCKF_USED_IN_IRQ_READ = · 4 unmodified lines
102	103	* addresses. Protected by the hash_lock.
103	104	*/
104	105	#define MAX_STACK_TRACE_ENTRIES 524288UL
	106	+#define STACK_TRACE_HASH_SIZE 16384
105	107	#endif
106	108
107	109	#define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS)
@@ -116,7 +118,8 @@ extern struct lock_chain lock_chains[]; · 8 unmodified lines
116	118	extern void get_usage_chars(struct lock_class *class,
117	119	char usage[LOCK_USAGE_CHARS]);
118	120
119		−extern const char * __get_key_name(struct lockdep_subclass_key key, char str);
	121	+extern const char __get_key_name(const struct lockdep_subclass_key key,
	122	+ char *str);
120	123
121	124	struct lock_class lock_chain_get_class(struct lock_chain chain, int i);
122	125
@@ -137,6 +140,10 @@ extern unsigned int max_bfs_queue_depth; · 14 unmodified lines
137	140	#ifdef CONFIG_PROVE_LOCKING
138	141	extern unsigned long lockdep_count_forward_deps(struct lock_class *);
139	142	extern unsigned long lockdep_count_backward_deps(struct lock_class *);
	143	+#ifdef CONFIG_TRACE_IRQFLAGS
	144	+u64 lockdep_stack_trace_count(void);
	145	+u64 lockdep_stack_hash_count(void);
	146	+#endif
140	147	#else
141	148	static inline unsigned long
142	149	lockdep_count_forward_deps(struct lock_class *class)

kernel/locking/lockdep_proc.c+7−1

@@ -285,6 +285,12 @@ static int lockdep_stats_show(struct seq_file m, void v) · 284 unmodified lines
285	285	nr_process_chains);
286	286	seq_printf(m, " stack-trace entries: %11lu [max: %lu]\n",
287	287	nr_stack_trace_entries, MAX_STACK_TRACE_ENTRIES);
	288	+#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
	289	+ seq_printf(m, " number of stack traces: %llu\n",
	290	+ lockdep_stack_trace_count());
	291	+ seq_printf(m, " number of stack hash chains: %llu\n",
	292	+ lockdep_stack_hash_count());
	293	+#endif
288	294	seq_printf(m, " combined max dependencies: %11u\n",
289	295	(nr_hardirq_chains + 1) *
290	296	(nr_softirq_chains + 1) *
@@ -399,7 +405,7 @@ static void seq_lock_time(struct seq_file m, struct lock_time lt) · 108 unmodified lines
399	405
400	406	static void seq_stats(struct seq_file m, struct lock_stat_data data)
401	407	{
402		− struct lockdep_subclass_key *ckey;
	408	+ const struct lockdep_subclass_key *ckey;
403	409	struct lock_class_stats *stats;
404	410	struct lock_class *class;
405	411	const char *cname;

kernel/locking/mutex.c+26−0

@@ -65,11 +65,37 @@ EXPORT_SYMBOL(__mutex_init); · 64 unmodified lines
65	65
66	66	#define MUTEX_FLAGS 0x07
67	67
	68	+/*
	69	+ * Internal helper function; C doesn't allow us to hide it :/
	70	+ *
	71	+ * DO NOT USE (outside of mutex code).
	72	+ */
	73	+static inline struct task_struct __mutex_owner(struct mutex lock)
	74	+{
	75	+ return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);
	76	+}
	77	+
68	78	static inline struct task_struct *__owner_task(unsigned long owner)
69	79	{
70	80	return (struct task_struct *)(owner & ~MUTEX_FLAGS);
71	81	}
72	82
	83	+bool mutex_is_locked(struct mutex *lock)
	84	+{
	85	+ return __mutex_owner(lock) != NULL;
	86	+}
	87	+EXPORT_SYMBOL(mutex_is_locked);
	88	+
	89	+__must_check enum mutex_trylock_recursive_enum
	90	+mutex_trylock_recursive(struct mutex *lock)
	91	+{
	92	+ if (unlikely(__mutex_owner(lock) == current))
	93	+ return MUTEX_TRYLOCK_RECURSIVE;
	94	+
	95	+ return mutex_trylock(lock);
	96	+}
	97	+EXPORT_SYMBOL(mutex_trylock_recursive);
	98	+
73	99	static inline unsigned long __owner_flags(unsigned long owner)
74	100	{
75	101	return owner & MUTEX_FLAGS;

kernel/locking/rwsem.c+44−10

@@ -105,8 +105,9 @@ · 104 unmodified lines
105	105	#ifdef CONFIG_DEBUG_RWSEMS
106	106	# define DEBUG_RWSEMS_WARN_ON(c, sem) do { \
107	107	if (!debug_locks_silent && \
108		− WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\
	108	+ WARN_ONCE(c, "DEBUG_RWSEMS_WARN_ON(%s): count = 0x%lx, magic = 0x%lx, owner = 0x%lx, curr 0x%lx, list %sempty\n",\
109	109	#c, atomic_long_read(&(sem)->count), \
	110	+ (unsigned long) sem->magic, \
110	111	atomic_long_read(&(sem)->owner), (long)current, \
111	112	list_empty(&(sem)->wait_list) ? "" : "not ")) \
112	113	debug_locks_off(); \
@@ -329,6 +330,9 @@ void __init_rwsem(struct rw_semaphore sem, const char name, · 216 unmodified lines
329	330	*/
330	331	debug_check_no_locks_freed((void )sem, sizeof(sem));
331	332	lockdep_init_map(&sem->dep_map, name, key, 0);
	333	+#endif
	334	+#ifdef CONFIG_DEBUG_RWSEMS
	335	+ sem->magic = sem;
332	336	#endif
333	337	atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
334	338	raw_spin_lock_init(&sem->wait_lock);
@@ -724,11 +728,12 @@ rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable) · 389 unmodified lines
724	728
725	729	rcu_read_lock();
726	730	for (;;) {
727		− if (atomic_long_read(&sem->count) & RWSEM_FLAG_HANDOFF) {
728		− state = OWNER_NONSPINNABLE;
729		− break;
730		− }
731		−
	731	+ /*
	732	+ * When a waiting writer set the handoff flag, it may spin
	733	+ * on the owner as well. Once that writer acquires the lock,
	734	+ * we can spin on it. So we don't need to quit even when the
	735	+ * handoff bit is set.
	736	+ */
732	737	new = rwsem_owner_flags(sem, &new_flags);
733	738	if ((new != owner) \|\| (new_flags != flags)) {
734	739	state = rwsem_owner_state(new, new_flags, nonspinnable);
@@ -974,6 +979,13 @@ static inline bool rwsem_reader_phase_trylock(struct rw_semaphore *sem, · 239 unmodified lines
974	979	{
975	980	return false;
976	981	}
	982	+
	983	+static inline int
	984	+rwsem_spin_on_owner(struct rw_semaphore *sem, unsigned long nonspinnable)
	985	+{
	986	+ return 0;
	987	+}
	988	+#define OWNER_NULL 1
977	989	#endif
978	990
979	991	/*
@@ -1206,6 +1218,18 @@ wait: · 226 unmodified lines
1206	1218
1207	1219	raw_spin_unlock_irq(&sem->wait_lock);
1208	1220
	1221	+ /*
	1222	+ * After setting the handoff bit and failing to acquire
	1223	+ * the lock, attempt to spin on owner to accelerate lock
	1224	+ * transfer. If the previous owner is a on-cpu writer and it
	1225	+ * has just released the lock, OWNER_NULL will be returned.
	1226	+ * In this case, we attempt to acquire the lock again
	1227	+ * without sleeping.
	1228	+ */
	1229	+ if ((wstate == WRITER_HANDOFF) &&
	1230	+ (rwsem_spin_on_owner(sem, 0) == OWNER_NULL))
	1231	+ goto trylock_again;
	1232	+
1209	1233	/* Block until there are no active lockers. */
1210	1234	for (;;) {
1211	1235	if (signal_pending_state(state, current))
@@ -1240,7 +1264,7 @@ wait: · 28 unmodified lines
1240	1264	break;
1241	1265	}
1242	1266	}
1243		−
	1267	+trylock_again:
1244	1268	raw_spin_lock_irq(&sem->wait_lock);
1245	1269	}
1246	1270	__set_current_state(TASK_RUNNING);
@@ -1338,11 +1362,14 @@ static inline int __down_read_killable(struct rw_semaphore *sem) · 91 unmodified lines
1338	1362
1339	1363	static inline int __down_read_trylock(struct rw_semaphore *sem)
1340	1364	{
	1365	+ long tmp;
	1366	+
	1367	+ DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
	1368	+
1341	1369	/*
1342	1370	* Optimize for the case when the rwsem is not locked at all.
1343	1371	*/
1344		− long tmp = RWSEM_UNLOCKED_VALUE;
1345		−
	1372	+ tmp = RWSEM_UNLOCKED_VALUE;
1346	1373	do {
1347	1374	if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,
1348	1375	tmp + RWSEM_READER_BIAS)) {
@@ -1383,8 +1410,11 @@ static inline int __down_write_killable(struct rw_semaphore *sem) · 34 unmodified lines
1383	1410
1384	1411	static inline int __down_write_trylock(struct rw_semaphore *sem)
1385	1412	{
1386		− long tmp = RWSEM_UNLOCKED_VALUE;
	1413	+ long tmp;
1387	1414
	1415	+ DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
	1416	+
	1417	+ tmp = RWSEM_UNLOCKED_VALUE;
1388	1418	if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,
1389	1419	RWSEM_WRITER_LOCKED)) {
1390	1420	rwsem_set_owner(sem);
@@ -1400,7 +1430,9 @@ inline void __up_read(struct rw_semaphore *sem) · 9 unmodified lines
1400	1430	{
1401	1431	long tmp;
1402	1432
	1433	+ DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
1403	1434	DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
	1435	+
1404	1436	rwsem_clear_reader_owned(sem);
1405	1437	tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);
1406	1438	DEBUG_RWSEMS_WARN_ON(tmp < 0, sem);
@@ -1418,12 +1450,14 @@ static inline void __up_write(struct rw_semaphore *sem) · 11 unmodified lines
1418	1450	{
1419	1451	long tmp;
1420	1452
	1453	+ DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);
1421	1454	/*
1422	1455	* sem->owner may differ from current if the ownership is transferred
1423	1456	* to an anonymous writer by setting the RWSEM_NONSPINNABLE bits.
1424	1457	*/
1425	1458	DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) &&
1426	1459	!rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);
	1460	+
1427	1461	rwsem_clear_owner(sem);
1428	1462	tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
1429	1463	if (unlikely(tmp & RWSEM_FLAG_WAITERS))

kernel/stacktrace.c+2−2

@@ -20,7 +20,7 @@ · 19 unmodified lines
20	20	* @nr_entries: Number of entries in the storage array
21	21	* @spaces: Number of leading spaces to print
22	22	*/
23		−void stack_trace_print(unsigned long *entries, unsigned int nr_entries,
	23	+void stack_trace_print(const unsigned long *entries, unsigned int nr_entries,
24	24	int spaces)
25	25	{
26	26	unsigned int i;
@@ -43,7 +43,7 @@ EXPORT_SYMBOL_GPL(stack_trace_print); · 16 unmodified lines
43	43	*
44	44	* Return: Number of bytes printed.
45	45	*/
46		−int stack_trace_snprint(char buf, size_t size, unsigned long entries,
	46	+int stack_trace_snprint(char buf, size_t size, const unsigned long entries,
47	47	unsigned int nr_entries, int spaces)
48	48	{
49	49	unsigned int generated, i, total = 0;