History

Merge tag 'locking_core_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Borislav Petkov: "Lots of cleanups and preparation. Highlights: - futex: Cleanup and remove runtime futex_cmpxchg detection - rtmutex: Some fixes for the PREEMPT_RT locking infrastructure - kcsan: Share owner_on_cpu() between mutex,rtmutex and rwsem and annotate the racy owner->on_cpu access *once*. - atomic64: Dead-Code-Elemination" [ Description above by Peter Zijlstra ] * tag 'locking_core_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/atomic: atomic64: Remove unusable atomic ops futex: Fix additional regressions locking: Allow to include asm/spinlock_types.h from linux/spinlock_types_raw.h x86/mm: Include spinlock_t definition in pgtable. locking: Mark racy reads of owner->on_cpu locking: Make owner_on_cpu() into <linux/sched.h> lockdep/selftests: Adapt ww-tests for PREEMPT_RT lockdep/selftests: Skip the softirq related tests on PREEMPT_RT lockdep/selftests: Unbalanced migrate_disable() & rcu_read_lock(). lockdep/selftests: Avoid using local_lock_{acquire|release}(). lockdep: Remove softirq accounting on PREEMPT_RT. locking/rtmutex: Add rt_mutex_lock_nest_lock() and rt_mutex_lock_killable(). locking/rtmutex: Squash self-deadlock check for ww_rt_mutex. locking: Remove rt_rwlock_is_contended(). sched: Trigger warning if ->migration_disabled counter underflows. futex: Fix sparc32/m68k/nds32 build regression futex: Remove futex_cmpxchg detection futex: Ensure futex_atomic_cmpxchg_inatomic() is present kernel/locking: Use a pointer in ww_mutex_trylock().

Linus Torvalds committed 4 years ago

Commit daadb3b2 parents 6ae7143 f16cc9845 files · +240 −225Browse files

Showing the changes to kernel (11 of 45 files). Show all

Unified · Split

11 files
  1. kernel/futex/core.c +0 −35
  2. kernel/futex/futex.h +0 −6
  3. kernel/futex/syscalls.c +0 −22
  4. kernel/locking/lockdep.c +2 −0
  5. kernel/locking/mutex.c +2 −9
  6. kernel/locking/rtmutex.c +6 −4
  7. kernel/locking/rtmutex_api.c +26 −4
  8. kernel/locking/rwsem.c +0 −9
  9. kernel/locking/spinlock_rt.c +0 −6
  10. kernel/locking/ww_rt_mutex.c +1 −1
  11. kernel/sched/core.c +3 −0
kernel/futex/core.c+0−35
@@ -41,11 +41,6 @@ · 40 unmodified lines
4141 #include "futex.h"
4242 #include "../locking/rtmutex_common.h"
4343
44−#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
45−int __read_mostly futex_cmpxchg_enabled;
46−#endif
47−
48−
4944 /*
5045 * The base of the bucket array and its size are always used together
5146 * (after initialization only in futex_hash()), so ensure that they
@@ -776,9 +771,6 @@ static void exit_robust_list(struct task_struct *curr) · 724 unmodified lines
776771 unsigned long futex_offset;
777772 int rc;
778773
779− if (!futex_cmpxchg_enabled)
780− return;
781−
782774 /*
783775 * Fetch the list head (which was registered earlier, via
784776 * sys_set_robust_list()):
@@ -874,9 +866,6 @@ static void compat_exit_robust_list(struct task_struct *curr) · 89 unmodified lines
874866 compat_long_t futex_offset;
875867 int rc;
876868
877− if (!futex_cmpxchg_enabled)
878− return;
879−
880869 /*
881870 * Fetch the list head (which was registered earlier, via
882871 * sys_set_robust_list()):
@@ -950,8 +939,6 @@ static void exit_pi_state_list(struct task_struct *curr) · 67 unmodified lines
950939 struct futex_hash_bucket *hb;
951940 union futex_key key = FUTEX_KEY_INIT;
952941
953− if (!futex_cmpxchg_enabled)
954− return;
955942 /*
956943 * We are a ZOMBIE and nobody can enqueue itself on
957944 * pi_state_list anymore, but we have to be careful
@@ -1125,26 +1112,6 @@ void futex_exit_release(struct task_struct *tsk) · 167 unmodified lines
11251112 futex_cleanup_end(tsk, FUTEX_STATE_DEAD);
11261113 }
11271114
1128−static void __init futex_detect_cmpxchg(void)
1129−{
1130−#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
1131− u32 curval;
1132−
1133− /*
1134− * This will fail and we want it. Some arch implementations do
1135− * runtime detection of the futex_atomic_cmpxchg_inatomic()
1136− * functionality. We want to know that before we call in any
1137− * of the complex code paths. Also we want to prevent
1138− * registration of robust lists in that case. NULL is
1139− * guaranteed to fault and we get -EFAULT on functional
1140− * implementation, the non-functional ones will return
1141− * -ENOSYS.
1142− */
1143− if (futex_cmpxchg_value_locked(&curval, NULL, 0, 0) == -EFAULT)
1144− futex_cmpxchg_enabled = 1;
1145−#endif
1146−}
1147−
11481115 static int __init futex_init(void)
11491116 {
11501117 unsigned int futex_shift;
@@ -1163,8 +1130,6 @@ static int __init futex_init(void) · 12 unmodified lines
11631130 futex_hashsize, futex_hashsize);
11641131 futex_hashsize = 1UL << futex_shift;
11651132
1166− futex_detect_cmpxchg();
1167−
11681133 for (i = 0; i < futex_hashsize; i++) {
11691134 atomic_set(&futex_queues[i].waiters, 0);
11701135 plist_head_init(&futex_queues[i].chain);
kernel/futex/futex.h+0−6
@@ -27,12 +27,6 @@ · 26 unmodified lines
2727 #define FLAGS_CLOCKRT 0x02
2828 #define FLAGS_HAS_TIMEOUT 0x04
2929
30−#ifdef CONFIG_HAVE_FUTEX_CMPXCHG
31−#define futex_cmpxchg_enabled 1
32−#else
33−extern int __read_mostly futex_cmpxchg_enabled;
34−#endif
35−
3630 #ifdef CONFIG_FAIL_FUTEX
3731 extern bool should_fail_futex(bool fshared);
3832 #else
kernel/futex/syscalls.c+0−22
@@ -29,8 +29,6 @@ · 28 unmodified lines
2929 SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,
3030 size_t, len)
3131 {
32− if (!futex_cmpxchg_enabled)
33− return -ENOSYS;
3432 /*
3533 * The kernel knows only one size for now:
3634 */
@@ -56,9 +54,6 @@ SYSCALL_DEFINE3(get_robust_list, int, pid, · 19 unmodified lines
5654 unsigned long ret;
5755 struct task_struct *p;
5856
59− if (!futex_cmpxchg_enabled)
60− return -ENOSYS;
61−
6257 rcu_read_lock();
6358
6459 ret = -ESRCH;
@@ -103,17 +98,6 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, · 38 unmodified lines
10398 return -ENOSYS;
10499 }
105100
106− switch (cmd) {
107− case FUTEX_LOCK_PI:
108− case FUTEX_LOCK_PI2:
109− case FUTEX_UNLOCK_PI:
110− case FUTEX_TRYLOCK_PI:
111− case FUTEX_WAIT_REQUEUE_PI:
112− case FUTEX_CMP_REQUEUE_PI:
113− if (!futex_cmpxchg_enabled)
114− return -ENOSYS;
115− }
116−
117101 switch (cmd) {
118102 case FUTEX_WAIT:
119103 val3 = FUTEX_BITSET_MATCH_ANY;
@@ -323,9 +307,6 @@ COMPAT_SYSCALL_DEFINE2(set_robust_list, · 203 unmodified lines
323307 struct compat_robust_list_head __user *, head,
324308 compat_size_t, len)
325309 {
326− if (!futex_cmpxchg_enabled)
327− return -ENOSYS;
328−
329310 if (unlikely(len != sizeof(*head)))
330311 return -EINVAL;
331312
@@ -342,9 +323,6 @@ COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid, · 10 unmodified lines
342323 unsigned long ret;
343324 struct task_struct *p;
344325
345− if (!futex_cmpxchg_enabled)
346− return -ENOSYS;
347−
348326 rcu_read_lock();
349327
350328 ret = -ESRCH;
kernel/locking/lockdep.c+2−0
@@ -5485,6 +5485,7 @@ static noinstr void check_flags(unsigned long flags) · 5484 unmodified lines
54855485 }
54865486 }
54875487
5488+#ifndef CONFIG_PREEMPT_RT
54885489 /*
54895490 * We dont accurately track softirq state in e.g.
54905491 * hardirq contexts (such as on 4KSTACKS), so only
@@ -5499,6 +5500,7 @@ static noinstr void check_flags(unsigned long flags) · 8 unmodified lines
54995500 DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
55005501 }
55015502 }
5503+#endif
55025504
55035505 if (!debug_locks)
55045506 print_irqtrace_events(current);
kernel/locking/mutex.c+2−9
@@ -367,8 +367,7 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, · 366 unmodified lines
367367 /*
368368 * Use vcpu_is_preempted to detect lock holder preemption issue.
369369 */
370− if (!owner->on_cpu || need_resched() ||
371− vcpu_is_preempted(task_cpu(owner))) {
370+ if (!owner_on_cpu(owner) || need_resched()) {
372371 ret = false;
373372 break;
374373 }
@@ -403,14 +402,8 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock) · 28 unmodified lines
403402 * structure won't go away during the spinning period.
404403 */
405404 owner = __mutex_owner(lock);
406−
407− /*
408− * As lock holder preemption issue, we both skip spinning if task is not
409− * on cpu or its cpu is preempted
410− */
411−
412405 if (owner)
413− retval = owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));
406+ retval = owner_on_cpu(owner);
414407
415408 /*
416409 * If lock->owner is not set, the mutex has been released. Return true
kernel/locking/rtmutex.c+6−4
@@ -1103,8 +1103,11 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock, · 1102 unmodified lines
11031103 * the other will detect the deadlock and return -EDEADLOCK,
11041104 * which is wrong, as the other waiter is not in a deadlock
11051105 * situation.
1106+ *
1107+ * Except for ww_mutex, in that case the chain walk must already deal
1108+ * with spurious cycles, see the comments at [3] and [6].
11061109 */
1107− if (owner == task)
1110+ if (owner == task && !(build_ww_mutex() && ww_ctx))
11081111 return -EDEADLK;
11091112
11101113 raw_spin_lock(&task->pi_lock);
@@ -1379,9 +1382,8 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock, · 268 unmodified lines
13791382 * for CONFIG_PREEMPT_RCU=y)
13801383 * - the VCPU on which owner runs is preempted
13811384 */
1382− if (!owner->on_cpu || need_resched() ||
1383− !rt_mutex_waiter_is_top_waiter(lock, waiter) ||
1384− vcpu_is_preempted(task_cpu(owner))) {
1385+ if (!owner_on_cpu(owner) || need_resched() ||
1386+ !rt_mutex_waiter_is_top_waiter(lock, waiter)) {
13851387 res = false;
13861388 break;
13871389 }
kernel/locking/rtmutex_api.c+26−4
@@ -21,12 +21,13 @@ int max_lock_depth = 1024; · 20 unmodified lines
2121 */
2222 static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,
2323 unsigned int state,
24+ struct lockdep_map *nest_lock,
2425 unsigned int subclass)
2526 {
2627 int ret;
2728
2829 might_sleep();
29− mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
30+ mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, _RET_IP_);
3031 ret = __rt_mutex_lock(&lock->rtmutex, state);
3132 if (ret)
3233 mutex_release(&lock->dep_map, _RET_IP_);
@@ -48,10 +49,16 @@ EXPORT_SYMBOL(rt_mutex_base_init); · 15 unmodified lines
4849 */
4950 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
5051 {
51− __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass);
52+ __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass);
5253 }
5354 EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
5455
56+void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)
57+{
58+ __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0);
59+}
60+EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
61+
5562 #else /* !CONFIG_DEBUG_LOCK_ALLOC */
5663
5764 /**
@@ -61,7 +68,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); · 3 unmodified lines
6168 */
6269 void __sched rt_mutex_lock(struct rt_mutex *lock)
6370 {
64− __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0);
71+ __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0);
6572 }
6673 EXPORT_SYMBOL_GPL(rt_mutex_lock);
6774 #endif
@@ -77,10 +84,25 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock); · 9 unmodified lines
7784 */
7885 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
7986 {
80− return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0);
87+ return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, NULL, 0);
8188 }
8289 EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
8390
91+/**
92+ * rt_mutex_lock_killable - lock a rt_mutex killable
93+ *
94+ * @lock: the rt_mutex to be locked
95+ *
96+ * Returns:
97+ * 0 on success
98+ * -EINTR when interrupted by a signal
99+ */
100+int __sched rt_mutex_lock_killable(struct rt_mutex *lock)
101+{
102+ return __rt_mutex_lock_common(lock, TASK_KILLABLE, NULL, 0);
103+}
104+EXPORT_SYMBOL_GPL(rt_mutex_lock_killable);
105+
84106 /**
85107 * rt_mutex_trylock - try to lock a rt_mutex
86108 *
kernel/locking/rwsem.c+0−9
@@ -658,15 +658,6 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) · 657 unmodified lines
658658 return false;
659659 }
660660
661−static inline bool owner_on_cpu(struct task_struct *owner)
662−{
663− /*
664− * As lock holder preemption issue, we both skip spinning if
665− * task is not on cpu or its cpu is preempted
666− */
667− return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));
668−}
669−
670661 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
671662 {
672663 struct task_struct *owner;
kernel/locking/spinlock_rt.c+0−6
@@ -257,12 +257,6 @@ void __sched rt_write_unlock(rwlock_t *rwlock) · 256 unmodified lines
257257 }
258258 EXPORT_SYMBOL(rt_write_unlock);
259259
260−int __sched rt_rwlock_is_contended(rwlock_t *rwlock)
261−{
262− return rw_base_is_contended(&rwlock->rwbase);
263−}
264−EXPORT_SYMBOL(rt_rwlock_is_contended);
265−
266260 #ifdef CONFIG_DEBUG_LOCK_ALLOC
267261 void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
268262 struct lock_class_key *key)
kernel/locking/ww_rt_mutex.c+1−1
@@ -26,7 +26,7 @@ int ww_mutex_trylock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) · 25 unmodified lines
2626
2727 if (__rt_mutex_trylock(&rtm->rtmutex)) {
2828 ww_mutex_set_context_fastpath(lock, ww_ctx);
29− mutex_acquire_nest(&rtm->dep_map, 0, 1, ww_ctx->dep_map, _RET_IP_);
29+ mutex_acquire_nest(&rtm->dep_map, 0, 1, &ww_ctx->dep_map, _RET_IP_);
3030 return 1;
3131 }
3232
kernel/sched/core.c+3−0
@@ -2184,6 +2184,9 @@ void migrate_enable(void) · 2183 unmodified lines
21842184 return;
21852185 }
21862186
2187+ if (WARN_ON_ONCE(!p->migration_disabled))
2188+ return;
2189+
21872190 /*
21882191 * Ensure stop_task runs either before or after this, and that
21892192 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t