History

Merge tag 'locking_core_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking updates from Borislav Petkov: "Lots of cleanups and preparation. Highlights: - futex: Cleanup and remove runtime futex_cmpxchg detection - rtmutex: Some fixes for the PREEMPT_RT locking infrastructure - kcsan: Share owner_on_cpu() between mutex,rtmutex and rwsem and annotate the racy owner->on_cpu access *once*. - atomic64: Dead-Code-Elemination" [ Description above by Peter Zijlstra ] * tag 'locking_core_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/atomic: atomic64: Remove unusable atomic ops futex: Fix additional regressions locking: Allow to include asm/spinlock_types.h from linux/spinlock_types_raw.h x86/mm: Include spinlock_t definition in pgtable. locking: Mark racy reads of owner->on_cpu locking: Make owner_on_cpu() into <linux/sched.h> lockdep/selftests: Adapt ww-tests for PREEMPT_RT lockdep/selftests: Skip the softirq related tests on PREEMPT_RT lockdep/selftests: Unbalanced migrate_disable() & rcu_read_lock(). lockdep/selftests: Avoid using local_lock_{acquire|release}(). lockdep: Remove softirq accounting on PREEMPT_RT. locking/rtmutex: Add rt_mutex_lock_nest_lock() and rt_mutex_lock_killable(). locking/rtmutex: Squash self-deadlock check for ww_rt_mutex. locking: Remove rt_rwlock_is_contended(). sched: Trigger warning if ->migration_disabled counter underflows. futex: Fix sparc32/m68k/nds32 build regression futex: Remove futex_cmpxchg detection futex: Ensure futex_atomic_cmpxchg_inatomic() is present kernel/locking: Use a pointer in ww_mutex_trylock().

Linus Torvalds committed 4 years ago

Commit daadb3b2 parents 6ae7143 f16cc9845 files · +240 −225Browse files

Showing the changes to kernel (11 of 45 files). Show all

Unified · Split

11 files
  1. kernel/futex/core.c +0 −35
  2. kernel/futex/futex.h +0 −6
  3. kernel/futex/syscalls.c +0 −22
  4. kernel/locking/lockdep.c +2 −0
  5. kernel/locking/mutex.c +2 −9
  6. kernel/locking/rtmutex.c +6 −4
  7. kernel/locking/rtmutex_api.c +26 −4
  8. kernel/locking/rwsem.c +0 −9
  9. kernel/locking/spinlock_rt.c +0 −6
  10. kernel/locking/ww_rt_mutex.c +1 −1
  11. kernel/sched/core.c +3 −0
kernel/futex/core.c+0−35
@@ -41,11 +41,6 @@ · 40 unmodified lines
41#include "futex.h"41#include "futex.h"
42#include "../locking/rtmutex_common.h"42#include "../locking/rtmutex_common.h"
43 43
44#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
45int __read_mostly futex_cmpxchg_enabled;
46#endif
47
48
49/*44/*
50 * The base of the bucket array and its size are always used together45 * The base of the bucket array and its size are always used together
51 * (after initialization only in futex_hash()), so ensure that they46 * (after initialization only in futex_hash()), so ensure that they
@@ -776,9 +771,6 @@ static void exit_robust_list(struct task_struct *curr) · 724 unmodified lines
776 unsigned long futex_offset;771 unsigned long futex_offset;
777 int rc;772 int rc;
778 773
779 if (!futex_cmpxchg_enabled)
780 return;
781
782 /*774 /*
783 * Fetch the list head (which was registered earlier, via775 * Fetch the list head (which was registered earlier, via
784 * sys_set_robust_list()):776 * sys_set_robust_list()):
@@ -874,9 +866,6 @@ static void compat_exit_robust_list(struct task_struct *curr) · 89 unmodified lines
874 compat_long_t futex_offset;866 compat_long_t futex_offset;
875 int rc;867 int rc;
876 868
877 if (!futex_cmpxchg_enabled)
878 return;
879
880 /*869 /*
881 * Fetch the list head (which was registered earlier, via870 * Fetch the list head (which was registered earlier, via
882 * sys_set_robust_list()):871 * sys_set_robust_list()):
@@ -950,8 +939,6 @@ static void exit_pi_state_list(struct task_struct *curr) · 67 unmodified lines
950 struct futex_hash_bucket *hb;939 struct futex_hash_bucket *hb;
951 union futex_key key = FUTEX_KEY_INIT;940 union futex_key key = FUTEX_KEY_INIT;
952 941
953 if (!futex_cmpxchg_enabled)
954 return;
955 /*942 /*
956 * We are a ZOMBIE and nobody can enqueue itself on943 * We are a ZOMBIE and nobody can enqueue itself on
957 * pi_state_list anymore, but we have to be careful944 * pi_state_list anymore, but we have to be careful
@@ -1125,26 +1112,6 @@ void futex_exit_release(struct task_struct *tsk) · 167 unmodified lines
1125 futex_cleanup_end(tsk, FUTEX_STATE_DEAD);1112 futex_cleanup_end(tsk, FUTEX_STATE_DEAD);
1126}1113}
1127 1114
1128static void __init futex_detect_cmpxchg(void)
1129{
1130#ifndef CONFIG_HAVE_FUTEX_CMPXCHG
1131 u32 curval;
1132
1133 /*
1134 * This will fail and we want it. Some arch implementations do
1135 * runtime detection of the futex_atomic_cmpxchg_inatomic()
1136 * functionality. We want to know that before we call in any
1137 * of the complex code paths. Also we want to prevent
1138 * registration of robust lists in that case. NULL is
1139 * guaranteed to fault and we get -EFAULT on functional
1140 * implementation, the non-functional ones will return
1141 * -ENOSYS.
1142 */
1143 if (futex_cmpxchg_value_locked(&curval, NULL, 0, 0) == -EFAULT)
1144 futex_cmpxchg_enabled = 1;
1145#endif
1146}
1147
1148static int __init futex_init(void)1115static int __init futex_init(void)
1149{1116{
1150 unsigned int futex_shift;1117 unsigned int futex_shift;
@@ -1163,8 +1130,6 @@ static int __init futex_init(void) · 12 unmodified lines
1163 futex_hashsize, futex_hashsize);1130 futex_hashsize, futex_hashsize);
1164 futex_hashsize = 1UL << futex_shift;1131 futex_hashsize = 1UL << futex_shift;
1165 1132
1166 futex_detect_cmpxchg();
1167
1168 for (i = 0; i < futex_hashsize; i++) {1133 for (i = 0; i < futex_hashsize; i++) {
1169 atomic_set(&futex_queues[i].waiters, 0);1134 atomic_set(&futex_queues[i].waiters, 0);
1170 plist_head_init(&futex_queues[i].chain);1135 plist_head_init(&futex_queues[i].chain);
kernel/futex/futex.h+0−6
@@ -27,12 +27,6 @@ · 26 unmodified lines
27#define FLAGS_CLOCKRT 0x0227#define FLAGS_CLOCKRT 0x02
28#define FLAGS_HAS_TIMEOUT 0x0428#define FLAGS_HAS_TIMEOUT 0x04
29 29
30#ifdef CONFIG_HAVE_FUTEX_CMPXCHG
31#define futex_cmpxchg_enabled 1
32#else
33extern int __read_mostly futex_cmpxchg_enabled;
34#endif
35
36#ifdef CONFIG_FAIL_FUTEX30#ifdef CONFIG_FAIL_FUTEX
37extern bool should_fail_futex(bool fshared);31extern bool should_fail_futex(bool fshared);
38#else32#else
kernel/futex/syscalls.c+0−22
@@ -29,8 +29,6 @@ · 28 unmodified lines
29SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,29SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,
30 size_t, len)30 size_t, len)
31{31{
32 if (!futex_cmpxchg_enabled)
33 return -ENOSYS;
34 /*32 /*
35 * The kernel knows only one size for now:33 * The kernel knows only one size for now:
36 */34 */
@@ -56,9 +54,6 @@ SYSCALL_DEFINE3(get_robust_list, int, pid, · 19 unmodified lines
56 unsigned long ret;54 unsigned long ret;
57 struct task_struct *p;55 struct task_struct *p;
58 56
59 if (!futex_cmpxchg_enabled)
60 return -ENOSYS;
61
62 rcu_read_lock();57 rcu_read_lock();
63 58
64 ret = -ESRCH;59 ret = -ESRCH;
@@ -103,17 +98,6 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, · 38 unmodified lines
103 return -ENOSYS;98 return -ENOSYS;
104 }99 }
105 100
106 switch (cmd) {
107 case FUTEX_LOCK_PI:
108 case FUTEX_LOCK_PI2:
109 case FUTEX_UNLOCK_PI:
110 case FUTEX_TRYLOCK_PI:
111 case FUTEX_WAIT_REQUEUE_PI:
112 case FUTEX_CMP_REQUEUE_PI:
113 if (!futex_cmpxchg_enabled)
114 return -ENOSYS;
115 }
116
117 switch (cmd) {101 switch (cmd) {
118 case FUTEX_WAIT:102 case FUTEX_WAIT:
119 val3 = FUTEX_BITSET_MATCH_ANY;103 val3 = FUTEX_BITSET_MATCH_ANY;
@@ -323,9 +307,6 @@ COMPAT_SYSCALL_DEFINE2(set_robust_list, · 203 unmodified lines
323 struct compat_robust_list_head __user *, head,307 struct compat_robust_list_head __user *, head,
324 compat_size_t, len)308 compat_size_t, len)
325{309{
326 if (!futex_cmpxchg_enabled)
327 return -ENOSYS;
328
329 if (unlikely(len != sizeof(*head)))310 if (unlikely(len != sizeof(*head)))
330 return -EINVAL;311 return -EINVAL;
331 312
@@ -342,9 +323,6 @@ COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid, · 10 unmodified lines
342 unsigned long ret;323 unsigned long ret;
343 struct task_struct *p;324 struct task_struct *p;
344 325
345 if (!futex_cmpxchg_enabled)
346 return -ENOSYS;
347
348 rcu_read_lock();326 rcu_read_lock();
349 327
350 ret = -ESRCH;328 ret = -ESRCH;
kernel/locking/lockdep.c+2−0
@@ -5485,6 +5485,7 @@ static noinstr void check_flags(unsigned long flags) · 5484 unmodified lines
5485 }5485 }
5486 }5486 }
5487 5487
5488#ifndef CONFIG_PREEMPT_RT
5488 /*5489 /*
5489 * We dont accurately track softirq state in e.g.5490 * We dont accurately track softirq state in e.g.
5490 * hardirq contexts (such as on 4KSTACKS), so only5491 * hardirq contexts (such as on 4KSTACKS), so only
@@ -5499,6 +5500,7 @@ static noinstr void check_flags(unsigned long flags) · 8 unmodified lines
5499 DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);5500 DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
5500 }5501 }
5501 }5502 }
5503#endif
5502 5504
5503 if (!debug_locks)5505 if (!debug_locks)
5504 print_irqtrace_events(current);5506 print_irqtrace_events(current);
kernel/locking/mutex.c+2−9
@@ -367,8 +367,7 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, · 366 unmodified lines
367 /*367 /*
368 * Use vcpu_is_preempted to detect lock holder preemption issue.368 * Use vcpu_is_preempted to detect lock holder preemption issue.
369 */369 */
370 if (!owner->on_cpu || need_resched() ||370 if (!owner_on_cpu(owner) || need_resched()) {
371 vcpu_is_preempted(task_cpu(owner))) {
372 ret = false;371 ret = false;
373 break;372 break;
374 }373 }
@@ -403,14 +402,8 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock) · 28 unmodified lines
403 * structure won't go away during the spinning period.402 * structure won't go away during the spinning period.
404 */403 */
405 owner = __mutex_owner(lock);404 owner = __mutex_owner(lock);
406
407 /*
408 * As lock holder preemption issue, we both skip spinning if task is not
409 * on cpu or its cpu is preempted
410 */
411
412 if (owner)405 if (owner)
413 retval = owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));406 retval = owner_on_cpu(owner);
414 407
415 /*408 /*
416 * If lock->owner is not set, the mutex has been released. Return true409 * If lock->owner is not set, the mutex has been released. Return true
kernel/locking/rtmutex.c+6−4
@@ -1103,8 +1103,11 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock, · 1102 unmodified lines
1103 * the other will detect the deadlock and return -EDEADLOCK,1103 * the other will detect the deadlock and return -EDEADLOCK,
1104 * which is wrong, as the other waiter is not in a deadlock1104 * which is wrong, as the other waiter is not in a deadlock
1105 * situation.1105 * situation.
1106 *
1107 * Except for ww_mutex, in that case the chain walk must already deal
1108 * with spurious cycles, see the comments at [3] and [6].
1106 */1109 */
1107 if (owner == task)1110 if (owner == task && !(build_ww_mutex() && ww_ctx))
1108 return -EDEADLK;1111 return -EDEADLK;
1109 1112
1110 raw_spin_lock(&task->pi_lock);1113 raw_spin_lock(&task->pi_lock);
@@ -1379,9 +1382,8 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock, · 268 unmodified lines
1379 * for CONFIG_PREEMPT_RCU=y)1382 * for CONFIG_PREEMPT_RCU=y)
1380 * - the VCPU on which owner runs is preempted1383 * - the VCPU on which owner runs is preempted
1381 */1384 */
1382 if (!owner->on_cpu || need_resched() ||1385 if (!owner_on_cpu(owner) || need_resched() ||
1383 !rt_mutex_waiter_is_top_waiter(lock, waiter) ||1386 !rt_mutex_waiter_is_top_waiter(lock, waiter)) {
1384 vcpu_is_preempted(task_cpu(owner))) {
1385 res = false;1387 res = false;
1386 break;1388 break;
1387 }1389 }
kernel/locking/rtmutex_api.c+26−4
@@ -21,12 +21,13 @@ int max_lock_depth = 1024; · 20 unmodified lines
21 */21 */
22static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,22static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,
23 unsigned int state,23 unsigned int state,
24 struct lockdep_map *nest_lock,
24 unsigned int subclass)25 unsigned int subclass)
25{26{
26 int ret;27 int ret;
27 28
28 might_sleep();29 might_sleep();
29 mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);30 mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, _RET_IP_);
30 ret = __rt_mutex_lock(&lock->rtmutex, state);31 ret = __rt_mutex_lock(&lock->rtmutex, state);
31 if (ret)32 if (ret)
32 mutex_release(&lock->dep_map, _RET_IP_);33 mutex_release(&lock->dep_map, _RET_IP_);
@@ -48,10 +49,16 @@ EXPORT_SYMBOL(rt_mutex_base_init); · 15 unmodified lines
48 */49 */
49void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)50void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
50{51{
51 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass);52 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass);
52}53}
53EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);54EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
54 55
56void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)
57{
58 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0);
59}
60EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
61
55#else /* !CONFIG_DEBUG_LOCK_ALLOC */62#else /* !CONFIG_DEBUG_LOCK_ALLOC */
56 63
57/**64/**
@@ -61,7 +68,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); · 3 unmodified lines
61 */68 */
62void __sched rt_mutex_lock(struct rt_mutex *lock)69void __sched rt_mutex_lock(struct rt_mutex *lock)
63{70{
64 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0);71 __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0);
65}72}
66EXPORT_SYMBOL_GPL(rt_mutex_lock);73EXPORT_SYMBOL_GPL(rt_mutex_lock);
67#endif74#endif
@@ -77,10 +84,25 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock); · 9 unmodified lines
77 */84 */
78int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)85int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
79{86{
80 return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0);87 return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, NULL, 0);
81}88}
82EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);89EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
83 90
91/**
92 * rt_mutex_lock_killable - lock a rt_mutex killable
93 *
94 * @lock: the rt_mutex to be locked
95 *
96 * Returns:
97 * 0 on success
98 * -EINTR when interrupted by a signal
99 */
100int __sched rt_mutex_lock_killable(struct rt_mutex *lock)
101{
102 return __rt_mutex_lock_common(lock, TASK_KILLABLE, NULL, 0);
103}
104EXPORT_SYMBOL_GPL(rt_mutex_lock_killable);
105
84/**106/**
85 * rt_mutex_trylock - try to lock a rt_mutex107 * rt_mutex_trylock - try to lock a rt_mutex
86 *108 *
kernel/locking/rwsem.c+0−9
@@ -658,15 +658,6 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) · 657 unmodified lines
658 return false;658 return false;
659}659}
660 660
661static inline bool owner_on_cpu(struct task_struct *owner)
662{
663 /*
664 * As lock holder preemption issue, we both skip spinning if
665 * task is not on cpu or its cpu is preempted
666 */
667 return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));
668}
669
670static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)661static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)
671{662{
672 struct task_struct *owner;663 struct task_struct *owner;
kernel/locking/spinlock_rt.c+0−6
@@ -257,12 +257,6 @@ void __sched rt_write_unlock(rwlock_t *rwlock) · 256 unmodified lines
257}257}
258EXPORT_SYMBOL(rt_write_unlock);258EXPORT_SYMBOL(rt_write_unlock);
259 259
260int __sched rt_rwlock_is_contended(rwlock_t *rwlock)
261{
262 return rw_base_is_contended(&rwlock->rwbase);
263}
264EXPORT_SYMBOL(rt_rwlock_is_contended);
265
266#ifdef CONFIG_DEBUG_LOCK_ALLOC260#ifdef CONFIG_DEBUG_LOCK_ALLOC
267void __rt_rwlock_init(rwlock_t *rwlock, const char *name,261void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
268 struct lock_class_key *key)262 struct lock_class_key *key)
kernel/locking/ww_rt_mutex.c+1−1
@@ -26,7 +26,7 @@ int ww_mutex_trylock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) · 25 unmodified lines
26 26
27 if (__rt_mutex_trylock(&rtm->rtmutex)) {27 if (__rt_mutex_trylock(&rtm->rtmutex)) {
28 ww_mutex_set_context_fastpath(lock, ww_ctx);28 ww_mutex_set_context_fastpath(lock, ww_ctx);
29 mutex_acquire_nest(&rtm->dep_map, 0, 1, ww_ctx->dep_map, _RET_IP_);29 mutex_acquire_nest(&rtm->dep_map, 0, 1, &ww_ctx->dep_map, _RET_IP_);
30 return 1;30 return 1;
31 }31 }
32 32
kernel/sched/core.c+3−0
@@ -2184,6 +2184,9 @@ void migrate_enable(void) · 2183 unmodified lines
2184 return;2184 return;
2185 }2185 }
2186 2186
2187 if (WARN_ON_ONCE(!p->migration_disabled))
2188 return;
2189
2187 /*2190 /*
2188 * Ensure stop_task runs either before or after this, and that2191 * Ensure stop_task runs either before or after this, and that
2189 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().2192 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule().

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t