Pull locking updates from Borislav Petkov: "Lots of cleanups and preparation. Highlights: - futex: Cleanup and remove runtime futex_cmpxchg detection - rtmutex: Some fixes for the PREEMPT_RT locking infrastructure - kcsan: Share owner_on_cpu() between mutex,rtmutex and rwsem and annotate the racy owner->on_cpu access *once*. - atomic64: Dead-Code-Elemination" [ Description above by Peter Zijlstra ] * tag 'locking_core_for_v5.17_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/atomic: atomic64: Remove unusable atomic ops futex: Fix additional regressions locking: Allow to include asm/spinlock_types.h from linux/spinlock_types_raw.h x86/mm: Include spinlock_t definition in pgtable. locking: Mark racy reads of owner->on_cpu locking: Make owner_on_cpu() into <linux/sched.h> lockdep/selftests: Adapt ww-tests for PREEMPT_RT lockdep/selftests: Skip the softirq related tests on PREEMPT_RT lockdep/selftests: Unbalanced migrate_disable() & rcu_read_lock(). lockdep/selftests: Avoid using local_lock_{acquire|release}(). lockdep: Remove softirq accounting on PREEMPT_RT. locking/rtmutex: Add rt_mutex_lock_nest_lock() and rt_mutex_lock_killable(). locking/rtmutex: Squash self-deadlock check for ww_rt_mutex. locking: Remove rt_rwlock_is_contended(). sched: Trigger warning if ->migration_disabled counter underflows. futex: Fix sparc32/m68k/nds32 build regression futex: Remove futex_cmpxchg detection futex: Ensure futex_atomic_cmpxchg_inatomic() is present kernel/locking: Use a pointer in ww_mutex_trylock().
Showing the changes to kernel (11 of 45 files). Show all
Unified · Split
| @@ -41,11 +41,6 @@ · 40 unmodified lines | ||
| 41 | 41 | #include "futex.h" |
| 42 | 42 | #include "../locking/rtmutex_common.h" |
| 43 | 43 | |
| 44 | −#ifndef CONFIG_HAVE_FUTEX_CMPXCHG | |
| 45 | −int __read_mostly futex_cmpxchg_enabled; | |
| 46 | −#endif | |
| 47 | − | |
| 48 | − | |
| 49 | 44 | /* |
| 50 | 45 | * The base of the bucket array and its size are always used together |
| 51 | 46 | * (after initialization only in futex_hash()), so ensure that they |
| @@ -776,9 +771,6 @@ static void exit_robust_list(struct task_struct *curr) · 724 unmodified lines | ||
| 776 | 771 | unsigned long futex_offset; |
| 777 | 772 | int rc; |
| 778 | 773 | |
| 779 | − if (!futex_cmpxchg_enabled) | |
| 780 | − return; | |
| 781 | − | |
| 782 | 774 | /* |
| 783 | 775 | * Fetch the list head (which was registered earlier, via |
| 784 | 776 | * sys_set_robust_list()): |
| @@ -874,9 +866,6 @@ static void compat_exit_robust_list(struct task_struct *curr) · 89 unmodified lines | ||
| 874 | 866 | compat_long_t futex_offset; |
| 875 | 867 | int rc; |
| 876 | 868 | |
| 877 | − if (!futex_cmpxchg_enabled) | |
| 878 | − return; | |
| 879 | − | |
| 880 | 869 | /* |
| 881 | 870 | * Fetch the list head (which was registered earlier, via |
| 882 | 871 | * sys_set_robust_list()): |
| @@ -950,8 +939,6 @@ static void exit_pi_state_list(struct task_struct *curr) · 67 unmodified lines | ||
| 950 | 939 | struct futex_hash_bucket *hb; |
| 951 | 940 | union futex_key key = FUTEX_KEY_INIT; |
| 952 | 941 | |
| 953 | − if (!futex_cmpxchg_enabled) | |
| 954 | − return; | |
| 955 | 942 | /* |
| 956 | 943 | * We are a ZOMBIE and nobody can enqueue itself on |
| 957 | 944 | * pi_state_list anymore, but we have to be careful |
| @@ -1125,26 +1112,6 @@ void futex_exit_release(struct task_struct *tsk) · 167 unmodified lines | ||
| 1125 | 1112 | futex_cleanup_end(tsk, FUTEX_STATE_DEAD); |
| 1126 | 1113 | } |
| 1127 | 1114 | |
| 1128 | −static void __init futex_detect_cmpxchg(void) | |
| 1129 | −{ | |
| 1130 | −#ifndef CONFIG_HAVE_FUTEX_CMPXCHG | |
| 1131 | − u32 curval; | |
| 1132 | − | |
| 1133 | − /* | |
| 1134 | − * This will fail and we want it. Some arch implementations do | |
| 1135 | − * runtime detection of the futex_atomic_cmpxchg_inatomic() | |
| 1136 | − * functionality. We want to know that before we call in any | |
| 1137 | − * of the complex code paths. Also we want to prevent | |
| 1138 | − * registration of robust lists in that case. NULL is | |
| 1139 | − * guaranteed to fault and we get -EFAULT on functional | |
| 1140 | − * implementation, the non-functional ones will return | |
| 1141 | − * -ENOSYS. | |
| 1142 | − */ | |
| 1143 | − if (futex_cmpxchg_value_locked(&curval, NULL, 0, 0) == -EFAULT) | |
| 1144 | − futex_cmpxchg_enabled = 1; | |
| 1145 | −#endif | |
| 1146 | −} | |
| 1147 | − | |
| 1148 | 1115 | static int __init futex_init(void) |
| 1149 | 1116 | { |
| 1150 | 1117 | unsigned int futex_shift; |
| @@ -1163,8 +1130,6 @@ static int __init futex_init(void) · 12 unmodified lines | ||
| 1163 | 1130 | futex_hashsize, futex_hashsize); |
| 1164 | 1131 | futex_hashsize = 1UL << futex_shift; |
| 1165 | 1132 | |
| 1166 | − futex_detect_cmpxchg(); | |
| 1167 | − | |
| 1168 | 1133 | for (i = 0; i < futex_hashsize; i++) { |
| 1169 | 1134 | atomic_set(&futex_queues[i].waiters, 0); |
| 1170 | 1135 | plist_head_init(&futex_queues[i].chain); |
| @@ -27,12 +27,6 @@ · 26 unmodified lines | ||
| 27 | 27 | #define FLAGS_CLOCKRT 0x02 |
| 28 | 28 | #define FLAGS_HAS_TIMEOUT 0x04 |
| 29 | 29 | |
| 30 | −#ifdef CONFIG_HAVE_FUTEX_CMPXCHG | |
| 31 | −#define futex_cmpxchg_enabled 1 | |
| 32 | −#else | |
| 33 | −extern int __read_mostly futex_cmpxchg_enabled; | |
| 34 | −#endif | |
| 35 | − | |
| 36 | 30 | #ifdef CONFIG_FAIL_FUTEX |
| 37 | 31 | extern bool should_fail_futex(bool fshared); |
| 38 | 32 | #else |
| @@ -29,8 +29,6 @@ · 28 unmodified lines | ||
| 29 | 29 | SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head, |
| 30 | 30 | size_t, len) |
| 31 | 31 | { |
| 32 | − if (!futex_cmpxchg_enabled) | |
| 33 | − return -ENOSYS; | |
| 34 | 32 | /* |
| 35 | 33 | * The kernel knows only one size for now: |
| 36 | 34 | */ |
| @@ -56,9 +54,6 @@ SYSCALL_DEFINE3(get_robust_list, int, pid, · 19 unmodified lines | ||
| 56 | 54 | unsigned long ret; |
| 57 | 55 | struct task_struct *p; |
| 58 | 56 | |
| 59 | − if (!futex_cmpxchg_enabled) | |
| 60 | − return -ENOSYS; | |
| 61 | − | |
| 62 | 57 | rcu_read_lock(); |
| 63 | 58 | |
| 64 | 59 | ret = -ESRCH; |
| @@ -103,17 +98,6 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, · 38 unmodified lines | ||
| 103 | 98 | return -ENOSYS; |
| 104 | 99 | } |
| 105 | 100 | |
| 106 | − switch (cmd) { | |
| 107 | − case FUTEX_LOCK_PI: | |
| 108 | − case FUTEX_LOCK_PI2: | |
| 109 | − case FUTEX_UNLOCK_PI: | |
| 110 | − case FUTEX_TRYLOCK_PI: | |
| 111 | − case FUTEX_WAIT_REQUEUE_PI: | |
| 112 | − case FUTEX_CMP_REQUEUE_PI: | |
| 113 | − if (!futex_cmpxchg_enabled) | |
| 114 | − return -ENOSYS; | |
| 115 | − } | |
| 116 | − | |
| 117 | 101 | switch (cmd) { |
| 118 | 102 | case FUTEX_WAIT: |
| 119 | 103 | val3 = FUTEX_BITSET_MATCH_ANY; |
| @@ -323,9 +307,6 @@ COMPAT_SYSCALL_DEFINE2(set_robust_list, · 203 unmodified lines | ||
| 323 | 307 | struct compat_robust_list_head __user *, head, |
| 324 | 308 | compat_size_t, len) |
| 325 | 309 | { |
| 326 | − if (!futex_cmpxchg_enabled) | |
| 327 | − return -ENOSYS; | |
| 328 | − | |
| 329 | 310 | if (unlikely(len != sizeof(*head))) |
| 330 | 311 | return -EINVAL; |
| 331 | 312 | |
| @@ -342,9 +323,6 @@ COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid, · 10 unmodified lines | ||
| 342 | 323 | unsigned long ret; |
| 343 | 324 | struct task_struct *p; |
| 344 | 325 | |
| 345 | − if (!futex_cmpxchg_enabled) | |
| 346 | − return -ENOSYS; | |
| 347 | − | |
| 348 | 326 | rcu_read_lock(); |
| 349 | 327 | |
| 350 | 328 | ret = -ESRCH; |
| @@ -5485,6 +5485,7 @@ static noinstr void check_flags(unsigned long flags) · 5484 unmodified lines | ||
| 5485 | 5485 | } |
| 5486 | 5486 | } |
| 5487 | 5487 | |
| 5488 | +#ifndef CONFIG_PREEMPT_RT | |
| 5488 | 5489 | /* |
| 5489 | 5490 | * We dont accurately track softirq state in e.g. |
| 5490 | 5491 | * hardirq contexts (such as on 4KSTACKS), so only |
| @@ -5499,6 +5500,7 @@ static noinstr void check_flags(unsigned long flags) · 8 unmodified lines | ||
| 5499 | 5500 | DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled); |
| 5500 | 5501 | } |
| 5501 | 5502 | } |
| 5503 | +#endif | |
| 5502 | 5504 | |
| 5503 | 5505 | if (!debug_locks) |
| 5504 | 5506 | print_irqtrace_events(current); |
| @@ -367,8 +367,7 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, · 366 unmodified lines | ||
| 367 | 367 | /* |
| 368 | 368 | * Use vcpu_is_preempted to detect lock holder preemption issue. |
| 369 | 369 | */ |
| 370 | − if (!owner->on_cpu || need_resched() || | |
| 371 | − vcpu_is_preempted(task_cpu(owner))) { | |
| 370 | + if (!owner_on_cpu(owner) || need_resched()) { | |
| 372 | 371 | ret = false; |
| 373 | 372 | break; |
| 374 | 373 | } |
| @@ -403,14 +402,8 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock) · 28 unmodified lines | ||
| 403 | 402 | * structure won't go away during the spinning period. |
| 404 | 403 | */ |
| 405 | 404 | owner = __mutex_owner(lock); |
| 406 | − | |
| 407 | − /* | |
| 408 | − * As lock holder preemption issue, we both skip spinning if task is not | |
| 409 | − * on cpu or its cpu is preempted | |
| 410 | − */ | |
| 411 | − | |
| 412 | 405 | if (owner) |
| 413 | − retval = owner->on_cpu && !vcpu_is_preempted(task_cpu(owner)); | |
| 406 | + retval = owner_on_cpu(owner); | |
| 414 | 407 | |
| 415 | 408 | /* |
| 416 | 409 | * If lock->owner is not set, the mutex has been released. Return true |
| @@ -1103,8 +1103,11 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock, · 1102 unmodified lines | ||
| 1103 | 1103 | * the other will detect the deadlock and return -EDEADLOCK, |
| 1104 | 1104 | * which is wrong, as the other waiter is not in a deadlock |
| 1105 | 1105 | * situation. |
| 1106 | + * | |
| 1107 | + * Except for ww_mutex, in that case the chain walk must already deal | |
| 1108 | + * with spurious cycles, see the comments at [3] and [6]. | |
| 1106 | 1109 | */ |
| 1107 | − if (owner == task) | |
| 1110 | + if (owner == task && !(build_ww_mutex() && ww_ctx)) | |
| 1108 | 1111 | return -EDEADLK; |
| 1109 | 1112 | |
| 1110 | 1113 | raw_spin_lock(&task->pi_lock); |
| @@ -1379,9 +1382,8 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock, · 268 unmodified lines | ||
| 1379 | 1382 | * for CONFIG_PREEMPT_RCU=y) |
| 1380 | 1383 | * - the VCPU on which owner runs is preempted |
| 1381 | 1384 | */ |
| 1382 | − if (!owner->on_cpu || need_resched() || | |
| 1383 | − !rt_mutex_waiter_is_top_waiter(lock, waiter) || | |
| 1384 | − vcpu_is_preempted(task_cpu(owner))) { | |
| 1385 | + if (!owner_on_cpu(owner) || need_resched() || | |
| 1386 | + !rt_mutex_waiter_is_top_waiter(lock, waiter)) { | |
| 1385 | 1387 | res = false; |
| 1386 | 1388 | break; |
| 1387 | 1389 | } |
| @@ -21,12 +21,13 @@ int max_lock_depth = 1024; · 20 unmodified lines | ||
| 21 | 21 | */ |
| 22 | 22 | static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock, |
| 23 | 23 | unsigned int state, |
| 24 | + struct lockdep_map *nest_lock, | |
| 24 | 25 | unsigned int subclass) |
| 25 | 26 | { |
| 26 | 27 | int ret; |
| 27 | 28 | |
| 28 | 29 | might_sleep(); |
| 29 | − mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
| 30 | + mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, _RET_IP_); | |
| 30 | 31 | ret = __rt_mutex_lock(&lock->rtmutex, state); |
| 31 | 32 | if (ret) |
| 32 | 33 | mutex_release(&lock->dep_map, _RET_IP_); |
| @@ -48,10 +49,16 @@ EXPORT_SYMBOL(rt_mutex_base_init); · 15 unmodified lines | ||
| 48 | 49 | */ |
| 49 | 50 | void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) |
| 50 | 51 | { |
| 51 | − __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass); | |
| 52 | + __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass); | |
| 52 | 53 | } |
| 53 | 54 | EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); |
| 54 | 55 | |
| 56 | +void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) | |
| 57 | +{ | |
| 58 | + __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0); | |
| 59 | +} | |
| 60 | +EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock); | |
| 61 | + | |
| 55 | 62 | #else /* !CONFIG_DEBUG_LOCK_ALLOC */ |
| 56 | 63 | |
| 57 | 64 | /** |
| @@ -61,7 +68,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_nested); · 3 unmodified lines | ||
| 61 | 68 | */ |
| 62 | 69 | void __sched rt_mutex_lock(struct rt_mutex *lock) |
| 63 | 70 | { |
| 64 | − __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0); | |
| 71 | + __rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0); | |
| 65 | 72 | } |
| 66 | 73 | EXPORT_SYMBOL_GPL(rt_mutex_lock); |
| 67 | 74 | #endif |
| @@ -77,10 +84,25 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock); · 9 unmodified lines | ||
| 77 | 84 | */ |
| 78 | 85 | int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock) |
| 79 | 86 | { |
| 80 | − return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0); | |
| 87 | + return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, NULL, 0); | |
| 81 | 88 | } |
| 82 | 89 | EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); |
| 83 | 90 | |
| 91 | +/** | |
| 92 | + * rt_mutex_lock_killable - lock a rt_mutex killable | |
| 93 | + * | |
| 94 | + * @lock: the rt_mutex to be locked | |
| 95 | + * | |
| 96 | + * Returns: | |
| 97 | + * 0 on success | |
| 98 | + * -EINTR when interrupted by a signal | |
| 99 | + */ | |
| 100 | +int __sched rt_mutex_lock_killable(struct rt_mutex *lock) | |
| 101 | +{ | |
| 102 | + return __rt_mutex_lock_common(lock, TASK_KILLABLE, NULL, 0); | |
| 103 | +} | |
| 104 | +EXPORT_SYMBOL_GPL(rt_mutex_lock_killable); | |
| 105 | + | |
| 84 | 106 | /** |
| 85 | 107 | * rt_mutex_trylock - try to lock a rt_mutex |
| 86 | 108 | * |
| @@ -658,15 +658,6 @@ static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) · 657 unmodified lines | ||
| 658 | 658 | return false; |
| 659 | 659 | } |
| 660 | 660 | |
| 661 | −static inline bool owner_on_cpu(struct task_struct *owner) | |
| 662 | −{ | |
| 663 | − /* | |
| 664 | − * As lock holder preemption issue, we both skip spinning if | |
| 665 | − * task is not on cpu or its cpu is preempted | |
| 666 | − */ | |
| 667 | − return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner)); | |
| 668 | −} | |
| 669 | − | |
| 670 | 661 | static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) |
| 671 | 662 | { |
| 672 | 663 | struct task_struct *owner; |
| @@ -257,12 +257,6 @@ void __sched rt_write_unlock(rwlock_t *rwlock) · 256 unmodified lines | ||
| 257 | 257 | } |
| 258 | 258 | EXPORT_SYMBOL(rt_write_unlock); |
| 259 | 259 | |
| 260 | −int __sched rt_rwlock_is_contended(rwlock_t *rwlock) | |
| 261 | −{ | |
| 262 | − return rw_base_is_contended(&rwlock->rwbase); | |
| 263 | −} | |
| 264 | −EXPORT_SYMBOL(rt_rwlock_is_contended); | |
| 265 | − | |
| 266 | 260 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
| 267 | 261 | void __rt_rwlock_init(rwlock_t *rwlock, const char *name, |
| 268 | 262 | struct lock_class_key *key) |
| @@ -26,7 +26,7 @@ int ww_mutex_trylock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) · 25 unmodified lines | ||
| 26 | 26 | |
| 27 | 27 | if (__rt_mutex_trylock(&rtm->rtmutex)) { |
| 28 | 28 | ww_mutex_set_context_fastpath(lock, ww_ctx); |
| 29 | − mutex_acquire_nest(&rtm->dep_map, 0, 1, ww_ctx->dep_map, _RET_IP_); | |
| 29 | + mutex_acquire_nest(&rtm->dep_map, 0, 1, &ww_ctx->dep_map, _RET_IP_); | |
| 30 | 30 | return 1; |
| 31 | 31 | } |
| 32 | 32 | |
| @@ -2184,6 +2184,9 @@ void migrate_enable(void) · 2183 unmodified lines | ||
| 2184 | 2184 | return; |
| 2185 | 2185 | } |
| 2186 | 2186 | |
| 2187 | + if (WARN_ON_ONCE(!p->migration_disabled)) | |
| 2188 | + return; | |
| 2189 | + | |
| 2187 | 2190 | /* |
| 2188 | 2191 | * Ensure stop_task runs either before or after this, and that |
| 2189 | 2192 | * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). |