Merge branch 'locking/core' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip into io_uring-futex

Pull in locking/core from the tip tree, to get the futex2 dependencies from Peter Zijlstra. * 'locking/core' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (24 commits) locking/ww_mutex/test: Make sure we bail out instead of livelock locking/ww_mutex/test: Fix potential workqueue corruption locking/ww_mutex/test: Use prng instead of rng to avoid hangs at bootup futex: Add sys_futex_requeue() futex: Add flags2 argument to futex_requeue() futex: Propagate flags into get_futex_key() futex: Add sys_futex_wait() futex: FLAGS_STRICT futex: Add sys_futex_wake() futex: Validate futex value against futex size futex: Flag conversion futex: Extend the FUTEX2 flags futex: Clarify FUTEX2 flags asm-generic: ticket-lock: Optimize arch_spin_value_unlocked() futex/pi: Fix recursive rt_mutex waiter state locking/rtmutex: Add a lockdep assert to catch potential nested blocking locking/rtmutex: Use rt_mutex specific scheduler helpers sched: Provide rt_mutex specific scheduler helpers sched: Extract __schedule_loop() locking/rtmutex: Avoid unconditional slowpath for DEBUG_RT_MUTEXES ...

Jens Axboe committed 3 years ago

Commit 52e856c2 parents 73c7e7a cfa92b640 files · +708 −207Browse files

Unified · Split

40 files

arch/alpha/kernel/syscalls/syscall.tbl +3 −0
arch/arm/tools/syscall.tbl +3 −0
arch/arm64/include/asm/unistd.h +1 −1
arch/arm64/include/asm/unistd32.h +6 −0
arch/ia64/kernel/syscalls/syscall.tbl +3 −0
arch/m68k/kernel/syscalls/syscall.tbl +3 −0
arch/microblaze/kernel/syscalls/syscall.tbl +3 −0
arch/mips/kernel/syscalls/syscall_n32.tbl +3 −0
arch/mips/kernel/syscalls/syscall_n64.tbl +3 −0
arch/mips/kernel/syscalls/syscall_o32.tbl +3 −0
arch/parisc/kernel/syscalls/syscall.tbl +3 −0
arch/powerpc/kernel/syscalls/syscall.tbl +3 −0
arch/s390/kernel/syscalls/syscall.tbl +3 −0
arch/sh/kernel/syscalls/syscall.tbl +3 −0
arch/sparc/kernel/syscalls/syscall.tbl +3 −0
arch/x86/Kconfig +1 −1
arch/x86/entry/syscalls/syscall_32.tbl +3 −0
arch/x86/entry/syscalls/syscall_64.tbl +3 −0
arch/xtensa/kernel/syscalls/syscall.tbl +3 −0
include/asm-generic/spinlock.h +9 −7
include/linux/cleanup.h +36 −3
include/linux/sched.h +3 −0
include/linux/sched/rt.h +4 −0
include/linux/syscalls.h +10 −0
include/uapi/asm-generic/unistd.h +7 −1
include/uapi/linux/futex.h +28 −3
kernel/futex/core.c +38 −36
kernel/futex/futex.h +79 −7
kernel/futex/pi.c +61 −30
kernel/futex/requeue.c +12 −8
kernel/futex/syscalls.c +175 −46
kernel/futex/waitwake.c +47 −33
kernel/locking/rtmutex.c +34 −3
kernel/locking/rwbase_rt.c +8 −0
kernel/locking/rwsem.c +7 −1
kernel/locking/spinlock_rt.c +6 −0
kernel/locking/test-ww_mutex.c +34 −14
kernel/locking/ww_rt_mutex.c +1 −1
kernel/sched/core.c +50 −12
kernel/sys_ni.c +3 −0

arch/alpha/kernel/syscalls/syscall.tbl+3−0

@@ -492,3 +492,6 @@ · 491 unmodified lines
492	492	560 common set_mempolicy_home_node sys_ni_syscall
493	493	561 common cachestat sys_cachestat
494	494	562 common fchmodat2 sys_fchmodat2
	495	+563 common futex_wake sys_futex_wake
	496	+564 common futex_wait sys_futex_wait
	497	+565 common futex_requeue sys_futex_requeue

arch/arm/tools/syscall.tbl+3−0

@@ -466,3 +466,6 @@ · 465 unmodified lines
466	466	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
467	467	451 common cachestat sys_cachestat
468	468	452 common fchmodat2 sys_fchmodat2
	469	+454 common futex_wake sys_futex_wake
	470	+455 common futex_wait sys_futex_wait
	471	+456 common futex_requeue sys_futex_requeue

arch/arm64/include/asm/unistd.h+1−1

@@ -39,7 +39,7 @@ · 38 unmodified lines
39	39	#define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5)
40	40	#define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800)
41	41
42		−#define __NR_compat_syscalls 453
	42	+#define __NR_compat_syscalls 457
43	43	#endif
44	44
45	45	#define __ARCH_WANT_SYS_CLONE

arch/arm64/include/asm/unistd32.h+6−0

@@ -911,6 +911,12 @@ __SYSCALL(__NR_set_mempolicy_home_node, sys_set_mempolicy_home_node) · 910 unmodified lines
911	911	__SYSCALL(__NR_cachestat, sys_cachestat)
912	912	#define __NR_fchmodat2 452
913	913	__SYSCALL(__NR_fchmodat2, sys_fchmodat2)
	914	+#define __NR_futex_wake 454
	915	+__SYSCALL(__NR_futex_wake, sys_futex_wake)
	916	+#define __NR_futex_wait 455
	917	+__SYSCALL(__NR_futex_wait, sys_futex_wait)
	918	+#define __NR_futex_requeue 456
	919	+__SYSCALL(__NR_futex_requeue, sys_futex_requeue)
914	920
915	921	/*
916	922	* Please add new compat syscalls above this comment and update

arch/ia64/kernel/syscalls/syscall.tbl+3−0

@@ -373,3 +373,6 @@ · 372 unmodified lines
373	373	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
374	374	451 common cachestat sys_cachestat
375	375	452 common fchmodat2 sys_fchmodat2
	376	+454 common futex_wake sys_futex_wake
	377	+455 common futex_wait sys_futex_wait
	378	+456 common futex_requeue sys_futex_requeue

arch/m68k/kernel/syscalls/syscall.tbl+3−0

@@ -452,3 +452,6 @@ · 451 unmodified lines
452	452	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
453	453	451 common cachestat sys_cachestat
454	454	452 common fchmodat2 sys_fchmodat2
	455	+454 common futex_wake sys_futex_wake
	456	+455 common futex_wait sys_futex_wait
	457	+456 common futex_requeue sys_futex_requeue

arch/microblaze/kernel/syscalls/syscall.tbl+3−0

@@ -458,3 +458,6 @@ · 457 unmodified lines
458	458	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
459	459	451 common cachestat sys_cachestat
460	460	452 common fchmodat2 sys_fchmodat2
	461	+454 common futex_wake sys_futex_wake
	462	+455 common futex_wait sys_futex_wait
	463	+456 common futex_requeue sys_futex_requeue

arch/mips/kernel/syscalls/syscall_n32.tbl+3−0

@@ -391,3 +391,6 @@ · 390 unmodified lines
391	391	450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node
392	392	451 n32 cachestat sys_cachestat
393	393	452 n32 fchmodat2 sys_fchmodat2
	394	+454 n32 futex_wake sys_futex_wake
	395	+455 n32 futex_wait sys_futex_wait
	396	+456 n32 futex_requeue sys_futex_requeue

arch/mips/kernel/syscalls/syscall_n64.tbl+3−0

@@ -367,3 +367,6 @@ · 366 unmodified lines
367	367	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
368	368	451 n64 cachestat sys_cachestat
369	369	452 n64 fchmodat2 sys_fchmodat2
	370	+454 n64 futex_wake sys_futex_wake
	371	+455 n64 futex_wait sys_futex_wait
	372	+456 n64 futex_requeue sys_futex_requeue

arch/mips/kernel/syscalls/syscall_o32.tbl+3−0

@@ -440,3 +440,6 @@ · 439 unmodified lines
440	440	450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node
441	441	451 o32 cachestat sys_cachestat
442	442	452 o32 fchmodat2 sys_fchmodat2
	443	+454 o32 futex_wake sys_futex_wake
	444	+455 o32 futex_wait sys_futex_wait
	445	+456 o32 futex_requeue sys_futex_requeue

arch/parisc/kernel/syscalls/syscall.tbl+3−0

@@ -451,3 +451,6 @@ · 450 unmodified lines
451	451	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
452	452	451 common cachestat sys_cachestat
453	453	452 common fchmodat2 sys_fchmodat2
	454	+454 common futex_wake sys_futex_wake
	455	+455 common futex_wait sys_futex_wait
	456	+456 common futex_requeue sys_futex_requeue

arch/powerpc/kernel/syscalls/syscall.tbl+3−0

@@ -539,3 +539,6 @@ · 538 unmodified lines
539	539	450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node
540	540	451 common cachestat sys_cachestat
541	541	452 common fchmodat2 sys_fchmodat2
	542	+454 common futex_wake sys_futex_wake
	543	+455 common futex_wait sys_futex_wait
	544	+456 common futex_requeue sys_futex_requeue

arch/s390/kernel/syscalls/syscall.tbl+3−0

@@ -455,3 +455,6 @@ · 454 unmodified lines
455	455	450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node
456	456	451 common cachestat sys_cachestat sys_cachestat
457	457	452 common fchmodat2 sys_fchmodat2 sys_fchmodat2
	458	+454 common futex_wake sys_futex_wake sys_futex_wake
	459	+455 common futex_wait sys_futex_wait sys_futex_wait
	460	+456 common futex_requeue sys_futex_requeue sys_futex_requeue

arch/sh/kernel/syscalls/syscall.tbl+3−0

@@ -455,3 +455,6 @@ · 454 unmodified lines
455	455	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
456	456	451 common cachestat sys_cachestat
457	457	452 common fchmodat2 sys_fchmodat2
	458	+454 common futex_wake sys_futex_wake
	459	+455 common futex_wait sys_futex_wait
	460	+456 common futex_requeue sys_futex_requeue

arch/sparc/kernel/syscalls/syscall.tbl+3−0

@@ -498,3 +498,6 @@ · 497 unmodified lines
498	498	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
499	499	451 common cachestat sys_cachestat
500	500	452 common fchmodat2 sys_fchmodat2
	501	+454 common futex_wake sys_futex_wake
	502	+455 common futex_wait sys_futex_wait
	503	+456 common futex_requeue sys_futex_requeue

arch/x86/Kconfig+1−1

@@ -28,7 +28,6 @@ config X86_64 · 27 unmodified lines
28	28	select ARCH_HAS_GIGANTIC_PAGE
29	29	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
30	30	select ARCH_SUPPORTS_PER_VMA_LOCK
31		− select ARCH_USE_CMPXCHG_LOCKREF
32	31	select HAVE_ARCH_SOFT_DIRTY
33	32	select MODULES_USE_ELF_RELA
34	33	select NEED_DMA_MAP_STATE
@@ -118,6 +117,7 @@ config X86 · 83 unmodified lines
118	117	select ARCH_SUPPORTS_LTO_CLANG
119	118	select ARCH_SUPPORTS_LTO_CLANG_THIN
120	119	select ARCH_USE_BUILTIN_BSWAP
	120	+ select ARCH_USE_CMPXCHG_LOCKREF if X86_CMPXCHG64
121	121	select ARCH_USE_MEMTEST
122	122	select ARCH_USE_QUEUED_RWLOCKS
123	123	select ARCH_USE_QUEUED_SPINLOCKS

arch/x86/entry/syscalls/syscall_32.tbl+3−0

@@ -457,3 +457,6 @@ · 456 unmodified lines
457	457	450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node
458	458	451 i386 cachestat sys_cachestat
459	459	452 i386 fchmodat2 sys_fchmodat2
	460	+454 i386 futex_wake sys_futex_wake
	461	+455 i386 futex_wait sys_futex_wait
	462	+456 i386 futex_requeue sys_futex_requeue

arch/x86/entry/syscalls/syscall_64.tbl+3−0

@@ -375,6 +375,9 @@ · 374 unmodified lines
375	375	451 common cachestat sys_cachestat
376	376	452 common fchmodat2 sys_fchmodat2
377	377	453 64 map_shadow_stack sys_map_shadow_stack
	378	+454 common futex_wake sys_futex_wake
	379	+455 common futex_wait sys_futex_wait
	380	+456 common futex_requeue sys_futex_requeue
378	381
379	382	#
380	383	# Due to a historical design error, certain syscalls are numbered differently

arch/xtensa/kernel/syscalls/syscall.tbl+3−0

@@ -423,3 +423,6 @@ · 422 unmodified lines
423	423	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
424	424	451 common cachestat sys_cachestat
425	425	452 common fchmodat2 sys_fchmodat2
	426	+454 common futex_wake sys_futex_wake
	427	+455 common futex_wait sys_futex_wait
	428	+456 common futex_requeue sys_futex_requeue

include/asm-generic/spinlock.h+9−7

@@ -68,11 +68,18 @@ static __always_inline void arch_spin_unlock(arch_spinlock_t *lock) · 67 unmodified lines
68	68	smp_store_release(ptr, (u16)val + 1);
69	69	}
70	70
	71	+static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
	72	+{
	73	+ u32 val = lock.counter;
	74	+
	75	+ return ((val >> 16) == (val & 0xffff));
	76	+}
	77	+
71	78	static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock)
72	79	{
73		− u32 val = atomic_read(lock);
	80	+ arch_spinlock_t val = READ_ONCE(*lock);
74	81
75		− return ((val >> 16) != (val & 0xffff));
	82	+ return !arch_spin_value_unlocked(val);
76	83	}
77	84
78	85	static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock)
@@ -82,11 +89,6 @@ static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock) · 3 unmodified lines
82	89	return (s16)((val >> 16) - (val & 0xffff)) > 1;
83	90	}
84	91
85		−static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
86		−{
87		− return !arch_spin_is_locked(&lock);
88		−}
89		−
90	92	#include <asm/qrwlock.h>
91	93
92	94	#endif /* __ASM_GENERIC_SPINLOCK_H */

include/linux/cleanup.h+36−3

@@ -7,8 +7,9 @@ · 6 unmodified lines
7	7	/*
8	8	* DEFINE_FREE(name, type, free):
9	9	* simple helper macro that defines the required wrapper for a __free()
10		− * based cleanup function. @free is an expression using '_T' to access
11		− * the variable.
	10	+ * based cleanup function. @free is an expression using '_T' to access the
	11	+ * variable. @free should typically include a NULL test before calling a
	12	+ * function, see the example below.
12	13	*
13	14	* __free(name):
14	15	* variable attribute to add a scoped based cleanup to the variable.
@@ -17,6 +18,9 @@ · 2 unmodified lines
17	18	* like a non-atomic xchg(var, NULL), such that the cleanup function will
18	19	* be inhibited -- provided it sanely deals with a NULL value.
19	20	*
	21	+ * NOTE: this has __must_check semantics so that it is harder to accidentally
	22	+ * leak the resource.
	23	+ *
20	24	* return_ptr(p):
21	25	* returns p while inhibiting the __free().
22	26	*
@@ -24,6 +28,8 @@ · 1 unmodified line
24	28	*
25	29	* DEFINE_FREE(kfree, void *, if (_T) kfree(_T))
26	30	*
	31	+ * void *alloc_obj(...)
	32	+ * {
27	33	* struct obj *p __free(kfree) = kmalloc(...);
28	34	* if (!p)
29	35	* return NULL;
@@ -32,6 +38,24 @@ · 2 unmodified lines
32	38	* return NULL;
33	39	*
34	40	* return_ptr(p);
	41	+ * }
	42	+ *
	43	+ * NOTE: the DEFINE_FREE()'s @free expression includes a NULL test even though
	44	+ * kfree() is fine to be called with a NULL value. This is on purpose. This way
	45	+ * the compiler sees the end of our alloc_obj() function as:
	46	+ *
	47	+ * tmp = p;
	48	+ * p = NULL;
	49	+ * if (p)
	50	+ * kfree(p);
	51	+ * return tmp;
	52	+ *
	53	+ * And through the magic of value-propagation and dead-code-elimination, it
	54	+ * eliminates the actual cleanup call and compiles into:
	55	+ *
	56	+ * return p;
	57	+ *
	58	+ * Without the NULL test it turns into a mess and the compiler can't help us.
35	59	*/
36	60
37	61	#define DEFINE_FREE(_name, _type, _free) \
@@ -39,8 +63,17 @@ · 1 unmodified line
39	63
40	64	#define __free(_name) __cleanup(__free_##_name)
41	65
	66	+#define __get_and_null_ptr(p) \
	67	+ ({ __auto_type __ptr = &(p); \
	68	+ __auto_type __val = *__ptr; \
	69	+ *__ptr = NULL; __val; })
	70	+
	71	+static inline __must_check
	72	+const volatile void * __must_check_fn(const volatile void *val)
	73	+{ return val; }
	74	+
42	75	#define no_free_ptr(p) \
43		− ({ __auto_type __ptr = (p); (p) = NULL; __ptr; })
	76	+ ((typeof(p)) __must_check_fn(__get_and_null_ptr(p)))
44	77
45	78	#define return_ptr(p) return no_free_ptr(p)
46	79

include/linux/sched.h+3−0

@@ -911,6 +911,9 @@ struct task_struct { · 910 unmodified lines
911	911	* ->sched_remote_wakeup gets used, so it can be in this word.
912	912	*/
913	913	unsigned sched_remote_wakeup:1;
	914	+#ifdef CONFIG_RT_MUTEXES
	915	+ unsigned sched_rt_mutex:1;
	916	+#endif
914	917
915	918	/* Bit to tell LSMs we're in execve(): */
916	919	unsigned in_execve:1;

include/linux/sched/rt.h+4−0

@@ -30,6 +30,10 @@ static inline bool task_is_realtime(struct task_struct *tsk) · 29 unmodified lines
30	30	}
31	31
32	32	#ifdef CONFIG_RT_MUTEXES
	33	+extern void rt_mutex_pre_schedule(void);
	34	+extern void rt_mutex_schedule(void);
	35	+extern void rt_mutex_post_schedule(void);
	36	+
33	37	/*
34	38	* Must hold either p->pi_lock or task_rq(p)->lock.
35	39	*/

include/linux/syscalls.h+10−0

@@ -549,6 +549,16 @@ asmlinkage long sys_set_robust_list(struct robust_list_head __user *head, · 548 unmodified lines
549	549	asmlinkage long sys_futex_waitv(struct futex_waitv *waiters,
550	550	unsigned int nr_futexes, unsigned int flags,
551	551	struct __kernel_timespec __user *timeout, clockid_t clockid);
	552	+
	553	+asmlinkage long sys_futex_wake(void __user *uaddr, unsigned long mask, int nr, unsigned int flags);
	554	+
	555	+asmlinkage long sys_futex_wait(void __user *uaddr, unsigned long val, unsigned long mask,
	556	+ unsigned int flags, struct __kernel_timespec __user *timespec,
	557	+ clockid_t clockid);
	558	+
	559	+asmlinkage long sys_futex_requeue(struct futex_waitv __user *waiters,
	560	+ unsigned int flags, int nr_wake, int nr_requeue);
	561	+
552	562	asmlinkage long sys_nanosleep(struct __kernel_timespec __user *rqtp,
553	563	struct __kernel_timespec __user *rmtp);
554	564	asmlinkage long sys_nanosleep_time32(struct old_timespec32 __user *rqtp,

include/uapi/asm-generic/unistd.h+7−1

@@ -822,9 +822,15 @@ __SYSCALL(__NR_cachestat, sys_cachestat) · 821 unmodified lines
822	822
823	823	#define __NR_fchmodat2 452
824	824	__SYSCALL(__NR_fchmodat2, sys_fchmodat2)
	825	+#define __NR_futex_wake 454
	826	+__SYSCALL(__NR_futex_wake, sys_futex_wake)
	827	+#define __NR_futex_wait 455
	828	+__SYSCALL(__NR_futex_wait, sys_futex_wait)
	829	+#define __NR_futex_requeue 456
	830	+__SYSCALL(__NR_futex_requeue, sys_futex_requeue)
825	831
826	832	#undef __NR_syscalls
827		−#define __NR_syscalls 453
	833	+#define __NR_syscalls 457
828	834
829	835	/*
830	836	* 32 bit systems traditionally used different

include/uapi/linux/futex.h+28−3

@@ -44,10 +44,35 @@ · 43 unmodified lines
44	44	FUTEX_PRIVATE_FLAG)
45	45
46	46	/*
47		− * Flags to specify the bit length of the futex word for futex2 syscalls.
48		− * Currently, only 32 is supported.
	47	+ * Flags for futex2 syscalls.
	48	+ *
	49	+ * NOTE: these are not pure flags, they can also be seen as:
	50	+ *
	51	+ * union {
	52	+ * u32 flags;
	53	+ * struct {
	54	+ * u32 size : 2,
	55	+ * numa : 1,
	56	+ * : 4,
	57	+ * private : 1;
	58	+ * };
	59	+ * };
49	60	*/
50		−#define FUTEX_32 2
	61	+#define FUTEX2_SIZE_U8 0x00
	62	+#define FUTEX2_SIZE_U16 0x01
	63	+#define FUTEX2_SIZE_U32 0x02
	64	+#define FUTEX2_SIZE_U64 0x03
	65	+#define FUTEX2_NUMA 0x04
	66	+ /* 0x08 */
	67	+ /* 0x10 */
	68	+ /* 0x20 */
	69	+ /* 0x40 */
	70	+#define FUTEX2_PRIVATE FUTEX_PRIVATE_FLAG
	71	+
	72	+#define FUTEX2_SIZE_MASK 0x03
	73	+
	74	+/* do not use */
	75	+#define FUTEX_32 FUTEX2_SIZE_U32 /* historical accident :-( */
51	76
52	77	/*
53	78	* Max numbers of elements in a futex_waitv array

kernel/futex/core.c+38−36

@@ -193,7 +193,7 @@ static u64 get_inode_sequence_number(struct inode *inode) · 192 unmodified lines
193	193	/**
194	194	* get_futex_key() - Get parameters which are the keys for a futex
195	195	* @uaddr: virtual address of the futex
196		− * @fshared: false for a PROCESS_PRIVATE futex, true for PROCESS_SHARED
	196	+ * @flags: FLAGS_*
197	197	* @key: address where result is stored.
198	198	* @rw: mapping needs to be read/write (values: FUTEX_READ,
199	199	* FUTEX_WRITE)
@@ -217,14 +217,18 @@ static u64 get_inode_sequence_number(struct inode *inode) · 17 unmodified lines
217	217	*
218	218	* lock_page() might sleep, the caller should not hold a spinlock.
219	219	*/
220		−int get_futex_key(u32 __user uaddr, bool fshared, union futex_key key,
	220	+int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key,
221	221	enum futex_access rw)
222	222	{
223	223	unsigned long address = (unsigned long)uaddr;
224	224	struct mm_struct *mm = current->mm;
225		− struct page page, tail;
	225	+ struct page *page;
	226	+ struct folio *folio;
226	227	struct address_space *mapping;
227	228	int err, ro = 0;
	229	+ bool fshared;
	230	+
	231	+ fshared = flags & FLAGS_SHARED;
228	232
229	233	/*
230	234	* The futex address must be "naturally" aligned.
@@ -273,54 +277,52 @@ again: · 42 unmodified lines
273	277	err = 0;
274	278
275	279	/*
276		− * The treatment of mapping from this point on is critical. The page
277		− * lock protects many things but in this context the page lock
	280	+ * The treatment of mapping from this point on is critical. The folio
	281	+ * lock protects many things but in this context the folio lock
278	282	* stabilizes mapping, prevents inode freeing in the shared
279	283	* file-backed region case and guards against movement to swap cache.
280	284	*
281		− * Strictly speaking the page lock is not needed in all cases being
282		− * considered here and page lock forces unnecessarily serialization
	285	+ * Strictly speaking the folio lock is not needed in all cases being
	286	+ * considered here and folio lock forces unnecessarily serialization.
283	287	* From this point on, mapping will be re-verified if necessary and
284		− * page lock will be acquired only if it is unavoidable
	288	+ * folio lock will be acquired only if it is unavoidable
285	289	*
286		− * Mapping checks require the head page for any compound page so the
287		− * head page and mapping is looked up now. For anonymous pages, it
288		− * does not matter if the page splits in the future as the key is
289		− * based on the address. For filesystem-backed pages, the tail is
290		− * required as the index of the page determines the key. For
291		− * base pages, there is no tail page and tail == page.
	290	+ * Mapping checks require the folio so it is looked up now. For
	291	+ * anonymous pages, it does not matter if the folio is split
	292	+ * in the future as the key is based on the address. For
	293	+ * filesystem-backed pages, the precise page is required as the
	294	+ * index of the page determines the key.
292	295	*/
293		− tail = page;
294		− page = compound_head(page);
295		− mapping = READ_ONCE(page->mapping);
	296	+ folio = page_folio(page);
	297	+ mapping = READ_ONCE(folio->mapping);
296	298
297	299	/*
298		− * If page->mapping is NULL, then it cannot be a PageAnon
	300	+ * If folio->mapping is NULL, then it cannot be an anonymous
299	301	* page; but it might be the ZERO_PAGE or in the gate area or
300	302	* in a special mapping (all cases which we are happy to fail);
301	303	* or it may have been a good file page when get_user_pages_fast
302	304	* found it, but truncated or holepunched or subjected to
303		− * invalidate_complete_page2 before we got the page lock (also
	305	+ * invalidate_complete_page2 before we got the folio lock (also
304	306	* cases which we are happy to fail). And we hold a reference,
305	307	* so refcount care in invalidate_inode_page's remove_mapping
306	308	* prevents drop_caches from setting mapping to NULL beneath us.
307	309	*
308	310	* The case we do have to guard against is when memory pressure made
309	311	* shmem_writepage move it from filecache to swapcache beneath us:
310		− * an unlikely race, but we do need to retry for page->mapping.
	312	+ * an unlikely race, but we do need to retry for folio->mapping.
311	313	*/
312	314	if (unlikely(!mapping)) {
313	315	int shmem_swizzled;
314	316
315	317	/*
316		− * Page lock is required to identify which special case above
317		− * applies. If this is really a shmem page then the page lock
	318	+ * Folio lock is required to identify which special case above
	319	+ * applies. If this is really a shmem page then the folio lock
318	320	* will prevent unexpected transitions.
319	321	*/
320		− lock_page(page);
321		− shmem_swizzled = PageSwapCache(page) \|\| page->mapping;
322		− unlock_page(page);
323		− put_page(page);
	322	+ folio_lock(folio);
	323	+ shmem_swizzled = folio_test_swapcache(folio) \|\| folio->mapping;
	324	+ folio_unlock(folio);
	325	+ folio_put(folio);
324	326
325	327	if (shmem_swizzled)
326	328	goto again;
@@ -331,14 +333,14 @@ again: · 4 unmodified lines
331	333	/*
332	334	* Private mappings are handled in a simple way.
333	335	*
334		− * If the futex key is stored on an anonymous page, then the associated
	336	+ * If the futex key is stored in anonymous memory, then the associated
335	337	* object is the mm which is implicitly pinned by the calling process.
336	338	*
337	339	* NOTE: When userspace waits on a MAP_SHARED mapping, even if
338	340	* it's a read-only handle, it's expected that futexes attach to
339	341	* the object not the particular process.
340	342	*/
341		− if (PageAnon(page)) {
	343	+ if (folio_test_anon(folio)) {
342	344	/*
343	345	* A RO anonymous page will never change and thus doesn't make
344	346	* sense for futex operations.
@@ -357,10 +359,10 @@ again: · 12 unmodified lines
357	359
358	360	/*
359	361	* The associated futex object in this case is the inode and
360		− * the page->mapping must be traversed. Ordinarily this should
361		− * be stabilised under page lock but it's not strictly
	362	+ * the folio->mapping must be traversed. Ordinarily this should
	363	+ * be stabilised under folio lock but it's not strictly
362	364	* necessary in this case as we just want to pin the inode, not
363		− * update the radix tree or anything like that.
	365	+ * update i_pages or anything like that.
364	366	*
365	367	* The RCU read lock is taken as the inode is finally freed
366	368	* under RCU. If the mapping still matches expectations then the
@@ -368,9 +370,9 @@ again: · 1 unmodified line
368	370	*/
369	371	rcu_read_lock();
370	372
371		− if (READ_ONCE(page->mapping) != mapping) {
	373	+ if (READ_ONCE(folio->mapping) != mapping) {
372	374	rcu_read_unlock();
373		− put_page(page);
	375	+ folio_put(folio);
374	376
375	377	goto again;
376	378	}
@@ -378,19 +380,19 @@ again: · 1 unmodified line
378	380	inode = READ_ONCE(mapping->host);
379	381	if (!inode) {
380	382	rcu_read_unlock();
381		− put_page(page);
	383	+ folio_put(folio);
382	384
383	385	goto again;
384	386	}
385	387
386	388	key->both.offset \|= FUT_OFF_INODE; /* inode-based key */
387	389	key->shared.i_seq = get_inode_sequence_number(inode);
388		− key->shared.pgoff = page_to_pgoff(tail);
	390	+ key->shared.pgoff = folio->index + folio_page_idx(folio, page);
389	391	rcu_read_unlock();
390	392	}
391	393
392	394	out:
393		− put_page(page);
	395	+ folio_put(folio);
394	396	return err;
395	397	}
396	398

kernel/futex/futex.h+79−7

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	#include <linux/futex.h>
6	6	#include <linux/rtmutex.h>
7	7	#include <linux/sched/wake_q.h>
	8	+#include <linux/compat.h>
8	9
9	10	#ifdef CONFIG_PREEMPT_RT
10	11	#include <linux/rcuwait.h>
@@ -16,17 +17,84 @@ · 5 unmodified lines
16	17	* Futex flags used to encode options to functions and preserve them across
17	18	* restarts.
18	19	*/
	20	+#define FLAGS_SIZE_8 0x0000
	21	+#define FLAGS_SIZE_16 0x0001
	22	+#define FLAGS_SIZE_32 0x0002
	23	+#define FLAGS_SIZE_64 0x0003
	24	+
	25	+#define FLAGS_SIZE_MASK 0x0003
	26	+
19	27	#ifdef CONFIG_MMU
20		−# define FLAGS_SHARED 0x01
	28	+# define FLAGS_SHARED 0x0010
21	29	#else
22	30	/*
23	31	* NOMMU does not have per process address space. Let the compiler optimize
24	32	* code away.
25	33	*/
26		−# define FLAGS_SHARED 0x00
	34	+# define FLAGS_SHARED 0x0000
27	35	#endif
28		−#define FLAGS_CLOCKRT 0x02
29		−#define FLAGS_HAS_TIMEOUT 0x04
	36	+#define FLAGS_CLOCKRT 0x0020
	37	+#define FLAGS_HAS_TIMEOUT 0x0040
	38	+#define FLAGS_NUMA 0x0080
	39	+#define FLAGS_STRICT 0x0100
	40	+
	41	+/* FUTEX_ to FLAGS_ */
	42	+static inline unsigned int futex_to_flags(unsigned int op)
	43	+{
	44	+ unsigned int flags = FLAGS_SIZE_32;
	45	+
	46	+ if (!(op & FUTEX_PRIVATE_FLAG))
	47	+ flags \|= FLAGS_SHARED;
	48	+
	49	+ if (op & FUTEX_CLOCK_REALTIME)
	50	+ flags \|= FLAGS_CLOCKRT;
	51	+
	52	+ return flags;
	53	+}
	54	+
	55	+/* FUTEX2_ to FLAGS_ */
	56	+static inline unsigned int futex2_to_flags(unsigned int flags2)
	57	+{
	58	+ unsigned int flags = flags2 & FUTEX2_SIZE_MASK;
	59	+
	60	+ if (!(flags2 & FUTEX2_PRIVATE))
	61	+ flags \|= FLAGS_SHARED;
	62	+
	63	+ if (flags2 & FUTEX2_NUMA)
	64	+ flags \|= FLAGS_NUMA;
	65	+
	66	+ return flags;
	67	+}
	68	+
	69	+static inline unsigned int futex_size(unsigned int flags)
	70	+{
	71	+ return 1 << (flags & FLAGS_SIZE_MASK);
	72	+}
	73	+
	74	+static inline bool futex_flags_valid(unsigned int flags)
	75	+{
	76	+ /* Only 64bit futexes for 64bit code */
	77	+ if (!IS_ENABLED(CONFIG_64BIT) \|\| in_compat_syscall()) {
	78	+ if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64)
	79	+ return false;
	80	+ }
	81	+
	82	+ /* Only 32bit futexes are implemented -- for now */
	83	+ if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
	84	+ return false;
	85	+
	86	+ return true;
	87	+}
	88	+
	89	+static inline bool futex_validate_input(unsigned int flags, u64 val)
	90	+{
	91	+ int bits = 8 * futex_size(flags);
	92	+
	93	+ if (bits < 64 && (val >> bits))
	94	+ return false;
	95	+
	96	+ return true;
	97	+}
30	98
31	99	#ifdef CONFIG_FAIL_FUTEX
32	100	extern bool should_fail_futex(bool fshared);
@@ -116,7 +184,7 @@ enum futex_access { · 83 unmodified lines
116	184	FUTEX_WRITE
117	185	};
118	186
119		−extern int get_futex_key(u32 __user uaddr, bool fshared, union futex_key key,
	187	+extern int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key,
120	188	enum futex_access rw);
121	189
122	190	extern struct hrtimer_sleeper *
@@ -260,10 +328,14 @@ extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32 · 137 unmodified lines
260	328	val, ktime_t *abs_time, u32 bitset, u32 __user
261	329	*uaddr2);
262	330
263		−extern int futex_requeue(u32 __user *uaddr1, unsigned int flags,
264		− u32 __user *uaddr2, int nr_wake, int nr_requeue,
	331	+extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
	332	+ u32 __user *uaddr2, unsigned int flags2,
	333	+ int nr_wake, int nr_requeue,
265	334	u32 *cmpval, int requeue_pi);
266	335
	336	+extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
	337	+ struct hrtimer_sleeper *to, u32 bitset);
	338	+
267	339	extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
268	340	ktime_t *abs_time, u32 bitset);
269	341

kernel/futex/pi.c+61−30

@@ -1,6 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0-or-later
2	2
3	3	#include <linux/slab.h>
	4	+#include <linux/sched/rt.h>
4	5	#include <linux/sched/task.h>
5	6
6	7	#include "futex.h"
@@ -610,29 +611,16 @@ int futex_lock_pi_atomic(u32 __user uaddr, struct futex_hash_bucket hb, · 603 unmodified lines
610	611	/*
611	612	* Caller must hold a reference on @pi_state.
612	613	*/
613		−static int wake_futex_pi(u32 __user uaddr, u32 uval, struct futex_pi_state pi_state)
	614	+static int wake_futex_pi(u32 __user *uaddr, u32 uval,
	615	+ struct futex_pi_state *pi_state,
	616	+ struct rt_mutex_waiter *top_waiter)
614	617	{
615		− struct rt_mutex_waiter *top_waiter;
616	618	struct task_struct *new_owner;
617	619	bool postunlock = false;
618	620	DEFINE_RT_WAKE_Q(wqh);
619	621	u32 curval, newval;
620	622	int ret = 0;
621	623
622		− top_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
623		− if (WARN_ON_ONCE(!top_waiter)) {
624		− /*
625		− * As per the comment in futex_unlock_pi() this should not happen.
626		− *
627		− * When this happens, give up our locks and try again, giving
628		− * the futex_lock_pi() instance time to complete, either by
629		− * waiting on the rtmutex or removing itself from the futex
630		− * queue.
631		− */
632		− ret = -EAGAIN;
633		− goto out_unlock;
634		− }
635		−
636	624	new_owner = top_waiter->task;
637	625
638	626	/*
@@ -945,7 +933,7 @@ int futex_lock_pi(u32 __user uaddr, unsigned int flags, ktime_t time, int tryl · 306 unmodified lines
945	933	to = futex_setup_timer(time, &timeout, flags, 0);
946	934
947	935	retry:
948		− ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE);
	936	+ ret = get_futex_key(uaddr, flags, &q.key, FUTEX_WRITE);
949	937	if (unlikely(ret != 0))
950	938	goto out;
951	939
@@ -1002,6 +990,12 @@ retry_private: · 50 unmodified lines
1002	990	goto no_block;
1003	991	}
1004	992
	993	+ /*
	994	+ * Must be done before we enqueue the waiter, here is unfortunately
	995	+ * under the hb lock, but that should work because it does nothing.
	996	+ */
	997	+ rt_mutex_pre_schedule();
	998	+
1005	999	rt_mutex_init_waiter(&rt_waiter);
1006	1000
1007	1001	/*
@@ -1039,19 +1033,37 @@ retry_private: · 31 unmodified lines
1039	1033	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
1040	1034
1041	1035	cleanup:
1042		− spin_lock(q.lock_ptr);
1043	1036	/*
1044	1037	* If we failed to acquire the lock (deadlock/signal/timeout), we must
1045		− * first acquire the hb->lock before removing the lock from the
1046		− * rt_mutex waitqueue, such that we can keep the hb and rt_mutex wait
1047		− * lists consistent.
	1038	+ * must unwind the above, however we canont lock hb->lock because
	1039	+ * rt_mutex already has a waiter enqueued and hb->lock can itself try
	1040	+ * and enqueue an rt_waiter through rtlock.
	1041	+ *
	1042	+ * Doing the cleanup without holding hb->lock can cause inconsistent
	1043	+ * state between hb and pi_state, but only in the direction of not
	1044	+ * seeing a waiter that is leaving.
	1045	+ *
	1046	+ * See futex_unlock_pi(), it deals with this inconsistency.
	1047	+ *
	1048	+ * There be dragons here, since we must deal with the inconsistency on
	1049	+ * the way out (here), it is impossible to detect/warn about the race
	1050	+ * the other way around (missing an incoming waiter).
1048	1051	*
1049		− * In particular; it is important that futex_unlock_pi() can not
1050		− * observe this inconsistency.
	1052	+ * What could possibly go wrong...
1051	1053	*/
1052	1054	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
1053	1055	ret = 0;
1054	1056
	1057	+ /*
	1058	+ * Now that the rt_waiter has been dequeued, it is safe to use
	1059	+ * spinlock/rtlock (which might enqueue its own rt_waiter) and fix up
	1060	+ * the
	1061	+ */
	1062	+ spin_lock(q.lock_ptr);
	1063	+ /*
	1064	+ * Waiter is unqueued.
	1065	+ */
	1066	+ rt_mutex_post_schedule();
1055	1067	no_block:
1056	1068	/*
1057	1069	* Fixup the pi_state owner and possibly acquire the lock if we
@@ -1117,7 +1129,7 @@ retry: · 59 unmodified lines
1117	1129	if ((uval & FUTEX_TID_MASK) != vpid)
1118	1130	return -EPERM;
1119	1131
1120		− ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_WRITE);
	1132	+ ret = get_futex_key(uaddr, flags, &key, FUTEX_WRITE);
1121	1133	if (ret)
1122	1134	return ret;
1123	1135
@@ -1132,6 +1144,7 @@ retry: · 8 unmodified lines
1132	1144	top_waiter = futex_top_waiter(hb, &key);
1133	1145	if (top_waiter) {
1134	1146	struct futex_pi_state *pi_state = top_waiter->pi_state;
	1147	+ struct rt_mutex_waiter *rt_waiter;
1135	1148
1136	1149	ret = -EINVAL;
1137	1150	if (!pi_state)
@@ -1144,22 +1157,39 @@ retry: · 6 unmodified lines
1144	1157	if (pi_state->owner != current)
1145	1158	goto out_unlock;
1146	1159
1147		− get_pi_state(pi_state);
1148	1160	/*
1149	1161	* By taking wait_lock while still holding hb->lock, we ensure
1150		− * there is no point where we hold neither; and therefore
1151		− * wake_futex_p() must observe a state consistent with what we
1152		− * observed.
	1162	+ * there is no point where we hold neither; and thereby
	1163	+ * wake_futex_pi() must observe any new waiters.
	1164	+ *
	1165	+ * Since the cleanup: case in futex_lock_pi() removes the
	1166	+ * rt_waiter without holding hb->lock, it is possible for
	1167	+ * wake_futex_pi() to not find a waiter while the above does,
	1168	+ * in this case the waiter is on the way out and it can be
	1169	+ * ignored.
1153	1170	*
1154	1171	* In particular; this forces __rt_mutex_start_proxy() to
1155	1172	* complete such that we're guaranteed to observe the
1156		− * rt_waiter. Also see the WARN in wake_futex_pi().
	1173	+ * rt_waiter.
1157	1174	*/
1158	1175	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
	1176	+
	1177	+ /*
	1178	+ * Futex vs rt_mutex waiter state -- if there are no rt_mutex
	1179	+ * waiters even though futex thinks there are, then the waiter
	1180	+ * is leaving and the uncontended path is safe to take.
	1181	+ */
	1182	+ rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
	1183	+ if (!rt_waiter) {
	1184	+ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
	1185	+ goto do_uncontended;
	1186	+ }
	1187	+
	1188	+ get_pi_state(pi_state);
1159	1189	spin_unlock(&hb->lock);
1160	1190
1161	1191	/* drops pi_state->pi_mutex.wait_lock */
1162		− ret = wake_futex_pi(uaddr, uval, pi_state);
	1192	+ ret = wake_futex_pi(uaddr, uval, pi_state, rt_waiter);
1163	1193
1164	1194	put_pi_state(pi_state);
1165	1195
@@ -1187,6 +1217,7 @@ retry: · 21 unmodified lines
1187	1217	return ret;
1188	1218	}
1189	1219
	1220	+do_uncontended:
1190	1221	/*
1191	1222	* We have no kernel internal state, i.e. no waiters in the
1192	1223	* kernel. Waiters which are about to queue themselves are stuck

kernel/futex/requeue.c+12−8

@@ -346,8 +346,9 @@ futex_proxy_trylock_atomic(u32 __user pifutex, struct futex_hash_bucket hb1, · 345 unmodified lines
346	346	/**
347	347	* futex_requeue() - Requeue waiters from uaddr1 to uaddr2
348	348	* @uaddr1: source futex user address
349		− * @flags: futex flags (FLAGS_SHARED, etc.)
	349	+ * @flags1: futex flags (FLAGS_SHARED, etc.)
350	350	* @uaddr2: target futex user address
	351	+ * @flags2: futex flags (FLAGS_SHARED, etc.)
351	352	* @nr_wake: number of waiters to wake (must be 1 for requeue_pi)
352	353	* @nr_requeue: number of waiters to requeue (0-INT_MAX)
353	354	* @cmpval: @uaddr1 expected value (or %NULL)
@@ -361,7 +362,8 @@ futex_proxy_trylock_atomic(u32 __user pifutex, struct futex_hash_bucket hb1, · 7 unmodified lines
361	362	* - >=0 - on success, the number of tasks requeued or woken;
362	363	* - <0 - on error
363	364	*/
364		−int futex_requeue(u32 __user uaddr1, unsigned int flags, u32 __user uaddr2,
	365	+int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
	366	+ u32 __user *uaddr2, unsigned int flags2,
365	367	int nr_wake, int nr_requeue, u32 *cmpval, int requeue_pi)
366	368	{
367	369	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
@@ -424,10 +426,10 @@ int futex_requeue(u32 __user uaddr1, unsigned int flags, u32 __user uaddr2, · 56 unmodified lines
424	426	}
425	427
426	428	retry:
427		− ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
	429	+ ret = get_futex_key(uaddr1, flags1, &key1, FUTEX_READ);
428	430	if (unlikely(ret != 0))
429	431	return ret;
430		− ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2,
	432	+ ret = get_futex_key(uaddr2, flags2, &key2,
431	433	requeue_pi ? FUTEX_WRITE : FUTEX_READ);
432	434	if (unlikely(ret != 0))
433	435	return ret;
@@ -459,7 +461,7 @@ retry_private: · 25 unmodified lines
459	461	if (ret)
460	462	return ret;
461	463
462		− if (!(flags & FLAGS_SHARED))
	464	+ if (!(flags1 & FLAGS_SHARED))
463	465	goto retry_private;
464	466
465	467	goto retry;
@@ -789,7 +791,7 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 323 unmodified lines
789	791	*/
790	792	rt_mutex_init_waiter(&rt_waiter);
791	793
792		− ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
	794	+ ret = get_futex_key(uaddr2, flags, &key2, FUTEX_WRITE);
793	795	if (unlikely(ret != 0))
794	796	goto out;
795	797
@@ -850,11 +852,13 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 54 unmodified lines
850	852	pi_mutex = &q.pi_state->pi_mutex;
851	853	ret = rt_mutex_wait_proxy_lock(pi_mutex, to, &rt_waiter);
852	854
853		− /* Current is not longer pi_blocked_on */
854		− spin_lock(q.lock_ptr);
	855	+ /*
	856	+ * See futex_unlock_pi()'s cleanup: comment.
	857	+ */
855	858	if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
856	859	ret = 0;
857	860
	861	+ spin_lock(q.lock_ptr);
858	862	debug_rt_mutex_free_waiter(&rt_waiter);
859	863	/*
860	864	* Fixup the pi_state owner and possibly acquire the lock if we

kernel/futex/syscalls.c+175−46

@@ -1,6 +1,5 @@
1	1	// SPDX-License-Identifier: GPL-2.0-or-later
2	2
3		−#include <linux/compat.h>
4	3	#include <linux/syscalls.h>
5	4	#include <linux/time_namespace.h>
6	5
@@ -85,15 +84,12 @@ err_unlock: · 78 unmodified lines
85	84	long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout,
86	85	u32 __user *uaddr2, u32 val2, u32 val3)
87	86	{
	87	+ unsigned int flags = futex_to_flags(op);
88	88	int cmd = op & FUTEX_CMD_MASK;
89		− unsigned int flags = 0;
90	89
91		− if (!(op & FUTEX_PRIVATE_FLAG))
92		− flags \|= FLAGS_SHARED;
93		−
94		− if (op & FUTEX_CLOCK_REALTIME) {
95		− flags \|= FLAGS_CLOCKRT;
96		− if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI &&
	90	+ if (flags & FLAGS_CLOCKRT) {
	91	+ if (cmd != FUTEX_WAIT_BITSET &&
	92	+ cmd != FUTEX_WAIT_REQUEUE_PI &&
97	93	cmd != FUTEX_LOCK_PI2)
98	94	return -ENOSYS;
99	95	}
@@ -110,9 +106,9 @@ long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout, · 10 unmodified lines
110	106	case FUTEX_WAKE_BITSET:
111	107	return futex_wake(uaddr, flags, val, val3);
112	108	case FUTEX_REQUEUE:
113		− return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);
	109	+ return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, NULL, 0);
114	110	case FUTEX_CMP_REQUEUE:
115		− return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);
	111	+ return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, &val3, 0);
116	112	case FUTEX_WAKE_OP:
117	113	return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);
118	114	case FUTEX_LOCK_PI:
@@ -129,7 +125,7 @@ long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout, · 10 unmodified lines
129	125	return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
130	126	uaddr2);
131	127	case FUTEX_CMP_REQUEUE_PI:
132		− return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
	128	+ return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, &val3, 1);
133	129	}
134	130	return -ENOSYS;
135	131	}
@@ -183,8 +179,7 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, · 47 unmodified lines
183	179	return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
184	180	}
185	181
186		−/* Mask of available flags for each futex in futex_waitv list */
187		−#define FUTEXV_WAITER_MASK (FUTEX_32 \| FUTEX_PRIVATE_FLAG)
	182	+#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_PRIVATE)
188	183
189	184	/**
190	185	* futex_parse_waitv - Parse a waitv array from userspace
@@ -202,16 +197,22 @@ static int futex_parse_waitv(struct futex_vector *futexv, · 11 unmodified lines
202	197	unsigned int i;
203	198
204	199	for (i = 0; i < nr_futexes; i++) {
	200	+ unsigned int flags;
	201	+
205	202	if (copy_from_user(&aux, &uwaitv[i], sizeof(aux)))
206	203	return -EFAULT;
207	204
208		− if ((aux.flags & ~FUTEXV_WAITER_MASK) \|\| aux.__reserved)
	205	+ if ((aux.flags & ~FUTEX2_VALID_MASK) \|\| aux.__reserved)
209	206	return -EINVAL;
210	207
211		− if (!(aux.flags & FUTEX_32))
	208	+ flags = futex2_to_flags(aux.flags);
	209	+ if (!futex_flags_valid(flags))
212	210	return -EINVAL;
213	211
214		− futexv[i].w.flags = aux.flags;
	212	+ if (!futex_validate_input(flags, aux.val))
	213	+ return -EINVAL;
	214	+
	215	+ futexv[i].w.flags = flags;
215	216	futexv[i].w.val = aux.val;
216	217	futexv[i].w.uaddr = aux.uaddr;
217	218	futexv[i].q = futex_q_init;
@@ -220,6 +221,46 @@ static int futex_parse_waitv(struct futex_vector *futexv, · 2 unmodified lines
220	221	return 0;
221	222	}
222	223
	224	+static int futex2_setup_timeout(struct __kernel_timespec __user *timeout,
	225	+ clockid_t clockid, struct hrtimer_sleeper *to)
	226	+{
	227	+ int flag_clkid = 0, flag_init = 0;
	228	+ struct timespec64 ts;
	229	+ ktime_t time;
	230	+ int ret;
	231	+
	232	+ if (!timeout)
	233	+ return 0;
	234	+
	235	+ if (clockid == CLOCK_REALTIME) {
	236	+ flag_clkid = FLAGS_CLOCKRT;
	237	+ flag_init = FUTEX_CLOCK_REALTIME;
	238	+ }
	239	+
	240	+ if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC)
	241	+ return -EINVAL;
	242	+
	243	+ if (get_timespec64(&ts, timeout))
	244	+ return -EFAULT;
	245	+
	246	+ /*
	247	+ * Since there's no opcode for futex_waitv, use
	248	+ * FUTEX_WAIT_BITSET that uses absolute timeout as well
	249	+ */
	250	+ ret = futex_init_timeout(FUTEX_WAIT_BITSET, flag_init, &ts, &time);
	251	+ if (ret)
	252	+ return ret;
	253	+
	254	+ futex_setup_timer(&time, to, flag_clkid, 0);
	255	+ return 0;
	256	+}
	257	+
	258	+static inline void futex2_destroy_timeout(struct hrtimer_sleeper *to)
	259	+{
	260	+ hrtimer_cancel(&to->timer);
	261	+ destroy_hrtimer_on_stack(&to->timer);
	262	+}
	263	+
223	264	/**
224	265	* sys_futex_waitv - Wait on a list of futexes
225	266	* @waiters: List of futexes to wait on
@@ -249,8 +290,6 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 23 unmodified lines
249	290	{
250	291	struct hrtimer_sleeper to;
251	292	struct futex_vector *futexv;
252		− struct timespec64 ts;
253		− ktime_t time;
254	293	int ret;
255	294
256	295	/* This syscall supports no flags for now */
@@ -260,30 +299,8 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 3 unmodified lines
260	299	if (!nr_futexes \|\| nr_futexes > FUTEX_WAITV_MAX \|\| !waiters)
261	300	return -EINVAL;
262	301
263		− if (timeout) {
264		− int flag_clkid = 0, flag_init = 0;
265		−
266		− if (clockid == CLOCK_REALTIME) {
267		− flag_clkid = FLAGS_CLOCKRT;
268		− flag_init = FUTEX_CLOCK_REALTIME;
269		− }
270		−
271		− if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC)
272		− return -EINVAL;
273		−
274		− if (get_timespec64(&ts, timeout))
275		− return -EFAULT;
276		−
277		− /*
278		− * Since there's no opcode for futex_waitv, use
279		− * FUTEX_WAIT_BITSET that uses absolute timeout as well
280		− */
281		− ret = futex_init_timeout(FUTEX_WAIT_BITSET, flag_init, &ts, &time);
282		− if (ret)
283		− return ret;
284		−
285		− futex_setup_timer(&time, &to, flag_clkid, 0);
286		− }
	302	+ if (timeout && (ret = futex2_setup_timeout(timeout, clockid, &to)))
	303	+ return ret;
287	304
288	305	futexv = kcalloc(nr_futexes, sizeof(*futexv), GFP_KERNEL);
289	306	if (!futexv) {
@@ -298,13 +315,125 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 8 unmodified lines
298	315	kfree(futexv);
299	316
300	317	destroy_timer:
301		− if (timeout) {
302		− hrtimer_cancel(&to.timer);
303		− destroy_hrtimer_on_stack(&to.timer);
304		− }
	318	+ if (timeout)
	319	+ futex2_destroy_timeout(&to);
	320	+ return ret;
	321	+}
	322	+
	323	+/*
	324	+ * sys_futex_wake - Wake a number of futexes
	325	+ * @uaddr: Address of the futex(es) to wake
	326	+ * @mask: bitmask
	327	+ * @nr: Number of the futexes to wake
	328	+ * @flags: FUTEX2 flags
	329	+ *
	330	+ * Identical to the traditional FUTEX_WAKE_BITSET op, except it is part of the
	331	+ * futex2 family of calls.
	332	+ */
	333	+
	334	+SYSCALL_DEFINE4(futex_wake,
	335	+ void __user *, uaddr,
	336	+ unsigned long, mask,
	337	+ int, nr,
	338	+ unsigned int, flags)
	339	+{
	340	+ if (flags & ~FUTEX2_VALID_MASK)
	341	+ return -EINVAL;
	342	+
	343	+ flags = futex2_to_flags(flags);
	344	+ if (!futex_flags_valid(flags))
	345	+ return -EINVAL;
	346	+
	347	+ if (!futex_validate_input(flags, mask))
	348	+ return -EINVAL;
	349	+
	350	+ return futex_wake(uaddr, FLAGS_STRICT \| flags, nr, mask);
	351	+}
	352	+
	353	+/*
	354	+ * sys_futex_wait - Wait on a futex
	355	+ * @uaddr: Address of the futex to wait on
	356	+ * @val: Value of @uaddr
	357	+ * @mask: bitmask
	358	+ * @flags: FUTEX2 flags
	359	+ * @timeout: Optional absolute timeout
	360	+ * @clockid: Clock to be used for the timeout, realtime or monotonic
	361	+ *
	362	+ * Identical to the traditional FUTEX_WAIT_BITSET op, except it is part of the
	363	+ * futex2 familiy of calls.
	364	+ */
	365	+
	366	+SYSCALL_DEFINE6(futex_wait,
	367	+ void __user *, uaddr,
	368	+ unsigned long, val,
	369	+ unsigned long, mask,
	370	+ unsigned int, flags,
	371	+ struct __kernel_timespec __user *, timeout,
	372	+ clockid_t, clockid)
	373	+{
	374	+ struct hrtimer_sleeper to;
	375	+ int ret;
	376	+
	377	+ if (flags & ~FUTEX2_VALID_MASK)
	378	+ return -EINVAL;
	379	+
	380	+ flags = futex2_to_flags(flags);
	381	+ if (!futex_flags_valid(flags))
	382	+ return -EINVAL;
	383	+
	384	+ if (!futex_validate_input(flags, val) \|\|
	385	+ !futex_validate_input(flags, mask))
	386	+ return -EINVAL;
	387	+
	388	+ if (timeout && (ret = futex2_setup_timeout(timeout, clockid, &to)))
	389	+ return ret;
	390	+
	391	+ ret = __futex_wait(uaddr, flags, val, timeout ? &to : NULL, mask);
	392	+
	393	+ if (timeout)
	394	+ futex2_destroy_timeout(&to);
	395	+
305	396	return ret;
306	397	}
307	398
	399	+/*
	400	+ * sys_futex_requeue - Requeue a waiter from one futex to another
	401	+ * @waiters: array describing the source and destination futex
	402	+ * @flags: unused
	403	+ * @nr_wake: number of futexes to wake
	404	+ * @nr_requeue: number of futexes to requeue
	405	+ *
	406	+ * Identical to the traditional FUTEX_CMP_REQUEUE op, except it is part of the
	407	+ * futex2 family of calls.
	408	+ */
	409	+
	410	+SYSCALL_DEFINE4(futex_requeue,
	411	+ struct futex_waitv __user *, waiters,
	412	+ unsigned int, flags,
	413	+ int, nr_wake,
	414	+ int, nr_requeue)
	415	+{
	416	+ struct futex_vector futexes[2];
	417	+ u32 cmpval;
	418	+ int ret;
	419	+
	420	+ if (flags)
	421	+ return -EINVAL;
	422	+
	423	+ if (!waiters)
	424	+ return -EINVAL;
	425	+
	426	+ ret = futex_parse_waitv(futexes, waiters, 2);
	427	+ if (ret)
	428	+ return ret;
	429	+
	430	+ cmpval = futexes[0].w.val;
	431	+
	432	+ return futex_requeue(u64_to_user_ptr(futexes[0].w.uaddr), futexes[0].w.flags,
	433	+ u64_to_user_ptr(futexes[1].w.uaddr), futexes[1].w.flags,
	434	+ nr_wake, nr_requeue, &cmpval, 0);
	435	+}
	436	+
308	437	#ifdef CONFIG_COMPAT
309	438	COMPAT_SYSCALL_DEFINE2(set_robust_list,
310	439	struct compat_robust_list_head __user *, head,

kernel/futex/waitwake.c+47−33

@@ -145,16 +145,19 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) · 144 unmodified lines
145	145	struct futex_hash_bucket *hb;
146	146	struct futex_q this, next;
147	147	union futex_key key = FUTEX_KEY_INIT;
148		− int ret;
149	148	DEFINE_WAKE_Q(wake_q);
	149	+ int ret;
150	150
151	151	if (!bitset)
152	152	return -EINVAL;
153	153
154		− ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);
	154	+ ret = get_futex_key(uaddr, flags, &key, FUTEX_READ);
155	155	if (unlikely(ret != 0))
156	156	return ret;
157	157
	158	+ if ((flags & FLAGS_STRICT) && !nr_wake)
	159	+ return 0;
	160	+
158	161	hb = futex_hash(&key);
159	162
160	163	/* Make sure we really have tasks to wakeup */
@@ -245,10 +248,10 @@ int futex_wake_op(u32 __user uaddr1, unsigned int flags, u32 __user uaddr2, · 84 unmodified lines
245	248	DEFINE_WAKE_Q(wake_q);
246	249
247	250	retry:
248		− ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
	251	+ ret = get_futex_key(uaddr1, flags, &key1, FUTEX_READ);
249	252	if (unlikely(ret != 0))
250	253	return ret;
251		− ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
	254	+ ret = get_futex_key(uaddr2, flags, &key2, FUTEX_WRITE);
252	255	if (unlikely(ret != 0))
253	256	return ret;
254	257
@@ -419,11 +422,11 @@ static int futex_wait_multiple_setup(struct futex_vector vs, int count, int wo · 164 unmodified lines
419	422	*/
420	423	retry:
421	424	for (i = 0; i < count; i++) {
422		− if ((vs[i].w.flags & FUTEX_PRIVATE_FLAG) && retry)
	425	+ if (!(vs[i].w.flags & FLAGS_SHARED) && retry)
423	426	continue;
424	427
425	428	ret = get_futex_key(u64_to_user_ptr(vs[i].w.uaddr),
426		− !(vs[i].w.flags & FUTEX_PRIVATE_FLAG),
	429	+ vs[i].w.flags,
427	430	&vs[i].q.key, FUTEX_READ);
428	431
429	432	if (unlikely(ret))
@@ -435,7 +438,7 @@ retry: · 5 unmodified lines
435	438	for (i = 0; i < count; i++) {
436	439	u32 __user uaddr = (u32 __user )(unsigned long)vs[i].w.uaddr;
437	440	struct futex_q *q = &vs[i].q;
438		− u32 val = (u32)vs[i].w.val;
	441	+ u32 val = vs[i].w.val;
439	442
440	443	hb = futex_q_lock(q);
441	444	ret = futex_get_value_locked(&uval, uaddr);
@@ -599,7 +602,7 @@ int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, · 157 unmodified lines
599	602	* while the syscall executes.
600	603	*/
601	604	retry:
602		− ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
	605	+ ret = get_futex_key(uaddr, flags, &q->key, FUTEX_READ);
603	606	if (unlikely(ret != 0))
604	607	return ret;
605	608
@@ -629,20 +632,18 @@ retry_private: · 23 unmodified lines
629	632	return ret;
630	633	}
631	634
632		−int futex_wait(u32 __user uaddr, unsigned int flags, u32 val, ktime_t abs_time, u32 bitset)
	635	+int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
	636	+ struct hrtimer_sleeper *to, u32 bitset)
633	637	{
634		− struct hrtimer_sleeper timeout, *to;
635		− struct restart_block *restart;
636		− struct futex_hash_bucket *hb;
637	638	struct futex_q q = futex_q_init;
	639	+ struct futex_hash_bucket *hb;
638	640	int ret;
639	641
640	642	if (!bitset)
641	643	return -EINVAL;
	644	+
642	645	q.bitset = bitset;
643	646
644		− to = futex_setup_timer(abs_time, &timeout, flags,
645		− current->timer_slack_ns);
646	647	retry:
647	648	/*
648	649	* Prepare to wait on uaddr. On success, it holds hb->lock and q
@@ -650,18 +651,17 @@ retry: · 1 unmodified line
650	651	*/
651	652	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
652	653	if (ret)
653		− goto out;
	654	+ return ret;
654	655
655	656	/* futex_queue and wait for wakeup, timeout, or a signal. */
656	657	futex_wait_queue(hb, &q, to);
657	658
658	659	/* If we were woken (and unqueued), we succeeded, whatever. */
659		− ret = 0;
660	660	if (!futex_unqueue(&q))
661		− goto out;
662		− ret = -ETIMEDOUT;
	661	+ return 0;
	662	+
663	663	if (to && !to->task)
664		− goto out;
	664	+ return -ETIMEDOUT;
665	665
666	666	/*
667	667	* We expect signal_pending(current), but we might be the
@@ -670,24 +670,38 @@ retry: · 2 unmodified lines
670	670	if (!signal_pending(current))
671	671	goto retry;
672	672
673		− ret = -ERESTARTSYS;
674		− if (!abs_time)
675		− goto out;
	673	+ return -ERESTARTSYS;
	674	+}
676	675
677		− restart = &current->restart_block;
678		− restart->futex.uaddr = uaddr;
679		− restart->futex.val = val;
680		− restart->futex.time = *abs_time;
681		− restart->futex.bitset = bitset;
682		− restart->futex.flags = flags \| FLAGS_HAS_TIMEOUT;
	676	+int futex_wait(u32 __user uaddr, unsigned int flags, u32 val, ktime_t abs_time, u32 bitset)
	677	+{
	678	+ struct hrtimer_sleeper timeout, *to;
	679	+ struct restart_block *restart;
	680	+ int ret;
683	681
684		− ret = set_restart_fn(restart, futex_wait_restart);
	682	+ to = futex_setup_timer(abs_time, &timeout, flags,
	683	+ current->timer_slack_ns);
685	684
686		−out:
687		− if (to) {
688		− hrtimer_cancel(&to->timer);
689		− destroy_hrtimer_on_stack(&to->timer);
	685	+ ret = __futex_wait(uaddr, flags, val, to, bitset);
	686	+
	687	+ /* No timeout, nothing to clean up. */
	688	+ if (!to)
	689	+ return ret;
	690	+
	691	+ hrtimer_cancel(&to->timer);
	692	+ destroy_hrtimer_on_stack(&to->timer);
	693	+
	694	+ if (ret == -ERESTARTSYS) {
	695	+ restart = &current->restart_block;
	696	+ restart->futex.uaddr = uaddr;
	697	+ restart->futex.val = val;
	698	+ restart->futex.time = *abs_time;
	699	+ restart->futex.bitset = bitset;
	700	+ restart->futex.flags = flags \| FLAGS_HAS_TIMEOUT;
	701	+
	702	+ return set_restart_fn(restart, futex_wait_restart);
690	703	}
	704	+
691	705	return ret;
692	706	}
693	707

kernel/locking/rtmutex.c+34−3

@@ -218,6 +218,11 @@ static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock, · 217 unmodified lines
218	218	return try_cmpxchg_acquire(&lock->owner, &old, new);
219	219	}
220	220
	221	+static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
	222	+{
	223	+ return rt_mutex_cmpxchg_acquire(lock, NULL, current);
	224	+}
	225	+
221	226	static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
222	227	struct task_struct *old,
223	228	struct task_struct *new)
@@ -297,6 +302,20 @@ static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock, · 73 unmodified lines
297	302
298	303	}
299	304
	305	+static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock);
	306	+
	307	+static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
	308	+{
	309	+ /*
	310	+ * With debug enabled rt_mutex_cmpxchg trylock() will always fail.
	311	+ *
	312	+ * Avoid unconditionally taking the slow path by using
	313	+ * rt_mutex_slow_trylock() which is covered by the debug code and can
	314	+ * acquire a non-contended rtmutex.
	315	+ */
	316	+ return rt_mutex_slowtrylock(lock);
	317	+}
	318	+
300	319	static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
301	320	struct task_struct *old,
302	321	struct task_struct *new)
@@ -1613,7 +1632,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock, · 1310 unmodified lines
1613	1632	raw_spin_unlock_irq(&lock->wait_lock);
1614	1633
1615	1634	if (!owner \|\| !rtmutex_spin_on_owner(lock, waiter, owner))
1616		− schedule();
	1635	+ rt_mutex_schedule();
1617	1636
1618	1637	raw_spin_lock_irq(&lock->wait_lock);
1619	1638	set_current_state(state);
@@ -1642,7 +1661,7 @@ static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock, · 22 unmodified lines
1642	1661	WARN(1, "rtmutex deadlock detected\n");
1643	1662	while (1) {
1644	1663	set_current_state(TASK_INTERRUPTIBLE);
1645		− schedule();
	1664	+ rt_mutex_schedule();
1646	1665	}
1647	1666	}
1648	1667
@@ -1737,6 +1756,15 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock, · 88 unmodified lines
1737	1756	unsigned long flags;
1738	1757	int ret;
1739	1758
	1759	+ /*
	1760	+ * Do all pre-schedule work here, before we queue a waiter and invoke
	1761	+ * PI -- any such work that trips on rtlock (PREEMPT_RT spinlock) would
	1762	+ * otherwise recurse back into task_blocks_on_rt_mutex() through
	1763	+ * rtlock_slowlock() and will then enqueue a second waiter for this
	1764	+ * same task and things get really confusing real fast.
	1765	+ */
	1766	+ rt_mutex_pre_schedule();
	1767	+
1740	1768	/*
1741	1769	* Technically we could use raw_spin_[un]lock_irq() here, but this can
1742	1770	* be called in early boot if the cmpxchg() fast path is disabled
@@ -1748,6 +1776,7 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock, · 5 unmodified lines
1748	1776	raw_spin_lock_irqsave(&lock->wait_lock, flags);
1749	1777	ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);
1750	1778	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
	1779	+ rt_mutex_post_schedule();
1751	1780
1752	1781	return ret;
1753	1782	}
@@ -1755,7 +1784,9 @@ static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock, · 1 unmodified line
1755	1784	static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
1756	1785	unsigned int state)
1757	1786	{
1758		− if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
	1787	+ lockdep_assert(!current->pi_blocked_on);
	1788	+
	1789	+ if (likely(rt_mutex_try_acquire(lock)))
1759	1790	return 0;
1760	1791
1761	1792	return rt_mutex_slowlock(lock, NULL, state);

kernel/locking/rwbase_rt.c+8−0

@@ -71,6 +71,7 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb, · 70 unmodified lines
71	71	struct rt_mutex_base *rtm = &rwb->rtmutex;
72	72	int ret;
73	73
	74	+ rwbase_pre_schedule();
74	75	raw_spin_lock_irq(&rtm->wait_lock);
75	76
76	77	/*
@@ -125,12 +126,15 @@ static int __sched __rwbase_read_lock(struct rwbase_rt *rwb, · 48 unmodified lines
125	126	rwbase_rtmutex_unlock(rtm);
126	127
127	128	trace_contention_end(rwb, ret);
	129	+ rwbase_post_schedule();
128	130	return ret;
129	131	}
130	132
131	133	static __always_inline int rwbase_read_lock(struct rwbase_rt *rwb,
132	134	unsigned int state)
133	135	{
	136	+ lockdep_assert(!current->pi_blocked_on);
	137	+
134	138	if (rwbase_read_trylock(rwb))
135	139	return 0;
136	140
@@ -237,6 +241,8 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb, · 100 unmodified lines
237	241	/* Force readers into slow path */
238	242	atomic_sub(READER_BIAS, &rwb->readers);
239	243
	244	+ rwbase_pre_schedule();
	245	+
240	246	raw_spin_lock_irqsave(&rtm->wait_lock, flags);
241	247	if (__rwbase_write_trylock(rwb))
242	248	goto out_unlock;
@@ -248,6 +254,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb, · 5 unmodified lines
248	254	if (rwbase_signal_pending_state(state, current)) {
249	255	rwbase_restore_current_state();
250	256	__rwbase_write_unlock(rwb, 0, flags);
	257	+ rwbase_post_schedule();
251	258	trace_contention_end(rwb, -EINTR);
252	259	return -EINTR;
253	260	}
@@ -266,6 +273,7 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb, · 12 unmodified lines
266	273
267	274	out_unlock:
268	275	raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
	276	+ rwbase_post_schedule();
269	277	return 0;
270	278	}
271	279

kernel/locking/rwsem.c+7−1

@@ -1427,8 +1427,14 @@ static inline void __downgrade_write(struct rw_semaphore *sem) · 1426 unmodified lines
1427	1427	#define rwbase_signal_pending_state(state, current) \
1428	1428	signal_pending_state(state, current)
1429	1429
	1430	+#define rwbase_pre_schedule() \
	1431	+ rt_mutex_pre_schedule()
	1432	+
1430	1433	#define rwbase_schedule() \
1431		− schedule()
	1434	+ rt_mutex_schedule()
	1435	+
	1436	+#define rwbase_post_schedule() \
	1437	+ rt_mutex_post_schedule()
1432	1438
1433	1439	#include "rwbase_rt.c"
1434	1440

kernel/locking/spinlock_rt.c+6−0

@@ -37,6 +37,8 @@ · 36 unmodified lines
37	37
38	38	static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
39	39	{
	40	+ lockdep_assert(!current->pi_blocked_on);
	41	+
40	42	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
41	43	rtlock_slowlock(rtm);
42	44	}
@@ -184,9 +186,13 @@ static __always_inline int rwbase_rtmutex_trylock(struct rt_mutex_base *rtm) · 141 unmodified lines
184	186
185	187	#define rwbase_signal_pending_state(state, current) (0)
186	188
	189	+#define rwbase_pre_schedule()
	190	+
187	191	#define rwbase_schedule() \
188	192	schedule_rtlock()
189	193
	194	+#define rwbase_post_schedule()
	195	+
190	196	#include "rwbase_rt.c"
191	197	/*
192	198	* The common functions which get wrapped into the rwlock API.

kernel/locking/test-ww_mutex.c+34−14

@@ -9,7 +9,7 @@ · 8 unmodified lines
9	9	#include <linux/delay.h>
10	10	#include <linux/kthread.h>
11	11	#include <linux/module.h>
12		−#include <linux/random.h>
	12	+#include <linux/prandom.h>
13	13	#include <linux/slab.h>
14	14	#include <linux/ww_mutex.h>
15	15
@@ -386,6 +386,19 @@ struct stress { · 370 unmodified lines
386	386	int nlocks;
387	387	};
388	388
	389	+struct rnd_state rng;
	390	+DEFINE_SPINLOCK(rng_lock);
	391	+
	392	+static inline u32 prandom_u32_below(u32 ceil)
	393	+{
	394	+ u32 ret;
	395	+
	396	+ spin_lock(&rng_lock);
	397	+ ret = prandom_u32_state(&rng) % ceil;
	398	+ spin_unlock(&rng_lock);
	399	+ return ret;
	400	+}
	401	+
389	402	static int *get_random_order(int count)
390	403	{
391	404	int *order;
@@ -399,7 +412,7 @@ static int *get_random_order(int count) · 7 unmodified lines
399	412	order[n] = n;
400	413
401	414	for (n = count - 1; n > 1; n--) {
402		− r = get_random_u32_below(n + 1);
	415	+ r = prandom_u32_below(n + 1);
403	416	if (r != n) {
404	417	tmp = order[n];
405	418	order[n] = order[r];
@@ -452,21 +465,21 @@ retry: · 46 unmodified lines
452	465	ww_mutex_unlock(&locks[order[n]]);
453	466
454	467	if (err == -EDEADLK) {
455		− ww_mutex_lock_slow(&locks[order[contended]], &ctx);
456		− goto retry;
	468	+ if (!time_after(jiffies, stress->timeout)) {
	469	+ ww_mutex_lock_slow(&locks[order[contended]], &ctx);
	470	+ goto retry;
	471	+ }
457	472	}
458	473
	474	+ ww_acquire_fini(&ctx);
459	475	if (err) {
460	476	pr_err_once("stress (%s) failed with %d\n",
461	477	__func__, err);
462	478	break;
463	479	}
464		−
465		− ww_acquire_fini(&ctx);
466	480	} while (!time_after(jiffies, stress->timeout));
467	481
468	482	kfree(order);
469		− kfree(stress);
470	483	}
471	484
472	485	struct reorder_lock {
@@ -531,7 +544,6 @@ out: · 58 unmodified lines
531	544	list_for_each_entry_safe(ll, ln, &locks, link)
532	545	kfree(ll);
533	546	kfree(order);
534		− kfree(stress);
535	547	}
536	548
537	549	static void stress_one_work(struct work_struct *work)
@@ -552,8 +564,6 @@ static void stress_one_work(struct work_struct *work) · 14 unmodified lines
552	564	break;
553	565	}
554	566	} while (!time_after(jiffies, stress->timeout));
555		−
556		− kfree(stress);
557	567	}
558	568
559	569	#define STRESS_INORDER BIT(0)
@@ -564,15 +574,24 @@ static void stress_one_work(struct work_struct *work) · 4 unmodified lines
564	574	static int stress(int nlocks, int nthreads, unsigned int flags)
565	575	{
566	576	struct ww_mutex *locks;
567		− int n;
	577	+ struct stress *stress_array;
	578	+ int n, count;
568	579
569	580	locks = kmalloc_array(nlocks, sizeof(*locks), GFP_KERNEL);
570	581	if (!locks)
571	582	return -ENOMEM;
572	583
	584	+ stress_array = kmalloc_array(nthreads, sizeof(*stress_array),
	585	+ GFP_KERNEL);
	586	+ if (!stress_array) {
	587	+ kfree(locks);
	588	+ return -ENOMEM;
	589	+ }
	590	+
573	591	for (n = 0; n < nlocks; n++)
574	592	ww_mutex_init(&locks[n], &ww_class);
575	593
	594	+ count = 0;
576	595	for (n = 0; nthreads; n++) {
577	596	struct stress *stress;
578	597	void (fn)(struct work_struct work);
@@ -596,9 +615,7 @@ static int stress(int nlocks, int nthreads, unsigned int flags) · 17 unmodified lines
596	615	if (!fn)
597	616	continue;
598	617
599		− stress = kmalloc(sizeof(*stress), GFP_KERNEL);
600		− if (!stress)
601		− break;
	618	+ stress = &stress_array[count++];
602	619
603	620	INIT_WORK(&stress->work, fn);
604	621	stress->locks = locks;
@@ -613,6 +630,7 @@ static int stress(int nlocks, int nthreads, unsigned int flags) · 8 unmodified lines
613	630
614	631	for (n = 0; n < nlocks; n++)
615	632	ww_mutex_destroy(&locks[n]);
	633	+ kfree(stress_array);
616	634	kfree(locks);
617	635
618	636	return 0;
@@ -625,6 +643,8 @@ static int __init test_ww_mutex_init(void) · 6 unmodified lines
625	643
626	644	printk(KERN_INFO "Beginning ww mutex selftests\n");
627	645
	646	+ prandom_seed_state(&rng, get_random_u64());
	647	+
628	648	wq = alloc_workqueue("test-ww_mutex", WQ_UNBOUND, 0);
629	649	if (!wq)
630	650	return -ENOMEM;

kernel/locking/ww_rt_mutex.c+1−1

@@ -62,7 +62,7 @@ __ww_rt_mutex_lock(struct ww_mutex lock, struct ww_acquire_ctx ww_ctx, · 61 unmodified lines
62	62	}
63	63	mutex_acquire_nest(&rtm->dep_map, 0, 0, nest_lock, ip);
64	64
65		− if (likely(rt_mutex_cmpxchg_acquire(&rtm->rtmutex, NULL, current))) {
	65	+ if (likely(rt_mutex_try_acquire(&rtm->rtmutex))) {
66	66	if (ww_ctx)
67	67	ww_mutex_set_context_fastpath(lock, ww_ctx);
68	68	return 0;

kernel/sched/core.c+50−12

@@ -6720,10 +6720,14 @@ void __noreturn do_task_dead(void) · 6719 unmodified lines
6720	6720
6721	6721	static inline void sched_submit_work(struct task_struct *tsk)
6722	6722	{
	6723	+ static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG);
6723	6724	unsigned int task_flags;
6724	6725
6725		− if (task_is_running(tsk))
6726		− return;
	6726	+ /*
	6727	+ * Establish LD_WAIT_CONFIG context to ensure none of the code called
	6728	+ * will use a blocking primitive -- which would lead to recursion.
	6729	+ */
	6730	+ lock_map_acquire_try(&sched_map);
6727	6731
6728	6732	task_flags = tsk->flags;
6729	6733	/*
@@ -6749,6 +6753,8 @@ static inline void sched_submit_work(struct task_struct *tsk) · 19 unmodified lines
6749	6753	* make sure to submit it to avoid deadlocks.
6750	6754	*/
6751	6755	blk_flush_plug(tsk->plug, true);
	6756	+
	6757	+ lock_map_release(&sched_map);
6752	6758	}
6753	6759
6754	6760	static void sched_update_worker(struct task_struct *tsk)
@@ -6761,16 +6767,26 @@ static void sched_update_worker(struct task_struct *tsk) · 6 unmodified lines
6761	6767	}
6762	6768	}
6763	6769
6764		−asmlinkage __visible void __sched schedule(void)
	6770	+static __always_inline void __schedule_loop(unsigned int sched_mode)
6765	6771	{
6766		− struct task_struct *tsk = current;
6767		−
6768		− sched_submit_work(tsk);
6769	6772	do {
6770	6773	preempt_disable();
6771		− __schedule(SM_NONE);
	6774	+ __schedule(sched_mode);
6772	6775	sched_preempt_enable_no_resched();
6773	6776	} while (need_resched());
	6777	+}
	6778	+
	6779	+asmlinkage __visible void __sched schedule(void)
	6780	+{
	6781	+ struct task_struct *tsk = current;
	6782	+
	6783	+#ifdef CONFIG_RT_MUTEXES
	6784	+ lockdep_assert(!tsk->sched_rt_mutex);
	6785	+#endif
	6786	+
	6787	+ if (!task_is_running(tsk))
	6788	+ sched_submit_work(tsk);
	6789	+ __schedule_loop(SM_NONE);
6774	6790	sched_update_worker(tsk);
6775	6791	}
6776	6792	EXPORT_SYMBOL(schedule);
@@ -6834,11 +6850,7 @@ void __sched schedule_preempt_disabled(void) · 57 unmodified lines
6834	6850	#ifdef CONFIG_PREEMPT_RT
6835	6851	void __sched notrace schedule_rtlock(void)
6836	6852	{
6837		− do {
6838		− preempt_disable();
6839		− __schedule(SM_RTLOCK_WAIT);
6840		− sched_preempt_enable_no_resched();
6841		− } while (need_resched());
	6853	+ __schedule_loop(SM_RTLOCK_WAIT);
6842	6854	}
6843	6855	NOKPROBE_SYMBOL(schedule_rtlock);
6844	6856	#endif
@@ -7034,6 +7046,32 @@ static void __setscheduler_prio(struct task_struct *p, int prio) · 189 unmodified lines
7034	7046
7035	7047	#ifdef CONFIG_RT_MUTEXES
7036	7048
	7049	+/*
	7050	+ * Would be more useful with typeof()/auto_type but they don't mix with
	7051	+ * bit-fields. Since it's a local thing, use int. Keep the generic sounding
	7052	+ * name such that if someone were to implement this function we get to compare
	7053	+ * notes.
	7054	+ */
	7055	+#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; })
	7056	+
	7057	+void rt_mutex_pre_schedule(void)
	7058	+{
	7059	+ lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
	7060	+ sched_submit_work(current);
	7061	+}
	7062	+
	7063	+void rt_mutex_schedule(void)
	7064	+{
	7065	+ lockdep_assert(current->sched_rt_mutex);
	7066	+ __schedule_loop(SM_NONE);
	7067	+}
	7068	+
	7069	+void rt_mutex_post_schedule(void)
	7070	+{
	7071	+ sched_update_worker(current);
	7072	+ lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
	7073	+}
	7074	+
7037	7075	static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
7038	7076	{
7039	7077	if (pi_task)

kernel/sys_ni.c+3−0

@@ -87,6 +87,9 @@ COND_SYSCALL_COMPAT(set_robust_list); · 86 unmodified lines
87	87	COND_SYSCALL(get_robust_list);
88	88	COND_SYSCALL_COMPAT(get_robust_list);
89	89	COND_SYSCALL(futex_waitv);
	90	+COND_SYSCALL(futex_wake);
	91	+COND_SYSCALL(futex_wait);
	92	+COND_SYSCALL(futex_requeue);
90	93	COND_SYSCALL(kexec_load);
91	94	COND_SYSCALL_COMPAT(kexec_load);
92	95	COND_SYSCALL(init_module);

Merge branch 'locking/core' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip into io_uring-futex

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code