lael-labs/linux

arch/alpha/kernel/syscalls/syscall.tbl+3−0

@@ -492,3 +492,6 @@ · 491 unmodified lines
492	560 common set_mempolicy_home_node sys_ni_syscall	492	560 common set_mempolicy_home_node sys_ni_syscall
493	561 common cachestat sys_cachestat	493	561 common cachestat sys_cachestat
494	562 common fchmodat2 sys_fchmodat2	494	562 common fchmodat2 sys_fchmodat2
		495	563 common futex_wake sys_futex_wake
		496	564 common futex_wait sys_futex_wait
		497	565 common futex_requeue sys_futex_requeue

arch/arm/tools/syscall.tbl+3−0

@@ -466,3 +466,6 @@ · 465 unmodified lines
466	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	466	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
467	451 common cachestat sys_cachestat	467	451 common cachestat sys_cachestat
468	452 common fchmodat2 sys_fchmodat2	468	452 common fchmodat2 sys_fchmodat2
		469	454 common futex_wake sys_futex_wake
		470	455 common futex_wait sys_futex_wait
		471	456 common futex_requeue sys_futex_requeue

arch/arm64/include/asm/unistd.h+1−1

@@ -39,7 +39,7 @@ · 38 unmodified lines
39	#define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5)	39	#define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5)
40	#define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800)	40	#define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800)
41		41
42	#define __NR_compat_syscalls 453	42	#define __NR_compat_syscalls 457
43	#endif	43	#endif
44		44
45	#define __ARCH_WANT_SYS_CLONE	45	#define __ARCH_WANT_SYS_CLONE

arch/arm64/include/asm/unistd32.h+6−0

 __SYSCALL(__NR_cachestat, sys_cachestat)
 #define __NR_fchmodat2 452
 __SYSCALL(__NR_fchmodat2, sys_fchmodat2)
+#define __NR_futex_wake 454
+__SYSCALL(__NR_futex_wake, sys_futex_wake)
+#define __NR_futex_wait 455
+__SYSCALL(__NR_futex_wait, sys_futex_wait)
+#define __NR_futex_requeue 456
+__SYSCALL(__NR_futex_requeue, sys_futex_requeue)
 /*
  * Please add new compat syscalls above this comment and update

arch/ia64/kernel/syscalls/syscall.tbl+3−0

@@ -373,3 +373,6 @@ · 372 unmodified lines
373	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	373	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
374	451 common cachestat sys_cachestat	374	451 common cachestat sys_cachestat
375	452 common fchmodat2 sys_fchmodat2	375	452 common fchmodat2 sys_fchmodat2
		376	454 common futex_wake sys_futex_wake
		377	455 common futex_wait sys_futex_wait
		378	456 common futex_requeue sys_futex_requeue

arch/m68k/kernel/syscalls/syscall.tbl+3−0

@@ -452,3 +452,6 @@ · 451 unmodified lines
452	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	452	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
453	451 common cachestat sys_cachestat	453	451 common cachestat sys_cachestat
454	452 common fchmodat2 sys_fchmodat2	454	452 common fchmodat2 sys_fchmodat2
		455	454 common futex_wake sys_futex_wake
		456	455 common futex_wait sys_futex_wait
		457	456 common futex_requeue sys_futex_requeue

arch/microblaze/kernel/syscalls/syscall.tbl+3−0

@@ -458,3 +458,6 @@ · 457 unmodified lines
458	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	458	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
459	451 common cachestat sys_cachestat	459	451 common cachestat sys_cachestat
460	452 common fchmodat2 sys_fchmodat2	460	452 common fchmodat2 sys_fchmodat2
		461	454 common futex_wake sys_futex_wake
		462	455 common futex_wait sys_futex_wait
		463	456 common futex_requeue sys_futex_requeue

arch/mips/kernel/syscalls/syscall_n32.tbl+3−0

@@ -391,3 +391,6 @@ · 390 unmodified lines
391	450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node	391	450 n32 set_mempolicy_home_node sys_set_mempolicy_home_node
392	451 n32 cachestat sys_cachestat	392	451 n32 cachestat sys_cachestat
393	452 n32 fchmodat2 sys_fchmodat2	393	452 n32 fchmodat2 sys_fchmodat2
		394	454 n32 futex_wake sys_futex_wake
		395	455 n32 futex_wait sys_futex_wait
		396	456 n32 futex_requeue sys_futex_requeue

arch/mips/kernel/syscalls/syscall_n64.tbl+3−0

@@ -367,3 +367,6 @@ · 366 unmodified lines
367	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	367	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
368	451 n64 cachestat sys_cachestat	368	451 n64 cachestat sys_cachestat
369	452 n64 fchmodat2 sys_fchmodat2	369	452 n64 fchmodat2 sys_fchmodat2
		370	454 n64 futex_wake sys_futex_wake
		371	455 n64 futex_wait sys_futex_wait
		372	456 n64 futex_requeue sys_futex_requeue

arch/mips/kernel/syscalls/syscall_o32.tbl+3−0

@@ -440,3 +440,6 @@ · 439 unmodified lines
440	450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node	440	450 o32 set_mempolicy_home_node sys_set_mempolicy_home_node
441	451 o32 cachestat sys_cachestat	441	451 o32 cachestat sys_cachestat
442	452 o32 fchmodat2 sys_fchmodat2	442	452 o32 fchmodat2 sys_fchmodat2
		443	454 o32 futex_wake sys_futex_wake
		444	455 o32 futex_wait sys_futex_wait
		445	456 o32 futex_requeue sys_futex_requeue

arch/parisc/kernel/syscalls/syscall.tbl+3−0

@@ -451,3 +451,6 @@ · 450 unmodified lines
451	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	451	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
452	451 common cachestat sys_cachestat	452	451 common cachestat sys_cachestat
453	452 common fchmodat2 sys_fchmodat2	453	452 common fchmodat2 sys_fchmodat2
		454	454 common futex_wake sys_futex_wake
		455	455 common futex_wait sys_futex_wait
		456	456 common futex_requeue sys_futex_requeue

arch/powerpc/kernel/syscalls/syscall.tbl+3−0

@@ -539,3 +539,6 @@ · 538 unmodified lines
539	450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node	539	450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node
540	451 common cachestat sys_cachestat	540	451 common cachestat sys_cachestat
541	452 common fchmodat2 sys_fchmodat2	541	452 common fchmodat2 sys_fchmodat2
		542	454 common futex_wake sys_futex_wake
		543	455 common futex_wait sys_futex_wait
		544	456 common futex_requeue sys_futex_requeue

arch/s390/kernel/syscalls/syscall.tbl+3−0

@@ -455,3 +455,6 @@ · 454 unmodified lines
455	450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node	455	450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node
456	451 common cachestat sys_cachestat sys_cachestat	456	451 common cachestat sys_cachestat sys_cachestat
457	452 common fchmodat2 sys_fchmodat2 sys_fchmodat2	457	452 common fchmodat2 sys_fchmodat2 sys_fchmodat2
		458	454 common futex_wake sys_futex_wake sys_futex_wake
		459	455 common futex_wait sys_futex_wait sys_futex_wait
		460	456 common futex_requeue sys_futex_requeue sys_futex_requeue

arch/sh/kernel/syscalls/syscall.tbl+3−0

@@ -455,3 +455,6 @@ · 454 unmodified lines
455	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	455	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
456	451 common cachestat sys_cachestat	456	451 common cachestat sys_cachestat
457	452 common fchmodat2 sys_fchmodat2	457	452 common fchmodat2 sys_fchmodat2
		458	454 common futex_wake sys_futex_wake
		459	455 common futex_wait sys_futex_wait
		460	456 common futex_requeue sys_futex_requeue

arch/sparc/kernel/syscalls/syscall.tbl+3−0

@@ -498,3 +498,6 @@ · 497 unmodified lines
498	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	498	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
499	451 common cachestat sys_cachestat	499	451 common cachestat sys_cachestat
500	452 common fchmodat2 sys_fchmodat2	500	452 common fchmodat2 sys_fchmodat2
		501	454 common futex_wake sys_futex_wake
		502	455 common futex_wait sys_futex_wait
		503	456 common futex_requeue sys_futex_requeue

arch/x86/Kconfig+1−1

 	select ARCH_HAS_GIGANTIC_PAGE
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
 	select ARCH_SUPPORTS_PER_VMA_LOCK
-	select ARCH_USE_CMPXCHG_LOCKREF
 	select HAVE_ARCH_SOFT_DIRTY
 	select MODULES_USE_ELF_RELA
 	select NEED_DMA_MAP_STATE
 	select ARCH_SUPPORTS_LTO_CLANG
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
 	select ARCH_USE_BUILTIN_BSWAP
+	select ARCH_USE_CMPXCHG_LOCKREF		if X86_CMPXCHG64
 	select ARCH_USE_MEMTEST
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_USE_QUEUED_SPINLOCKS

arch/x86/entry/syscalls/syscall_32.tbl+3−0

@@ -457,3 +457,6 @@ · 456 unmodified lines
457	450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node	457	450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node
458	451 i386 cachestat sys_cachestat	458	451 i386 cachestat sys_cachestat
459	452 i386 fchmodat2 sys_fchmodat2	459	452 i386 fchmodat2 sys_fchmodat2
		460	454 i386 futex_wake sys_futex_wake
		461	455 i386 futex_wait sys_futex_wait
		462	456 i386 futex_requeue sys_futex_requeue

arch/x86/entry/syscalls/syscall_64.tbl+3−0

 451	common	cachestat		sys_cachestat
 452	common	fchmodat2		sys_fchmodat2
 453	64	map_shadow_stack	sys_map_shadow_stack
+454	common	futex_wake		sys_futex_wake
+455	common	futex_wait		sys_futex_wait
+456	common	futex_requeue		sys_futex_requeue
 #
 # Due to a historical design error, certain syscalls are numbered differently

arch/xtensa/kernel/syscalls/syscall.tbl+3−0

@@ -423,3 +423,6 @@ · 422 unmodified lines
423	450 common set_mempolicy_home_node sys_set_mempolicy_home_node	423	450 common set_mempolicy_home_node sys_set_mempolicy_home_node
424	451 common cachestat sys_cachestat	424	451 common cachestat sys_cachestat
425	452 common fchmodat2 sys_fchmodat2	425	452 common fchmodat2 sys_fchmodat2
		426	454 common futex_wake sys_futex_wake
		427	455 common futex_wait sys_futex_wait
		428	456 common futex_requeue sys_futex_requeue

include/asm-generic/spinlock.h+9−7

 	smp_store_release(ptr, (u16)val + 1);
 }
+static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
+{
+	u32 val = lock.counter;
+	return ((val >> 16) == (val & 0xffff));
+}
 static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock)
 {
-	u32 val = atomic_read(lock);
+	arch_spinlock_t val = READ_ONCE(*lock);
-	return ((val >> 16) != (val & 0xffff));
+	return !arch_spin_value_unlocked(val);
 }
 static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock)
 	return (s16)((val >> 16) - (val & 0xffff)) > 1;
 }
-static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
-{
-	return !arch_spin_is_locked(&lock);
-}
 #include <asm/qrwlock.h>
 #endif /* __ASM_GENERIC_SPINLOCK_H */

include/linux/cleanup.h+36−3

 /*
  * DEFINE_FREE(name, type, free):
  *	simple helper macro that defines the required wrapper for a __free()
- *	based cleanup function. @free is an expression using '_T' to access
+ *	based cleanup function. @free is an expression using '_T' to access the
- *	the variable.
+ *	variable. @free should typically include a NULL test before calling a
+ *	function, see the example below.
  *
  * __free(name):
  *	variable attribute to add a scoped based cleanup to the variable.
  *	like a non-atomic xchg(var, NULL), such that the cleanup function will
  *	be inhibited -- provided it sanely deals with a NULL value.
  *
+ *	NOTE: this has __must_check semantics so that it is harder to accidentally
+ *	leak the resource.
+ *
  * return_ptr(p):
  *	returns p while inhibiting the __free().
  *
  *
  * DEFINE_FREE(kfree, void *, if (_T) kfree(_T))
  *
+ * void *alloc_obj(...)
+ * {
  *	struct obj *p __free(kfree) = kmalloc(...);
  *	if (!p)
  *		return NULL;
  *		return NULL;
  *
  *	return_ptr(p);
+ * }
+ *
+ * NOTE: the DEFINE_FREE()'s @free expression includes a NULL test even though
+ * kfree() is fine to be called with a NULL value. This is on purpose. This way
+ * the compiler sees the end of our alloc_obj() function as:
+ *
+ *	tmp = p;
+ *	p = NULL;
+ *	if (p)
+ *		kfree(p);
+ *	return tmp;
+ *
+ * And through the magic of value-propagation and dead-code-elimination, it
+ * eliminates the actual cleanup call and compiles into:
+ *
+ *	return p;
+ *
+ * Without the NULL test it turns into a mess and the compiler can't help us.
  */
 #define DEFINE_FREE(_name, _type, _free) \
 #define __free(_name)	__cleanup(__free_##_name)
+#define __get_and_null_ptr(p) \
+	({ __auto_type __ptr = &(p); \
+	   __auto_type __val = *__ptr; \
+	   *__ptr = NULL;  __val; })
+static inline __must_check
+const volatile void * __must_check_fn(const volatile void *val)
+{ return val; }
 #define no_free_ptr(p) \
-	({ __auto_type __ptr = (p); (p) = NULL; __ptr; })
+	((typeof(p)) __must_check_fn(__get_and_null_ptr(p)))
 #define return_ptr(p)	return no_free_ptr(p)

include/linux/sched.h+3−0

 	 * ->sched_remote_wakeup gets used, so it can be in this word.
 	 */
 	unsigned			sched_remote_wakeup:1;
+#ifdef CONFIG_RT_MUTEXES
+	unsigned			sched_rt_mutex:1;
+#endif
 	/* Bit to tell LSMs we're in execve(): */
 	unsigned			in_execve:1;

include/linux/sched/rt.h+4−0

 }
 #ifdef CONFIG_RT_MUTEXES
+extern void rt_mutex_pre_schedule(void);
+extern void rt_mutex_schedule(void);
+extern void rt_mutex_post_schedule(void);
 /*
  * Must hold either p->pi_lock or task_rq(p)->lock.
  */

include/linux/syscalls.h+10−0

@@ -549,6 +549,16 @@ asmlinkage long sys_set_robust_list(struct robust_list_head __user *head, · 548 unmodified lines
549	asmlinkage long sys_futex_waitv(struct futex_waitv *waiters,	549	asmlinkage long sys_futex_waitv(struct futex_waitv *waiters,
550	unsigned int nr_futexes, unsigned int flags,	550	unsigned int nr_futexes, unsigned int flags,
551	struct __kernel_timespec __user *timeout, clockid_t clockid);	551	struct __kernel_timespec __user *timeout, clockid_t clockid);
		552
		553	asmlinkage long sys_futex_wake(void __user *uaddr, unsigned long mask, int nr, unsigned int flags);
		554
		555	asmlinkage long sys_futex_wait(void __user *uaddr, unsigned long val, unsigned long mask,
		556	unsigned int flags, struct __kernel_timespec __user *timespec,
		557	clockid_t clockid);
		558
		559	asmlinkage long sys_futex_requeue(struct futex_waitv __user *waiters,
		560	unsigned int flags, int nr_wake, int nr_requeue);
		561
552	asmlinkage long sys_nanosleep(struct __kernel_timespec __user *rqtp,	562	asmlinkage long sys_nanosleep(struct __kernel_timespec __user *rqtp,
553	struct __kernel_timespec __user *rmtp);	563	struct __kernel_timespec __user *rmtp);
554	asmlinkage long sys_nanosleep_time32(struct old_timespec32 __user *rqtp,	564	asmlinkage long sys_nanosleep_time32(struct old_timespec32 __user *rqtp,

include/uapi/asm-generic/unistd.h+7−1

 #define __NR_fchmodat2 452
 __SYSCALL(__NR_fchmodat2, sys_fchmodat2)
+#define __NR_futex_wake 454
+__SYSCALL(__NR_futex_wake, sys_futex_wake)
+#define __NR_futex_wait 455
+__SYSCALL(__NR_futex_wait, sys_futex_wait)
+#define __NR_futex_requeue 456
+__SYSCALL(__NR_futex_requeue, sys_futex_requeue)
 #undef __NR_syscalls
-#define __NR_syscalls 453
+#define __NR_syscalls 457
 /*
  * 32 bit systems traditionally used different

include/uapi/linux/futex.h+28−3

@@ -44,10 +44,35 @@ · 43 unmodified lines
44	FUTEX_PRIVATE_FLAG)	44	FUTEX_PRIVATE_FLAG)
45		45
46	/*	46	/*
47	* Flags to specify the bit length of the futex word for futex2 syscalls.	47	* Flags for futex2 syscalls.
48	* Currently, only 32 is supported.	48	*
		49	* NOTE: these are not pure flags, they can also be seen as:
		50	*
		51	* union {
		52	* u32 flags;
		53	* struct {
		54	* u32 size : 2,
		55	* numa : 1,
		56	* : 4,
		57	* private : 1;
		58	* };
		59	* };
49	*/	60	*/
50	#define FUTEX_32 2	61	#define FUTEX2_SIZE_U8 0x00
		62	#define FUTEX2_SIZE_U16 0x01
		63	#define FUTEX2_SIZE_U32 0x02
		64	#define FUTEX2_SIZE_U64 0x03
		65	#define FUTEX2_NUMA 0x04
		66	/* 0x08 */
		67	/* 0x10 */
		68	/* 0x20 */
		69	/* 0x40 */
		70	#define FUTEX2_PRIVATE FUTEX_PRIVATE_FLAG
		71
		72	#define FUTEX2_SIZE_MASK 0x03
		73
		74	/* do not use */
		75	#define FUTEX_32 FUTEX2_SIZE_U32 /* historical accident :-( */
51		76
52	/*	77	/*
53	* Max numbers of elements in a futex_waitv array	78	* Max numbers of elements in a futex_waitv array

kernel/futex/core.c+38−36

 /**
  * get_futex_key() - Get parameters which are the keys for a futex
  * @uaddr:	virtual address of the futex
- * @fshared:	false for a PROCESS_PRIVATE futex, true for PROCESS_SHARED
+ * @flags:	FLAGS_*
  * @key:	address where result is stored.
  * @rw:		mapping needs to be read/write (values: FUTEX_READ,
  *              FUTEX_WRITE)
  *
  * lock_page() might sleep, the caller should not hold a spinlock.
  */
-int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,
+int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key,
 		  enum futex_access rw)
 {
 	unsigned long address = (unsigned long)uaddr;
 	struct mm_struct *mm = current->mm;
-	struct page *page, *tail;
+	struct page *page;
+	struct folio *folio;
 	struct address_space *mapping;
 	int err, ro = 0;
+	bool fshared;
+	fshared = flags & FLAGS_SHARED;
 	/*
 	 * The futex address must be "naturally" aligned.
 		err = 0;
 	/*
-	 * The treatment of mapping from this point on is critical. The page
+	 * The treatment of mapping from this point on is critical. The folio
-	 * lock protects many things but in this context the page lock
+	 * lock protects many things but in this context the folio lock
 	 * stabilizes mapping, prevents inode freeing in the shared
 	 * file-backed region case and guards against movement to swap cache.
 	 *
-	 * Strictly speaking the page lock is not needed in all cases being
+	 * Strictly speaking the folio lock is not needed in all cases being
-	 * considered here and page lock forces unnecessarily serialization
+	 * considered here and folio lock forces unnecessarily serialization.
 	 * From this point on, mapping will be re-verified if necessary and
-	 * page lock will be acquired only if it is unavoidable
+	 * folio lock will be acquired only if it is unavoidable
 	 *
-	 * Mapping checks require the head page for any compound page so the
+	 * Mapping checks require the folio so it is looked up now. For
-	 * head page and mapping is looked up now. For anonymous pages, it
+	 * anonymous pages, it does not matter if the folio is split
-	 * does not matter if the page splits in the future as the key is
+	 * in the future as the key is based on the address. For
-	 * based on the address. For filesystem-backed pages, the tail is
+	 * filesystem-backed pages, the precise page is required as the
-	 * required as the index of the page determines the key. For
+	 * index of the page determines the key.
-	 * base pages, there is no tail page and tail == page.
 	 */
-	tail = page;
+	folio = page_folio(page);
-	page = compound_head(page);
+	mapping = READ_ONCE(folio->mapping);
-	mapping = READ_ONCE(page->mapping);
 	/*
-	 * If page->mapping is NULL, then it cannot be a PageAnon
+	 * If folio->mapping is NULL, then it cannot be an anonymous
 	 * page; but it might be the ZERO_PAGE or in the gate area or
 	 * in a special mapping (all cases which we are happy to fail);
 	 * or it may have been a good file page when get_user_pages_fast
 	 * found it, but truncated or holepunched or subjected to
-	 * invalidate_complete_page2 before we got the page lock (also
+	 * invalidate_complete_page2 before we got the folio lock (also
 	 * cases which we are happy to fail).  And we hold a reference,
 	 * so refcount care in invalidate_inode_page's remove_mapping
 	 * prevents drop_caches from setting mapping to NULL beneath us.
 	 *
 	 * The case we do have to guard against is when memory pressure made
 	 * shmem_writepage move it from filecache to swapcache beneath us:
-	 * an unlikely race, but we do need to retry for page->mapping.
+	 * an unlikely race, but we do need to retry for folio->mapping.
 	 */
 	if (unlikely(!mapping)) {
 		int shmem_swizzled;
 		/*
-		 * Page lock is required to identify which special case above
+		 * Folio lock is required to identify which special case above
-		 * applies. If this is really a shmem page then the page lock
+		 * applies. If this is really a shmem page then the folio lock
 		 * will prevent unexpected transitions.
 		 */
-		lock_page(page);
+		folio_lock(folio);
-		shmem_swizzled = PageSwapCache(page) || page->mapping;
+		shmem_swizzled = folio_test_swapcache(folio) || folio->mapping;
-		unlock_page(page);
+		folio_unlock(folio);
-		put_page(page);
+		folio_put(folio);
 		if (shmem_swizzled)
 			goto again;
 	/*
 	 * Private mappings are handled in a simple way.
 	 *
-	 * If the futex key is stored on an anonymous page, then the associated
+	 * If the futex key is stored in anonymous memory, then the associated
 	 * object is the mm which is implicitly pinned by the calling process.
 	 *
 	 * NOTE: When userspace waits on a MAP_SHARED mapping, even if
 	 * it's a read-only handle, it's expected that futexes attach to
 	 * the object not the particular process.
 	 */
-	if (PageAnon(page)) {
+	if (folio_test_anon(folio)) {
 		/*
 		 * A RO anonymous page will never change and thus doesn't make
 		 * sense for futex operations.
 		/*
 		 * The associated futex object in this case is the inode and
-		 * the page->mapping must be traversed. Ordinarily this should
+		 * the folio->mapping must be traversed. Ordinarily this should
-		 * be stabilised under page lock but it's not strictly
+		 * be stabilised under folio lock but it's not strictly
 		 * necessary in this case as we just want to pin the inode, not
-		 * update the radix tree or anything like that.
+		 * update i_pages or anything like that.
 		 *
 		 * The RCU read lock is taken as the inode is finally freed
 		 * under RCU. If the mapping still matches expectations then the
 		 */
 		rcu_read_lock();
-		if (READ_ONCE(page->mapping) != mapping) {
+		if (READ_ONCE(folio->mapping) != mapping) {
 			rcu_read_unlock();
-			put_page(page);
+			folio_put(folio);
 			goto again;
 		}
 		inode = READ_ONCE(mapping->host);
 		if (!inode) {
 			rcu_read_unlock();
-			put_page(page);
+			folio_put(folio);
 			goto again;
 		}
 		key->both.offset |= FUT_OFF_INODE; /* inode-based key */
 		key->shared.i_seq = get_inode_sequence_number(inode);
-		key->shared.pgoff = page_to_pgoff(tail);
+		key->shared.pgoff = folio->index + folio_page_idx(folio, page);
 		rcu_read_unlock();
 	}
 out:
-	put_page(page);
+	folio_put(folio);
 	return err;
 }

kernel/futex/futex.h+79−7

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	#include <linux/futex.h>	5	#include <linux/futex.h>
6	#include <linux/rtmutex.h>	6	#include <linux/rtmutex.h>
7	#include <linux/sched/wake_q.h>	7	#include <linux/sched/wake_q.h>
		8	#include <linux/compat.h>
8		9
9	#ifdef CONFIG_PREEMPT_RT	10	#ifdef CONFIG_PREEMPT_RT
10	#include <linux/rcuwait.h>	11	#include <linux/rcuwait.h>
@@ -16,17 +17,84 @@ · 5 unmodified lines
16	* Futex flags used to encode options to functions and preserve them across	17	* Futex flags used to encode options to functions and preserve them across
17	* restarts.	18	* restarts.
18	*/	19	*/
		20	#define FLAGS_SIZE_8 0x0000
		21	#define FLAGS_SIZE_16 0x0001
		22	#define FLAGS_SIZE_32 0x0002
		23	#define FLAGS_SIZE_64 0x0003
		24
		25	#define FLAGS_SIZE_MASK 0x0003
		26
19	#ifdef CONFIG_MMU	27	#ifdef CONFIG_MMU
20	# define FLAGS_SHARED 0x01	28	# define FLAGS_SHARED 0x0010
21	#else	29	#else
22	/*	30	/*
23	* NOMMU does not have per process address space. Let the compiler optimize	31	* NOMMU does not have per process address space. Let the compiler optimize
24	* code away.	32	* code away.
25	*/	33	*/
26	# define FLAGS_SHARED 0x00	34	# define FLAGS_SHARED 0x0000
27	#endif	35	#endif
28	#define FLAGS_CLOCKRT 0x02	36	#define FLAGS_CLOCKRT 0x0020
29	#define FLAGS_HAS_TIMEOUT 0x04	37	#define FLAGS_HAS_TIMEOUT 0x0040
		38	#define FLAGS_NUMA 0x0080
		39	#define FLAGS_STRICT 0x0100
		40
		41	/* FUTEX_ to FLAGS_ */
		42	static inline unsigned int futex_to_flags(unsigned int op)
		43	{
		44	unsigned int flags = FLAGS_SIZE_32;
		45
		46	if (!(op & FUTEX_PRIVATE_FLAG))
		47	flags \|= FLAGS_SHARED;
		48
		49	if (op & FUTEX_CLOCK_REALTIME)
		50	flags \|= FLAGS_CLOCKRT;
		51
		52	return flags;
		53	}
		54
		55	/* FUTEX2_ to FLAGS_ */
		56	static inline unsigned int futex2_to_flags(unsigned int flags2)
		57	{
		58	unsigned int flags = flags2 & FUTEX2_SIZE_MASK;
		59
		60	if (!(flags2 & FUTEX2_PRIVATE))
		61	flags \|= FLAGS_SHARED;
		62
		63	if (flags2 & FUTEX2_NUMA)
		64	flags \|= FLAGS_NUMA;
		65
		66	return flags;
		67	}
		68
		69	static inline unsigned int futex_size(unsigned int flags)
		70	{
		71	return 1 << (flags & FLAGS_SIZE_MASK);
		72	}
		73
		74	static inline bool futex_flags_valid(unsigned int flags)
		75	{
		76	/* Only 64bit futexes for 64bit code */
		77	if (!IS_ENABLED(CONFIG_64BIT) \|\| in_compat_syscall()) {
		78	if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64)
		79	return false;
		80	}
		81
		82	/* Only 32bit futexes are implemented -- for now */
		83	if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
		84	return false;
		85
		86	return true;
		87	}
		88
		89	static inline bool futex_validate_input(unsigned int flags, u64 val)
		90	{
		91	int bits = 8 * futex_size(flags);
		92
		93	if (bits < 64 && (val >> bits))
		94	return false;
		95
		96	return true;
		97	}
30		98
31	#ifdef CONFIG_FAIL_FUTEX	99	#ifdef CONFIG_FAIL_FUTEX
32	extern bool should_fail_futex(bool fshared);	100	extern bool should_fail_futex(bool fshared);
@@ -116,7 +184,7 @@ enum futex_access { · 83 unmodified lines
116	FUTEX_WRITE	184	FUTEX_WRITE
117	};	185	};
118		186
119	extern int get_futex_key(u32 __user uaddr, bool fshared, union futex_key key,	187	extern int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key,
120	enum futex_access rw);	188	enum futex_access rw);
121		189
122	extern struct hrtimer_sleeper *	190	extern struct hrtimer_sleeper *
@@ -260,10 +328,14 @@ extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32 · 137 unmodified lines
260	val, ktime_t *abs_time, u32 bitset, u32 __user	328	val, ktime_t *abs_time, u32 bitset, u32 __user
261	*uaddr2);	329	*uaddr2);
262		330
263	extern int futex_requeue(u32 __user *uaddr1, unsigned int flags,	331	extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
264	u32 __user *uaddr2, int nr_wake, int nr_requeue,	332	u32 __user *uaddr2, unsigned int flags2,
		333	int nr_wake, int nr_requeue,
265	u32 *cmpval, int requeue_pi);	334	u32 *cmpval, int requeue_pi);
266		335
		336	extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
		337	struct hrtimer_sleeper *to, u32 bitset);
		338
267	extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,	339	extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
268	ktime_t *abs_time, u32 bitset);	340	ktime_t *abs_time, u32 bitset);
269		341

kernel/futex/pi.c+61−30

 // SPDX-License-Identifier: GPL-2.0-or-later
 #include <linux/slab.h>
+#include <linux/sched/rt.h>
 #include <linux/sched/task.h>
 #include "futex.h"
 /*
  * Caller must hold a reference on @pi_state.
  */
-static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)
+static int wake_futex_pi(u32 __user *uaddr, u32 uval,
+			 struct futex_pi_state *pi_state,
+			 struct rt_mutex_waiter *top_waiter)
 {
-	struct rt_mutex_waiter *top_waiter;
 	struct task_struct *new_owner;
 	bool postunlock = false;
 	DEFINE_RT_WAKE_Q(wqh);
 	u32 curval, newval;
 	int ret = 0;
-	top_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
-	if (WARN_ON_ONCE(!top_waiter)) {
-		/*
-		 * As per the comment in futex_unlock_pi() this should not happen.
-		 *
-		 * When this happens, give up our locks and try again, giving
-		 * the futex_lock_pi() instance time to complete, either by
-		 * waiting on the rtmutex or removing itself from the futex
-		 * queue.
-		 */
-		ret = -EAGAIN;
-		goto out_unlock;
-	}
 	new_owner = top_waiter->task;
 	/*
 	to = futex_setup_timer(time, &timeout, flags, 0);
 retry:
-	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE);
+	ret = get_futex_key(uaddr, flags, &q.key, FUTEX_WRITE);
 	if (unlikely(ret != 0))
 		goto out;
 		goto no_block;
 	}
+	/*
+	 * Must be done before we enqueue the waiter, here is unfortunately
+	 * under the hb lock, but that *should* work because it does nothing.
+	 */
+	rt_mutex_pre_schedule();
 	rt_mutex_init_waiter(&rt_waiter);
 	/*
 	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
 cleanup:
-	spin_lock(q.lock_ptr);
 	/*
 	 * If we failed to acquire the lock (deadlock/signal/timeout), we must
-	 * first acquire the hb->lock before removing the lock from the
+	 * must unwind the above, however we canont lock hb->lock because
-	 * rt_mutex waitqueue, such that we can keep the hb and rt_mutex wait
+	 * rt_mutex already has a waiter enqueued and hb->lock can itself try
-	 * lists consistent.
+	 * and enqueue an rt_waiter through rtlock.
+	 *
+	 * Doing the cleanup without holding hb->lock can cause inconsistent
+	 * state between hb and pi_state, but only in the direction of not
+	 * seeing a waiter that is leaving.
+	 *
+	 * See futex_unlock_pi(), it deals with this inconsistency.
+	 *
+	 * There be dragons here, since we must deal with the inconsistency on
+	 * the way out (here), it is impossible to detect/warn about the race
+	 * the other way around (missing an incoming waiter).
 	 *
-	 * In particular; it is important that futex_unlock_pi() can not
+	 * What could possibly go wrong...
-	 * observe this inconsistency.
 	 */
 	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
 		ret = 0;
+	/*
+	 * Now that the rt_waiter has been dequeued, it is safe to use
+	 * spinlock/rtlock (which might enqueue its own rt_waiter) and fix up
+	 * the
+	 */
+	spin_lock(q.lock_ptr);
+	/*
+	 * Waiter is unqueued.
+	 */
+	rt_mutex_post_schedule();
 no_block:
 	/*
 	 * Fixup the pi_state owner and possibly acquire the lock if we
 	if ((uval & FUTEX_TID_MASK) != vpid)
 		return -EPERM;
-	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_WRITE);
+	ret = get_futex_key(uaddr, flags, &key, FUTEX_WRITE);
 	if (ret)
 		return ret;
 	top_waiter = futex_top_waiter(hb, &key);
 	if (top_waiter) {
 		struct futex_pi_state *pi_state = top_waiter->pi_state;
+		struct rt_mutex_waiter *rt_waiter;
 		ret = -EINVAL;
 		if (!pi_state)
 		if (pi_state->owner != current)
 			goto out_unlock;
-		get_pi_state(pi_state);
 		/*
 		 * By taking wait_lock while still holding hb->lock, we ensure
-		 * there is no point where we hold neither; and therefore
+		 * there is no point where we hold neither; and thereby
-		 * wake_futex_p() must observe a state consistent with what we
+		 * wake_futex_pi() must observe any new waiters.
-		 * observed.
+		 *
+		 * Since the cleanup: case in futex_lock_pi() removes the
+		 * rt_waiter without holding hb->lock, it is possible for
+		 * wake_futex_pi() to not find a waiter while the above does,
+		 * in this case the waiter is on the way out and it can be
+		 * ignored.
 		 *
 		 * In particular; this forces __rt_mutex_start_proxy() to
 		 * complete such that we're guaranteed to observe the
-		 * rt_waiter. Also see the WARN in wake_futex_pi().
+		 * rt_waiter.
 		 */
 		raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
+		/*
+		 * Futex vs rt_mutex waiter state -- if there are no rt_mutex
+		 * waiters even though futex thinks there are, then the waiter
+		 * is leaving and the uncontended path is safe to take.
+		 */
+		rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
+		if (!rt_waiter) {
+			raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
+			goto do_uncontended;
+		}
+		get_pi_state(pi_state);
 		spin_unlock(&hb->lock);
 		/* drops pi_state->pi_mutex.wait_lock */
-		ret = wake_futex_pi(uaddr, uval, pi_state);
+		ret = wake_futex_pi(uaddr, uval, pi_state, rt_waiter);
 		put_pi_state(pi_state);
 		return ret;
 	}
+do_uncontended:
 	/*
 	 * We have no kernel internal state, i.e. no waiters in the
 	 * kernel. Waiters which are about to queue themselves are stuck

kernel/futex/requeue.c+12−8

 /**
  * futex_requeue() - Requeue waiters from uaddr1 to uaddr2
  * @uaddr1:	source futex user address
- * @flags:	futex flags (FLAGS_SHARED, etc.)
+ * @flags1:	futex flags (FLAGS_SHARED, etc.)
  * @uaddr2:	target futex user address
+ * @flags2:	futex flags (FLAGS_SHARED, etc.)
  * @nr_wake:	number of waiters to wake (must be 1 for requeue_pi)
  * @nr_requeue:	number of waiters to requeue (0-INT_MAX)
  * @cmpval:	@uaddr1 expected value (or %NULL)
  *  - >=0 - on success, the number of tasks requeued or woken;
  *  -  <0 - on error
  */
-int futex_requeue(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
+int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
+		  u32 __user *uaddr2, unsigned int flags2,
 		  int nr_wake, int nr_requeue, u32 *cmpval, int requeue_pi)
 {
 	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
 	}
 retry:
-	ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
+	ret = get_futex_key(uaddr1, flags1, &key1, FUTEX_READ);
 	if (unlikely(ret != 0))
 		return ret;
-	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2,
+	ret = get_futex_key(uaddr2, flags2, &key2,
 			    requeue_pi ? FUTEX_WRITE : FUTEX_READ);
 	if (unlikely(ret != 0))
 		return ret;
 			if (ret)
 				return ret;
-			if (!(flags & FLAGS_SHARED))
+			if (!(flags1 & FLAGS_SHARED))
 				goto retry_private;
 			goto retry;
 	 */
 	rt_mutex_init_waiter(&rt_waiter);
-	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
+	ret = get_futex_key(uaddr2, flags, &key2, FUTEX_WRITE);
 	if (unlikely(ret != 0))
 		goto out;
 		pi_mutex = &q.pi_state->pi_mutex;
 		ret = rt_mutex_wait_proxy_lock(pi_mutex, to, &rt_waiter);
-		/* Current is not longer pi_blocked_on */
+		/*
-		spin_lock(q.lock_ptr);
+		 * See futex_unlock_pi()'s cleanup: comment.
+		 */
 		if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
 			ret = 0;
+		spin_lock(q.lock_ptr);
 		debug_rt_mutex_free_waiter(&rt_waiter);
 		/*
 		 * Fixup the pi_state owner and possibly acquire the lock if we

kernel/futex/syscalls.c+175−46

@@ -1,6 +1,5 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later	1	// SPDX-License-Identifier: GPL-2.0-or-later
2		2
3	#include <linux/compat.h>
4	#include <linux/syscalls.h>	3	#include <linux/syscalls.h>
5	#include <linux/time_namespace.h>	4	#include <linux/time_namespace.h>
6		5
@@ -85,15 +84,12 @@ err_unlock: · 78 unmodified lines
85	long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout,	84	long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout,
86	u32 __user *uaddr2, u32 val2, u32 val3)	85	u32 __user *uaddr2, u32 val2, u32 val3)
87	{	86	{
		87	unsigned int flags = futex_to_flags(op);
88	int cmd = op & FUTEX_CMD_MASK;	88	int cmd = op & FUTEX_CMD_MASK;
89	unsigned int flags = 0;
90		89
91	if (!(op & FUTEX_PRIVATE_FLAG))	90	if (flags & FLAGS_CLOCKRT) {
92	flags \|= FLAGS_SHARED;	91	if (cmd != FUTEX_WAIT_BITSET &&
93		92	cmd != FUTEX_WAIT_REQUEUE_PI &&
94	if (op & FUTEX_CLOCK_REALTIME) {
95	flags \|= FLAGS_CLOCKRT;
96	if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI &&
97	cmd != FUTEX_LOCK_PI2)	93	cmd != FUTEX_LOCK_PI2)
98	return -ENOSYS;	94	return -ENOSYS;
99	}	95	}
@@ -110,9 +106,9 @@ long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout, · 10 unmodified lines
110	case FUTEX_WAKE_BITSET:	106	case FUTEX_WAKE_BITSET:
111	return futex_wake(uaddr, flags, val, val3);	107	return futex_wake(uaddr, flags, val, val3);
112	case FUTEX_REQUEUE:	108	case FUTEX_REQUEUE:
113	return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);	109	return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, NULL, 0);
114	case FUTEX_CMP_REQUEUE:	110	case FUTEX_CMP_REQUEUE:
115	return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);	111	return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, &val3, 0);
116	case FUTEX_WAKE_OP:	112	case FUTEX_WAKE_OP:
117	return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);	113	return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);
118	case FUTEX_LOCK_PI:	114	case FUTEX_LOCK_PI:
@@ -129,7 +125,7 @@ long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout, · 10 unmodified lines
129	return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,	125	return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
130	uaddr2);	126	uaddr2);
131	case FUTEX_CMP_REQUEUE_PI:	127	case FUTEX_CMP_REQUEUE_PI:
132	return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);	128	return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, &val3, 1);
133	}	129	}
134	return -ENOSYS;	130	return -ENOSYS;
135	}	131	}
@@ -183,8 +179,7 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, · 47 unmodified lines
183	return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);	179	return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
184	}	180	}
185		181
186	/* Mask of available flags for each futex in futex_waitv list */	182	#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_PRIVATE)
187	#define FUTEXV_WAITER_MASK (FUTEX_32 \| FUTEX_PRIVATE_FLAG)
188		183
189	/**	184	/**
190	* futex_parse_waitv - Parse a waitv array from userspace	185	* futex_parse_waitv - Parse a waitv array from userspace
@@ -202,16 +197,22 @@ static int futex_parse_waitv(struct futex_vector *futexv, · 11 unmodified lines
202	unsigned int i;	197	unsigned int i;
203		198
204	for (i = 0; i < nr_futexes; i++) {	199	for (i = 0; i < nr_futexes; i++) {
		200	unsigned int flags;
		201
205	if (copy_from_user(&aux, &uwaitv[i], sizeof(aux)))	202	if (copy_from_user(&aux, &uwaitv[i], sizeof(aux)))
206	return -EFAULT;	203	return -EFAULT;
207		204
208	if ((aux.flags & ~FUTEXV_WAITER_MASK) \|\| aux.__reserved)	205	if ((aux.flags & ~FUTEX2_VALID_MASK) \|\| aux.__reserved)
209	return -EINVAL;	206	return -EINVAL;
210		207
211	if (!(aux.flags & FUTEX_32))	208	flags = futex2_to_flags(aux.flags);
		209	if (!futex_flags_valid(flags))
212	return -EINVAL;	210	return -EINVAL;
213		211
214	futexv[i].w.flags = aux.flags;	212	if (!futex_validate_input(flags, aux.val))
		213	return -EINVAL;
		214
		215	futexv[i].w.flags = flags;
215	futexv[i].w.val = aux.val;	216	futexv[i].w.val = aux.val;
216	futexv[i].w.uaddr = aux.uaddr;	217	futexv[i].w.uaddr = aux.uaddr;
217	futexv[i].q = futex_q_init;	218	futexv[i].q = futex_q_init;
@@ -220,6 +221,46 @@ static int futex_parse_waitv(struct futex_vector *futexv, · 2 unmodified lines
220	return 0;	221	return 0;
221	}	222	}
222		223
		224	static int futex2_setup_timeout(struct __kernel_timespec __user *timeout,
		225	clockid_t clockid, struct hrtimer_sleeper *to)
		226	{
		227	int flag_clkid = 0, flag_init = 0;
		228	struct timespec64 ts;
		229	ktime_t time;
		230	int ret;
		231
		232	if (!timeout)
		233	return 0;
		234
		235	if (clockid == CLOCK_REALTIME) {
		236	flag_clkid = FLAGS_CLOCKRT;
		237	flag_init = FUTEX_CLOCK_REALTIME;
		238	}
		239
		240	if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC)
		241	return -EINVAL;
		242
		243	if (get_timespec64(&ts, timeout))
		244	return -EFAULT;
		245
		246	/*
		247	* Since there's no opcode for futex_waitv, use
		248	* FUTEX_WAIT_BITSET that uses absolute timeout as well
		249	*/
		250	ret = futex_init_timeout(FUTEX_WAIT_BITSET, flag_init, &ts, &time);
		251	if (ret)
		252	return ret;
		253
		254	futex_setup_timer(&time, to, flag_clkid, 0);
		255	return 0;
		256	}
		257
		258	static inline void futex2_destroy_timeout(struct hrtimer_sleeper *to)
		259	{
		260	hrtimer_cancel(&to->timer);
		261	destroy_hrtimer_on_stack(&to->timer);
		262	}
		263
223	/**	264	/**
224	* sys_futex_waitv - Wait on a list of futexes	265	* sys_futex_waitv - Wait on a list of futexes
225	* @waiters: List of futexes to wait on	266	* @waiters: List of futexes to wait on
@@ -249,8 +290,6 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 23 unmodified lines
249	{	290	{
250	struct hrtimer_sleeper to;	291	struct hrtimer_sleeper to;
251	struct futex_vector *futexv;	292	struct futex_vector *futexv;
252	struct timespec64 ts;
253	ktime_t time;
254	int ret;	293	int ret;
255		294
256	/* This syscall supports no flags for now */	295	/* This syscall supports no flags for now */
@@ -260,30 +299,8 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 3 unmodified lines
260	if (!nr_futexes \|\| nr_futexes > FUTEX_WAITV_MAX \|\| !waiters)	299	if (!nr_futexes \|\| nr_futexes > FUTEX_WAITV_MAX \|\| !waiters)
261	return -EINVAL;	300	return -EINVAL;
262		301
263	if (timeout) {	302	if (timeout && (ret = futex2_setup_timeout(timeout, clockid, &to)))
264	int flag_clkid = 0, flag_init = 0;	303	return ret;
265
266	if (clockid == CLOCK_REALTIME) {
267	flag_clkid = FLAGS_CLOCKRT;
268	flag_init = FUTEX_CLOCK_REALTIME;
269	}
270
271	if (clockid != CLOCK_REALTIME && clockid != CLOCK_MONOTONIC)
272	return -EINVAL;
273
274	if (get_timespec64(&ts, timeout))
275	return -EFAULT;
276
277	/*
278	* Since there's no opcode for futex_waitv, use
279	* FUTEX_WAIT_BITSET that uses absolute timeout as well
280	*/
281	ret = futex_init_timeout(FUTEX_WAIT_BITSET, flag_init, &ts, &time);
282	if (ret)
283	return ret;
284
285	futex_setup_timer(&time, &to, flag_clkid, 0);
286	}
287		304
288	futexv = kcalloc(nr_futexes, sizeof(*futexv), GFP_KERNEL);	305	futexv = kcalloc(nr_futexes, sizeof(*futexv), GFP_KERNEL);
289	if (!futexv) {	306	if (!futexv) {
@@ -298,13 +315,125 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, · 8 unmodified lines
298	kfree(futexv);	315	kfree(futexv);
299		316
300	destroy_timer:	317	destroy_timer:
301	if (timeout) {	318	if (timeout)
302	hrtimer_cancel(&to.timer);	319	futex2_destroy_timeout(&to);
303	destroy_hrtimer_on_stack(&to.timer);	320	return ret;
304	}	321	}
		322
		323	/*
		324	* sys_futex_wake - Wake a number of futexes
		325	* @uaddr: Address of the futex(es) to wake
		326	* @mask: bitmask
		327	* @nr: Number of the futexes to wake
		328	* @flags: FUTEX2 flags
		329	*
		330	* Identical to the traditional FUTEX_WAKE_BITSET op, except it is part of the
		331	* futex2 family of calls.
		332	*/
		333
		334	SYSCALL_DEFINE4(futex_wake,
		335	void __user *, uaddr,
		336	unsigned long, mask,
		337	int, nr,
		338	unsigned int, flags)
		339	{
		340	if (flags & ~FUTEX2_VALID_MASK)
		341	return -EINVAL;
		342
		343	flags = futex2_to_flags(flags);
		344	if (!futex_flags_valid(flags))
		345	return -EINVAL;
		346
		347	if (!futex_validate_input(flags, mask))
		348	return -EINVAL;
		349
		350	return futex_wake(uaddr, FLAGS_STRICT \| flags, nr, mask);
		351	}
		352
		353	/*
		354	* sys_futex_wait - Wait on a futex
		355	* @uaddr: Address of the futex to wait on
		356	* @val: Value of @uaddr
		357	* @mask: bitmask
		358	* @flags: FUTEX2 flags
		359	* @timeout: Optional absolute timeout
		360	* @clockid: Clock to be used for the timeout, realtime or monotonic
		361	*
		362	* Identical to the traditional FUTEX_WAIT_BITSET op, except it is part of the
		363	* futex2 familiy of calls.
		364	*/
		365
		366	SYSCALL_DEFINE6(futex_wait,
		367	void __user *, uaddr,
		368	unsigned long, val,
		369	unsigned long, mask,
		370	unsigned int, flags,
		371	struct __kernel_timespec __user *, timeout,
		372	clockid_t, clockid)
		373	{
		374	struct hrtimer_sleeper to;
		375	int ret;
		376
		377	if (flags & ~FUTEX2_VALID_MASK)
		378	return -EINVAL;
		379
		380	flags = futex2_to_flags(flags);
		381	if (!futex_flags_valid(flags))
		382	return -EINVAL;
		383
		384	if (!futex_validate_input(flags, val) \|\|
		385	!futex_validate_input(flags, mask))
		386	return -EINVAL;
		387
		388	if (timeout && (ret = futex2_setup_timeout(timeout, clockid, &to)))
		389	return ret;
		390
		391	ret = __futex_wait(uaddr, flags, val, timeout ? &to : NULL, mask);
		392
		393	if (timeout)
		394	futex2_destroy_timeout(&to);
		395
305	return ret;	396	return ret;
306	}	397	}
307		398
		399	/*
		400	* sys_futex_requeue - Requeue a waiter from one futex to another
		401	* @waiters: array describing the source and destination futex
		402	* @flags: unused
		403	* @nr_wake: number of futexes to wake
		404	* @nr_requeue: number of futexes to requeue
		405	*
		406	* Identical to the traditional FUTEX_CMP_REQUEUE op, except it is part of the
		407	* futex2 family of calls.
		408	*/
		409
		410	SYSCALL_DEFINE4(futex_requeue,
		411	struct futex_waitv __user *, waiters,
		412	unsigned int, flags,
		413	int, nr_wake,
		414	int, nr_requeue)
		415	{
		416	struct futex_vector futexes[2];
		417	u32 cmpval;
		418	int ret;
		419
		420	if (flags)
		421	return -EINVAL;
		422
		423	if (!waiters)
		424	return -EINVAL;
		425
		426	ret = futex_parse_waitv(futexes, waiters, 2);
		427	if (ret)
		428	return ret;
		429
		430	cmpval = futexes[0].w.val;
		431
		432	return futex_requeue(u64_to_user_ptr(futexes[0].w.uaddr), futexes[0].w.flags,
		433	u64_to_user_ptr(futexes[1].w.uaddr), futexes[1].w.flags,
		434	nr_wake, nr_requeue, &cmpval, 0);
		435	}
		436
308	#ifdef CONFIG_COMPAT	437	#ifdef CONFIG_COMPAT
309	COMPAT_SYSCALL_DEFINE2(set_robust_list,	438	COMPAT_SYSCALL_DEFINE2(set_robust_list,
310	struct compat_robust_list_head __user *, head,	439	struct compat_robust_list_head __user *, head,

kernel/futex/waitwake.c+47−33

@@ -145,16 +145,19 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) · 144 unmodified lines
145	struct futex_hash_bucket *hb;	145	struct futex_hash_bucket *hb;
146	struct futex_q this, next;	146	struct futex_q this, next;
147	union futex_key key = FUTEX_KEY_INIT;	147	union futex_key key = FUTEX_KEY_INIT;
148	int ret;
149	DEFINE_WAKE_Q(wake_q);	148	DEFINE_WAKE_Q(wake_q);
		149	int ret;
150		150
151	if (!bitset)	151	if (!bitset)
152	return -EINVAL;	152	return -EINVAL;
153		153
154	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);	154	ret = get_futex_key(uaddr, flags, &key, FUTEX_READ);
155	if (unlikely(ret != 0))	155	if (unlikely(ret != 0))
156	return ret;	156	return ret;
157		157
		158	if ((flags & FLAGS_STRICT) && !nr_wake)
		159	return 0;
		160
158	hb = futex_hash(&key);	161	hb = futex_hash(&key);
159		162
160	/* Make sure we really have tasks to wakeup */	163	/* Make sure we really have tasks to wakeup */
@@ -245,10 +248,10 @@ int futex_wake_op(u32 __user uaddr1, unsigned int flags, u32 __user uaddr2, · 84 unmodified lines
245	DEFINE_WAKE_Q(wake_q);	248	DEFINE_WAKE_Q(wake_q);
246		249
247	retry:	250	retry:
248	ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);	251	ret = get_futex_key(uaddr1, flags, &key1, FUTEX_READ);
249	if (unlikely(ret != 0))	252	if (unlikely(ret != 0))
250	return ret;	253	return ret;
251	ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);	254	ret = get_futex_key(uaddr2, flags, &key2, FUTEX_WRITE);
252	if (unlikely(ret != 0))	255	if (unlikely(ret != 0))
253	return ret;	256	return ret;
254		257
@@ -419,11 +422,11 @@ static int futex_wait_multiple_setup(struct futex_vector vs, int count, int wo · 164 unmodified lines
419	*/	422	*/
420	retry:	423	retry:
421	for (i = 0; i < count; i++) {	424	for (i = 0; i < count; i++) {
422	if ((vs[i].w.flags & FUTEX_PRIVATE_FLAG) && retry)	425	if (!(vs[i].w.flags & FLAGS_SHARED) && retry)
423	continue;	426	continue;
424		427
425	ret = get_futex_key(u64_to_user_ptr(vs[i].w.uaddr),	428	ret = get_futex_key(u64_to_user_ptr(vs[i].w.uaddr),
426	!(vs[i].w.flags & FUTEX_PRIVATE_FLAG),	429	vs[i].w.flags,
427	&vs[i].q.key, FUTEX_READ);	430	&vs[i].q.key, FUTEX_READ);
428		431
429	if (unlikely(ret))	432	if (unlikely(ret))
@@ -435,7 +438,7 @@ retry: · 5 unmodified lines
435	for (i = 0; i < count; i++) {	438	for (i = 0; i < count; i++) {
436	u32 __user uaddr = (u32 __user )(unsigned long)vs[i].w.uaddr;	439	u32 __user uaddr = (u32 __user )(unsigned long)vs[i].w.uaddr;
437	struct futex_q *q = &vs[i].q;	440	struct futex_q *q = &vs[i].q;
438	u32 val = (u32)vs[i].w.val;	441	u32 val = vs[i].w.val;
439		442
440	hb = futex_q_lock(q);	443	hb = futex_q_lock(q);
441	ret = futex_get_value_locked(&uval, uaddr);	444	ret = futex_get_value_locked(&uval, uaddr);
@@ -599,7 +602,7 @@ int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, · 157 unmodified lines
599	* while the syscall executes.	602	* while the syscall executes.
600	*/	603	*/
601	retry:	604	retry:
602	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);	605	ret = get_futex_key(uaddr, flags, &q->key, FUTEX_READ);
603	if (unlikely(ret != 0))	606	if (unlikely(ret != 0))
604	return ret;	607	return ret;
605		608
@@ -629,20 +632,18 @@ retry_private: · 23 unmodified lines
629	return ret;	632	return ret;
630	}	633	}
631		634
632	int futex_wait(u32 __user uaddr, unsigned int flags, u32 val, ktime_t abs_time, u32 bitset)	635	int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
		636	struct hrtimer_sleeper *to, u32 bitset)
633	{	637	{
634	struct hrtimer_sleeper timeout, *to;
635	struct restart_block *restart;
636	struct futex_hash_bucket *hb;
637	struct futex_q q = futex_q_init;	638	struct futex_q q = futex_q_init;
		639	struct futex_hash_bucket *hb;
638	int ret;	640	int ret;
639		641
640	if (!bitset)	642	if (!bitset)
641	return -EINVAL;	643	return -EINVAL;
		644
642	q.bitset = bitset;	645	q.bitset = bitset;
643		646
644	to = futex_setup_timer(abs_time, &timeout, flags,
645	current->timer_slack_ns);
646	retry:	647	retry:
647	/*	648	/*
648	* Prepare to wait on uaddr. On success, it holds hb->lock and q	649	* Prepare to wait on uaddr. On success, it holds hb->lock and q
@@ -650,18 +651,17 @@ retry: · 1 unmodified line
650	*/	651	*/
651	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);	652	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
652	if (ret)	653	if (ret)
653	goto out;	654	return ret;
654		655
655	/* futex_queue and wait for wakeup, timeout, or a signal. */	656	/* futex_queue and wait for wakeup, timeout, or a signal. */
656	futex_wait_queue(hb, &q, to);	657	futex_wait_queue(hb, &q, to);
657		658
658	/* If we were woken (and unqueued), we succeeded, whatever. */	659	/* If we were woken (and unqueued), we succeeded, whatever. */
659	ret = 0;
660	if (!futex_unqueue(&q))	660	if (!futex_unqueue(&q))
661	goto out;	661	return 0;
662	ret = -ETIMEDOUT;	662
663	if (to && !to->task)	663	if (to && !to->task)
664	goto out;	664	return -ETIMEDOUT;
665		665
666	/*	666	/*
667	* We expect signal_pending(current), but we might be the	667	* We expect signal_pending(current), but we might be the
@@ -670,24 +670,38 @@ retry: · 2 unmodified lines
670	if (!signal_pending(current))	670	if (!signal_pending(current))
671	goto retry;	671	goto retry;
672		672
673	ret = -ERESTARTSYS;	673	return -ERESTARTSYS;
674	if (!abs_time)	674	}
675	goto out;
676		675
677	restart = &current->restart_block;	676	int futex_wait(u32 __user uaddr, unsigned int flags, u32 val, ktime_t abs_time, u32 bitset)
678	restart->futex.uaddr = uaddr;	677	{
679	restart->futex.val = val;	678	struct hrtimer_sleeper timeout, *to;
680	restart->futex.time = *abs_time;	679	struct restart_block *restart;
681	restart->futex.bitset = bitset;	680	int ret;
682	restart->futex.flags = flags \| FLAGS_HAS_TIMEOUT;
683		681
684	ret = set_restart_fn(restart, futex_wait_restart);	682	to = futex_setup_timer(abs_time, &timeout, flags,
		683	current->timer_slack_ns);
685		684
686	out:	685	ret = __futex_wait(uaddr, flags, val, to, bitset);
687	if (to) {	686
688	hrtimer_cancel(&to->timer);	687	/* No timeout, nothing to clean up. */
689	destroy_hrtimer_on_stack(&to->timer);	688	if (!to)
		689	return ret;
		690
		691	hrtimer_cancel(&to->timer);
		692	destroy_hrtimer_on_stack(&to->timer);
		693
		694	if (ret == -ERESTARTSYS) {
		695	restart = &current->restart_block;
		696	restart->futex.uaddr = uaddr;
		697	restart->futex.val = val;
		698	restart->futex.time = *abs_time;
		699	restart->futex.bitset = bitset;
		700	restart->futex.flags = flags \| FLAGS_HAS_TIMEOUT;
		701
		702	return set_restart_fn(restart, futex_wait_restart);
690	}	703	}
		704
691	return ret;	705	return ret;
692	}	706	}
693		707

kernel/locking/rtmutex.c+34−3

 	return try_cmpxchg_acquire(&lock->owner, &old, new);
 }
+static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
+{
+	return rt_mutex_cmpxchg_acquire(lock, NULL, current);
+}
 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
 						     struct task_struct *old,
 						     struct task_struct *new)
 }
+static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock);
+static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock)
+{
+	/*
+	 * With debug enabled rt_mutex_cmpxchg trylock() will always fail.
+	 *
+	 * Avoid unconditionally taking the slow path by using
+	 * rt_mutex_slow_trylock() which is covered by the debug code and can
+	 * acquire a non-contended rtmutex.
+	 */
+	return rt_mutex_slowtrylock(lock);
+}
 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
 						     struct task_struct *old,
 						     struct task_struct *new)
 		raw_spin_unlock_irq(&lock->wait_lock);
 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))
-			schedule();
+			rt_mutex_schedule();
 		raw_spin_lock_irq(&lock->wait_lock);
 		set_current_state(state);
 	WARN(1, "rtmutex deadlock detected\n");
 	while (1) {
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule();
+		rt_mutex_schedule();
 	}
 }
 	unsigned long flags;
 	int ret;
+	/*
+	 * Do all pre-schedule work here, before we queue a waiter and invoke
+	 * PI -- any such work that trips on rtlock (PREEMPT_RT spinlock) would
+	 * otherwise recurse back into task_blocks_on_rt_mutex() through
+	 * rtlock_slowlock() and will then enqueue a second waiter for this
+	 * same task and things get really confusing real fast.
+	 */
+	rt_mutex_pre_schedule();
 	/*
 	 * Technically we could use raw_spin_[un]lock_irq() here, but this can
 	 * be called in early boot if the cmpxchg() fast path is disabled
 	raw_spin_lock_irqsave(&lock->wait_lock, flags);
 	ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);
 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
+	rt_mutex_post_schedule();
 	return ret;
 }
 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
 					   unsigned int state)
 {
-	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
+	lockdep_assert(!current->pi_blocked_on);
+	if (likely(rt_mutex_try_acquire(lock)))
 		return 0;
 	return rt_mutex_slowlock(lock, NULL, state);

kernel/locking/rwbase_rt.c+8−0

 	struct rt_mutex_base *rtm = &rwb->rtmutex;
 	int ret;
+	rwbase_pre_schedule();
 	raw_spin_lock_irq(&rtm->wait_lock);
 	/*
 		rwbase_rtmutex_unlock(rtm);
 	trace_contention_end(rwb, ret);
+	rwbase_post_schedule();
 	return ret;
 }
 static __always_inline int rwbase_read_lock(struct rwbase_rt *rwb,
 					    unsigned int state)
 {
+	lockdep_assert(!current->pi_blocked_on);
 	if (rwbase_read_trylock(rwb))
 		return 0;
 	/* Force readers into slow path */
 	atomic_sub(READER_BIAS, &rwb->readers);
+	rwbase_pre_schedule();
 	raw_spin_lock_irqsave(&rtm->wait_lock, flags);
 	if (__rwbase_write_trylock(rwb))
 		goto out_unlock;
 		if (rwbase_signal_pending_state(state, current)) {
 			rwbase_restore_current_state();
 			__rwbase_write_unlock(rwb, 0, flags);
+			rwbase_post_schedule();
 			trace_contention_end(rwb, -EINTR);
 			return -EINTR;
 		}
 out_unlock:
 	raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
+	rwbase_post_schedule();
 	return 0;
 }

kernel/locking/rwsem.c+7−1

 #define rwbase_signal_pending_state(state, current)	\
 	signal_pending_state(state, current)
+#define rwbase_pre_schedule()				\
+	rt_mutex_pre_schedule()
 #define rwbase_schedule()				\
-	schedule()
+	rt_mutex_schedule()
+#define rwbase_post_schedule()				\
+	rt_mutex_post_schedule()
 #include "rwbase_rt.c"

kernel/locking/spinlock_rt.c+6−0

 static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
 {
+	lockdep_assert(!current->pi_blocked_on);
 	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
 		rtlock_slowlock(rtm);
 }
 #define rwbase_signal_pending_state(state, current)	(0)
+#define rwbase_pre_schedule()
 #define rwbase_schedule()				\
 	schedule_rtlock()
+#define rwbase_post_schedule()
 #include "rwbase_rt.c"
 /*
  * The common functions which get wrapped into the rwlock API.

kernel/locking/test-ww_mutex.c+34−14

 #include <linux/delay.h>
 #include <linux/kthread.h>
 #include <linux/module.h>
-#include <linux/random.h>
+#include <linux/prandom.h>
 #include <linux/slab.h>
 #include <linux/ww_mutex.h>
 	int nlocks;
 };
+struct rnd_state rng;
+DEFINE_SPINLOCK(rng_lock);
+static inline u32 prandom_u32_below(u32 ceil)
+{
+	u32 ret;
+	spin_lock(&rng_lock);
+	ret = prandom_u32_state(&rng) % ceil;
+	spin_unlock(&rng_lock);
+	return ret;
+}
 static int *get_random_order(int count)
 {
 	int *order;
 		order[n] = n;
 	for (n = count - 1; n > 1; n--) {
-		r = get_random_u32_below(n + 1);
+		r = prandom_u32_below(n + 1);
 		if (r != n) {
 			tmp = order[n];
 			order[n] = order[r];
 			ww_mutex_unlock(&locks[order[n]]);
 		if (err == -EDEADLK) {
-			ww_mutex_lock_slow(&locks[order[contended]], &ctx);
+			if (!time_after(jiffies, stress->timeout)) {
-			goto retry;
+				ww_mutex_lock_slow(&locks[order[contended]], &ctx);
+				goto retry;
+			}
 		}
+		ww_acquire_fini(&ctx);
 		if (err) {
 			pr_err_once("stress (%s) failed with %d\n",
 				    __func__, err);
 			break;
 		}
-		ww_acquire_fini(&ctx);
 	} while (!time_after(jiffies, stress->timeout));
 	kfree(order);
-	kfree(stress);
 }
 struct reorder_lock {
 	list_for_each_entry_safe(ll, ln, &locks, link)
 		kfree(ll);
 	kfree(order);
-	kfree(stress);
 }
 static void stress_one_work(struct work_struct *work)
 			break;
 		}
 	} while (!time_after(jiffies, stress->timeout));
-	kfree(stress);
 }
 #define STRESS_INORDER BIT(0)
 static int stress(int nlocks, int nthreads, unsigned int flags)
 {
 	struct ww_mutex *locks;
-	int n;
+	struct stress *stress_array;
+	int n, count;
 	locks = kmalloc_array(nlocks, sizeof(*locks), GFP_KERNEL);
 	if (!locks)
 		return -ENOMEM;
+	stress_array = kmalloc_array(nthreads, sizeof(*stress_array),
+				     GFP_KERNEL);
+	if (!stress_array) {
+		kfree(locks);
+		return -ENOMEM;
+	}
 	for (n = 0; n < nlocks; n++)
 		ww_mutex_init(&locks[n], &ww_class);
+	count = 0;
 	for (n = 0; nthreads; n++) {
 		struct stress *stress;
 		void (*fn)(struct work_struct *work);
 		if (!fn)
 			continue;
-		stress = kmalloc(sizeof(*stress), GFP_KERNEL);
+		stress = &stress_array[count++];
-		if (!stress)
-			break;
 		INIT_WORK(&stress->work, fn);
 		stress->locks = locks;
 	for (n = 0; n < nlocks; n++)
 		ww_mutex_destroy(&locks[n]);
+	kfree(stress_array);
 	kfree(locks);
 	return 0;
 	printk(KERN_INFO "Beginning ww mutex selftests\n");
+	prandom_seed_state(&rng, get_random_u64());
 	wq = alloc_workqueue("test-ww_mutex", WQ_UNBOUND, 0);
 	if (!wq)
 		return -ENOMEM;

kernel/locking/ww_rt_mutex.c+1−1

@@ -62,7 +62,7 @@ __ww_rt_mutex_lock(struct ww_mutex lock, struct ww_acquire_ctx ww_ctx, · 61 unmodified lines
62	}	62	}
63	mutex_acquire_nest(&rtm->dep_map, 0, 0, nest_lock, ip);	63	mutex_acquire_nest(&rtm->dep_map, 0, 0, nest_lock, ip);
64		64
65	if (likely(rt_mutex_cmpxchg_acquire(&rtm->rtmutex, NULL, current))) {	65	if (likely(rt_mutex_try_acquire(&rtm->rtmutex))) {
66	if (ww_ctx)	66	if (ww_ctx)
67	ww_mutex_set_context_fastpath(lock, ww_ctx);	67	ww_mutex_set_context_fastpath(lock, ww_ctx);
68	return 0;	68	return 0;

kernel/sched/core.c+50−12

 static inline void sched_submit_work(struct task_struct *tsk)
 {
+	static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG);
 	unsigned int task_flags;
-	if (task_is_running(tsk))
+	/*
-		return;
+	 * Establish LD_WAIT_CONFIG context to ensure none of the code called
+	 * will use a blocking primitive -- which would lead to recursion.
+	 */
+	lock_map_acquire_try(&sched_map);
 	task_flags = tsk->flags;
 	/*
 	 * make sure to submit it to avoid deadlocks.
 	 */
 	blk_flush_plug(tsk->plug, true);
+	lock_map_release(&sched_map);
 }
 static void sched_update_worker(struct task_struct *tsk)
 	}
 }
-asmlinkage __visible void __sched schedule(void)
+static __always_inline void __schedule_loop(unsigned int sched_mode)
 {
-	struct task_struct *tsk = current;
-	sched_submit_work(tsk);
 	do {
 		preempt_disable();
-		__schedule(SM_NONE);
+		__schedule(sched_mode);
 		sched_preempt_enable_no_resched();
 	} while (need_resched());
+}
+asmlinkage __visible void __sched schedule(void)
+{
+	struct task_struct *tsk = current;
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(!tsk->sched_rt_mutex);
+#endif
+	if (!task_is_running(tsk))
+		sched_submit_work(tsk);
+	__schedule_loop(SM_NONE);
 	sched_update_worker(tsk);
 }
 EXPORT_SYMBOL(schedule);
 #ifdef CONFIG_PREEMPT_RT
 void __sched notrace schedule_rtlock(void)
 {
-	do {
+	__schedule_loop(SM_RTLOCK_WAIT);
-		preempt_disable();
-		__schedule(SM_RTLOCK_WAIT);
-		sched_preempt_enable_no_resched();
-	} while (need_resched());
 }
 NOKPROBE_SYMBOL(schedule_rtlock);
 #endif
 #ifdef CONFIG_RT_MUTEXES
+/*
+ * Would be more useful with typeof()/auto_type but they don't mix with
+ * bit-fields. Since it's a local thing, use int. Keep the generic sounding
+ * name such that if someone were to implement this function we get to compare
+ * notes.
+ */
+#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; })
+void rt_mutex_pre_schedule(void)
+{
+	lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
+	sched_submit_work(current);
+}
+void rt_mutex_schedule(void)
+{
+	lockdep_assert(current->sched_rt_mutex);
+	__schedule_loop(SM_NONE);
+}
+void rt_mutex_post_schedule(void)
+{
+	sched_update_worker(current);
+	lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
+}
 static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
 {
 	if (pi_task)

kernel/sys_ni.c+3−0

 COND_SYSCALL(get_robust_list);
 COND_SYSCALL_COMPAT(get_robust_list);
 COND_SYSCALL(futex_waitv);
+COND_SYSCALL(futex_wake);
+COND_SYSCALL(futex_wait);
+COND_SYSCALL(futex_requeue);
 COND_SYSCALL(kexec_load);
 COND_SYSCALL_COMPAT(kexec_load);
 COND_SYSCALL(init_module);

Merge branch 'locking/core' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip into io_uring-futex

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code