lael-labs/linux

include/linux/futex.h+34−2

 #include <linux/sched.h>
 #include <linux/ktime.h>
+#include <linux/mm_types.h>
 #include <uapi/linux/futex.h>
 struct inode;
-struct mm_struct;
 struct task_struct;
 /*
 		u64 i_seq;
 		unsigned long pgoff;
 		unsigned int offset;
+		/* unsigned int node; */
 	} shared;
 	struct {
 		union {
 		};
 		unsigned long address;
 		unsigned int offset;
+		/* unsigned int node; */
 	} private;
 	struct {
 		u64 ptr;
 		unsigned long word;
 		unsigned int offset;
+		unsigned int node;	/* NOT hashed! */
 	} both;
 };
 long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
 	      u32 __user *uaddr2, u32 val2, u32 val3);
-#else
+int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4);
+#ifdef CONFIG_FUTEX_PRIVATE_HASH
+int futex_hash_allocate_default(void);
+void futex_hash_free(struct mm_struct *mm);
+static inline void futex_mm_init(struct mm_struct *mm)
+{
+	RCU_INIT_POINTER(mm->futex_phash, NULL);
+	mutex_init(&mm->futex_hash_lock);
+}
+#else /* !CONFIG_FUTEX_PRIVATE_HASH */
+static inline int futex_hash_allocate_default(void) { return 0; }
+static inline void futex_hash_free(struct mm_struct *mm) { }
+static inline void futex_mm_init(struct mm_struct *mm) { }
+#endif /* CONFIG_FUTEX_PRIVATE_HASH */
+#else /* !CONFIG_FUTEX */
 static inline void futex_init_task(struct task_struct *tsk) { }
 static inline void futex_exit_recursive(struct task_struct *tsk) { }
 static inline void futex_exit_release(struct task_struct *tsk) { }
 {
 	return -EINVAL;
 }
+static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
+{
+	return -EINVAL;
+}
+static inline int futex_hash_allocate_default(void)
+{
+	return 0;
+}
+static inline void futex_hash_free(struct mm_struct *mm) { }
+static inline void futex_mm_init(struct mm_struct *mm) { }
 #endif
 #endif

include/linux/mm_types.h+6−1

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	#define INIT_PASID 0	31	#define INIT_PASID 0
32		32
33	struct address_space;	33	struct address_space;
		34	struct futex_private_hash;
34	struct mem_cgroup;	35	struct mem_cgroup;
35		36
36	/*	37	/*
@@ -1031,7 +1032,11 @@ struct mm_struct { · 994 unmodified lines
1031	*/	1032	*/
1032	seqcount_t mm_lock_seq;	1033	seqcount_t mm_lock_seq;
1033	#endif	1034	#endif
1034		1035	#ifdef CONFIG_FUTEX_PRIVATE_HASH
		1036	struct mutex futex_hash_lock;
		1037	struct futex_private_hash __rcu *futex_phash;
		1038	struct futex_private_hash *futex_phash_new;
		1039	#endif
1035		1040
1036	unsigned long hiwater_rss; /* High-watermark of RSS usage */	1041	unsigned long hiwater_rss; /* High-watermark of RSS usage */
1037	unsigned long hiwater_vm; /* High-water virtual memory usage */	1042	unsigned long hiwater_vm; /* High-water virtual memory usage */

include/linux/mmap_lock.h+4−0

 #include <linux/rwsem.h>
 #include <linux/tracepoint-defs.h>
 #include <linux/types.h>
+#include <linux/cleanup.h>
 #define MMAP_LOCK_INITIALIZER(name) \
 	.mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),
 	up_read(&mm->mmap_lock);
 }
+DEFINE_GUARD(mmap_read_lock, struct mm_struct *,
+	     mmap_read_lock(_T), mmap_read_unlock(_T))
 static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
 {
 	__mmap_lock_trace_released(mm, false);

include/linux/rcuref.h+21−1

@@ -30,7 +30,11 @@ static inline void rcuref_init(rcuref_t *ref, unsigned int cnt) · 29 unmodified lines
30	* rcuref_read - Read the number of held reference counts of a rcuref	30	* rcuref_read - Read the number of held reference counts of a rcuref
31	* @ref: Pointer to the reference count	31	* @ref: Pointer to the reference count
32	*	32	*
33	* Return: The number of held references (0 ... N)	33	* Return: The number of held references (0 ... N). The value 0 does not
		34	* indicate that it is safe to schedule the object, protected by this reference
		35	* counter, for deconstruction.
		36	* If you want to know if the reference counter has been marked DEAD (as
		37	* signaled by rcuref_put()) please use rcuread_is_dead().
34	*/	38	*/
35	static inline unsigned int rcuref_read(rcuref_t *ref)	39	static inline unsigned int rcuref_read(rcuref_t *ref)
36	{	40	{
@@ -40,6 +44,22 @@ static inline unsigned int rcuref_read(rcuref_t *ref) · 3 unmodified lines
40	return c >= RCUREF_RELEASED ? 0 : c + 1;	44	return c >= RCUREF_RELEASED ? 0 : c + 1;
41	}	45	}
42		46
		47	/**
		48	* rcuref_is_dead - Check if the rcuref has been already marked dead
		49	* @ref: Pointer to the reference count
		50	*
		51	* Return: True if the object has been marked DEAD. This signals that a previous
		52	* invocation of rcuref_put() returned true on this reference counter meaning
		53	* the protected object can safely be scheduled for deconstruction.
		54	* Otherwise, returns false.
		55	*/
		56	static inline bool rcuref_is_dead(rcuref_t *ref)
		57	{
		58	unsigned int c = atomic_read(&ref->refcnt);
		59
		60	return (c >= RCUREF_RELEASED) && (c < RCUREF_NOREF);
		61	}
		62
43	extern __must_check bool rcuref_get_slowpath(rcuref_t *ref);	63	extern __must_check bool rcuref_get_slowpath(rcuref_t *ref);
44		64
45	/**	65	/**

include/linux/restart_block.h+1−1

@@ -26,7 +26,7 @@ struct restart_block { · 25 unmodified lines
26	unsigned long arch_data;	26	unsigned long arch_data;
27	long (fn)(struct restart_block );	27	long (fn)(struct restart_block );
28	union {	28	union {
29	/* For futex_wait and futex_wait_requeue_pi */	29	/* For futex_wait() */
30	struct {	30	struct {
31	u32 __user *uaddr;	31	u32 __user *uaddr;
32	u32 val;	32	u32 val;

include/linux/vmalloc.h+7−2

 		int node, const void *caller) __alloc_size(1);
 #define __vmalloc_node(...)	alloc_hooks(__vmalloc_node_noprof(__VA_ARGS__))
-void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) __alloc_size(1);
+void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node) __alloc_size(1);
-#define vmalloc_huge(...)	alloc_hooks(vmalloc_huge_noprof(__VA_ARGS__))
+#define vmalloc_huge_node(...)	alloc_hooks(vmalloc_huge_node_noprof(__VA_ARGS__))
+static inline void *vmalloc_huge(unsigned long size, gfp_t gfp_mask)
+{
+	return vmalloc_huge_node(size, gfp_mask, NUMA_NO_NODE);
+}
 extern void *__vmalloc_array_noprof(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2);
 #define __vmalloc_array(...)	alloc_hooks(__vmalloc_array_noprof(__VA_ARGS__))

include/uapi/linux/futex.h+8−1

 #define FUTEX2_SIZE_U32		0x02
 #define FUTEX2_SIZE_U64		0x03
 #define FUTEX2_NUMA		0x04
-			/*	0x08 */
+#define FUTEX2_MPOL		0x08
 			/*	0x10 */
 			/*	0x20 */
 			/*	0x40 */
 /* do not use */
 #define FUTEX_32		FUTEX2_SIZE_U32 /* historical accident :-( */
+/*
+ * When FUTEX2_NUMA doubles the futex word, the second word is a node value.
+ * The special value -1 indicates no-node. This is the same value as
+ * NUMA_NO_NODE, except that value is not ABI, this is.
+ */
+#define FUTEX_NO_NODE		(-1)
 /*
  * Max numbers of elements in a futex_waitv array
  */

include/uapi/linux/prctl.h+7−0

@@ -364,4 +364,11 @@ struct prctl_mm_map { · 363 unmodified lines
364	# define PR_TIMER_CREATE_RESTORE_IDS_ON 1	364	# define PR_TIMER_CREATE_RESTORE_IDS_ON 1
365	# define PR_TIMER_CREATE_RESTORE_IDS_GET 2	365	# define PR_TIMER_CREATE_RESTORE_IDS_GET 2
366		366
		367	/* FUTEX hash management */
		368	#define PR_FUTEX_HASH 78
		369	# define PR_FUTEX_HASH_SET_SLOTS 1
		370	# define FH_FLAG_IMMUTABLE (1ULL << 0)
		371	# define PR_FUTEX_HASH_GET_SLOTS 2
		372	# define PR_FUTEX_HASH_GET_IMMUTABLE 3
		373
367	#endif /* _LINUX_PRCTL_H */	374	#endif /* _LINUX_PRCTL_H */

init/Kconfig+10−0

@@ -1699,6 +1699,16 @@ config FUTEX_PI · 1698 unmodified lines
1699	depends on FUTEX && RT_MUTEXES	1699	depends on FUTEX && RT_MUTEXES
1700	default y	1700	default y
1701		1701
		1702	config FUTEX_PRIVATE_HASH
		1703	bool
		1704	depends on FUTEX && !BASE_SMALL && MMU
		1705	default y
		1706
		1707	config FUTEX_MPOL
		1708	bool
		1709	depends on FUTEX && NUMA
		1710	default y
		1711
1702	config EPOLL	1712	config EPOLL
1703	bool "Enable eventpoll support" if EXPERT	1713	bool "Enable eventpoll support" if EXPERT
1704	default y	1714	default y

io_uring/futex.c+1−3

 	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_futex_data *ifd = NULL;
-	struct futex_hash_bucket *hb;
 	int ret;
 	if (!iof->futex_mask) {
 	ifd->req = req;
 	ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags,
-			       &ifd->q, &hb);
+			       &ifd->q, NULL, NULL);
 	if (!ret) {
 		hlist_add_head(&req->hash_node, &ctx->futex_list);
 		io_ring_submit_unlock(ctx, issue_flags);
-		futex_queue(&ifd->q, hb, NULL);
 		return IOU_ISSUE_SKIP_COMPLETE;
 	}

kernel/fork.c+24−0

 	RCU_INIT_POINTER(mm->exe_file, NULL);
 	mmu_notifier_subscriptions_init(mm);
 	init_tlb_flush_pending(mm);
+	futex_mm_init(mm);
 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
 	mm->pmd_huge_pte = NULL;
 #endif
 	if (mm->binfmt)
 		module_put(mm->binfmt->module);
 	lru_gen_del_mm(mm);
+	futex_hash_free(mm);
 	mmdrop(mm);
 }
 #define rv_task_fork(p) do {} while (0)
 #endif
+static bool need_futex_hash_allocate_default(u64 clone_flags)
+{
+	if ((clone_flags & (CLONE_THREAD | CLONE_VM)) != (CLONE_THREAD | CLONE_VM))
+		return false;
+	return true;
+}
 /*
  * This creates a new process as a copy of the old one,
  * but does not actually start it yet.
 	if (retval)
 		goto bad_fork_cancel_cgroup;
+	/*
+	 * Allocate a default futex hash for the user process once the first
+	 * thread spawns.
+	 */
+	if (need_futex_hash_allocate_default(clone_flags)) {
+		retval = futex_hash_allocate_default();
+		if (retval)
+			goto bad_fork_core_free;
+		/*
+		 * If we fail beyond this point we don't free the allocated
+		 * futex hash map. We assume that another thread will be created
+		 * and makes use of it. The hash map will be freed once the main
+		 * thread terminates.
+		 */
+	}
 	/*
 	 * From this point on we must avoid any synchronous user-space
 	 * communication until we take the tasklist-lock. In particular, we do

kernel/futex/core.c+732−71

@@ -36,9 +36,15 @@ · 35 unmodified lines
36	#include <linux/pagemap.h>	36	#include <linux/pagemap.h>
37	#include <linux/debugfs.h>	37	#include <linux/debugfs.h>
38	#include <linux/plist.h>	38	#include <linux/plist.h>
		39	#include <linux/gfp.h>
		40	#include <linux/vmalloc.h>
39	#include <linux/memblock.h>	41	#include <linux/memblock.h>
40	#include <linux/fault-inject.h>	42	#include <linux/fault-inject.h>
41	#include <linux/slab.h>	43	#include <linux/slab.h>
		44	#include <linux/prctl.h>
		45	#include <linux/rcuref.h>
		46	#include <linux/mempolicy.h>
		47	#include <linux/mmap_lock.h>
42		48
43	#include "futex.h"	49	#include "futex.h"
44	#include "../locking/rtmutex_common.h"	50	#include "../locking/rtmutex_common.h"
@@ -49,12 +55,24 @@ · 4 unmodified lines
49	* reside in the same cacheline.	55	* reside in the same cacheline.
50	*/	56	*/
51	static struct {	57	static struct {
52	struct futex_hash_bucket *queues;
53	unsigned long hashmask;	58	unsigned long hashmask;
		59	unsigned int hashshift;
		60	struct futex_hash_bucket *queues[MAX_NUMNODES];
54	} __futex_data __read_mostly __aligned(2*sizeof(long));	61	} __futex_data __read_mostly __aligned(2*sizeof(long));
55	#define futex_queues (__futex_data.queues)
56	#define futex_hashmask (__futex_data.hashmask)
57		62
		63	#define futex_hashmask (__futex_data.hashmask)
		64	#define futex_hashshift (__futex_data.hashshift)
		65	#define futex_queues (__futex_data.queues)
		66
		67	struct futex_private_hash {
		68	rcuref_t users;
		69	unsigned int hash_mask;
		70	struct rcu_head rcu;
		71	void *mm;
		72	bool custom;
		73	bool immutable;
		74	struct futex_hash_bucket queues[];
		75	};
58		76
59	/*	77	/*
60	* Fault injections for futexes.	78	* Fault injections for futexes.
@@ -107,21 +125,328 @@ late_initcall(fail_futex_debugfs); · 46 unmodified lines
107		125
108	#endif /* CONFIG_FAIL_FUTEX */	126	#endif /* CONFIG_FAIL_FUTEX */
109		127
		128	static struct futex_hash_bucket *
		129	__futex_hash(union futex_key key, struct futex_private_hash fph);
		130
		131	#ifdef CONFIG_FUTEX_PRIVATE_HASH
		132	static inline bool futex_key_is_private(union futex_key *key)
		133	{
		134	/*
		135	* Relies on get_futex_key() to set either bit for shared
		136	* futexes -- see comment with union futex_key.
		137	*/
		138	return !(key->both.offset & (FUT_OFF_INODE \| FUT_OFF_MMSHARED));
		139	}
		140
		141	bool futex_private_hash_get(struct futex_private_hash *fph)
		142	{
		143	if (fph->immutable)
		144	return true;
		145	return rcuref_get(&fph->users);
		146	}
		147
		148	void futex_private_hash_put(struct futex_private_hash *fph)
		149	{
		150	/* Ignore return value, last put is verified via rcuref_is_dead() */
		151	if (fph->immutable)
		152	return;
		153	if (rcuref_put(&fph->users))
		154	wake_up_var(fph->mm);
		155	}
		156
110	/**	157	/**
111	* futex_hash - Return the hash bucket in the global hash	158	* futex_hash_get - Get an additional reference for the local hash.
112	* @key: Pointer to the futex key for which the hash is calculated	159	* @hb: ptr to the private local hash.
113	*	160	*
114	* We hash on the keys returned from get_futex_key (see below) and return the	161	* Obtain an additional reference for the already obtained hash bucket. The
115	* corresponding hash bucket in the global hash.	162	* caller must already own an reference.
116	*/	163	*/
		164	void futex_hash_get(struct futex_hash_bucket *hb)
		165	{
		166	struct futex_private_hash *fph = hb->priv;
		167
		168	if (!fph)
		169	return;
		170	WARN_ON_ONCE(!futex_private_hash_get(fph));
		171	}
		172
		173	void futex_hash_put(struct futex_hash_bucket *hb)
		174	{
		175	struct futex_private_hash *fph = hb->priv;
		176
		177	if (!fph)
		178	return;
		179	futex_private_hash_put(fph);
		180	}
		181
		182	static struct futex_hash_bucket *
		183	__futex_hash_private(union futex_key key, struct futex_private_hash fph)
		184	{
		185	u32 hash;
		186
		187	if (!futex_key_is_private(key))
		188	return NULL;
		189
		190	if (!fph)
		191	fph = rcu_dereference(key->private.mm->futex_phash);
		192	if (!fph \|\| !fph->hash_mask)
		193	return NULL;
		194
		195	hash = jhash2((void *)&key->private.address,
		196	sizeof(key->private.address) / 4,
		197	key->both.offset);
		198	return &fph->queues[hash & fph->hash_mask];
		199	}
		200
		201	static void futex_rehash_private(struct futex_private_hash *old,
		202	struct futex_private_hash *new)
		203	{
		204	struct futex_hash_bucket hb_old, hb_new;
		205	unsigned int slots = old->hash_mask + 1;
		206	unsigned int i;
		207
		208	for (i = 0; i < slots; i++) {
		209	struct futex_q this, tmp;
		210
		211	hb_old = &old->queues[i];
		212
		213	spin_lock(&hb_old->lock);
		214	plist_for_each_entry_safe(this, tmp, &hb_old->chain, list) {
		215
		216	plist_del(&this->list, &hb_old->chain);
		217	futex_hb_waiters_dec(hb_old);
		218
		219	WARN_ON_ONCE(this->lock_ptr != &hb_old->lock);
		220
		221	hb_new = __futex_hash(&this->key, new);
		222	futex_hb_waiters_inc(hb_new);
		223	/*
		224	* The new pointer isn't published yet but an already
		225	* moved user can be unqueued due to timeout or signal.
		226	*/
		227	spin_lock_nested(&hb_new->lock, SINGLE_DEPTH_NESTING);
		228	plist_add(&this->list, &hb_new->chain);
		229	this->lock_ptr = &hb_new->lock;
		230	spin_unlock(&hb_new->lock);
		231	}
		232	spin_unlock(&hb_old->lock);
		233	}
		234	}
		235
		236	static bool __futex_pivot_hash(struct mm_struct *mm,
		237	struct futex_private_hash *new)
		238	{
		239	struct futex_private_hash *fph;
		240
		241	WARN_ON_ONCE(mm->futex_phash_new);
		242
		243	fph = rcu_dereference_protected(mm->futex_phash,
		244	lockdep_is_held(&mm->futex_hash_lock));
		245	if (fph) {
		246	if (!rcuref_is_dead(&fph->users)) {
		247	mm->futex_phash_new = new;
		248	return false;
		249	}
		250
		251	futex_rehash_private(fph, new);
		252	}
		253	rcu_assign_pointer(mm->futex_phash, new);
		254	kvfree_rcu(fph, rcu);
		255	return true;
		256	}
		257
		258	static void futex_pivot_hash(struct mm_struct *mm)
		259	{
		260	scoped_guard(mutex, &mm->futex_hash_lock) {
		261	struct futex_private_hash *fph;
		262
		263	fph = mm->futex_phash_new;
		264	if (fph) {
		265	mm->futex_phash_new = NULL;
		266	__futex_pivot_hash(mm, fph);
		267	}
		268	}
		269	}
		270
		271	struct futex_private_hash *futex_private_hash(void)
		272	{
		273	struct mm_struct *mm = current->mm;
		274	/*
		275	* Ideally we don't loop. If there is a replacement in progress
		276	* then a new private hash is already prepared and a reference can't be
		277	* obtained once the last user dropped it's.
		278	* In that case we block on mm_struct::futex_hash_lock and either have
		279	* to perform the replacement or wait while someone else is doing the
		280	* job. Eitherway, on the second iteration we acquire a reference on the
		281	* new private hash or loop again because a new replacement has been
		282	* requested.
		283	*/
		284	again:
		285	scoped_guard(rcu) {
		286	struct futex_private_hash *fph;
		287
		288	fph = rcu_dereference(mm->futex_phash);
		289	if (!fph)
		290	return NULL;
		291
		292	if (fph->immutable)
		293	return fph;
		294	if (rcuref_get(&fph->users))
		295	return fph;
		296	}
		297	futex_pivot_hash(mm);
		298	goto again;
		299	}
		300
117	struct futex_hash_bucket futex_hash(union futex_key key)	301	struct futex_hash_bucket futex_hash(union futex_key key)
118	{	302	{
119	u32 hash = jhash2((u32 )key, offsetof(typeof(key), both.offset) / 4,	303	struct futex_private_hash *fph;
120	key->both.offset);	304	struct futex_hash_bucket *hb;
		305
		306	again:
		307	scoped_guard(rcu) {
		308	hb = __futex_hash(key, NULL);
		309	fph = hb->priv;
		310
		311	if (!fph \|\| futex_private_hash_get(fph))
		312	return hb;
		313	}
		314	futex_pivot_hash(key->private.mm);
		315	goto again;
		316	}
		317
		318	#else /* !CONFIG_FUTEX_PRIVATE_HASH */
		319
		320	static struct futex_hash_bucket *
		321	__futex_hash_private(union futex_key key, struct futex_private_hash fph)
		322	{
		323	return NULL;
		324	}
		325
		326	struct futex_hash_bucket futex_hash(union futex_key key)
		327	{
		328	return __futex_hash(key, NULL);
		329	}
		330
		331	#endif /* CONFIG_FUTEX_PRIVATE_HASH */
		332
		333	#ifdef CONFIG_FUTEX_MPOL
		334
		335	static int __futex_key_to_node(struct mm_struct *mm, unsigned long addr)
		336	{
		337	struct vm_area_struct *vma = vma_lookup(mm, addr);
		338	struct mempolicy *mpol;
		339	int node = FUTEX_NO_NODE;
		340
		341	if (!vma)
		342	return FUTEX_NO_NODE;
		343
		344	mpol = vma_policy(vma);
		345	if (!mpol)
		346	return FUTEX_NO_NODE;
		347
		348	switch (mpol->mode) {
		349	case MPOL_PREFERRED:
		350	node = first_node(mpol->nodes);
		351	break;
		352	case MPOL_PREFERRED_MANY:
		353	case MPOL_BIND:
		354	if (mpol->home_node != NUMA_NO_NODE)
		355	node = mpol->home_node;
		356	break;
		357	default:
		358	break;
		359	}
		360
		361	return node;
		362	}
		363
		364	static int futex_key_to_node_opt(struct mm_struct *mm, unsigned long addr)
		365	{
		366	int seq, node;
		367
		368	guard(rcu)();
		369
		370	if (!mmap_lock_speculate_try_begin(mm, &seq))
		371	return -EBUSY;
		372
		373	node = __futex_key_to_node(mm, addr);
		374
		375	if (mmap_lock_speculate_retry(mm, seq))
		376	return -EAGAIN;
		377
		378	return node;
		379	}
		380
		381	static int futex_mpol(struct mm_struct *mm, unsigned long addr)
		382	{
		383	int node;
		384
		385	node = futex_key_to_node_opt(mm, addr);
		386	if (node >= FUTEX_NO_NODE)
		387	return node;
121		388
122	return &futex_queues[hash & futex_hashmask];	389	guard(mmap_read_lock)(mm);
		390	return __futex_key_to_node(mm, addr);
123	}	391	}
124		392
		393	#else /* !CONFIG_FUTEX_MPOL */
		394
		395	static int futex_mpol(struct mm_struct *mm, unsigned long addr)
		396	{
		397	return FUTEX_NO_NODE;
		398	}
		399
		400	#endif /* CONFIG_FUTEX_MPOL */
		401
		402	/**
		403	* __futex_hash - Return the hash bucket
		404	* @key: Pointer to the futex key for which the hash is calculated
		405	* @fph: Pointer to private hash if known
		406	*
		407	* We hash on the keys returned from get_futex_key (see below) and return the
		408	* corresponding hash bucket.
		409	* If the FUTEX is PROCESS_PRIVATE then a per-process hash bucket (from the
		410	* private hash) is returned if existing. Otherwise a hash bucket from the
		411	* global hash is returned.
		412	*/
		413	static struct futex_hash_bucket *
		414	__futex_hash(union futex_key key, struct futex_private_hash fph)
		415	{
		416	int node = key->both.node;
		417	u32 hash;
		418
		419	if (node == FUTEX_NO_NODE) {
		420	struct futex_hash_bucket *hb;
		421
		422	hb = __futex_hash_private(key, fph);
		423	if (hb)
		424	return hb;
		425	}
		426
		427	hash = jhash2((u32 *)key,
		428	offsetof(typeof(*key), both.offset) / sizeof(u32),
		429	key->both.offset);
		430
		431	if (node == FUTEX_NO_NODE) {
		432	/*
		433	* In case of !FLAGS_NUMA, use some unused hash bits to pick a
		434	* node -- this ensures regular futexes are interleaved across
		435	* the nodes and avoids having to allocate multiple
		436	* hash-tables.
		437	*
		438	* NOTE: this isn't perfectly uniform, but it is fast and
		439	* handles sparse node masks.
		440	*/
		441	node = (hash >> futex_hashshift) % nr_node_ids;
		442	if (!node_possible(node)) {
		443	node = find_next_bit_wrap(node_possible_map.bits,
		444	nr_node_ids, node);
		445	}
		446	}
		447
		448	return &futex_queues[node][hash & futex_hashmask];
		449	}
125		450
126	/**	451	/**
127	* futex_setup_timer - set up the sleeping hrtimer.	452	* futex_setup_timer - set up the sleeping hrtimer.
@@ -227,25 +552,60 @@ int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key, · 99 unmodified lines
227	struct page *page;	552	struct page *page;
228	struct folio *folio;	553	struct folio *folio;
229	struct address_space *mapping;	554	struct address_space *mapping;
230	int err, ro = 0;	555	int node, err, size, ro = 0;
		556	bool node_updated = false;
231	bool fshared;	557	bool fshared;
232		558
233	fshared = flags & FLAGS_SHARED;	559	fshared = flags & FLAGS_SHARED;
		560	size = futex_size(flags);
		561	if (flags & FLAGS_NUMA)
		562	size *= 2;
234		563
235	/*	564	/*
236	* The futex address must be "naturally" aligned.	565	* The futex address must be "naturally" aligned.
237	*/	566	*/
238	key->both.offset = address % PAGE_SIZE;	567	key->both.offset = address % PAGE_SIZE;
239	if (unlikely((address % sizeof(u32)) != 0))	568	if (unlikely((address % size) != 0))
240	return -EINVAL;	569	return -EINVAL;
241	address -= key->both.offset;	570	address -= key->both.offset;
242		571
243	if (unlikely(!access_ok(uaddr, sizeof(u32))))	572	if (unlikely(!access_ok(uaddr, size)))
244	return -EFAULT;	573	return -EFAULT;
245		574
246	if (unlikely(should_fail_futex(fshared)))	575	if (unlikely(should_fail_futex(fshared)))
247	return -EFAULT;	576	return -EFAULT;
248		577
		578	node = FUTEX_NO_NODE;
		579
		580	if (flags & FLAGS_NUMA) {
		581	u32 __user naddr = (void )uaddr + size / 2;
		582
		583	if (futex_get_value(&node, naddr))
		584	return -EFAULT;
		585
		586	if (node != FUTEX_NO_NODE &&
		587	(node >= MAX_NUMNODES \|\| !node_possible(node)))
		588	return -EINVAL;
		589	}
		590
		591	if (node == FUTEX_NO_NODE && (flags & FLAGS_MPOL)) {
		592	node = futex_mpol(mm, address);
		593	node_updated = true;
		594	}
		595
		596	if (flags & FLAGS_NUMA) {
		597	u32 __user naddr = (void )uaddr + size / 2;
		598
		599	if (node == FUTEX_NO_NODE) {
		600	node = numa_node_id();
		601	node_updated = true;
		602	}
		603	if (node_updated && futex_put_value(node, naddr))
		604	return -EFAULT;
		605	}
		606
		607	key->both.node = node;
		608
249	/*	609	/*
250	* PROCESS_PRIVATE futexes are fast.	610	* PROCESS_PRIVATE futexes are fast.
251	* As the mm cannot disappear under us and the 'key' only needs	611	* As the mm cannot disappear under us and the 'key' only needs
@@ -502,13 +862,9 @@ void __futex_unqueue(struct futex_q *q) · 250 unmodified lines
502	}	862	}
503		863
504	/* The key must be already stored in q->key. */	864	/* The key must be already stored in q->key. */
505	struct futex_hash_bucket futex_q_lock(struct futex_q q)	865	void futex_q_lock(struct futex_q q, struct futex_hash_bucket hb)
506	__acquires(&hb->lock)	866	__acquires(&hb->lock)
507	{	867	{
508	struct futex_hash_bucket *hb;
509
510	hb = futex_hash(&q->key);
511
512	/*	868	/*
513	* Increment the counter before taking the lock so that	869	* Increment the counter before taking the lock so that
514	* a potential waker won't miss a to-be-slept task that is	870	* a potential waker won't miss a to-be-slept task that is
@@ -522,14 +878,13 @@ struct futex_hash_bucket futex_q_lock(struct futex_q q) · 7 unmodified lines
522	q->lock_ptr = &hb->lock;	878	q->lock_ptr = &hb->lock;
523		879
524	spin_lock(&hb->lock);	880	spin_lock(&hb->lock);
525	return hb;
526	}	881	}
527		882
528	void futex_q_unlock(struct futex_hash_bucket *hb)	883	void futex_q_unlock(struct futex_hash_bucket *hb)
529	__releases(&hb->lock)	884	__releases(&hb->lock)
530	{	885	{
531	spin_unlock(&hb->lock);
532	futex_hb_waiters_dec(hb);	886	futex_hb_waiters_dec(hb);
		887	spin_unlock(&hb->lock);
533	}	888	}
534		889
535	void __futex_queue(struct futex_q q, struct futex_hash_bucket hb,	890	void __futex_queue(struct futex_q q, struct futex_hash_bucket hb,
@@ -568,6 +923,8 @@ int futex_unqueue(struct futex_q *q) · 32 unmodified lines
568	spinlock_t *lock_ptr;	923	spinlock_t *lock_ptr;
569	int ret = 0;	924	int ret = 0;
570		925
		926	/* RCU so lock_ptr is not going away during locking. */
		927	guard(rcu)();
571	/* In the common case we don't take the spinlock, which is nice. */	928	/* In the common case we don't take the spinlock, which is nice. */
572	retry:	929	retry:
573	/*	930	/*
@@ -606,6 +963,24 @@ retry: · 32 unmodified lines
606	return ret;	963	return ret;
607	}	964	}
608		965
		966	void futex_q_lockptr_lock(struct futex_q *q)
		967	{
		968	spinlock_t *lock_ptr;
		969
		970	/*
		971	* See futex_unqueue() why lock_ptr can change.
		972	*/
		973	guard(rcu)();
		974	retry:
		975	lock_ptr = READ_ONCE(q->lock_ptr);
		976	spin_lock(lock_ptr);
		977
		978	if (unlikely(lock_ptr != q->lock_ptr)) {
		979	spin_unlock(lock_ptr);
		980	goto retry;
		981	}
		982	}
		983
609	/*	984	/*
610	* PI futexes can not be requeued and must remove themselves from the hash	985	* PI futexes can not be requeued and must remove themselves from the hash
611	* bucket. The hash bucket lock (i.e. lock_ptr) is held.	986	* bucket. The hash bucket lock (i.e. lock_ptr) is held.
@@ -949,9 +1324,19 @@ static void exit_pi_state_list(struct task_struct *curr) · 337 unmodified lines
949	{	1324	{
950	struct list_head next, head = &curr->pi_state_list;	1325	struct list_head next, head = &curr->pi_state_list;
951	struct futex_pi_state *pi_state;	1326	struct futex_pi_state *pi_state;
952	struct futex_hash_bucket *hb;
953	union futex_key key = FUTEX_KEY_INIT;	1327	union futex_key key = FUTEX_KEY_INIT;
954		1328
		1329	/*
		1330	* The mutex mm_struct::futex_hash_lock might be acquired.
		1331	*/
		1332	might_sleep();
		1333	/*
		1334	* Ensure the hash remains stable (no resize) during the while loop
		1335	* below. The hb pointer is acquired under the pi_lock so we can't block
		1336	* on the mutex.
		1337	*/
		1338	WARN_ON(curr != current);
		1339	guard(private_hash)();
955	/*	1340	/*
956	* We are a ZOMBIE and nobody can enqueue itself on	1341	* We are a ZOMBIE and nobody can enqueue itself on
957	* pi_state_list anymore, but we have to be careful	1342	* pi_state_list anymore, but we have to be careful
@@ -962,50 +1347,52 @@ static void exit_pi_state_list(struct task_struct *curr) · 4 unmodified lines
962	next = head->next;	1347	next = head->next;
963	pi_state = list_entry(next, struct futex_pi_state, list);	1348	pi_state = list_entry(next, struct futex_pi_state, list);
964	key = pi_state->key;	1349	key = pi_state->key;
965	hb = futex_hash(&key);	1350	if (1) {
966		1351	CLASS(hb, hb)(&key);
967	/*	1352
968	* We can race against put_pi_state() removing itself from the	1353	/*
969	* list (a waiter going away). put_pi_state() will first	1354	* We can race against put_pi_state() removing itself from the
970	* decrement the reference count and then modify the list, so	1355	* list (a waiter going away). put_pi_state() will first
971	* its possible to see the list entry but fail this reference	1356	* decrement the reference count and then modify the list, so
972	* acquire.	1357	* its possible to see the list entry but fail this reference
973	*	1358	* acquire.
974	* In that case; drop the locks to let put_pi_state() make	1359	*
975	* progress and retry the loop.	1360	* In that case; drop the locks to let put_pi_state() make
976	*/	1361	* progress and retry the loop.
977	if (!refcount_inc_not_zero(&pi_state->refcount)) {	1362	*/
		1363	if (!refcount_inc_not_zero(&pi_state->refcount)) {
		1364	raw_spin_unlock_irq(&curr->pi_lock);
		1365	cpu_relax();
		1366	raw_spin_lock_irq(&curr->pi_lock);
		1367	continue;
		1368	}
978	raw_spin_unlock_irq(&curr->pi_lock);	1369	raw_spin_unlock_irq(&curr->pi_lock);
979	cpu_relax();
980	raw_spin_lock_irq(&curr->pi_lock);
981	continue;
982	}
983	raw_spin_unlock_irq(&curr->pi_lock);
984		1370
985	spin_lock(&hb->lock);	1371	spin_lock(&hb->lock);
986	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);	1372	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
987	raw_spin_lock(&curr->pi_lock);	1373	raw_spin_lock(&curr->pi_lock);
988	/*	1374	/*
989	* We dropped the pi-lock, so re-check whether this	1375	* We dropped the pi-lock, so re-check whether this
990	* task still owns the PI-state:	1376	* task still owns the PI-state:
991	*/	1377	*/
992	if (head->next != next) {	1378	if (head->next != next) {
993	/* retain curr->pi_lock for the loop invariant */	1379	/* retain curr->pi_lock for the loop invariant */
994	raw_spin_unlock(&pi_state->pi_mutex.wait_lock);	1380	raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
		1381	spin_unlock(&hb->lock);
		1382	put_pi_state(pi_state);
		1383	continue;
		1384	}
		1385
		1386	WARN_ON(pi_state->owner != curr);
		1387	WARN_ON(list_empty(&pi_state->list));
		1388	list_del_init(&pi_state->list);
		1389	pi_state->owner = NULL;
		1390
		1391	raw_spin_unlock(&curr->pi_lock);
		1392	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
995	spin_unlock(&hb->lock);	1393	spin_unlock(&hb->lock);
996	put_pi_state(pi_state);
997	continue;
998	}	1394	}
999		1395
1000	WARN_ON(pi_state->owner != curr);
1001	WARN_ON(list_empty(&pi_state->list));
1002	list_del_init(&pi_state->list);
1003	pi_state->owner = NULL;
1004
1005	raw_spin_unlock(&curr->pi_lock);
1006	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1007	spin_unlock(&hb->lock);
1008
1009	rt_mutex_futex_unlock(&pi_state->pi_mutex);	1396	rt_mutex_futex_unlock(&pi_state->pi_mutex);
1010	put_pi_state(pi_state);	1397	put_pi_state(pi_state);
1011		1398
@@ -1125,30 +1512,304 @@ void futex_exit_release(struct task_struct *tsk) · 113 unmodified lines
1125	futex_cleanup_end(tsk, FUTEX_STATE_DEAD);	1512	futex_cleanup_end(tsk, FUTEX_STATE_DEAD);
1126	}	1513	}
1127		1514
		1515	static void futex_hash_bucket_init(struct futex_hash_bucket *fhb,
		1516	struct futex_private_hash *fph)
		1517	{
		1518	#ifdef CONFIG_FUTEX_PRIVATE_HASH
		1519	fhb->priv = fph;
		1520	#endif
		1521	atomic_set(&fhb->waiters, 0);
		1522	plist_head_init(&fhb->chain);
		1523	spin_lock_init(&fhb->lock);
		1524	}
		1525
		1526	#define FH_CUSTOM 0x01
		1527	#define FH_IMMUTABLE 0x02
		1528
		1529	#ifdef CONFIG_FUTEX_PRIVATE_HASH
		1530	void futex_hash_free(struct mm_struct *mm)
		1531	{
		1532	struct futex_private_hash *fph;
		1533
		1534	kvfree(mm->futex_phash_new);
		1535	fph = rcu_dereference_raw(mm->futex_phash);
		1536	if (fph) {
		1537	WARN_ON_ONCE(rcuref_read(&fph->users) > 1);
		1538	kvfree(fph);
		1539	}
		1540	}
		1541
		1542	static bool futex_pivot_pending(struct mm_struct *mm)
		1543	{
		1544	struct futex_private_hash *fph;
		1545
		1546	guard(rcu)();
		1547
		1548	if (!mm->futex_phash_new)
		1549	return true;
		1550
		1551	fph = rcu_dereference(mm->futex_phash);
		1552	return rcuref_is_dead(&fph->users);
		1553	}
		1554
		1555	static bool futex_hash_less(struct futex_private_hash *a,
		1556	struct futex_private_hash *b)
		1557	{
		1558	/* user provided always wins */
		1559	if (!a->custom && b->custom)
		1560	return true;
		1561	if (a->custom && !b->custom)
		1562	return false;
		1563
		1564	/* zero-sized hash wins */
		1565	if (!b->hash_mask)
		1566	return true;
		1567	if (!a->hash_mask)
		1568	return false;
		1569
		1570	/* keep the biggest */
		1571	if (a->hash_mask < b->hash_mask)
		1572	return true;
		1573	if (a->hash_mask > b->hash_mask)
		1574	return false;
		1575
		1576	return false; /* equal */
		1577	}
		1578
		1579	static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
		1580	{
		1581	struct mm_struct *mm = current->mm;
		1582	struct futex_private_hash *fph;
		1583	bool custom = flags & FH_CUSTOM;
		1584	int i;
		1585
		1586	if (hash_slots && (hash_slots == 1 \|\| !is_power_of_2(hash_slots)))
		1587	return -EINVAL;
		1588
		1589	/*
		1590	* Once we've disabled the global hash there is no way back.
		1591	*/
		1592	scoped_guard(rcu) {
		1593	fph = rcu_dereference(mm->futex_phash);
		1594	if (fph && (!fph->hash_mask \|\| fph->immutable)) {
		1595	if (custom)
		1596	return -EBUSY;
		1597	return 0;
		1598	}
		1599	}
		1600
		1601	fph = kvzalloc(struct_size(fph, queues, hash_slots), GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
		1602	if (!fph)
		1603	return -ENOMEM;
		1604
		1605	rcuref_init(&fph->users, 1);
		1606	fph->hash_mask = hash_slots ? hash_slots - 1 : 0;
		1607	fph->custom = custom;
		1608	fph->immutable = !!(flags & FH_IMMUTABLE);
		1609	fph->mm = mm;
		1610
		1611	for (i = 0; i < hash_slots; i++)
		1612	futex_hash_bucket_init(&fph->queues[i], fph);
		1613
		1614	if (custom) {
		1615	/*
		1616	* Only let prctl() wait / retry; don't unduly delay clone().
		1617	*/
		1618	again:
		1619	wait_var_event(mm, futex_pivot_pending(mm));
		1620	}
		1621
		1622	scoped_guard(mutex, &mm->futex_hash_lock) {
		1623	struct futex_private_hash *free __free(kvfree) = NULL;
		1624	struct futex_private_hash cur, new;
		1625
		1626	cur = rcu_dereference_protected(mm->futex_phash,
		1627	lockdep_is_held(&mm->futex_hash_lock));
		1628	new = mm->futex_phash_new;
		1629	mm->futex_phash_new = NULL;
		1630
		1631	if (fph) {
		1632	if (cur && !new) {
		1633	/*
		1634	* If we have an existing hash, but do not yet have
		1635	* allocated a replacement hash, drop the initial
		1636	* reference on the existing hash.
		1637	*/
		1638	futex_private_hash_put(cur);
		1639	}
		1640
		1641	if (new) {
		1642	/*
		1643	* Two updates raced; throw out the lesser one.
		1644	*/
		1645	if (futex_hash_less(new, fph)) {
		1646	free = new;
		1647	new = fph;
		1648	} else {
		1649	free = fph;
		1650	}
		1651	} else {
		1652	new = fph;
		1653	}
		1654	fph = NULL;
		1655	}
		1656
		1657	if (new) {
		1658	/*
		1659	* Will set mm->futex_phash_new on failure;
		1660	* futex_private_hash_get() will try again.
		1661	*/
		1662	if (!__futex_pivot_hash(mm, new) && custom)
		1663	goto again;
		1664	}
		1665	}
		1666	return 0;
		1667	}
		1668
		1669	int futex_hash_allocate_default(void)
		1670	{
		1671	unsigned int threads, buckets, current_buckets = 0;
		1672	struct futex_private_hash *fph;
		1673
		1674	if (!current->mm)
		1675	return 0;
		1676
		1677	scoped_guard(rcu) {
		1678	threads = min_t(unsigned int,
		1679	get_nr_threads(current),
		1680	num_online_cpus());
		1681
		1682	fph = rcu_dereference(current->mm->futex_phash);
		1683	if (fph) {
		1684	if (fph->custom)
		1685	return 0;
		1686
		1687	current_buckets = fph->hash_mask + 1;
		1688	}
		1689	}
		1690
		1691	/*
		1692	* The default allocation will remain within
		1693	* 16 <= threads * 4 <= global hash size
		1694	*/
		1695	buckets = roundup_pow_of_two(4 * threads);
		1696	buckets = clamp(buckets, 16, futex_hashmask + 1);
		1697
		1698	if (current_buckets >= buckets)
		1699	return 0;
		1700
		1701	return futex_hash_allocate(buckets, 0);
		1702	}
		1703
		1704	static int futex_hash_get_slots(void)
		1705	{
		1706	struct futex_private_hash *fph;
		1707
		1708	guard(rcu)();
		1709	fph = rcu_dereference(current->mm->futex_phash);
		1710	if (fph && fph->hash_mask)
		1711	return fph->hash_mask + 1;
		1712	return 0;
		1713	}
		1714
		1715	static int futex_hash_get_immutable(void)
		1716	{
		1717	struct futex_private_hash *fph;
		1718
		1719	guard(rcu)();
		1720	fph = rcu_dereference(current->mm->futex_phash);
		1721	if (fph && fph->immutable)
		1722	return 1;
		1723	if (fph && !fph->hash_mask)
		1724	return 1;
		1725	return 0;
		1726	}
		1727
		1728	#else
		1729
		1730	static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
		1731	{
		1732	return -EINVAL;
		1733	}
		1734
		1735	static int futex_hash_get_slots(void)
		1736	{
		1737	return 0;
		1738	}
		1739
		1740	static int futex_hash_get_immutable(void)
		1741	{
		1742	return 0;
		1743	}
		1744	#endif
		1745
		1746	int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
		1747	{
		1748	unsigned int flags = FH_CUSTOM;
		1749	int ret;
		1750
		1751	switch (arg2) {
		1752	case PR_FUTEX_HASH_SET_SLOTS:
		1753	if (arg4 & ~FH_FLAG_IMMUTABLE)
		1754	return -EINVAL;
		1755	if (arg4 & FH_FLAG_IMMUTABLE)
		1756	flags \|= FH_IMMUTABLE;
		1757	ret = futex_hash_allocate(arg3, flags);
		1758	break;
		1759
		1760	case PR_FUTEX_HASH_GET_SLOTS:
		1761	ret = futex_hash_get_slots();
		1762	break;
		1763
		1764	case PR_FUTEX_HASH_GET_IMMUTABLE:
		1765	ret = futex_hash_get_immutable();
		1766	break;
		1767
		1768	default:
		1769	ret = -EINVAL;
		1770	break;
		1771	}
		1772	return ret;
		1773	}
		1774
1128	static int __init futex_init(void)	1775	static int __init futex_init(void)
1129	{	1776	{
1130	unsigned long hashsize, i;	1777	unsigned long hashsize, i;
1131	unsigned int futex_shift;	1778	unsigned int order, n;
		1779	unsigned long size;
1132		1780
1133	#ifdef CONFIG_BASE_SMALL	1781	#ifdef CONFIG_BASE_SMALL
1134	hashsize = 16;	1782	hashsize = 16;
1135	#else	1783	#else
1136	hashsize = roundup_pow_of_two(256 * num_possible_cpus());	1784	hashsize = 256 * num_possible_cpus();
		1785	hashsize /= num_possible_nodes();
		1786	hashsize = max(4, hashsize);
		1787	hashsize = roundup_pow_of_two(hashsize);
1137	#endif	1788	#endif
		1789	futex_hashshift = ilog2(hashsize);
		1790	size = sizeof(struct futex_hash_bucket) * hashsize;
		1791	order = get_order(size);
		1792
		1793	for_each_node(n) {
		1794	struct futex_hash_bucket *table;
		1795
		1796	if (order > MAX_PAGE_ORDER)
		1797	table = vmalloc_huge_node(size, GFP_KERNEL, n);
		1798	else
		1799	table = alloc_pages_exact_nid(n, size, GFP_KERNEL);
		1800
		1801	BUG_ON(!table);
1138		1802
1139	futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),	1803	for (i = 0; i < hashsize; i++)
1140	hashsize, 0, 0,	1804	futex_hash_bucket_init(&table[i], NULL);
1141	&futex_shift, NULL,
1142	hashsize, hashsize);
1143	hashsize = 1UL << futex_shift;
1144		1805
1145	for (i = 0; i < hashsize; i++) {	1806	futex_queues[n] = table;
1146	atomic_set(&futex_queues[i].waiters, 0);
1147	plist_head_init(&futex_queues[i].chain);
1148	spin_lock_init(&futex_queues[i].lock);
1149	}	1807	}
1150		1808
1151	futex_hashmask = hashsize - 1;	1809	futex_hashmask = hashsize - 1;
		1810	pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n",
		1811	hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024,
		1812	order > MAX_PAGE_ORDER ? "vmalloc" : "linear");
1152	return 0;	1813	return 0;
1153	}	1814	}
1154	core_initcall(futex_init);	1815	core_initcall(futex_init);

kernel/futex/futex.h+66−8

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include <linux/sched/wake_q.h>	7	#include <linux/sched/wake_q.h>
8	#include <linux/compat.h>	8	#include <linux/compat.h>
9	#include <linux/uaccess.h>	9	#include <linux/uaccess.h>
		10	#include <linux/cleanup.h>
10		11
11	#ifdef CONFIG_PREEMPT_RT	12	#ifdef CONFIG_PREEMPT_RT
12	#include <linux/rcuwait.h>	13	#include <linux/rcuwait.h>
@@ -38,6 +39,7 @@ · 25 unmodified lines
38	#define FLAGS_HAS_TIMEOUT 0x0040	39	#define FLAGS_HAS_TIMEOUT 0x0040
39	#define FLAGS_NUMA 0x0080	40	#define FLAGS_NUMA 0x0080
40	#define FLAGS_STRICT 0x0100	41	#define FLAGS_STRICT 0x0100
		42	#define FLAGS_MPOL 0x0200
41		43
42	/* FUTEX_ to FLAGS_ */	44	/* FUTEX_ to FLAGS_ */
43	static inline unsigned int futex_to_flags(unsigned int op)	45	static inline unsigned int futex_to_flags(unsigned int op)
@@ -53,7 +55,7 @@ static inline unsigned int futex_to_flags(unsigned int op) · 9 unmodified lines
53	return flags;	55	return flags;
54	}	56	}
55		57
56	#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_PRIVATE)	58	#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_NUMA \| FUTEX2_MPOL \| FUTEX2_PRIVATE)
57		59
58	/* FUTEX2_ to FLAGS_ */	60	/* FUTEX2_ to FLAGS_ */
59	static inline unsigned int futex2_to_flags(unsigned int flags2)	61	static inline unsigned int futex2_to_flags(unsigned int flags2)
@@ -66,6 +68,9 @@ static inline unsigned int futex2_to_flags(unsigned int flags2) · 6 unmodified lines
66	if (flags2 & FUTEX2_NUMA)	68	if (flags2 & FUTEX2_NUMA)
67	flags \|= FLAGS_NUMA;	69	flags \|= FLAGS_NUMA;
68		70
		71	if (flags2 & FUTEX2_MPOL)
		72	flags \|= FLAGS_MPOL;
		73
69	return flags;	74	return flags;
70	}	75	}
71		76
@@ -86,6 +91,19 @@ static inline bool futex_flags_valid(unsigned int flags) · 14 unmodified lines
86	if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)	91	if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
87	return false;	92	return false;
88		93
		94	/*
		95	* Must be able to represent both FUTEX_NO_NODE and every valid nodeid
		96	* in a futex word.
		97	*/
		98	if (flags & FLAGS_NUMA) {
		99	int bits = 8 * futex_size(flags);
		100	u64 max = ~0ULL;
		101
		102	max >>= 64 - bits;
		103	if (nr_node_ids >= max)
		104	return false;
		105	}
		106
89	return true;	107	return true;
90	}	108	}
91		109
@@ -117,6 +135,7 @@ struct futex_hash_bucket { · 25 unmodified lines
117	atomic_t waiters;	135	atomic_t waiters;
118	spinlock_t lock;	136	spinlock_t lock;
119	struct plist_head chain;	137	struct plist_head chain;
		138	struct futex_private_hash *priv;
120	} ____cacheline_aligned_in_smp;	139	} ____cacheline_aligned_in_smp;
121		140
122	/*	141	/*
@@ -156,6 +175,7 @@ typedef void (futex_wake_fn)(struct wake_q_head wake_q, struct futex_q q); · 33 unmodified lines
156	* @requeue_pi_key: the requeue_pi target futex key	175	* @requeue_pi_key: the requeue_pi target futex key
157	* @bitset: bitset for the optional bitmasked wakeup	176	* @bitset: bitset for the optional bitmasked wakeup
158	* @requeue_state: State field for futex_requeue_pi()	177	* @requeue_state: State field for futex_requeue_pi()
		178	* @drop_hb_ref: Waiter should drop the extra hash bucket reference if true
159	* @requeue_wait: RCU wait for futex_requeue_pi() (RT only)	179	* @requeue_wait: RCU wait for futex_requeue_pi() (RT only)
160	*	180	*
161	* We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so	181	* We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
@@ -182,6 +202,7 @@ struct futex_q { · 20 unmodified lines
182	union futex_key *requeue_pi_key;	202	union futex_key *requeue_pi_key;
183	u32 bitset;	203	u32 bitset;
184	atomic_t requeue_state;	204	atomic_t requeue_state;
		205	bool drop_hb_ref;
185	#ifdef CONFIG_PREEMPT_RT	206	#ifdef CONFIG_PREEMPT_RT
186	struct rcuwait requeue_wait;	207	struct rcuwait requeue_wait;
187	#endif	208	#endif
@@ -196,12 +217,35 @@ enum futex_access { · 8 unmodified lines
196		217
197	extern int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key,	218	extern int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key,
198	enum futex_access rw);	219	enum futex_access rw);
199		220	extern void futex_q_lockptr_lock(struct futex_q *q);
200	extern struct hrtimer_sleeper *	221	extern struct hrtimer_sleeper *
201	futex_setup_timer(ktime_t time, struct hrtimer_sleeper timeout,	222	futex_setup_timer(ktime_t time, struct hrtimer_sleeper timeout,
202	int flags, u64 range_ns);	223	int flags, u64 range_ns);
203		224
204	extern struct futex_hash_bucket futex_hash(union futex_key key);	225	extern struct futex_hash_bucket futex_hash(union futex_key key);
		226	#ifdef CONFIG_FUTEX_PRIVATE_HASH
		227	extern void futex_hash_get(struct futex_hash_bucket *hb);
		228	extern void futex_hash_put(struct futex_hash_bucket *hb);
		229
		230	extern struct futex_private_hash *futex_private_hash(void);
		231	extern bool futex_private_hash_get(struct futex_private_hash *fph);
		232	extern void futex_private_hash_put(struct futex_private_hash *fph);
		233
		234	#else /* !CONFIG_FUTEX_PRIVATE_HASH */
		235	static inline void futex_hash_get(struct futex_hash_bucket *hb) { }
		236	static inline void futex_hash_put(struct futex_hash_bucket *hb) { }
		237	static inline struct futex_private_hash *futex_private_hash(void) { return NULL; }
		238	static inline bool futex_private_hash_get(void) { return false; }
		239	static inline void futex_private_hash_put(struct futex_private_hash *fph) { }
		240	#endif
		241
		242	DEFINE_CLASS(hb, struct futex_hash_bucket *,
		243	if (_T) futex_hash_put(_T),
		244	futex_hash(key), union futex_key *key);
		245
		246	DEFINE_CLASS(private_hash, struct futex_private_hash *,
		247	if (_T) futex_private_hash_put(_T),
		248	futex_private_hash(), void);
205		249
206	/**	250	/**
207	* futex_match - Check whether two futex keys are equal	251	* futex_match - Check whether two futex keys are equal
@@ -219,9 +263,9 @@ static inline int futex_match(union futex_key key1, union futex_key key2) · 11 unmodified lines
219	}	263	}
220		264
221	extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,	265	extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
222	struct futex_q q, struct futex_hash_bucket *hb);	266	struct futex_q q, union futex_key key2,
223	extern void futex_wait_queue(struct futex_hash_bucket hb, struct futex_q q,	267	struct task_struct *task);
224	struct hrtimer_sleeper *timeout);	268	extern void futex_do_wait(struct futex_q q, struct hrtimer_sleeper timeout);
225	extern bool __futex_wake_mark(struct futex_q *q);	269	extern bool __futex_wake_mark(struct futex_q *q);
226	extern void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q);	270	extern void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q);
227		271
@@ -256,7 +300,7 @@ static inline int futex_cmpxchg_value_locked(u32 curval, u32 __user uaddr, u32 · 28 unmodified lines
256	* This looks a bit overkill, but generally just results in a couple	300	* This looks a bit overkill, but generally just results in a couple
257	* of instructions.	301	* of instructions.
258	*/	302	*/
259	static __always_inline int futex_read_inatomic(u32 dest, u32 __user from)	303	static __always_inline int futex_get_value(u32 dest, u32 __user from)
260	{	304	{
261	u32 val;	305	u32 val;
262		306
@@ -273,12 +317,26 @@ Efault: · 10 unmodified lines
273	return -EFAULT;	317	return -EFAULT;
274	}	318	}
275		319
		320	static __always_inline int futex_put_value(u32 val, u32 __user *to)
		321	{
		322	if (can_do_masked_user_access())
		323	to = masked_user_access_begin(to);
		324	else if (!user_read_access_begin(to, sizeof(*to)))
		325	return -EFAULT;
		326	unsafe_put_user(val, to, Efault);
		327	user_read_access_end();
		328	return 0;
		329	Efault:
		330	user_read_access_end();
		331	return -EFAULT;
		332	}
		333
276	static inline int futex_get_value_locked(u32 dest, u32 __user from)	334	static inline int futex_get_value_locked(u32 dest, u32 __user from)
277	{	335	{
278	int ret;	336	int ret;
279		337
280	pagefault_disable();	338	pagefault_disable();
281	ret = futex_read_inatomic(dest, from);	339	ret = futex_get_value(dest, from);
282	pagefault_enable();	340	pagefault_enable();
283		341
284	return ret;	342	return ret;
@@ -354,7 +412,7 @@ static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb) · 69 unmodified lines
354	#endif	412	#endif
355	}	413	}
356		414
357	extern struct futex_hash_bucket futex_q_lock(struct futex_q q);	415	extern void futex_q_lock(struct futex_q q, struct futex_hash_bucket hb);
358	extern void futex_q_unlock(struct futex_hash_bucket *hb);	416	extern void futex_q_unlock(struct futex_hash_bucket *hb);
359		417
360		418

kernel/futex/pi.c+166−142

@@ -806,7 +806,7 @@ handle_err: · 805 unmodified lines
806	break;	806	break;
807	}	807	}
808		808
809	spin_lock(q->lock_ptr);	809	futex_q_lockptr_lock(q);
810	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);	810	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
811		811
812	/*	812	/*
@@ -920,7 +920,6 @@ int futex_lock_pi(u32 __user uaddr, unsigned int flags, ktime_t time, int tryl · 107 unmodified lines
920	struct hrtimer_sleeper timeout, *to;	920	struct hrtimer_sleeper timeout, *to;
921	struct task_struct *exiting = NULL;	921	struct task_struct *exiting = NULL;
922	struct rt_mutex_waiter rt_waiter;	922	struct rt_mutex_waiter rt_waiter;
923	struct futex_hash_bucket *hb;
924	struct futex_q q = futex_q_init;	923	struct futex_q q = futex_q_init;
925	DEFINE_WAKE_Q(wake_q);	924	DEFINE_WAKE_Q(wake_q);
926	int res, ret;	925	int res, ret;
@@ -939,151 +938,183 @@ retry: · 12 unmodified lines
939	goto out;	938	goto out;
940		939
941	retry_private:	940	retry_private:
942	hb = futex_q_lock(&q);	941	if (1) {
		942	CLASS(hb, hb)(&q.key);
943		943
944	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,	944	futex_q_lock(&q, hb);
945	&exiting, 0);	945
946	if (unlikely(ret)) {	946	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,
947	/*	947	&exiting, 0);
948	* Atomic work succeeded and we got the lock,	948	if (unlikely(ret)) {
949	* or failed. Either way, we do _not_ block.
950	*/
951	switch (ret) {
952	case 1:
953	/* We got the lock. */
954	ret = 0;
955	goto out_unlock_put_key;
956	case -EFAULT:
957	goto uaddr_faulted;
958	case -EBUSY:
959	case -EAGAIN:
960	/*
961	* Two reasons for this:
962	* - EBUSY: Task is exiting and we just wait for the
963	* exit to complete.
964	* - EAGAIN: The user space value changed.
965	*/
966	futex_q_unlock(hb);
967	/*	949	/*
968	* Handle the case where the owner is in the middle of	950	* Atomic work succeeded and we got the lock,
969	* exiting. Wait for the exit to complete otherwise	951	* or failed. Either way, we do _not_ block.
970	* this task might loop forever, aka. live lock.
971	*/	952	*/
972	wait_for_owner_exiting(ret, exiting);	953	switch (ret) {
973	cond_resched();	954	case 1:
974	goto retry;	955	/* We got the lock. */
975	default:	956	ret = 0;
976	goto out_unlock_put_key;	957	goto out_unlock_put_key;
		958	case -EFAULT:
		959	goto uaddr_faulted;
		960	case -EBUSY:
		961	case -EAGAIN:
		962	/*
		963	* Two reasons for this:
		964	* - EBUSY: Task is exiting and we just wait for the
		965	* exit to complete.
		966	* - EAGAIN: The user space value changed.
		967	*/
		968	futex_q_unlock(hb);
		969	/*
		970	* Handle the case where the owner is in the middle of
		971	* exiting. Wait for the exit to complete otherwise
		972	* this task might loop forever, aka. live lock.
		973	*/
		974	wait_for_owner_exiting(ret, exiting);
		975	cond_resched();
		976	goto retry;
		977	default:
		978	goto out_unlock_put_key;
		979	}
977	}	980	}
978	}
979		981
980	WARN_ON(!q.pi_state);	982	WARN_ON(!q.pi_state);
981		983
982	/*	984	/*
983	* Only actually queue now that the atomic ops are done:	985	* Only actually queue now that the atomic ops are done:
984	*/	986	*/
985	__futex_queue(&q, hb, current);	987	__futex_queue(&q, hb, current);
986		988
987	if (trylock) {	989	if (trylock) {
988	ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);	990	ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
989	/* Fixup the trylock return value: */	991	/* Fixup the trylock return value: */
990	ret = ret ? 0 : -EWOULDBLOCK;	992	ret = ret ? 0 : -EWOULDBLOCK;
991	goto no_block;	993	goto no_block;
992	}	994	}
993		995
994	/*	996	/*
995	* Must be done before we enqueue the waiter, here is unfortunately	997	* Caution; releasing @hb in-scope. The hb->lock is still locked
996	* under the hb lock, but that should work because it does nothing.	998	* while the reference is dropped. The reference can not be dropped
997	*/	999	* after the unlock because if a user initiated resize is in progress
998	rt_mutex_pre_schedule();	1000	* then we might need to wake him. This can not be done after the
		1001	* rt_mutex_pre_schedule() invocation. The hb will remain valid because
		1002	* the thread, performing resize, will block on hb->lock during
		1003	* the requeue.
		1004	*/
		1005	futex_hash_put(no_free_ptr(hb));
		1006	/*
		1007	* Must be done before we enqueue the waiter, here is unfortunately
		1008	* under the hb lock, but that should work because it does nothing.
		1009	*/
		1010	rt_mutex_pre_schedule();
999		1011
1000	rt_mutex_init_waiter(&rt_waiter);	1012	rt_mutex_init_waiter(&rt_waiter);
1001		1013
1002	/*	1014	/*
1003	* On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not	1015	* On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not
1004	* hold it while doing rt_mutex_start_proxy(), because then it will	1016	* hold it while doing rt_mutex_start_proxy(), because then it will
1005	* include hb->lock in the blocking chain, even through we'll not in	1017	* include hb->lock in the blocking chain, even through we'll not in
1006	* fact hold it while blocking. This will lead it to report -EDEADLK	1018	* fact hold it while blocking. This will lead it to report -EDEADLK
1007	* and BUG when futex_unlock_pi() interleaves with this.	1019	* and BUG when futex_unlock_pi() interleaves with this.
1008	*	1020	*
1009	* Therefore acquire wait_lock while holding hb->lock, but drop the	1021	* Therefore acquire wait_lock while holding hb->lock, but drop the
1010	* latter before calling __rt_mutex_start_proxy_lock(). This	1022	* latter before calling __rt_mutex_start_proxy_lock(). This
1011	* interleaves with futex_unlock_pi() -- which does a similar lock	1023	* interleaves with futex_unlock_pi() -- which does a similar lock
1012	* handoff -- such that the latter can observe the futex_q::pi_state	1024	* handoff -- such that the latter can observe the futex_q::pi_state
1013	* before __rt_mutex_start_proxy_lock() is done.	1025	* before __rt_mutex_start_proxy_lock() is done.
1014	*/	1026	*/
1015	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);	1027	raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
1016	spin_unlock(q.lock_ptr);	1028	spin_unlock(q.lock_ptr);
1017	/*	1029	/*
1018	* __rt_mutex_start_proxy_lock() unconditionally enqueues the @rt_waiter	1030	* __rt_mutex_start_proxy_lock() unconditionally enqueues the @rt_waiter
1019	* such that futex_unlock_pi() is guaranteed to observe the waiter when	1031	* such that futex_unlock_pi() is guaranteed to observe the waiter when
1020	* it sees the futex_q::pi_state.	1032	* it sees the futex_q::pi_state.
1021	*/	1033	*/
1022	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);	1034	ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);
1023	raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);	1035	raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);
1024		1036
1025	if (ret) {	1037	if (ret) {
1026	if (ret == 1)	1038	if (ret == 1)
1027	ret = 0;	1039	ret = 0;
1028	goto cleanup;	1040	goto cleanup;
1029	}	1041	}
1030		1042
1031	if (unlikely(to))	1043	if (unlikely(to))
1032	hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);	1044	hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
1033		1045
1034	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);	1046	ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
1035		1047
1036	cleanup:	1048	cleanup:
1037	/*	1049	/*
1038	* If we failed to acquire the lock (deadlock/signal/timeout), we must	1050	* If we failed to acquire the lock (deadlock/signal/timeout), we must
1039	* must unwind the above, however we canont lock hb->lock because	1051	* unwind the above, however we canont lock hb->lock because
1040	* rt_mutex already has a waiter enqueued and hb->lock can itself try	1052	* rt_mutex already has a waiter enqueued and hb->lock can itself try
1041	* and enqueue an rt_waiter through rtlock.	1053	* and enqueue an rt_waiter through rtlock.
1042	*	1054	*
1043	* Doing the cleanup without holding hb->lock can cause inconsistent	1055	* Doing the cleanup without holding hb->lock can cause inconsistent
1044	* state between hb and pi_state, but only in the direction of not	1056	* state between hb and pi_state, but only in the direction of not
1045	* seeing a waiter that is leaving.	1057	* seeing a waiter that is leaving.
1046	*	1058	*
1047	* See futex_unlock_pi(), it deals with this inconsistency.	1059	* See futex_unlock_pi(), it deals with this inconsistency.
1048	*	1060	*
1049	* There be dragons here, since we must deal with the inconsistency on	1061	* There be dragons here, since we must deal with the inconsistency on
1050	* the way out (here), it is impossible to detect/warn about the race	1062	* the way out (here), it is impossible to detect/warn about the race
1051	* the other way around (missing an incoming waiter).	1063	* the other way around (missing an incoming waiter).
1052	*	1064	*
1053	* What could possibly go wrong...	1065	* What could possibly go wrong...
1054	*/	1066	*/
1055	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))	1067	if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
1056	ret = 0;	1068	ret = 0;
1057		1069
1058	/*	1070	/*
1059	* Now that the rt_waiter has been dequeued, it is safe to use	1071	* Now that the rt_waiter has been dequeued, it is safe to use
1060	* spinlock/rtlock (which might enqueue its own rt_waiter) and fix up	1072	* spinlock/rtlock (which might enqueue its own rt_waiter) and fix up
1061	* the	1073	* the
1062	*/	1074	*/
1063	spin_lock(q.lock_ptr);	1075	futex_q_lockptr_lock(&q);
1064	/*	1076	/*
1065	* Waiter is unqueued.	1077	* Waiter is unqueued.
1066	*/	1078	*/
1067	rt_mutex_post_schedule();	1079	rt_mutex_post_schedule();
1068	no_block:	1080	no_block:
1069	/*	1081	/*
1070	* Fixup the pi_state owner and possibly acquire the lock if we	1082	* Fixup the pi_state owner and possibly acquire the lock if we
1071	* haven't already.	1083	* haven't already.
1072	*/	1084	*/
1073	res = fixup_pi_owner(uaddr, &q, !ret);	1085	res = fixup_pi_owner(uaddr, &q, !ret);
1074	/*	1086	/*
1075	* If fixup_pi_owner() returned an error, propagate that. If it acquired	1087	* If fixup_pi_owner() returned an error, propagate that. If it acquired
1076	* the lock, clear our -ETIMEDOUT or -EINTR.	1088	* the lock, clear our -ETIMEDOUT or -EINTR.
1077	*/	1089	*/
1078	if (res)	1090	if (res)
1079	ret = (res < 0) ? res : 0;	1091	ret = (res < 0) ? res : 0;
1080		1092
1081	futex_unqueue_pi(&q);	1093	futex_unqueue_pi(&q);
1082	spin_unlock(q.lock_ptr);	1094	spin_unlock(q.lock_ptr);
1083	goto out;	1095	if (q.drop_hb_ref) {
		1096	CLASS(hb, hb)(&q.key);
		1097	/* Additional reference from futex_unlock_pi() */
		1098	futex_hash_put(hb);
		1099	}
		1100	goto out;
1084		1101
1085	out_unlock_put_key:	1102	out_unlock_put_key:
1086	futex_q_unlock(hb);	1103	futex_q_unlock(hb);
		1104	goto out;
		1105
		1106	uaddr_faulted:
		1107	futex_q_unlock(hb);
		1108
		1109	ret = fault_in_user_writeable(uaddr);
		1110	if (ret)
		1111	goto out;
		1112
		1113	if (!(flags & FLAGS_SHARED))
		1114	goto retry_private;
		1115
		1116	goto retry;
		1117	}
1087		1118
1088	out:	1119	out:
1089	if (to) {	1120	if (to) {
@@ -1091,18 +1122,6 @@ out: · 1 unmodified line
1091	destroy_hrtimer_on_stack(&to->timer);	1122	destroy_hrtimer_on_stack(&to->timer);
1092	}	1123	}
1093	return ret != -EINTR ? ret : -ERESTARTNOINTR;	1124	return ret != -EINTR ? ret : -ERESTARTNOINTR;
1094
1095	uaddr_faulted:
1096	futex_q_unlock(hb);
1097
1098	ret = fault_in_user_writeable(uaddr);
1099	if (ret)
1100	goto out;
1101
1102	if (!(flags & FLAGS_SHARED))
1103	goto retry_private;
1104
1105	goto retry;
1106	}	1125	}
1107		1126
1108	/*	1127	/*
@@ -1114,7 +1133,6 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) · 5 unmodified lines
1114	{	1133	{
1115	u32 curval, uval, vpid = task_pid_vnr(current);	1134	u32 curval, uval, vpid = task_pid_vnr(current);
1116	union futex_key key = FUTEX_KEY_INIT;	1135	union futex_key key = FUTEX_KEY_INIT;
1117	struct futex_hash_bucket *hb;
1118	struct futex_q *top_waiter;	1136	struct futex_q *top_waiter;
1119	int ret;	1137	int ret;
1120		1138
@@ -1134,7 +1152,7 @@ retry: · 13 unmodified lines
1134	if (ret)	1152	if (ret)
1135	return ret;	1153	return ret;
1136		1154
1137	hb = futex_hash(&key);	1155	CLASS(hb, hb)(&key);
1138	spin_lock(&hb->lock);	1156	spin_lock(&hb->lock);
1139	retry_hb:	1157	retry_hb:
1140		1158
@@ -1187,6 +1205,12 @@ retry_hb: · 46 unmodified lines
1187	*/	1205	*/
1188	rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);	1206	rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
1189	if (!rt_waiter) {	1207	if (!rt_waiter) {
		1208	/*
		1209	* Acquire a reference for the leaving waiter to ensure
		1210	* valid futex_q::lock_ptr.
		1211	*/
		1212	futex_hash_get(hb);
		1213	top_waiter->drop_hb_ref = true;
1190	__futex_unqueue(top_waiter);	1214	__futex_unqueue(top_waiter);
1191	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);	1215	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1192	goto retry_hb;	1216	goto retry_hb;

kernel/futex/requeue.c+234−226

@@ -87,6 +87,11 @@ void requeue_futex(struct futex_q q, struct futex_hash_bucket hb1, · 86 unmodified lines
87	futex_hb_waiters_inc(hb2);	87	futex_hb_waiters_inc(hb2);
88	plist_add(&q->list, &hb2->chain);	88	plist_add(&q->list, &hb2->chain);
89	q->lock_ptr = &hb2->lock;	89	q->lock_ptr = &hb2->lock;
		90	/*
		91	* hb1 and hb2 belong to the same futex_hash_bucket_private
		92	* because if we managed get a reference on hb1 then it can't be
		93	* replaced. Therefore we avoid put(hb1)+get(hb2) here.
		94	*/
90	}	95	}
91	q->key = *key2;	96	q->key = *key2;
92	}	97	}
@@ -231,7 +236,12 @@ void requeue_pi_wake_futex(struct futex_q q, union futex_key key, · 138 unmodified lines
231		236
232	WARN_ON(!q->rt_waiter);	237	WARN_ON(!q->rt_waiter);
233	q->rt_waiter = NULL;	238	q->rt_waiter = NULL;
234		239	/*
		240	* Acquire a reference for the waiter to ensure valid
		241	* futex_q::lock_ptr.
		242	*/
		243	futex_hash_get(hb);
		244	q->drop_hb_ref = true;
235	q->lock_ptr = &hb->lock;	245	q->lock_ptr = &hb->lock;
236		246
237	/* Signal locked state to the waiter */	247	/* Signal locked state to the waiter */
@@ -371,7 +381,6 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1, · 133 unmodified lines
371	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;	381	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
372	int task_count = 0, ret;	382	int task_count = 0, ret;
373	struct futex_pi_state *pi_state = NULL;	383	struct futex_pi_state *pi_state = NULL;
374	struct futex_hash_bucket hb1, hb2;
375	struct futex_q this, next;	384	struct futex_q this, next;
376	DEFINE_WAKE_Q(wake_q);	385	DEFINE_WAKE_Q(wake_q);
377		386
@@ -443,240 +452,242 @@ retry: · 65 unmodified lines
443	if (requeue_pi && futex_match(&key1, &key2))	452	if (requeue_pi && futex_match(&key1, &key2))
444	return -EINVAL;	453	return -EINVAL;
445		454
446	hb1 = futex_hash(&key1);
447	hb2 = futex_hash(&key2);
448
449	retry_private:	455	retry_private:
450	futex_hb_waiters_inc(hb2);	456	if (1) {
451	double_lock_hb(hb1, hb2);	457	CLASS(hb, hb1)(&key1);
		458	CLASS(hb, hb2)(&key2);
452		459
453	if (likely(cmpval != NULL)) {	460	futex_hb_waiters_inc(hb2);
454	u32 curval;	461	double_lock_hb(hb1, hb2);
455		462
456	ret = futex_get_value_locked(&curval, uaddr1);	463	if (likely(cmpval != NULL)) {
		464	u32 curval;
457		465
458	if (unlikely(ret)) {	466	ret = futex_get_value_locked(&curval, uaddr1);
459	double_unlock_hb(hb1, hb2);
460	futex_hb_waiters_dec(hb2);
461		467
462	ret = get_user(curval, uaddr1);	468	if (unlikely(ret)) {
463	if (ret)	469	futex_hb_waiters_dec(hb2);
464	return ret;	470	double_unlock_hb(hb1, hb2);
465		471
466	if (!(flags1 & FLAGS_SHARED))	472	ret = get_user(curval, uaddr1);
467	goto retry_private;	473	if (ret)
		474	return ret;
468		475
469	goto retry;	476	if (!(flags1 & FLAGS_SHARED))
470	}	477	goto retry_private;
471	if (curval != *cmpval) {
472	ret = -EAGAIN;
473	goto out_unlock;
474	}
475	}
476		478
477	if (requeue_pi) {	479	goto retry;
478	struct task_struct *exiting = NULL;	480	}
		481	if (curval != *cmpval) {
		482	ret = -EAGAIN;
		483	goto out_unlock;
		484	}
		485	}
479		486
480	/*	487	if (requeue_pi) {
481	* Attempt to acquire uaddr2 and wake the top waiter. If we	488	struct task_struct *exiting = NULL;
482	* intend to requeue waiters, force setting the FUTEX_WAITERS
483	* bit. We force this here where we are able to easily handle
484	* faults rather in the requeue loop below.
485	*
486	* Updates topwaiter::requeue_state if a top waiter exists.
487	*/
488	ret = futex_proxy_trylock_atomic(uaddr2, hb1, hb2, &key1,
489	&key2, &pi_state,
490	&exiting, nr_requeue);
491		489
492	/*
493	* At this point the top_waiter has either taken uaddr2 or
494	* is waiting on it. In both cases pi_state has been
495	* established and an initial refcount on it. In case of an
496	* error there's nothing.
497	*
498	* The top waiter's requeue_state is up to date:
499	*
500	* - If the lock was acquired atomically (ret == 1), then
501	* the state is Q_REQUEUE_PI_LOCKED.
502	*
503	* The top waiter has been dequeued and woken up and can
504	* return to user space immediately. The kernel/user
505	* space state is consistent. In case that there must be
506	* more waiters requeued the WAITERS bit in the user
507	* space futex is set so the top waiter task has to go
508	* into the syscall slowpath to unlock the futex. This
509	* will block until this requeue operation has been
510	* completed and the hash bucket locks have been
511	* dropped.
512	*
513	* - If the trylock failed with an error (ret < 0) then
514	* the state is either Q_REQUEUE_PI_NONE, i.e. "nothing
515	* happened", or Q_REQUEUE_PI_IGNORE when there was an
516	* interleaved early wakeup.
517	*
518	* - If the trylock did not succeed (ret == 0) then the
519	* state is either Q_REQUEUE_PI_IN_PROGRESS or
520	* Q_REQUEUE_PI_WAIT if an early wakeup interleaved.
521	* This will be cleaned up in the loop below, which
522	* cannot fail because futex_proxy_trylock_atomic() did
523	* the same sanity checks for requeue_pi as the loop
524	* below does.
525	*/
526	switch (ret) {
527	case 0:
528	/* We hold a reference on the pi state. */
529	break;
530
531	case 1:
532	/*	490	/*
533	* futex_proxy_trylock_atomic() acquired the user space	491	* Attempt to acquire uaddr2 and wake the top waiter. If we
534	* futex. Adjust task_count.	492	* intend to requeue waiters, force setting the FUTEX_WAITERS
		493	* bit. We force this here where we are able to easily handle
		494	* faults rather in the requeue loop below.
		495	*
		496	* Updates topwaiter::requeue_state if a top waiter exists.
535	*/	497	*/
536	task_count++;	498	ret = futex_proxy_trylock_atomic(uaddr2, hb1, hb2, &key1,
537	ret = 0;	499	&key2, &pi_state,
538	break;	500	&exiting, nr_requeue);
539		501
540	/*
541	* If the above failed, then pi_state is NULL and
542	* waiter::requeue_state is correct.
543	*/
544	case -EFAULT:
545	double_unlock_hb(hb1, hb2);
546	futex_hb_waiters_dec(hb2);
547	ret = fault_in_user_writeable(uaddr2);
548	if (!ret)
549	goto retry;
550	return ret;
551	case -EBUSY:
552	case -EAGAIN:
553	/*
554	* Two reasons for this:
555	* - EBUSY: Owner is exiting and we just wait for the
556	* exit to complete.
557	* - EAGAIN: The user space value changed.
558	*/
559	double_unlock_hb(hb1, hb2);
560	futex_hb_waiters_dec(hb2);
561	/*	502	/*
562	* Handle the case where the owner is in the middle of	503	* At this point the top_waiter has either taken uaddr2 or
563	* exiting. Wait for the exit to complete otherwise	504	* is waiting on it. In both cases pi_state has been
564	* this task might loop forever, aka. live lock.	505	* established and an initial refcount on it. In case of an
		506	* error there's nothing.
		507	*
		508	* The top waiter's requeue_state is up to date:
		509	*
		510	* - If the lock was acquired atomically (ret == 1), then
		511	* the state is Q_REQUEUE_PI_LOCKED.
		512	*
		513	* The top waiter has been dequeued and woken up and can
		514	* return to user space immediately. The kernel/user
		515	* space state is consistent. In case that there must be
		516	* more waiters requeued the WAITERS bit in the user
		517	* space futex is set so the top waiter task has to go
		518	* into the syscall slowpath to unlock the futex. This
		519	* will block until this requeue operation has been
		520	* completed and the hash bucket locks have been
		521	* dropped.
		522	*
		523	* - If the trylock failed with an error (ret < 0) then
		524	* the state is either Q_REQUEUE_PI_NONE, i.e. "nothing
		525	* happened", or Q_REQUEUE_PI_IGNORE when there was an
		526	* interleaved early wakeup.
		527	*
		528	* - If the trylock did not succeed (ret == 0) then the
		529	* state is either Q_REQUEUE_PI_IN_PROGRESS or
		530	* Q_REQUEUE_PI_WAIT if an early wakeup interleaved.
		531	* This will be cleaned up in the loop below, which
		532	* cannot fail because futex_proxy_trylock_atomic() did
		533	* the same sanity checks for requeue_pi as the loop
		534	* below does.
565	*/	535	*/
566	wait_for_owner_exiting(ret, exiting);	536	switch (ret) {
567	cond_resched();	537	case 0:
568	goto retry;	538	/* We hold a reference on the pi state. */
569	default:	539	break;
570	goto out_unlock;	540
571	}	541	case 1:
572	}	542	/*
573		543	* futex_proxy_trylock_atomic() acquired the user space
574	plist_for_each_entry_safe(this, next, &hb1->chain, list) {	544	* futex. Adjust task_count.
575	if (task_count - nr_wake >= nr_requeue)	545	*/
576	break;	546	task_count++;
577		547	ret = 0;
578	if (!futex_match(&this->key, &key1))	548	break;
579	continue;	549
580		550	/*
581	/*	551	* If the above failed, then pi_state is NULL and
582	* FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI should always	552	* waiter::requeue_state is correct.
583	* be paired with each other and no other futex ops.	553	*/
584	*	554	case -EFAULT:
585	* We should never be requeueing a futex_q with a pi_state,	555	futex_hb_waiters_dec(hb2);
586	* which is awaiting a futex_unlock_pi().	556	double_unlock_hb(hb1, hb2);
587	*/	557	ret = fault_in_user_writeable(uaddr2);
588	if ((requeue_pi && !this->rt_waiter) \|\|	558	if (!ret)
589	(!requeue_pi && this->rt_waiter) \|\|	559	goto retry;
590	this->pi_state) {	560	return ret;
591	ret = -EINVAL;	561	case -EBUSY:
592	break;	562	case -EAGAIN:
		563	/*
		564	* Two reasons for this:
		565	* - EBUSY: Owner is exiting and we just wait for the
		566	* exit to complete.
		567	* - EAGAIN: The user space value changed.
		568	*/
		569	futex_hb_waiters_dec(hb2);
		570	double_unlock_hb(hb1, hb2);
		571	/*
		572	* Handle the case where the owner is in the middle of
		573	* exiting. Wait for the exit to complete otherwise
		574	* this task might loop forever, aka. live lock.
		575	*/
		576	wait_for_owner_exiting(ret, exiting);
		577	cond_resched();
		578	goto retry;
		579	default:
		580	goto out_unlock;
		581	}
593	}	582	}
594		583
595	/* Plain futexes just wake or requeue and are done */	584	plist_for_each_entry_safe(this, next, &hb1->chain, list) {
596	if (!requeue_pi) {	585	if (task_count - nr_wake >= nr_requeue)
597	if (++task_count <= nr_wake)	586	break;
598	this->wake(&wake_q, this);
599	else
600	requeue_futex(this, hb1, hb2, &key2);
601	continue;
602	}
603		587
604	/* Ensure we requeue to the expected futex for requeue_pi. */	588	if (!futex_match(&this->key, &key1))
605	if (!futex_match(this->requeue_pi_key, &key2)) {	589	continue;
606	ret = -EINVAL;
607	break;
608	}
609		590
610	/*
611	* Requeue nr_requeue waiters and possibly one more in the case
612	* of requeue_pi if we couldn't acquire the lock atomically.
613	*
614	* Prepare the waiter to take the rt_mutex. Take a refcount
615	* on the pi_state and store the pointer in the futex_q
616	* object of the waiter.
617	*/
618	get_pi_state(pi_state);
619
620	/* Don't requeue when the waiter is already on the way out. */
621	if (!futex_requeue_pi_prepare(this, pi_state)) {
622	/*	591	/*
623	* Early woken waiter signaled that it is on the	592	* FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI should always
624	* way out. Drop the pi_state reference and try the	593	* be paired with each other and no other futex ops.
625	* next waiter. @this->pi_state is still NULL.	594	*
		595	* We should never be requeueing a futex_q with a pi_state,
		596	* which is awaiting a futex_unlock_pi().
626	*/	597	*/
627	put_pi_state(pi_state);	598	if ((requeue_pi && !this->rt_waiter) \|\|
628	continue;	599	(!requeue_pi && this->rt_waiter) \|\|
629	}	600	this->pi_state) {
		601	ret = -EINVAL;
		602	break;
		603	}
		604
		605	/* Plain futexes just wake or requeue and are done */
		606	if (!requeue_pi) {
		607	if (++task_count <= nr_wake)
		608	this->wake(&wake_q, this);
		609	else
		610	requeue_futex(this, hb1, hb2, &key2);
		611	continue;
		612	}
		613
		614	/* Ensure we requeue to the expected futex for requeue_pi. */
		615	if (!futex_match(this->requeue_pi_key, &key2)) {
		616	ret = -EINVAL;
		617	break;
		618	}
630		619
631	ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
632	this->rt_waiter,
633	this->task);
634
635	if (ret == 1) {
636	/*
637	* We got the lock. We do neither drop the refcount
638	* on pi_state nor clear this->pi_state because the
639	* waiter needs the pi_state for cleaning up the
640	* user space value. It will drop the refcount
641	* after doing so. this::requeue_state is updated
642	* in the wakeup as well.
643	*/
644	requeue_pi_wake_futex(this, &key2, hb2);
645	task_count++;
646	} else if (!ret) {
647	/* Waiter is queued, move it to hb2 */
648	requeue_futex(this, hb1, hb2, &key2);
649	futex_requeue_pi_complete(this, 0);
650	task_count++;
651	} else {
652	/*	620	/*
653	* rt_mutex_start_proxy_lock() detected a potential	621	* Requeue nr_requeue waiters and possibly one more in the case
654	* deadlock when we tried to queue that waiter.	622	* of requeue_pi if we couldn't acquire the lock atomically.
655	* Drop the pi_state reference which we took above	623	*
656	* and remove the pointer to the state from the	624	* Prepare the waiter to take the rt_mutex. Take a refcount
657	* waiters futex_q object.	625	* on the pi_state and store the pointer in the futex_q
		626	* object of the waiter.
658	*/	627	*/
659	this->pi_state = NULL;	628	get_pi_state(pi_state);
660	put_pi_state(pi_state);	629
661	futex_requeue_pi_complete(this, ret);	630	/* Don't requeue when the waiter is already on the way out. */
662	/*	631	if (!futex_requeue_pi_prepare(this, pi_state)) {
663	* We stop queueing more waiters and let user space	632	/*
664	* deal with the mess.	633	* Early woken waiter signaled that it is on the
665	*/	634	* way out. Drop the pi_state reference and try the
666	break;	635	* next waiter. @this->pi_state is still NULL.
		636	*/
		637	put_pi_state(pi_state);
		638	continue;
		639	}
		640
		641	ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
		642	this->rt_waiter,
		643	this->task);
		644
		645	if (ret == 1) {
		646	/*
		647	* We got the lock. We do neither drop the refcount
		648	* on pi_state nor clear this->pi_state because the
		649	* waiter needs the pi_state for cleaning up the
		650	* user space value. It will drop the refcount
		651	* after doing so. this::requeue_state is updated
		652	* in the wakeup as well.
		653	*/
		654	requeue_pi_wake_futex(this, &key2, hb2);
		655	task_count++;
		656	} else if (!ret) {
		657	/* Waiter is queued, move it to hb2 */
		658	requeue_futex(this, hb1, hb2, &key2);
		659	futex_requeue_pi_complete(this, 0);
		660	task_count++;
		661	} else {
		662	/*
		663	* rt_mutex_start_proxy_lock() detected a potential
		664	* deadlock when we tried to queue that waiter.
		665	* Drop the pi_state reference which we took above
		666	* and remove the pointer to the state from the
		667	* waiters futex_q object.
		668	*/
		669	this->pi_state = NULL;
		670	put_pi_state(pi_state);
		671	futex_requeue_pi_complete(this, ret);
		672	/*
		673	* We stop queueing more waiters and let user space
		674	* deal with the mess.
		675	*/
		676	break;
		677	}
667	}	678	}
668	}
669		679
670	/*	680	/*
671	* We took an extra initial reference to the pi_state in	681	* We took an extra initial reference to the pi_state in
672	* futex_proxy_trylock_atomic(). We need to drop it here again.	682	* futex_proxy_trylock_atomic(). We need to drop it here again.
673	*/	683	*/
674	put_pi_state(pi_state);	684	put_pi_state(pi_state);
675		685
676	out_unlock:	686	out_unlock:
677	double_unlock_hb(hb1, hb2);	687	futex_hb_waiters_dec(hb2);
		688	double_unlock_hb(hb1, hb2);
		689	}
678	wake_up_q(&wake_q);	690	wake_up_q(&wake_q);
679	futex_hb_waiters_dec(hb2);
680	return ret ? ret : task_count;	691	return ret ? ret : task_count;
681	}	692	}
682		693
@@ -769,7 +780,6 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 86 unmodified lines
769	{	780	{
770	struct hrtimer_sleeper timeout, *to;	781	struct hrtimer_sleeper timeout, *to;
771	struct rt_mutex_waiter rt_waiter;	782	struct rt_mutex_waiter rt_waiter;
772	struct futex_hash_bucket *hb;
773	union futex_key key2 = FUTEX_KEY_INIT;	783	union futex_key key2 = FUTEX_KEY_INIT;
774	struct futex_q q = futex_q_init;	784	struct futex_q q = futex_q_init;
775	struct rt_mutex_base *pi_mutex;	785	struct rt_mutex_base *pi_mutex;
@@ -805,35 +815,28 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 29 unmodified lines
805	* Prepare to wait on uaddr. On success, it holds hb->lock and q	815	* Prepare to wait on uaddr. On success, it holds hb->lock and q
806	* is initialized.	816	* is initialized.
807	*/	817	*/
808	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);	818	ret = futex_wait_setup(uaddr, val, flags, &q, &key2, current);
809	if (ret)	819	if (ret)
810	goto out;	820	goto out;
811		821
812	/*
813	* The check above which compares uaddrs is not sufficient for
814	* shared futexes. We need to compare the keys:
815	*/
816	if (futex_match(&q.key, &key2)) {
817	futex_q_unlock(hb);
818	ret = -EINVAL;
819	goto out;
820	}
821
822	/* Queue the futex_q, drop the hb lock, wait for wakeup. */	822	/* Queue the futex_q, drop the hb lock, wait for wakeup. */
823	futex_wait_queue(hb, &q, to);	823	futex_do_wait(&q, to);
824		824
825	switch (futex_requeue_pi_wakeup_sync(&q)) {	825	switch (futex_requeue_pi_wakeup_sync(&q)) {
826	case Q_REQUEUE_PI_IGNORE:	826	case Q_REQUEUE_PI_IGNORE:
827	/* The waiter is still on uaddr1 */	827	{
828	spin_lock(&hb->lock);	828	CLASS(hb, hb)(&q.key);
829	ret = handle_early_requeue_pi_wakeup(hb, &q, to);	829	/* The waiter is still on uaddr1 */
830	spin_unlock(&hb->lock);	830	spin_lock(&hb->lock);
		831	ret = handle_early_requeue_pi_wakeup(hb, &q, to);
		832	spin_unlock(&hb->lock);
		833	}
831	break;	834	break;
832		835
833	case Q_REQUEUE_PI_LOCKED:	836	case Q_REQUEUE_PI_LOCKED:
834	/* The requeue acquired the lock */	837	/* The requeue acquired the lock */
835	if (q.pi_state && (q.pi_state->owner != current)) {	838	if (q.pi_state && (q.pi_state->owner != current)) {
836	spin_lock(q.lock_ptr);	839	futex_q_lockptr_lock(&q);
837	ret = fixup_pi_owner(uaddr2, &q, true);	840	ret = fixup_pi_owner(uaddr2, &q, true);
838	/*	841	/*
839	* Drop the reference to the pi state which the	842	* Drop the reference to the pi state which the
@@ -860,7 +863,7 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 20 unmodified lines
860	if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))	863	if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
861	ret = 0;	864	ret = 0;
862		865
863	spin_lock(q.lock_ptr);	866	futex_q_lockptr_lock(&q);
864	debug_rt_mutex_free_waiter(&rt_waiter);	867	debug_rt_mutex_free_waiter(&rt_waiter);
865	/*	868	/*
866	* Fixup the pi_state owner and possibly acquire the lock if we	869	* Fixup the pi_state owner and possibly acquire the lock if we
@@ -892,6 +895,11 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 25 unmodified lines
892	default:	895	default:
893	BUG();	896	BUG();
894	}	897	}
		898	if (q.drop_hb_ref) {
		899	CLASS(hb, hb)(&q.key);
		900	/* Additional reference from requeue_pi_wake_futex() */
		901	futex_hash_put(hb);
		902	}
895		903
896	out:	904	out:
897	if (to) {	905	if (to) {

kernel/futex/waitwake.c+113−94

@@ -154,7 +154,6 @@ void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q) · 153 unmodified lines
154	*/	154	*/
155	int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)	155	int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
156	{	156	{
157	struct futex_hash_bucket *hb;
158	struct futex_q this, next;	157	struct futex_q this, next;
159	union futex_key key = FUTEX_KEY_INIT;	158	union futex_key key = FUTEX_KEY_INIT;
160	DEFINE_WAKE_Q(wake_q);	159	DEFINE_WAKE_Q(wake_q);
@@ -170,7 +169,7 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) · 9 unmodified lines
170	if ((flags & FLAGS_STRICT) && !nr_wake)	169	if ((flags & FLAGS_STRICT) && !nr_wake)
171	return 0;	170	return 0;
172		171
173	hb = futex_hash(&key);	172	CLASS(hb, hb)(&key);
174		173
175	/* Make sure we really have tasks to wakeup */	174	/* Make sure we really have tasks to wakeup */
176	if (!futex_hb_waiters_pending(hb))	175	if (!futex_hb_waiters_pending(hb))
@@ -253,7 +252,6 @@ int futex_wake_op(u32 __user uaddr1, unsigned int flags, u32 __user uaddr2, · 76 unmodified lines
253	int nr_wake, int nr_wake2, int op)	252	int nr_wake, int nr_wake2, int op)
254	{	253	{
255	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;	254	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
256	struct futex_hash_bucket hb1, hb2;
257	struct futex_q this, next;	255	struct futex_q this, next;
258	int ret, op_ret;	256	int ret, op_ret;
259	DEFINE_WAKE_Q(wake_q);	257	DEFINE_WAKE_Q(wake_q);
@@ -266,67 +264,69 @@ retry: · 6 unmodified lines
266	if (unlikely(ret != 0))	264	if (unlikely(ret != 0))
267	return ret;	265	return ret;
268		266
269	hb1 = futex_hash(&key1);
270	hb2 = futex_hash(&key2);
271
272	retry_private:	267	retry_private:
273	double_lock_hb(hb1, hb2);	268	if (1) {
274	op_ret = futex_atomic_op_inuser(op, uaddr2);	269	CLASS(hb, hb1)(&key1);
275	if (unlikely(op_ret < 0)) {	270	CLASS(hb, hb2)(&key2);
276	double_unlock_hb(hb1, hb2);	271
277		272	double_lock_hb(hb1, hb2);
278	if (!IS_ENABLED(CONFIG_MMU) \|\|	273	op_ret = futex_atomic_op_inuser(op, uaddr2);
279	unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {	274	if (unlikely(op_ret < 0)) {
280	/*	275	double_unlock_hb(hb1, hb2);
281	* we don't get EFAULT from MMU faults if we don't have	276
282	* an MMU, but we might get them from range checking	277	if (!IS_ENABLED(CONFIG_MMU) \|\|
283	*/	278	unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
284	ret = op_ret;	279	/*
285	return ret;	280	* we don't get EFAULT from MMU faults if we don't have
286	}	281	* an MMU, but we might get them from range checking
287		282	*/
288	if (op_ret == -EFAULT) {	283	ret = op_ret;
289	ret = fault_in_user_writeable(uaddr2);
290	if (ret)
291	return ret;	284	return ret;
292	}	285	}
293
294	cond_resched();
295	if (!(flags & FLAGS_SHARED))
296	goto retry_private;
297	goto retry;
298	}
299		286
300	plist_for_each_entry_safe(this, next, &hb1->chain, list) {	287	if (op_ret == -EFAULT) {
301	if (futex_match (&this->key, &key1)) {	288	ret = fault_in_user_writeable(uaddr2);
302	if (this->pi_state \|\| this->rt_waiter) {	289	if (ret)
303	ret = -EINVAL;	290	return ret;
304	goto out_unlock;
305	}	291	}
306	this->wake(&wake_q, this);	292
307	if (++ret >= nr_wake)	293	cond_resched();
308	break;	294	if (!(flags & FLAGS_SHARED))
		295	goto retry_private;
		296	goto retry;
309	}	297	}
310	}
311		298
312	if (op_ret > 0) {	299	plist_for_each_entry_safe(this, next, &hb1->chain, list) {
313	op_ret = 0;	300	if (futex_match(&this->key, &key1)) {
314	plist_for_each_entry_safe(this, next, &hb2->chain, list) {
315	if (futex_match (&this->key, &key2)) {
316	if (this->pi_state \|\| this->rt_waiter) {	301	if (this->pi_state \|\| this->rt_waiter) {
317	ret = -EINVAL;	302	ret = -EINVAL;
318	goto out_unlock;	303	goto out_unlock;
319	}	304	}
320	this->wake(&wake_q, this);	305	this->wake(&wake_q, this);
321	if (++op_ret >= nr_wake2)	306	if (++ret >= nr_wake)
322	break;	307	break;
323	}	308	}
324	}	309	}
325	ret += op_ret;	310
326	}	311	if (op_ret > 0) {
		312	op_ret = 0;
		313	plist_for_each_entry_safe(this, next, &hb2->chain, list) {
		314	if (futex_match(&this->key, &key2)) {
		315	if (this->pi_state \|\| this->rt_waiter) {
		316	ret = -EINVAL;
		317	goto out_unlock;
		318	}
		319	this->wake(&wake_q, this);
		320	if (++op_ret >= nr_wake2)
		321	break;
		322	}
		323	}
		324	ret += op_ret;
		325	}
327		326
328	out_unlock:	327	out_unlock:
329	double_unlock_hb(hb1, hb2);	328	double_unlock_hb(hb1, hb2);
		329	}
330	wake_up_q(&wake_q);	330	wake_up_q(&wake_q);
331	return ret;	331	return ret;
332	}	332	}
@@ -334,23 +334,12 @@ out_unlock: · 1 unmodified line
334	static long futex_wait_restart(struct restart_block *restart);	334	static long futex_wait_restart(struct restart_block *restart);
335		335
336	/**	336	/**
337	* futex_wait_queue() - futex_queue() and wait for wakeup, timeout, or signal	337	* futex_do_wait() - wait for wakeup, timeout, or signal
338	* @hb: the futex hash bucket, must be locked by the caller
339	* @q: the futex_q to queue up on	338	* @q: the futex_q to queue up on
340	* @timeout: the prepared hrtimer_sleeper, or null for no timeout	339	* @timeout: the prepared hrtimer_sleeper, or null for no timeout
341	*/	340	*/
342	void futex_wait_queue(struct futex_hash_bucket hb, struct futex_q q,	341	void futex_do_wait(struct futex_q q, struct hrtimer_sleeper timeout)
343	struct hrtimer_sleeper *timeout)
344	{	342	{
345	/*
346	* The task state is guaranteed to be set before another task can
347	* wake it. set_current_state() is implemented using smp_store_mb() and
348	* futex_queue() calls spin_unlock() upon completion, both serializing
349	* access to the hash list and forcing another memory barrier.
350	*/
351	set_current_state(TASK_INTERRUPTIBLE\|TASK_FREEZABLE);
352	futex_queue(q, hb, current);
353
354	/* Arm the timer */	343	/* Arm the timer */
355	if (timeout)	344	if (timeout)
356	hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);	345	hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
@@ -412,11 +401,16 @@ int futex_unqueue_multiple(struct futex_vector *v, int count) · 55 unmodified lines
412	*/	401	*/
413	int futex_wait_multiple_setup(struct futex_vector vs, int count, int woken)	402	int futex_wait_multiple_setup(struct futex_vector vs, int count, int woken)
414	{	403	{
415	struct futex_hash_bucket *hb;
416	bool retry = false;	404	bool retry = false;
417	int ret, i;	405	int ret, i;
418	u32 uval;	406	u32 uval;
419		407
		408	/*
		409	* Make sure to have a reference on the private_hash such that we
		410	* don't block on rehash after changing the task state below.
		411	*/
		412	guard(private_hash)();
		413
420	/*	414	/*
421	* Enqueuing multiple futexes is tricky, because we need to enqueue	415	* Enqueuing multiple futexes is tricky, because we need to enqueue
422	* each futex on the list before dealing with the next one to avoid	416	* each futex on the list before dealing with the next one to avoid
@@ -451,20 +445,24 @@ retry: · 28 unmodified lines
451	struct futex_q *q = &vs[i].q;	445	struct futex_q *q = &vs[i].q;
452	u32 val = vs[i].w.val;	446	u32 val = vs[i].w.val;
453		447
454	hb = futex_q_lock(q);	448	if (1) {
455	ret = futex_get_value_locked(&uval, uaddr);	449	CLASS(hb, hb)(&q->key);
456		450
457	if (!ret && uval == val) {	451	futex_q_lock(q, hb);
458	/*	452	ret = futex_get_value_locked(&uval, uaddr);
459	* The bucket lock can't be held while dealing with the	453
460	* next futex. Queue each futex at this moment so hb can	454	if (!ret && uval == val) {
461	* be unlocked.	455	/*
462	*/	456	* The bucket lock can't be held while dealing with the
463	futex_queue(q, hb, current);	457	* next futex. Queue each futex at this moment so hb can
464	continue;	458	* be unlocked.
465	}	459	*/
		460	futex_queue(q, hb, current);
		461	continue;
		462	}
466		463
467	futex_q_unlock(hb);	464	futex_q_unlock(hb);
		465	}
468	__set_current_state(TASK_RUNNING);	466	__set_current_state(TASK_RUNNING);
469		467
470	/*	468	/*
@@ -578,7 +576,8 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, · 107 unmodified lines
578	* @val: the expected value	576	* @val: the expected value
579	* @flags: futex flags (FLAGS_SHARED, etc.)	577	* @flags: futex flags (FLAGS_SHARED, etc.)
580	* @q: the associated futex_q	578	* @q: the associated futex_q
581	* @hb: storage for hash_bucket pointer to be returned to caller	579	* @key2: the second futex_key if used for requeue PI
		580	* @task: Task queueing this futex
582	*	581	*
583	* Setup the futex_q and locate the hash_bucket. Get the futex value and	582	* Setup the futex_q and locate the hash_bucket. Get the futex value and
584	* compare it with the expected value. Handle atomic faults internally.	583	* compare it with the expected value. Handle atomic faults internally.
@@ -586,10 +585,12 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, · 1 unmodified line
586	*	585	*
587	* Return:	586	* Return:
588	* - 0 - uaddr contains val and hb has been locked;	587	* - 0 - uaddr contains val and hb has been locked;
589	* - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked	588	* - <0 - On error and the hb is unlocked. A possible reason: the uaddr can not
		589	* be read, does not contain the expected value or is not properly aligned.
590	*/	590	*/
591	int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,	591	int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
592	struct futex_q q, struct futex_hash_bucket *hb)	592	struct futex_q q, union futex_key key2,
		593	struct task_struct *task)
593	{	594	{
594	u32 uval;	595	u32 uval;
595	int ret;	596	int ret;
@@ -618,26 +619,45 @@ retry: · 22 unmodified lines
618	return ret;	619	return ret;
619		620
620	retry_private:	621	retry_private:
621	*hb = futex_q_lock(q);	622	if (1) {
		623	CLASS(hb, hb)(&q->key);
622		624
623	ret = futex_get_value_locked(&uval, uaddr);	625	futex_q_lock(q, hb);
624		626
625	if (ret) {	627	ret = futex_get_value_locked(&uval, uaddr);
626	futex_q_unlock(*hb);
627		628
628	ret = get_user(uval, uaddr);	629	if (ret) {
629	if (ret)	630	futex_q_unlock(hb);
630	return ret;
631		631
632	if (!(flags & FLAGS_SHARED))	632	ret = get_user(uval, uaddr);
633	goto retry_private;	633	if (ret)
		634	return ret;
634		635
635	goto retry;	636	if (!(flags & FLAGS_SHARED))
636	}	637	goto retry_private;
		638
		639	goto retry;
		640	}
637		641
638	if (uval != val) {	642	if (uval != val) {
639	futex_q_unlock(*hb);	643	futex_q_unlock(hb);
640	ret = -EWOULDBLOCK;	644	return -EWOULDBLOCK;
		645	}
		646
		647	if (key2 && futex_match(&q->key, key2)) {
		648	futex_q_unlock(hb);
		649	return -EINVAL;
		650	}
		651
		652	/*
		653	* The task state is guaranteed to be set before another task can
		654	* wake it. set_current_state() is implemented using smp_store_mb() and
		655	* futex_queue() calls spin_unlock() upon completion, both serializing
		656	* access to the hash list and forcing another memory barrier.
		657	*/
		658	if (task == current)
		659	set_current_state(TASK_INTERRUPTIBLE\|TASK_FREEZABLE);
		660	futex_queue(q, hb, task);
641	}	661	}
642		662
643	return ret;	663	return ret;
@@ -647,7 +667,6 @@ int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, · 3 unmodified lines
647	struct hrtimer_sleeper *to, u32 bitset)	667	struct hrtimer_sleeper *to, u32 bitset)
648	{	668	{
649	struct futex_q q = futex_q_init;	669	struct futex_q q = futex_q_init;
650	struct futex_hash_bucket *hb;
651	int ret;	670	int ret;
652		671
653	if (!bitset)	672	if (!bitset)
@@ -660,12 +679,12 @@ retry: · 6 unmodified lines
660	* Prepare to wait on uaddr. On success, it holds hb->lock and q	679	* Prepare to wait on uaddr. On success, it holds hb->lock and q
661	* is initialized.	680	* is initialized.
662	*/	681	*/
663	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);	682	ret = futex_wait_setup(uaddr, val, flags, &q, NULL, current);
664	if (ret)	683	if (ret)
665	return ret;	684	return ret;
666		685
667	/* futex_queue and wait for wakeup, timeout, or a signal. */	686	/* futex_queue and wait for wakeup, timeout, or a signal. */
668	futex_wait_queue(hb, &q, to);	687	futex_do_wait(&q, to);
669		688
670	/* If we were woken (and unqueued), we succeeded, whatever. */	689	/* If we were woken (and unqueued), we succeeded, whatever. */
671	if (!futex_unqueue(&q))	690	if (!futex_unqueue(&q))

kernel/sys.c+4−0

 #include <linux/user_namespace.h>
 #include <linux/time_namespace.h>
 #include <linux/binfmts.h>
+#include <linux/futex.h>
 #include <linux/sched.h>
 #include <linux/sched/autogroup.h>
 			return -EINVAL;
 		error = posixtimer_create_prctl(arg2);
 		break;
+	case PR_FUTEX_HASH:
+		error = futex_hash_prctl(arg2, arg3, arg4);
+		break;
 	default:
 		trace_task_prctl_unknown(option, arg2, arg3, arg4, arg5);
 		error = -EINVAL;

mm/nommu.c+17−1

@@ -200,7 +200,23 @@ void *vmalloc_noprof(unsigned long size) · 199 unmodified lines
200	}	200	}
201	EXPORT_SYMBOL(vmalloc_noprof);	201	EXPORT_SYMBOL(vmalloc_noprof);
202		202
203	void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) __weak __alias(__vmalloc_noprof);	203	/*
		204	* vmalloc_huge_node - allocate virtually contiguous memory, on a node
		205	*
		206	* @size: allocation size
		207	* @gfp_mask: flags for the page level allocator
		208	* @node: node to use for allocation or NUMA_NO_NODE
		209	*
		210	* Allocate enough pages to cover @size from the page level
		211	* allocator and map them into contiguous kernel virtual space.
		212	*
		213	* Due to NOMMU implications the node argument and HUGE page attribute is
		214	* ignored.
		215	*/
		216	void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node)
		217	{
		218	return __vmalloc_noprof(size, gfp_mask);
		219	}
204		220
205	/*	221	/*
206	* vzalloc - allocate virtually contiguous memory with zero fill	222	* vzalloc - allocate virtually contiguous memory with zero fill

mm/vmalloc.c+6−5

 EXPORT_SYMBOL(vmalloc_noprof);
 /**
- * vmalloc_huge - allocate virtually contiguous memory, allow huge pages
+ * vmalloc_huge_node - allocate virtually contiguous memory, allow huge pages
  * @size:      allocation size
  * @gfp_mask:  flags for the page level allocator
+ * @node:	    node to use for allocation or NUMA_NO_NODE
  *
  * Allocate enough pages to cover @size from the page level
  * allocator and map them into contiguous kernel virtual space.
  *
  * Return: pointer to the allocated memory or %NULL on error
  */
-void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask)
+void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node)
 {
 	return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END,
-				    gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
+					   gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
-				    NUMA_NO_NODE, __builtin_return_address(0));
+					   node, __builtin_return_address(0));
 }
-EXPORT_SYMBOL_GPL(vmalloc_huge_noprof);
+EXPORT_SYMBOL_GPL(vmalloc_huge_node_noprof);
 /**
  * vzalloc - allocate virtually contiguous memory with zero fill

tools/include/uapi/linux/prctl.h+44−1

@@ -230,7 +230,7 @@ struct prctl_mm_map { · 229 unmodified lines
230	# define PR_PAC_APDBKEY (1UL << 3)	230	# define PR_PAC_APDBKEY (1UL << 3)
231	# define PR_PAC_APGAKEY (1UL << 4)	231	# define PR_PAC_APGAKEY (1UL << 4)
232		232
233	/* Tagged user address controls for arm64 */	233	/* Tagged user address controls for arm64 and RISC-V */
234	#define PR_SET_TAGGED_ADDR_CTRL 55	234	#define PR_SET_TAGGED_ADDR_CTRL 55
235	#define PR_GET_TAGGED_ADDR_CTRL 56	235	#define PR_GET_TAGGED_ADDR_CTRL 56
236	# define PR_TAGGED_ADDR_ENABLE (1UL << 0)	236	# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
@@ -244,6 +244,9 @@ struct prctl_mm_map { · 7 unmodified lines
244	# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)	244	# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
245	/* Unused; kept only for source compatibility */	245	/* Unused; kept only for source compatibility */
246	# define PR_MTE_TCF_SHIFT 1	246	# define PR_MTE_TCF_SHIFT 1
		247	/* RISC-V pointer masking tag length */
		248	# define PR_PMLEN_SHIFT 24
		249	# define PR_PMLEN_MASK (0x7fUL << PR_PMLEN_SHIFT)
247		250
248	/* Control reclaim behavior when allocating memory */	251	/* Control reclaim behavior when allocating memory */
249	#define PR_SET_IO_FLUSHER 57	252	#define PR_SET_IO_FLUSHER 57
@@ -328,4 +331,44 @@ struct prctl_mm_map { · 78 unmodified lines
328	# define PR_PPC_DEXCR_CTRL_CLEAR_ONEXEC 0x10 /* Clear the aspect on exec */	331	# define PR_PPC_DEXCR_CTRL_CLEAR_ONEXEC 0x10 /* Clear the aspect on exec */
329	# define PR_PPC_DEXCR_CTRL_MASK 0x1f	332	# define PR_PPC_DEXCR_CTRL_MASK 0x1f
330		333
		334	/*
		335	* Get the current shadow stack configuration for the current thread,
		336	* this will be the value configured via PR_SET_SHADOW_STACK_STATUS.
		337	*/
		338	#define PR_GET_SHADOW_STACK_STATUS 74
		339
		340	/*
		341	* Set the current shadow stack configuration. Enabling the shadow
		342	* stack will cause a shadow stack to be allocated for the thread.
		343	*/
		344	#define PR_SET_SHADOW_STACK_STATUS 75
		345	# define PR_SHADOW_STACK_ENABLE (1UL << 0)
		346	# define PR_SHADOW_STACK_WRITE (1UL << 1)
		347	# define PR_SHADOW_STACK_PUSH (1UL << 2)
		348
		349	/*
		350	* Prevent further changes to the specified shadow stack
		351	* configuration. All bits may be locked via this call, including
		352	* undefined bits.
		353	*/
		354	#define PR_LOCK_SHADOW_STACK_STATUS 76
		355
		356	/*
		357	* Controls the mode of timer_create() for CRIU restore operations.
		358	* Enabling this allows CRIU to restore timers with explicit IDs.
		359	*
		360	* Don't use for normal operations as the result might be undefined.
		361	*/
		362	#define PR_TIMER_CREATE_RESTORE_IDS 77
		363	# define PR_TIMER_CREATE_RESTORE_IDS_OFF 0
		364	# define PR_TIMER_CREATE_RESTORE_IDS_ON 1
		365	# define PR_TIMER_CREATE_RESTORE_IDS_GET 2
		366
		367	/* FUTEX hash management */
		368	#define PR_FUTEX_HASH 78
		369	# define PR_FUTEX_HASH_SET_SLOTS 1
		370	# define FH_FLAG_IMMUTABLE (1ULL << 0)
		371	# define PR_FUTEX_HASH_GET_SLOTS 2
		372	# define PR_FUTEX_HASH_GET_IMMUTABLE 3
		373
331	#endif /* _LINUX_PRCTL_H */	374	#endif /* _LINUX_PRCTL_H */

tools/perf/bench/Build+1−0

@@ -3,6 +3,7 @@ perf-bench-y += sched-pipe.o · 2 unmodified lines
3	perf-bench-y += sched-seccomp-notify.o	3	perf-bench-y += sched-seccomp-notify.o
4	perf-bench-y += syscall.o	4	perf-bench-y += syscall.o
5	perf-bench-y += mem-functions.o	5	perf-bench-y += mem-functions.o
		6	perf-bench-y += futex.o
6	perf-bench-y += futex-hash.o	7	perf-bench-y += futex-hash.o
7	perf-bench-y += futex-wake.o	8	perf-bench-y += futex-wake.o
8	perf-bench-y += futex-wake-parallel.o	9	perf-bench-y += futex-wake-parallel.o

tools/perf/bench/futex-hash.c+7−0

 #include <stdlib.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/prctl.h>
 #include <linux/zalloc.h>
 #include <sys/time.h>
 #include <sys/mman.h>
+#include <sys/prctl.h>
 #include <perf/cpumap.h>
 #include "../util/mutex.h"
 static struct bench_futex_parameters params = {
 	.nfutexes = 1024,
 	.runtime  = 10,
+	.nbuckets = -1,
 };
 static const struct option options[] = {
+	OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
+	OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
 	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
 	OPT_UINTEGER('r', "runtime", &params.runtime, "Specify runtime (in seconds)"),
 	OPT_UINTEGER('f', "futexes", &params.nfutexes, "Specify amount of futexes per threads"),
 	printf("%sAveraged %ld operations/sec (+- %.2f%%), total secs = %d\n",
 	       !params.silent ? "\n" : "", avg, rel_stddev_stats(stddev, avg),
 	       (int)bench__runtime.tv_sec);
+	futex_print_nbuckets(&params);
 }
 int bench_futex_hash(int argc, const char **argv)
 	if (!params.fshared)
 		futex_flag = FUTEX_PRIVATE_FLAG;
+	futex_set_nbuckets_param(&params);
 	printf("Run summary [PID %d]: %d threads, each operating on %d [%s] futexes for %d secs.\n\n",
 	       getpid(), params.nthreads, params.nfutexes, params.fshared ? "shared":"private", params.runtime);

tools/perf/bench/futex-lock-pi.c+5−0

 static struct cond thread_parent, thread_worker;
 static struct bench_futex_parameters params = {
+	.nbuckets = -1,
 	.runtime  = 10,
 };
 static const struct option options[] = {
+	OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
+	OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
 	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
 	OPT_UINTEGER('r', "runtime", &params.runtime, "Specify runtime (in seconds)"),
 	OPT_BOOLEAN( 'M', "multi",   &params.multi, "Use multiple futexes"),
 	printf("%sAveraged %ld operations/sec (+- %.2f%%), total secs = %d\n",
 	       !params.silent ? "\n" : "", avg, rel_stddev_stats(stddev, avg),
 	       (int)bench__runtime.tv_sec);
+	futex_print_nbuckets(&params);
 }
 static void toggle_done(int sig __maybe_unused,
 	mutex_init(&thread_lock);
 	cond_init(&thread_parent);
 	cond_init(&thread_worker);
+	futex_set_nbuckets_param(&params);
 	threads_starting = params.nthreads;
 	gettimeofday(&bench__start, NULL);

tools/perf/bench/futex-requeue.c+6−0

 static int futex_flag = 0;
 static struct bench_futex_parameters params = {
+	.nbuckets = -1,
 	/*
 	 * How many tasks to requeue at a time.
 	 * Default to 1 in order to make the kernel work more.
 };
 static const struct option options[] = {
+	OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
+	OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
 	OPT_UINTEGER('t', "threads",  &params.nthreads, "Specify amount of threads"),
 	OPT_UINTEGER('q', "nrequeue", &params.nrequeue, "Specify amount of threads to requeue at once"),
 	OPT_BOOLEAN( 's', "silent",   &params.silent, "Silent mode: do not display data/details"),
 	       params.nthreads,
 	       requeuetime_avg / USEC_PER_MSEC,
 	       rel_stddev_stats(requeuetime_stddev, requeuetime_avg));
+	futex_print_nbuckets(&params);
 }
 static void *workerfn(void *arg __maybe_unused)
 	if (params.broadcast)
 		params.nrequeue = params.nthreads;
+	futex_set_nbuckets_param(&params);
 	printf("Run summary [PID %d]: Requeuing %d threads (from [%s] %p to %s%p), "
 	       "%d at a time.\n\n",  getpid(), params.nthreads,
 	       params.fshared ? "shared":"private", &futex1,

tools/perf/bench/futex-wake-parallel.c+8−1

 static unsigned int threads_starting;
 static int futex_flag = 0;
-static struct bench_futex_parameters params;
+static struct bench_futex_parameters params = {
+	.nbuckets = -1,
+};
 static const struct option options[] = {
+	OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
+	OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
 	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
 	OPT_UINTEGER('w', "nwakers", &params.nwakes, "Specify amount of waking threads"),
 	OPT_BOOLEAN( 's', "silent",  &params.silent, "Silent mode: do not display data/details"),
 	       params.nthreads,
 	       waketime_avg / USEC_PER_MSEC,
 	       rel_stddev_stats(waketime_stddev, waketime_avg));
+	futex_print_nbuckets(&params);
 }
 	if (!params.fshared)
 		futex_flag = FUTEX_PRIVATE_FLAG;
+	futex_set_nbuckets_param(&params);
 	printf("Run summary [PID %d]: blocking on %d threads (at [%s] "
 	       "futex %p), %d threads waking up %d at a time.\n\n",
 	       getpid(), params.nthreads, params.fshared ? "shared":"private",

tools/perf/bench/futex-wake.c+4−0

 static int futex_flag = 0;
 static struct bench_futex_parameters params = {
+	.nbuckets = -1,
 	/*
 	 * How many wakeups to do at a time.
 	 * Default to 1 in order to make the kernel work more.
 };
 static const struct option options[] = {
+	OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
+	OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
 	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
 	OPT_UINTEGER('w', "nwakes",  &params.nwakes, "Specify amount of threads to wake at once"),
 	OPT_BOOLEAN( 's', "silent",  &params.silent, "Silent mode: do not display data/details"),
 	       params.nthreads,
 	       waketime_avg / USEC_PER_MSEC,
 	       rel_stddev_stats(waketime_stddev, waketime_avg));
+	futex_print_nbuckets(&params);
 }
 static void block_threads(pthread_t *w, struct perf_cpu_map *cpu)

tools/perf/bench/futex.c+67−0

@@ -0,0 +1,67 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <err.h>
	3	#include <stdio.h>
	4	#include <stdlib.h>
	5	#include <linux/prctl.h>
	6	#include <sys/prctl.h>
	7
	8	#include "futex.h"
	9
	10	void futex_set_nbuckets_param(struct bench_futex_parameters *params)
	11	{
	12	unsigned long flags;
	13	int ret;
	14
	15	if (params->nbuckets < 0)
	16	return;
	17
	18	flags = params->buckets_immutable ? FH_FLAG_IMMUTABLE : 0;
	19	ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, params->nbuckets, flags);
	20	if (ret) {
	21	printf("Requesting %d hash buckets failed: %d/%m\n",
	22	params->nbuckets, ret);
	23	err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	24	}
	25	}
	26
	27	void futex_print_nbuckets(struct bench_futex_parameters *params)
	28	{
	29	char *futex_hash_mode;
	30	int ret;
	31
	32	ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_SLOTS);
	33	if (params->nbuckets >= 0) {
	34	if (ret != params->nbuckets) {
	35	if (ret < 0) {
	36	printf("Can't query number of buckets: %m\n");
	37	err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	38	}
	39	printf("Requested number of hash buckets does not currently used.\n");
	40	printf("Requested: %d in usage: %d\n", params->nbuckets, ret);
	41	err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	42	}
	43	if (params->nbuckets == 0) {
	44	ret = asprintf(&futex_hash_mode, "Futex hashing: global hash");
	45	} else {
	46	ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_IMMUTABLE);
	47	if (ret < 0) {
	48	printf("Can't check if the hash is immutable: %m\n");
	49	err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	50	}
	51	ret = asprintf(&futex_hash_mode, "Futex hashing: %d hash buckets %s",
	52	params->nbuckets,
	53	ret == 1 ? "(immutable)" : "");
	54	}
	55	} else {
	56	if (ret <= 0) {
	57	ret = asprintf(&futex_hash_mode, "Futex hashing: global hash");
	58	} else {
	59	ret = asprintf(&futex_hash_mode, "Futex hashing: auto resized to %d buckets",
	60	ret);
	61	}
	62	}
	63	if (ret < 0)
	64	err(EXIT_FAILURE, "ENOMEM, futex_hash_mode");
	65	printf("%s\n", futex_hash_mode);
	66	free(futex_hash_mode);
	67	}

tools/perf/bench/futex.h+5−0

 	unsigned int nfutexes;
 	unsigned int nwakes;
 	unsigned int nrequeue;
+	int nbuckets;
+	bool buckets_immutable;
 };
 /**
 					val, opflags);
 }
+void futex_set_nbuckets_param(struct bench_futex_parameters *params);
+void futex_print_nbuckets(struct bench_futex_parameters *params);
 #endif /* _FUTEX_H */

tools/testing/selftests/futex/functional/.gitignore+4−2

 # SPDX-License-Identifier: GPL-2.0-only
+futex_numa_mpol
+futex_priv_hash
+futex_requeue
 futex_requeue_pi
 futex_requeue_pi_mismatched_ops
 futex_requeue_pi_signal_restart
+futex_wait
 futex_wait_private_mapped_file
 futex_wait_timeout
 futex_wait_uninitialized_heap
 futex_wait_wouldblock
-futex_wait
-futex_requeue
 futex_waitv

tools/testing/selftests/futex/functional/Makefile+5−2

 # SPDX-License-Identifier: GPL-2.0
 INCLUDES := -I../include -I../../ $(KHDR_INCLUDES)
 CFLAGS := $(CFLAGS) -g -O2 -Wall -pthread $(INCLUDES) $(KHDR_INCLUDES)
-LDLIBS := -lpthread -lrt
+LDLIBS := -lpthread -lrt -lnuma
 LOCAL_HDRS := \
 	../include/futextest.h \
 	futex_wait_private_mapped_file \
 	futex_wait \
 	futex_requeue \
-	futex_waitv
+	futex_priv_hash \
+	futex_numa_mpol \
+	futex_waitv \
+	futex_numa
 TEST_PROGS := run.sh

tools/testing/selftests/futex/functional/futex_numa.c+262−0

@@ -0,0 +1,262 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <pthread.h>
	4	#include <sys/shm.h>
	5	#include <sys/mman.h>
	6	#include <fcntl.h>
	7	#include <stdbool.h>
	8	#include <time.h>
	9	#include <assert.h>
	10	#include "logging.h"
	11	#include "futextest.h"
	12	#include "futex2test.h"
	13
	14	typedef u_int32_t u32;
	15	typedef int32_t s32;
	16	typedef u_int64_t u64;
	17
	18	static unsigned int fflags = (FUTEX2_SIZE_U32 \| FUTEX2_PRIVATE);
	19	static int fnode = FUTEX_NO_NODE;
	20
	21	/* fairly stupid test-and-set lock with a waiter flag */
	22
	23	#define N_LOCK 0x0000001
	24	#define N_WAITERS 0x0001000
	25
	26	struct futex_numa_32 {
	27	union {
	28	u64 full;
	29	struct {
	30	u32 val;
	31	u32 node;
	32	};
	33	};
	34	};
	35
	36	void futex_numa_32_lock(struct futex_numa_32 *lock)
	37	{
	38	for (;;) {
	39	struct futex_numa_32 new, old = {
	40	.full = __atomic_load_n(&lock->full, __ATOMIC_RELAXED),
	41	};
	42
	43	for (;;) {
	44	new = old;
	45	if (old.val == 0) {
	46	/* no waiter, no lock -> first lock, set no-node */
	47	new.node = fnode;
	48	}
	49	if (old.val & N_LOCK) {
	50	/* contention, set waiter */
	51	new.val \|= N_WAITERS;
	52	}
	53	new.val \|= N_LOCK;
	54
	55	/* nothing changed, ready to block */
	56	if (old.full == new.full)
	57	break;
	58
	59	/*
	60	* Use u64 cmpxchg to set the futex value and node in a
	61	* consistent manner.
	62	*/
	63	if (__atomic_compare_exchange_n(&lock->full,
	64	&old.full, new.full,
	65	/* .weak */ false,
	66	__ATOMIC_ACQUIRE,
	67	__ATOMIC_RELAXED)) {
	68
	69	/* if we just set N_LOCK, we own it */
	70	if (!(old.val & N_LOCK))
	71	return;
	72
	73	/* go block */
	74	break;
	75	}
	76	}
	77
	78	futex2_wait(lock, new.val, fflags, NULL, 0);
	79	}
	80	}
	81
	82	void futex_numa_32_unlock(struct futex_numa_32 *lock)
	83	{
	84	u32 val = __atomic_sub_fetch(&lock->val, N_LOCK, __ATOMIC_RELEASE);
	85	assert((s32)val >= 0);
	86	if (val & N_WAITERS) {
	87	int woken = futex2_wake(lock, 1, fflags);
	88	assert(val == N_WAITERS);
	89	if (!woken) {
	90	__atomic_compare_exchange_n(&lock->val, &val, 0U,
	91	false, __ATOMIC_RELAXED,
	92	__ATOMIC_RELAXED);
	93	}
	94	}
	95	}
	96
	97	static long nanos = 50000;
	98
	99	struct thread_args {
	100	pthread_t tid;
	101	volatile int * done;
	102	struct futex_numa_32 *lock;
	103	int val;
	104	int val1, val2;
	105	int node;
	106	};
	107
	108	static void threadfn(void _arg)
	109	{
	110	struct thread_args *args = _arg;
	111	struct timespec ts = {
	112	.tv_nsec = nanos,
	113	};
	114	int node;
	115
	116	while (!*args->done) {
	117
	118	futex_numa_32_lock(args->lock);
	119	args->val++;
	120
	121	assert(args->val1 == args->val2);
	122	(*args->val1)++;
	123	nanosleep(&ts, NULL);
	124	(*args->val2)++;
	125
	126	node = args->lock->node;
	127	futex_numa_32_unlock(args->lock);
	128
	129	if (node != args->node) {
	130	args->node = node;
	131	printf("node: %d\n", node);
	132	}
	133
	134	nanosleep(&ts, NULL);
	135	}
	136
	137	return NULL;
	138	}
	139
	140	static void contendfn(void _arg)
	141	{
	142	struct thread_args *args = _arg;
	143
	144	while (!*args->done) {
	145	/*
	146	* futex2_wait() will take hb-lock, verify *var == val and
	147	* queue/abort. By knowingly setting val 'wrong' this will
	148	* abort and thereby generate hb-lock contention.
	149	*/
	150	futex2_wait(&args->lock->val, ~0U, fflags, NULL, 0);
	151	args->val++;
	152	}
	153
	154	return NULL;
	155	}
	156
	157	static volatile int done = 0;
	158	static struct futex_numa_32 lock = { .val = 0, };
	159	static int val1, val2;
	160
	161	int main(int argc, char *argv[])
	162	{
	163	struct thread_args tas[512], cas[512];
	164	int c, t, threads = 2, contenders = 0;
	165	int sleeps = 10;
	166	int total = 0;
	167
	168	while ((c = getopt(argc, argv, "c:t:s:n:N::")) != -1) {
	169	switch (c) {
	170	case 'c':
	171	contenders = atoi(optarg);
	172	break;
	173	case 't':
	174	threads = atoi(optarg);
	175	break;
	176	case 's':
	177	sleeps = atoi(optarg);
	178	break;
	179	case 'n':
	180	nanos = atoi(optarg);
	181	break;
	182	case 'N':
	183	fflags \|= FUTEX2_NUMA;
	184	if (optarg)
	185	fnode = atoi(optarg);
	186	break;
	187	default:
	188	exit(1);
	189	break;
	190	}
	191	}
	192
	193	for (t = 0; t < contenders; t++) {
	194	struct thread_args args = calloc(1, sizeof(args));
	195	if (!args) {
	196	perror("thread_args");
	197	exit(-1);
	198	}
	199
	200	args->done = &done;
	201	args->lock = &lock;
	202	args->val1 = &val1;
	203	args->val2 = &val2;
	204	args->node = -1;
	205
	206	if (pthread_create(&args->tid, NULL, contendfn, args)) {
	207	perror("pthread_create");
	208	exit(-1);
	209	}
	210
	211	cas[t] = args;
	212	}
	213
	214	for (t = 0; t < threads; t++) {
	215	struct thread_args args = calloc(1, sizeof(args));
	216	if (!args) {
	217	perror("thread_args");
	218	exit(-1);
	219	}
	220
	221	args->done = &done;
	222	args->lock = &lock;
	223	args->val1 = &val1;
	224	args->val2 = &val2;
	225	args->node = -1;
	226
	227	if (pthread_create(&args->tid, NULL, threadfn, args)) {
	228	perror("pthread_create");
	229	exit(-1);
	230	}
	231
	232	tas[t] = args;
	233	}
	234
	235	sleep(sleeps);
	236
	237	done = true;
	238
	239	for (t = 0; t < threads; t++) {
	240	struct thread_args *args = tas[t];
	241
	242	pthread_join(args->tid, NULL);
	243	total += args->val;
	244	// printf("tval: %d\n", args->val);
	245	}
	246	printf("total: %d\n", total);
	247
	248	if (contenders) {
	249	total = 0;
	250	for (t = 0; t < contenders; t++) {
	251	struct thread_args *args = cas[t];
	252
	253	pthread_join(args->tid, NULL);
	254	total += args->val;
	255	// printf("tval: %d\n", args->val);
	256	}
	257	printf("contenders: %d\n", total);
	258	}
	259
	260	return 0;
	261	}
	262

tools/testing/selftests/futex/functional/futex_numa_mpol.c+231−0

@@ -0,0 +1,231 @@
	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	/*
	3	* Copyright (C) 2025 Sebastian Andrzej Siewior <bigeasy@linutronix.de>
	4	*/
	5
	6	#define _GNU_SOURCE
	7
	8	#include <errno.h>
	9	#include <pthread.h>
	10	#include <stdio.h>
	11	#include <stdlib.h>
	12	#include <unistd.h>
	13	#include <numa.h>
	14	#include <numaif.h>
	15
	16	#include <linux/futex.h>
	17	#include <sys/mman.h>
	18
	19	#include "logging.h"
	20	#include "futextest.h"
	21	#include "futex2test.h"
	22
	23	#define MAX_THREADS 64
	24
	25	static pthread_barrier_t barrier_main;
	26	static pthread_t threads[MAX_THREADS];
	27
	28	struct thread_args {
	29	void *futex_ptr;
	30	unsigned int flags;
	31	int result;
	32	};
	33
	34	static struct thread_args thread_args[MAX_THREADS];
	35
	36	#ifndef FUTEX_NO_NODE
	37	#define FUTEX_NO_NODE (-1)
	38	#endif
	39
	40	#ifndef FUTEX2_MPOL
	41	#define FUTEX2_MPOL 0x08
	42	#endif
	43
	44	static void thread_lock_fn(void arg)
	45	{
	46	struct thread_args *args = arg;
	47	int ret;
	48
	49	pthread_barrier_wait(&barrier_main);
	50	ret = futex2_wait(args->futex_ptr, 0, args->flags, NULL, 0);
	51	args->result = ret;
	52	return NULL;
	53	}
	54
	55	static void create_max_threads(void *futex_ptr)
	56	{
	57	int i, ret;
	58
	59	for (i = 0; i < MAX_THREADS; i++) {
	60	thread_args[i].futex_ptr = futex_ptr;
	61	thread_args[i].flags = FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA;
	62	thread_args[i].result = 0;
	63	ret = pthread_create(&threads[i], NULL, thread_lock_fn, &thread_args[i]);
	64	if (ret)
	65	ksft_exit_fail_msg("pthread_create failed\n");
	66	}
	67	}
	68
	69	static void join_max_threads(void)
	70	{
	71	int i, ret;
	72
	73	for (i = 0; i < MAX_THREADS; i++) {
	74	ret = pthread_join(threads[i], NULL);
	75	if (ret)
	76	ksft_exit_fail_msg("pthread_join failed for thread %d\n", i);
	77	}
	78	}
	79
	80	static void __test_futex(void *futex_ptr, int must_fail, unsigned int futex_flags)
	81	{
	82	int to_wake, ret, i, need_exit = 0;
	83
	84	pthread_barrier_init(&barrier_main, NULL, MAX_THREADS + 1);
	85	create_max_threads(futex_ptr);
	86	pthread_barrier_wait(&barrier_main);
	87	to_wake = MAX_THREADS;
	88
	89	do {
	90	ret = futex2_wake(futex_ptr, to_wake, futex_flags);
	91	if (must_fail) {
	92	if (ret < 0)
	93	break;
	94	ksft_exit_fail_msg("futex2_wake(%d, 0x%x) should fail, but didn't\n",
	95	to_wake, futex_flags);
	96	}
	97	if (ret < 0) {
	98	ksft_exit_fail_msg("Failed futex2_wake(%d, 0x%x): %m\n",
	99	to_wake, futex_flags);
	100	}
	101	if (!ret)
	102	usleep(50);
	103	to_wake -= ret;
	104
	105	} while (to_wake);
	106	join_max_threads();
	107
	108	for (i = 0; i < MAX_THREADS; i++) {
	109	if (must_fail && thread_args[i].result != -1) {
	110	ksft_print_msg("Thread %d should fail but succeeded (%d)\n",
	111	i, thread_args[i].result);
	112	need_exit = 1;
	113	}
	114	if (!must_fail && thread_args[i].result != 0) {
	115	ksft_print_msg("Thread %d failed (%d)\n", i, thread_args[i].result);
	116	need_exit = 1;
	117	}
	118	}
	119	if (need_exit)
	120	ksft_exit_fail_msg("Aborting due to earlier errors.\n");
	121	}
	122
	123	static void test_futex(void *futex_ptr, int must_fail)
	124	{
	125	__test_futex(futex_ptr, must_fail, FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA);
	126	}
	127
	128	static void test_futex_mpol(void *futex_ptr, int must_fail)
	129	{
	130	__test_futex(futex_ptr, must_fail, FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA \| FUTEX2_MPOL);
	131	}
	132
	133	static void usage(char *prog)
	134	{
	135	printf("Usage: %s\n", prog);
	136	printf(" -c Use color\n");
	137	printf(" -h Display this help message\n");
	138	printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n",
	139	VQUIET, VCRITICAL, VINFO);
	140	}
	141
	142	int main(int argc, char *argv[])
	143	{
	144	struct futex32_numa *futex_numa;
	145	int mem_size, i;
	146	void *futex_ptr;
	147	char c;
	148
	149	while ((c = getopt(argc, argv, "chv:")) != -1) {
	150	switch (c) {
	151	case 'c':
	152	log_color(1);
	153	break;
	154	case 'h':
	155	usage(basename(argv[0]));
	156	exit(0);
	157	break;
	158	case 'v':
	159	log_verbosity(atoi(optarg));
	160	break;
	161	default:
	162	usage(basename(argv[0]));
	163	exit(1);
	164	}
	165	}
	166
	167	ksft_print_header();
	168	ksft_set_plan(1);
	169
	170	mem_size = sysconf(_SC_PAGE_SIZE);
	171	futex_ptr = mmap(NULL, mem_size, PROT_READ \| PROT_WRITE, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
	172	if (futex_ptr == MAP_FAILED)
	173	ksft_exit_fail_msg("mmap() for %d bytes failed\n", mem_size);
	174
	175	futex_numa = futex_ptr;
	176
	177	ksft_print_msg("Regular test\n");
	178	futex_numa->futex = 0;
	179	futex_numa->numa = FUTEX_NO_NODE;
	180	test_futex(futex_ptr, 0);
	181
	182	if (futex_numa->numa == FUTEX_NO_NODE)
	183	ksft_exit_fail_msg("NUMA node is left uninitialized\n");
	184
	185	ksft_print_msg("Memory too small\n");
	186	test_futex(futex_ptr + mem_size - 4, 1);
	187
	188	ksft_print_msg("Memory out of range\n");
	189	test_futex(futex_ptr + mem_size, 1);
	190
	191	futex_numa->numa = FUTEX_NO_NODE;
	192	mprotect(futex_ptr, mem_size, PROT_READ);
	193	ksft_print_msg("Memory, RO\n");
	194	test_futex(futex_ptr, 1);
	195
	196	mprotect(futex_ptr, mem_size, PROT_NONE);
	197	ksft_print_msg("Memory, no access\n");
	198	test_futex(futex_ptr, 1);
	199
	200	mprotect(futex_ptr, mem_size, PROT_READ \| PROT_WRITE);
	201	ksft_print_msg("Memory back to RW\n");
	202	test_futex(futex_ptr, 0);
	203
	204	/* MPOL test. Does not work as expected */
	205	for (i = 0; i < 4; i++) {
	206	unsigned long nodemask;
	207	int ret;
	208
	209	nodemask = 1 << i;
	210	ret = mbind(futex_ptr, mem_size, MPOL_BIND, &nodemask,
	211	sizeof(nodemask) * 8, 0);
	212	if (ret == 0) {
	213	ksft_print_msg("Node %d test\n", i);
	214	futex_numa->futex = 0;
	215	futex_numa->numa = FUTEX_NO_NODE;
	216
	217	ret = futex2_wake(futex_ptr, 0, FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA \| FUTEX2_MPOL);
	218	if (ret < 0)
	219	ksft_test_result_fail("Failed to wake 0 with MPOL: %m\n");
	220	if (0)
	221	test_futex_mpol(futex_numa, 0);
	222	if (futex_numa->numa != i) {
	223	ksft_test_result_fail("Returned NUMA node is %d expected %d\n",
	224	futex_numa->numa, i);
	225	}
	226	}
	227	}
	228	ksft_test_result_pass("NUMA MPOL tests passed\n");
	229	ksft_finished();
	230	return 0;
	231	}

tools/testing/selftests/futex/functional/futex_priv_hash.c+292−0

@@ -0,0 +1,292 @@
	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	/*
	3	* Copyright (C) 2025 Sebastian Andrzej Siewior <bigeasy@linutronix.de>
	4	*/
	5
	6	#define _GNU_SOURCE
	7
	8	#include <errno.h>
	9	#include <pthread.h>
	10	#include <stdio.h>
	11	#include <stdlib.h>
	12	#include <unistd.h>
	13
	14	#include <linux/prctl.h>
	15	#include <sys/prctl.h>
	16
	17	#include "logging.h"
	18
	19	#define MAX_THREADS 64
	20
	21	static pthread_barrier_t barrier_main;
	22	static pthread_mutex_t global_lock;
	23	static pthread_t threads[MAX_THREADS];
	24	static int counter;
	25
	26	#ifndef PR_FUTEX_HASH
	27	#define PR_FUTEX_HASH 78
	28	# define PR_FUTEX_HASH_SET_SLOTS 1
	29	# define FH_FLAG_IMMUTABLE (1ULL << 0)
	30	# define PR_FUTEX_HASH_GET_SLOTS 2
	31	# define PR_FUTEX_HASH_GET_IMMUTABLE 3
	32	#endif
	33
	34	static int futex_hash_slots_set(unsigned int slots, int flags)
	35	{
	36	return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, slots, flags);
	37	}
	38
	39	static int futex_hash_slots_get(void)
	40	{
	41	return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_SLOTS);
	42	}
	43
	44	static int futex_hash_immutable_get(void)
	45	{
	46	return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_IMMUTABLE);
	47	}
	48
	49	static void futex_hash_slots_set_verify(int slots)
	50	{
	51	int ret;
	52
	53	ret = futex_hash_slots_set(slots, 0);
	54	if (ret != 0) {
	55	ksft_test_result_fail("Failed to set slots to %d: %m\n", slots);
	56	ksft_finished();
	57	}
	58	ret = futex_hash_slots_get();
	59	if (ret != slots) {
	60	ksft_test_result_fail("Set %d slots but PR_FUTEX_HASH_GET_SLOTS returns: %d, %m\n",
	61	slots, ret);
	62	ksft_finished();
	63	}
	64	ksft_test_result_pass("SET and GET slots %d passed\n", slots);
	65	}
	66
	67	static void futex_hash_slots_set_must_fail(int slots, int flags)
	68	{
	69	int ret;
	70
	71	ret = futex_hash_slots_set(slots, flags);
	72	ksft_test_result(ret < 0, "futex_hash_slots_set(%d, %d)\n",
	73	slots, flags);
	74	}
	75
	76	static void thread_return_fn(void arg)
	77	{
	78	return NULL;
	79	}
	80
	81	static void thread_lock_fn(void arg)
	82	{
	83	pthread_barrier_wait(&barrier_main);
	84
	85	pthread_mutex_lock(&global_lock);
	86	counter++;
	87	usleep(20);
	88	pthread_mutex_unlock(&global_lock);
	89	return NULL;
	90	}
	91
	92	static void create_max_threads(void (thread_fn)(void *))
	93	{
	94	int i, ret;
	95
	96	for (i = 0; i < MAX_THREADS; i++) {
	97	ret = pthread_create(&threads[i], NULL, thread_fn, NULL);
	98	if (ret)
	99	ksft_exit_fail_msg("pthread_create failed: %m\n");
	100	}
	101	}
	102
	103	static void join_max_threads(void)
	104	{
	105	int i, ret;
	106
	107	for (i = 0; i < MAX_THREADS; i++) {
	108	ret = pthread_join(threads[i], NULL);
	109	if (ret)
	110	ksft_exit_fail_msg("pthread_join failed for thread %d\n", i);
	111	}
	112	}
	113
	114	static void usage(char *prog)
	115	{
	116	printf("Usage: %s\n", prog);
	117	printf(" -c Use color\n");
	118	printf(" -g Test global hash instead intead local immutable \n");
	119	printf(" -h Display this help message\n");
	120	printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n",
	121	VQUIET, VCRITICAL, VINFO);
	122	}
	123
	124	static const char *test_msg_auto_create = "Automatic hash bucket init on thread creation.\n";
	125	static const char *test_msg_auto_inc = "Automatic increase with more than 16 CPUs\n";
	126
	127	int main(int argc, char *argv[])
	128	{
	129	int futex_slots1, futex_slotsn, online_cpus;
	130	pthread_mutexattr_t mutex_attr_pi;
	131	int use_global_hash = 0;
	132	int ret;
	133	char c;
	134
	135	while ((c = getopt(argc, argv, "cghv:")) != -1) {
	136	switch (c) {
	137	case 'c':
	138	log_color(1);
	139	break;
	140	case 'g':
	141	use_global_hash = 1;
	142	break;
	143	case 'h':
	144	usage(basename(argv[0]));
	145	exit(0);
	146	break;
	147	case 'v':
	148	log_verbosity(atoi(optarg));
	149	break;
	150	default:
	151	usage(basename(argv[0]));
	152	exit(1);
	153	}
	154	}
	155
	156	ksft_print_header();
	157	ksft_set_plan(22);
	158
	159	ret = pthread_mutexattr_init(&mutex_attr_pi);
	160	ret \|= pthread_mutexattr_setprotocol(&mutex_attr_pi, PTHREAD_PRIO_INHERIT);
	161	ret \|= pthread_mutex_init(&global_lock, &mutex_attr_pi);
	162	if (ret != 0) {
	163	ksft_exit_fail_msg("Failed to initialize pthread mutex.\n");
	164	}
	165	/* First thread, expect to be 0, not yet initialized */
	166	ret = futex_hash_slots_get();
	167	if (ret != 0)
	168	ksft_exit_fail_msg("futex_hash_slots_get() failed: %d, %m\n", ret);
	169
	170	ret = futex_hash_immutable_get();
	171	if (ret != 0)
	172	ksft_exit_fail_msg("futex_hash_immutable_get() failed: %d, %m\n", ret);
	173
	174	ksft_test_result_pass("Basic get slots and immutable status.\n");
	175	ret = pthread_create(&threads[0], NULL, thread_return_fn, NULL);
	176	if (ret != 0)
	177	ksft_exit_fail_msg("pthread_create() failed: %d, %m\n", ret);
	178
	179	ret = pthread_join(threads[0], NULL);
	180	if (ret != 0)
	181	ksft_exit_fail_msg("pthread_join() failed: %d, %m\n", ret);
	182
	183	/* First thread, has to initialiaze private hash */
	184	futex_slots1 = futex_hash_slots_get();
	185	if (futex_slots1 <= 0) {
	186	ksft_print_msg("Current hash buckets: %d\n", futex_slots1);
	187	ksft_exit_fail_msg(test_msg_auto_create);
	188	}
	189
	190	ksft_test_result_pass(test_msg_auto_create);
	191
	192	online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
	193	ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS + 1);
	194	if (ret != 0)
	195	ksft_exit_fail_msg("pthread_barrier_init failed: %m.\n");
	196
	197	ret = pthread_mutex_lock(&global_lock);
	198	if (ret != 0)
	199	ksft_exit_fail_msg("pthread_mutex_lock failed: %m.\n");
	200
	201	counter = 0;
	202	create_max_threads(thread_lock_fn);
	203	pthread_barrier_wait(&barrier_main);
	204
	205	/*
	206	* The current default size of hash buckets is 16. The auto increase
	207	* works only if more than 16 CPUs are available.
	208	*/
	209	ksft_print_msg("Online CPUs: %d\n", online_cpus);
	210	if (online_cpus > 16) {
	211	futex_slotsn = futex_hash_slots_get();
	212	if (futex_slotsn < 0 \|\| futex_slots1 == futex_slotsn) {
	213	ksft_print_msg("Expected increase of hash buckets but got: %d -> %d\n",
	214	futex_slots1, futex_slotsn);
	215	ksft_exit_fail_msg(test_msg_auto_inc);
	216	}
	217	ksft_test_result_pass(test_msg_auto_inc);
	218	} else {
	219	ksft_test_result_skip(test_msg_auto_inc);
	220	}
	221	ret = pthread_mutex_unlock(&global_lock);
	222
	223	/* Once the user changes it, it has to be what is set */
	224	futex_hash_slots_set_verify(2);
	225	futex_hash_slots_set_verify(4);
	226	futex_hash_slots_set_verify(8);
	227	futex_hash_slots_set_verify(32);
	228	futex_hash_slots_set_verify(16);
	229
	230	ret = futex_hash_slots_set(15, 0);
	231	ksft_test_result(ret < 0, "Use 15 slots\n");
	232
	233	futex_hash_slots_set_verify(2);
	234	join_max_threads();
	235	ksft_test_result(counter == MAX_THREADS, "Created of waited for %d of %d threads\n",
	236	counter, MAX_THREADS);
	237	counter = 0;
	238	/* Once the user set something, auto reisze must be disabled */
	239	ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS);
	240
	241	create_max_threads(thread_lock_fn);
	242	join_max_threads();
	243
	244	ret = futex_hash_slots_get();
	245	ksft_test_result(ret == 2, "No more auto-resize after manaul setting, got %d\n",
	246	ret);
	247
	248	futex_hash_slots_set_must_fail(1 << 29, 0);
	249
	250	/*
	251	* Once the private hash has been made immutable or global hash has been requested,
	252	* then this requested can not be undone.
	253	*/
	254	if (use_global_hash) {
	255	ret = futex_hash_slots_set(0, 0);
	256	ksft_test_result(ret == 0, "Global hash request\n");
	257	} else {
	258	ret = futex_hash_slots_set(4, FH_FLAG_IMMUTABLE);
	259	ksft_test_result(ret == 0, "Immutable resize to 4\n");
	260	}
	261	if (ret != 0)
	262	goto out;
	263
	264	futex_hash_slots_set_must_fail(4, 0);
	265	futex_hash_slots_set_must_fail(4, FH_FLAG_IMMUTABLE);
	266	futex_hash_slots_set_must_fail(8, 0);
	267	futex_hash_slots_set_must_fail(8, FH_FLAG_IMMUTABLE);
	268	futex_hash_slots_set_must_fail(0, FH_FLAG_IMMUTABLE);
	269	futex_hash_slots_set_must_fail(6, FH_FLAG_IMMUTABLE);
	270
	271	ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS);
	272	if (ret != 0) {
	273	ksft_exit_fail_msg("pthread_barrier_init failed: %m\n");
	274	return 1;
	275	}
	276	create_max_threads(thread_lock_fn);
	277	join_max_threads();
	278
	279	ret = futex_hash_slots_get();
	280	if (use_global_hash) {
	281	ksft_test_result(ret == 0, "Continue to use global hash\n");
	282	} else {
	283	ksft_test_result(ret == 4, "Continue to use the 4 hash buckets\n");
	284	}
	285
	286	ret = futex_hash_immutable_get();
	287	ksft_test_result(ret == 1, "Hash reports to be immutable\n");
	288
	289	out:
	290	ksft_finished();
	291	return 0;
	292	}

tools/testing/selftests/futex/functional/run.sh+7−0

@@ -82,3 +82,10 @@ echo · 81 unmodified lines
82		82
83	echo	83	echo
84	./futex_waitv $COLOR	84	./futex_waitv $COLOR
		85
		86	echo
		87	./futex_priv_hash $COLOR
		88	./futex_priv_hash -g $COLOR
		89
		90	echo
		91	./futex_numa_mpol $COLOR

tools/testing/selftests/futex/include/futex2test.h+70−0

@@ -8,6 +8,53 @@ · 7 unmodified lines
8		8
9	#define u64_to_ptr(x) ((void *)(uintptr_t)(x))	9	#define u64_to_ptr(x) ((void *)(uintptr_t)(x))
10		10
		11	#ifndef __NR_futex_waitv
		12	#define __NR_futex_waitv 449
		13	struct futex_waitv {
		14	__u64 val;
		15	__u64 uaddr;
		16	__u32 flags;
		17	__u32 __reserved;
		18	};
		19	#endif
		20
		21	#ifndef __NR_futex_wake
		22	#define __NR_futex_wake 454
		23	#endif
		24
		25	#ifndef __NR_futex_wait
		26	#define __NR_futex_wait 455
		27	#endif
		28
		29	#ifndef FUTEX2_SIZE_U32
		30	#define FUTEX2_SIZE_U32 0x02
		31	#endif
		32
		33	#ifndef FUTEX2_NUMA
		34	#define FUTEX2_NUMA 0x04
		35	#endif
		36
		37	#ifndef FUTEX2_MPOL
		38	#define FUTEX2_MPOL 0x08
		39	#endif
		40
		41	#ifndef FUTEX2_PRIVATE
		42	#define FUTEX2_PRIVATE FUTEX_PRIVATE_FLAG
		43	#endif
		44
		45	#ifndef FUTEX2_NO_NODE
		46	#define FUTEX_NO_NODE (-1)
		47	#endif
		48
		49	#ifndef FUTEX_32
		50	#define FUTEX_32 FUTEX2_SIZE_U32
		51	#endif
		52
		53	struct futex32_numa {
		54	futex_t futex;
		55	futex_t numa;
		56	};
		57
11	/**	58	/**
12	* futex_waitv - Wait at multiple futexes, wake on any	59	* futex_waitv - Wait at multiple futexes, wake on any
13	* @waiters: Array of waiters	60	* @waiters: Array of waiters
@@ -20,3 +67,26 @@ static inline int futex_waitv(volatile struct futex_waitv *waiters, unsigned lon · 6 unmodified lines
20	{	67	{
21	return syscall(__NR_futex_waitv, waiters, nr_waiters, flags, timo, clockid);	68	return syscall(__NR_futex_waitv, waiters, nr_waiters, flags, timo, clockid);
22	}	69	}
		70
		71	/*
		72	* futex_wait() - block on uaddr with optional timeout
		73	* @val: Expected value
		74	* @flags: FUTEX2 flags
		75	* @timeout: Relative timeout
		76	* @clockid: Clock id for the timeout
		77	*/
		78	static inline int futex2_wait(void *uaddr, long val, unsigned int flags,
		79	struct timespec *timeout, clockid_t clockid)
		80	{
		81	return syscall(__NR_futex_wait, uaddr, val, ~0U, flags, timeout, clockid);
		82	}
		83
		84	/*
		85	* futex2_wake() - Wake a number of futexes
		86	* @nr: Number of threads to wake at most
		87	* @flags: FUTEX2 flags
		88	*/
		89	static inline int futex2_wake(void *uaddr, int nr, unsigned int flags)
		90	{
		91	return syscall(__NR_futex_wake, uaddr, ~0U, nr, flags);
		92	}