Merge branch 'locking/futex' into locking/core, to pick up pending futex changes

Ingo Molnar committed 1 year ago

Commit 94ec7082 parents cdb7d2d 78272d435 files · +2479 −564Browse files

Unified · Split

35 files

include/linux/futex.h +34 −2
include/linux/mm_types.h +6 −1
include/linux/mmap_lock.h +4 −0
include/linux/rcuref.h +21 −1
include/linux/restart_block.h +1 −1
include/linux/vmalloc.h +7 −2
include/uapi/linux/futex.h +8 −1
include/uapi/linux/prctl.h +7 −0
init/Kconfig +10 −0
io_uring/futex.c +1 −3
kernel/fork.c +24 −0
kernel/futex/core.c +732 −71
kernel/futex/futex.h +66 −8
kernel/futex/pi.c +166 −142
kernel/futex/requeue.c +234 −226
kernel/futex/waitwake.c +113 −94
kernel/sys.c +4 −0
mm/nommu.c +17 −1
mm/vmalloc.c +6 −5
tools/include/uapi/linux/prctl.h +44 −1
tools/perf/bench/Build +1 −0
tools/perf/bench/futex-hash.c +7 −0
tools/perf/bench/futex-lock-pi.c +5 −0
tools/perf/bench/futex-requeue.c +6 −0
tools/perf/bench/futex-wake-parallel.c +8 −1
tools/perf/bench/futex-wake.c +4 −0
tools/perf/bench/futex.c +67 −0
tools/perf/bench/futex.h +5 −0
tools/testing/selftests/futex/functional/.gitignore +4 −2
tools/testing/selftests/futex/functional/Makefile +5 −2
tools/testing/selftests/futex/functional/futex_numa.c +262 −0
tools/testing/selftests/futex/functional/futex_numa_mpol.c +231 −0
tools/testing/selftests/futex/functional/futex_priv_hash.c +292 −0
tools/testing/selftests/futex/functional/run.sh +7 −0
tools/testing/selftests/futex/include/futex2test.h +70 −0

@@ -4,11 +4,11 @@ · 3 unmodified lines
4	4
5	5	#include <linux/sched.h>
6	6	#include <linux/ktime.h>
	7	+#include <linux/mm_types.h>
7	8
8	9	#include <uapi/linux/futex.h>
9	10
10	11	struct inode;
11		−struct mm_struct;
12	12	struct task_struct;
13	13
14	14	/*
@@ -34,6 +34,7 @@ union futex_key { · 19 unmodified lines
34	34	u64 i_seq;
35	35	unsigned long pgoff;
36	36	unsigned int offset;
	37	+ /* unsigned int node; */
37	38	} shared;
38	39	struct {
39	40	union {
@@ -42,11 +43,13 @@ union futex_key { · 2 unmodified lines
42	43	};
43	44	unsigned long address;
44	45	unsigned int offset;
	46	+ /* unsigned int node; */
45	47	} private;
46	48	struct {
47	49	u64 ptr;
48	50	unsigned long word;
49	51	unsigned int offset;
	52	+ unsigned int node; /* NOT hashed! */
50	53	} both;
51	54	};
52	55
@@ -77,7 +80,25 @@ void futex_exec_release(struct task_struct *tsk); · 24 unmodified lines
77	80
78	81	long do_futex(u32 __user uaddr, int op, u32 val, ktime_t timeout,
79	82	u32 __user *uaddr2, u32 val2, u32 val3);
80		−#else
	83	+int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4);
	84	+
	85	+#ifdef CONFIG_FUTEX_PRIVATE_HASH
	86	+int futex_hash_allocate_default(void);
	87	+void futex_hash_free(struct mm_struct *mm);
	88	+
	89	+static inline void futex_mm_init(struct mm_struct *mm)
	90	+{
	91	+ RCU_INIT_POINTER(mm->futex_phash, NULL);
	92	+ mutex_init(&mm->futex_hash_lock);
	93	+}
	94	+
	95	+#else /* !CONFIG_FUTEX_PRIVATE_HASH */
	96	+static inline int futex_hash_allocate_default(void) { return 0; }
	97	+static inline void futex_hash_free(struct mm_struct *mm) { }
	98	+static inline void futex_mm_init(struct mm_struct *mm) { }
	99	+#endif /* CONFIG_FUTEX_PRIVATE_HASH */
	100	+
	101	+#else /* !CONFIG_FUTEX */
81	102	static inline void futex_init_task(struct task_struct *tsk) { }
82	103	static inline void futex_exit_recursive(struct task_struct *tsk) { }
83	104	static inline void futex_exit_release(struct task_struct *tsk) { }
@@ -88,6 +109,17 @@ static inline long do_futex(u32 __user *uaddr, int op, u32 val, · 4 unmodified lines
88	109	{
89	110	return -EINVAL;
90	111	}
	112	+static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
	113	+{
	114	+ return -EINVAL;
	115	+}
	116	+static inline int futex_hash_allocate_default(void)
	117	+{
	118	+ return 0;
	119	+}
	120	+static inline void futex_hash_free(struct mm_struct *mm) { }
	121	+static inline void futex_mm_init(struct mm_struct *mm) { }
	122	+
91	123	#endif
92	124
93	125	#endif

include/linux/mm_types.h+6−1

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#define INIT_PASID 0
32	32
33	33	struct address_space;
	34	+struct futex_private_hash;
34	35	struct mem_cgroup;
35	36
36	37	/*
@@ -1031,7 +1032,11 @@ struct mm_struct { · 994 unmodified lines
1031	1032	*/
1032	1033	seqcount_t mm_lock_seq;
1033	1034	#endif
1034		−
	1035	+#ifdef CONFIG_FUTEX_PRIVATE_HASH
	1036	+ struct mutex futex_hash_lock;
	1037	+ struct futex_private_hash __rcu *futex_phash;
	1038	+ struct futex_private_hash *futex_phash_new;
	1039	+#endif
1035	1040
1036	1041	unsigned long hiwater_rss; /* High-watermark of RSS usage */
1037	1042	unsigned long hiwater_vm; /* High-water virtual memory usage */

include/linux/mmap_lock.h+4−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/rwsem.h>
8	8	#include <linux/tracepoint-defs.h>
9	9	#include <linux/types.h>
	10	+#include <linux/cleanup.h>
10	11
11	12	#define MMAP_LOCK_INITIALIZER(name) \
12	13	.mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),
@@ -211,6 +212,9 @@ static inline void mmap_read_unlock(struct mm_struct *mm) · 198 unmodified lines
211	212	up_read(&mm->mmap_lock);
212	213	}
213	214
	215	+DEFINE_GUARD(mmap_read_lock, struct mm_struct *,
	216	+ mmap_read_lock(_T), mmap_read_unlock(_T))
	217	+
214	218	static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
215	219	{
216	220	__mmap_lock_trace_released(mm, false);

include/linux/rcuref.h+21−1

@@ -30,7 +30,11 @@ static inline void rcuref_init(rcuref_t *ref, unsigned int cnt) · 29 unmodified lines
30	30	* rcuref_read - Read the number of held reference counts of a rcuref
31	31	* @ref: Pointer to the reference count
32	32	*
33		− * Return: The number of held references (0 ... N)
	33	+ * Return: The number of held references (0 ... N). The value 0 does not
	34	+ * indicate that it is safe to schedule the object, protected by this reference
	35	+ * counter, for deconstruction.
	36	+ * If you want to know if the reference counter has been marked DEAD (as
	37	+ * signaled by rcuref_put()) please use rcuread_is_dead().
34	38	*/
35	39	static inline unsigned int rcuref_read(rcuref_t *ref)
36	40	{
@@ -40,6 +44,22 @@ static inline unsigned int rcuref_read(rcuref_t *ref) · 3 unmodified lines
40	44	return c >= RCUREF_RELEASED ? 0 : c + 1;
41	45	}
42	46
	47	+/**
	48	+ * rcuref_is_dead - Check if the rcuref has been already marked dead
	49	+ * @ref: Pointer to the reference count
	50	+ *
	51	+ * Return: True if the object has been marked DEAD. This signals that a previous
	52	+ * invocation of rcuref_put() returned true on this reference counter meaning
	53	+ * the protected object can safely be scheduled for deconstruction.
	54	+ * Otherwise, returns false.
	55	+ */
	56	+static inline bool rcuref_is_dead(rcuref_t *ref)
	57	+{
	58	+ unsigned int c = atomic_read(&ref->refcnt);
	59	+
	60	+ return (c >= RCUREF_RELEASED) && (c < RCUREF_NOREF);
	61	+}
	62	+
43	63	extern __must_check bool rcuref_get_slowpath(rcuref_t *ref);
44	64
45	65	/**

include/linux/restart_block.h+1−1

@@ -26,7 +26,7 @@ struct restart_block { · 25 unmodified lines
26	26	unsigned long arch_data;
27	27	long (fn)(struct restart_block );
28	28	union {
29		− /* For futex_wait and futex_wait_requeue_pi */
	29	+ /* For futex_wait() */
30	30	struct {
31	31	u32 __user *uaddr;
32	32	u32 val;

include/linux/vmalloc.h+7−2

@@ -168,8 +168,13 @@ void *__vmalloc_node_noprof(unsigned long size, unsigned long align, gfp_t gfp_m · 167 unmodified lines
168	168	int node, const void *caller) __alloc_size(1);
169	169	#define __vmalloc_node(...) alloc_hooks(__vmalloc_node_noprof(__VA_ARGS__))
170	170
171		−void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) __alloc_size(1);
172		−#define vmalloc_huge(...) alloc_hooks(vmalloc_huge_noprof(__VA_ARGS__))
	171	+void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node) __alloc_size(1);
	172	+#define vmalloc_huge_node(...) alloc_hooks(vmalloc_huge_node_noprof(__VA_ARGS__))
	173	+
	174	+static inline void *vmalloc_huge(unsigned long size, gfp_t gfp_mask)
	175	+{
	176	+ return vmalloc_huge_node(size, gfp_mask, NUMA_NO_NODE);
	177	+}
173	178
174	179	extern void *__vmalloc_array_noprof(size_t n, size_t size, gfp_t flags) __alloc_size(1, 2);
175	180	#define __vmalloc_array(...) alloc_hooks(__vmalloc_array_noprof(__VA_ARGS__))

include/uapi/linux/futex.h+8−1

@@ -63,7 +63,7 @@ · 62 unmodified lines
63	63	#define FUTEX2_SIZE_U32 0x02
64	64	#define FUTEX2_SIZE_U64 0x03
65	65	#define FUTEX2_NUMA 0x04
66		− /* 0x08 */
	66	+#define FUTEX2_MPOL 0x08
67	67	/* 0x10 */
68	68	/* 0x20 */
69	69	/* 0x40 */
@@ -74,6 +74,13 @@ · 4 unmodified lines
74	74	/* do not use */
75	75	#define FUTEX_32 FUTEX2_SIZE_U32 /* historical accident :-( */
76	76
	77	+/*
	78	+ * When FUTEX2_NUMA doubles the futex word, the second word is a node value.
	79	+ * The special value -1 indicates no-node. This is the same value as
	80	+ * NUMA_NO_NODE, except that value is not ABI, this is.
	81	+ */
	82	+#define FUTEX_NO_NODE (-1)
	83	+
77	84	/*
78	85	* Max numbers of elements in a futex_waitv array
79	86	*/

include/uapi/linux/prctl.h+7−0

@@ -364,4 +364,11 @@ struct prctl_mm_map { · 363 unmodified lines
364	364	# define PR_TIMER_CREATE_RESTORE_IDS_ON 1
365	365	# define PR_TIMER_CREATE_RESTORE_IDS_GET 2
366	366
	367	+/* FUTEX hash management */
	368	+#define PR_FUTEX_HASH 78
	369	+# define PR_FUTEX_HASH_SET_SLOTS 1
	370	+# define FH_FLAG_IMMUTABLE (1ULL << 0)
	371	+# define PR_FUTEX_HASH_GET_SLOTS 2
	372	+# define PR_FUTEX_HASH_GET_IMMUTABLE 3
	373	+
367	374	#endif /* _LINUX_PRCTL_H */

init/Kconfig+10−0

@@ -1699,6 +1699,16 @@ config FUTEX_PI · 1698 unmodified lines
1699	1699	depends on FUTEX && RT_MUTEXES
1700	1700	default y
1701	1701
	1702	+config FUTEX_PRIVATE_HASH
	1703	+ bool
	1704	+ depends on FUTEX && !BASE_SMALL && MMU
	1705	+ default y
	1706	+
	1707	+config FUTEX_MPOL
	1708	+ bool
	1709	+ depends on FUTEX && NUMA
	1710	+ default y
	1711	+
1702	1712	config EPOLL
1703	1713	bool "Enable eventpoll support" if EXPERT
1704	1714	default y

io_uring/futex.c+1−3

@@ -273,7 +273,6 @@ int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) · 272 unmodified lines
273	273	struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
274	274	struct io_ring_ctx *ctx = req->ctx;
275	275	struct io_futex_data *ifd = NULL;
276		− struct futex_hash_bucket *hb;
277	276	int ret;
278	277
279	278	if (!iof->futex_mask) {
@@ -295,12 +294,11 @@ int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) · 15 unmodified lines
295	294	ifd->req = req;
296	295
297	296	ret = futex_wait_setup(iof->uaddr, iof->futex_val, iof->futex_flags,
298		− &ifd->q, &hb);
	297	+ &ifd->q, NULL, NULL);
299	298	if (!ret) {
300	299	hlist_add_head(&req->hash_node, &ctx->futex_list);
301	300	io_ring_submit_unlock(ctx, issue_flags);
302	301
303		− futex_queue(&ifd->q, hb, NULL);
304	302	return IOU_ISSUE_SKIP_COMPLETE;
305	303	}
306	304

kernel/fork.c+24−0

@@ -1305,6 +1305,7 @@ static struct mm_struct mm_init(struct mm_struct mm, struct task_struct *p, · 1304 unmodified lines
1305	1305	RCU_INIT_POINTER(mm->exe_file, NULL);
1306	1306	mmu_notifier_subscriptions_init(mm);
1307	1307	init_tlb_flush_pending(mm);
	1308	+ futex_mm_init(mm);
1308	1309	#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
1309	1310	mm->pmd_huge_pte = NULL;
1310	1311	#endif
@@ -1387,6 +1388,7 @@ static inline void __mmput(struct mm_struct *mm) · 76 unmodified lines
1387	1388	if (mm->binfmt)
1388	1389	module_put(mm->binfmt->module);
1389	1390	lru_gen_del_mm(mm);
	1391	+ futex_hash_free(mm);
1390	1392	mmdrop(mm);
1391	1393	}
1392	1394
@@ -2162,6 +2164,13 @@ static void rv_task_fork(struct task_struct *p) · 769 unmodified lines
2162	2164	#define rv_task_fork(p) do {} while (0)
2163	2165	#endif
2164	2166
	2167	+static bool need_futex_hash_allocate_default(u64 clone_flags)
	2168	+{
	2169	+ if ((clone_flags & (CLONE_THREAD \| CLONE_VM)) != (CLONE_THREAD \| CLONE_VM))
	2170	+ return false;
	2171	+ return true;
	2172	+}
	2173	+
2165	2174	/*
2166	2175	* This creates a new process as a copy of the old one,
2167	2176	* but does not actually start it yet.
@@ -2542,6 +2551,21 @@ __latent_entropy struct task_struct *copy_process( · 374 unmodified lines
2542	2551	if (retval)
2543	2552	goto bad_fork_cancel_cgroup;
2544	2553
	2554	+ /*
	2555	+ * Allocate a default futex hash for the user process once the first
	2556	+ * thread spawns.
	2557	+ */
	2558	+ if (need_futex_hash_allocate_default(clone_flags)) {
	2559	+ retval = futex_hash_allocate_default();
	2560	+ if (retval)
	2561	+ goto bad_fork_core_free;
	2562	+ /*
	2563	+ * If we fail beyond this point we don't free the allocated
	2564	+ * futex hash map. We assume that another thread will be created
	2565	+ * and makes use of it. The hash map will be freed once the main
	2566	+ * thread terminates.
	2567	+ */
	2568	+ }
2545	2569	/*
2546	2570	* From this point on we must avoid any synchronous user-space
2547	2571	* communication until we take the tasklist-lock. In particular, we do

kernel/futex/core.c+732−71

@@ -36,9 +36,15 @@ · 35 unmodified lines
36	36	#include <linux/pagemap.h>
37	37	#include <linux/debugfs.h>
38	38	#include <linux/plist.h>
	39	+#include <linux/gfp.h>
	40	+#include <linux/vmalloc.h>
39	41	#include <linux/memblock.h>
40	42	#include <linux/fault-inject.h>
41	43	#include <linux/slab.h>
	44	+#include <linux/prctl.h>
	45	+#include <linux/rcuref.h>
	46	+#include <linux/mempolicy.h>
	47	+#include <linux/mmap_lock.h>
42	48
43	49	#include "futex.h"
44	50	#include "../locking/rtmutex_common.h"
@@ -49,12 +55,24 @@ · 4 unmodified lines
49	55	* reside in the same cacheline.
50	56	*/
51	57	static struct {
52		− struct futex_hash_bucket *queues;
53	58	unsigned long hashmask;
	59	+ unsigned int hashshift;
	60	+ struct futex_hash_bucket *queues[MAX_NUMNODES];
54	61	} __futex_data __read_mostly __aligned(2*sizeof(long));
55		−#define futex_queues (__futex_data.queues)
56		−#define futex_hashmask (__futex_data.hashmask)
57	62
	63	+#define futex_hashmask (__futex_data.hashmask)
	64	+#define futex_hashshift (__futex_data.hashshift)
	65	+#define futex_queues (__futex_data.queues)
	66	+
	67	+struct futex_private_hash {
	68	+ rcuref_t users;
	69	+ unsigned int hash_mask;
	70	+ struct rcu_head rcu;
	71	+ void *mm;
	72	+ bool custom;
	73	+ bool immutable;
	74	+ struct futex_hash_bucket queues[];
	75	+};
58	76
59	77	/*
60	78	* Fault injections for futexes.
@@ -107,21 +125,328 @@ late_initcall(fail_futex_debugfs); · 46 unmodified lines
107	125
108	126	#endif /* CONFIG_FAIL_FUTEX */
109	127
	128	+static struct futex_hash_bucket *
	129	+__futex_hash(union futex_key key, struct futex_private_hash fph);
	130	+
	131	+#ifdef CONFIG_FUTEX_PRIVATE_HASH
	132	+static inline bool futex_key_is_private(union futex_key *key)
	133	+{
	134	+ /*
	135	+ * Relies on get_futex_key() to set either bit for shared
	136	+ * futexes -- see comment with union futex_key.
	137	+ */
	138	+ return !(key->both.offset & (FUT_OFF_INODE \| FUT_OFF_MMSHARED));
	139	+}
	140	+
	141	+bool futex_private_hash_get(struct futex_private_hash *fph)
	142	+{
	143	+ if (fph->immutable)
	144	+ return true;
	145	+ return rcuref_get(&fph->users);
	146	+}
	147	+
	148	+void futex_private_hash_put(struct futex_private_hash *fph)
	149	+{
	150	+ /* Ignore return value, last put is verified via rcuref_is_dead() */
	151	+ if (fph->immutable)
	152	+ return;
	153	+ if (rcuref_put(&fph->users))
	154	+ wake_up_var(fph->mm);
	155	+}
	156	+
110	157	/**
111		− * futex_hash - Return the hash bucket in the global hash
112		− * @key: Pointer to the futex key for which the hash is calculated
	158	+ * futex_hash_get - Get an additional reference for the local hash.
	159	+ * @hb: ptr to the private local hash.
113	160	*
114		− * We hash on the keys returned from get_futex_key (see below) and return the
115		− * corresponding hash bucket in the global hash.
	161	+ * Obtain an additional reference for the already obtained hash bucket. The
	162	+ * caller must already own an reference.
116	163	*/
	164	+void futex_hash_get(struct futex_hash_bucket *hb)
	165	+{
	166	+ struct futex_private_hash *fph = hb->priv;
	167	+
	168	+ if (!fph)
	169	+ return;
	170	+ WARN_ON_ONCE(!futex_private_hash_get(fph));
	171	+}
	172	+
	173	+void futex_hash_put(struct futex_hash_bucket *hb)
	174	+{
	175	+ struct futex_private_hash *fph = hb->priv;
	176	+
	177	+ if (!fph)
	178	+ return;
	179	+ futex_private_hash_put(fph);
	180	+}
	181	+
	182	+static struct futex_hash_bucket *
	183	+__futex_hash_private(union futex_key key, struct futex_private_hash fph)
	184	+{
	185	+ u32 hash;
	186	+
	187	+ if (!futex_key_is_private(key))
	188	+ return NULL;
	189	+
	190	+ if (!fph)
	191	+ fph = rcu_dereference(key->private.mm->futex_phash);
	192	+ if (!fph \|\| !fph->hash_mask)
	193	+ return NULL;
	194	+
	195	+ hash = jhash2((void *)&key->private.address,
	196	+ sizeof(key->private.address) / 4,
	197	+ key->both.offset);
	198	+ return &fph->queues[hash & fph->hash_mask];
	199	+}
	200	+
	201	+static void futex_rehash_private(struct futex_private_hash *old,
	202	+ struct futex_private_hash *new)
	203	+{
	204	+ struct futex_hash_bucket hb_old, hb_new;
	205	+ unsigned int slots = old->hash_mask + 1;
	206	+ unsigned int i;
	207	+
	208	+ for (i = 0; i < slots; i++) {
	209	+ struct futex_q this, tmp;
	210	+
	211	+ hb_old = &old->queues[i];
	212	+
	213	+ spin_lock(&hb_old->lock);
	214	+ plist_for_each_entry_safe(this, tmp, &hb_old->chain, list) {
	215	+
	216	+ plist_del(&this->list, &hb_old->chain);
	217	+ futex_hb_waiters_dec(hb_old);
	218	+
	219	+ WARN_ON_ONCE(this->lock_ptr != &hb_old->lock);
	220	+
	221	+ hb_new = __futex_hash(&this->key, new);
	222	+ futex_hb_waiters_inc(hb_new);
	223	+ /*
	224	+ * The new pointer isn't published yet but an already
	225	+ * moved user can be unqueued due to timeout or signal.
	226	+ */
	227	+ spin_lock_nested(&hb_new->lock, SINGLE_DEPTH_NESTING);
	228	+ plist_add(&this->list, &hb_new->chain);
	229	+ this->lock_ptr = &hb_new->lock;
	230	+ spin_unlock(&hb_new->lock);
	231	+ }
	232	+ spin_unlock(&hb_old->lock);
	233	+ }
	234	+}
	235	+
	236	+static bool __futex_pivot_hash(struct mm_struct *mm,
	237	+ struct futex_private_hash *new)
	238	+{
	239	+ struct futex_private_hash *fph;
	240	+
	241	+ WARN_ON_ONCE(mm->futex_phash_new);
	242	+
	243	+ fph = rcu_dereference_protected(mm->futex_phash,
	244	+ lockdep_is_held(&mm->futex_hash_lock));
	245	+ if (fph) {
	246	+ if (!rcuref_is_dead(&fph->users)) {
	247	+ mm->futex_phash_new = new;
	248	+ return false;
	249	+ }
	250	+
	251	+ futex_rehash_private(fph, new);
	252	+ }
	253	+ rcu_assign_pointer(mm->futex_phash, new);
	254	+ kvfree_rcu(fph, rcu);
	255	+ return true;
	256	+}
	257	+
	258	+static void futex_pivot_hash(struct mm_struct *mm)
	259	+{
	260	+ scoped_guard(mutex, &mm->futex_hash_lock) {
	261	+ struct futex_private_hash *fph;
	262	+
	263	+ fph = mm->futex_phash_new;
	264	+ if (fph) {
	265	+ mm->futex_phash_new = NULL;
	266	+ __futex_pivot_hash(mm, fph);
	267	+ }
	268	+ }
	269	+}
	270	+
	271	+struct futex_private_hash *futex_private_hash(void)
	272	+{
	273	+ struct mm_struct *mm = current->mm;
	274	+ /*
	275	+ * Ideally we don't loop. If there is a replacement in progress
	276	+ * then a new private hash is already prepared and a reference can't be
	277	+ * obtained once the last user dropped it's.
	278	+ * In that case we block on mm_struct::futex_hash_lock and either have
	279	+ * to perform the replacement or wait while someone else is doing the
	280	+ * job. Eitherway, on the second iteration we acquire a reference on the
	281	+ * new private hash or loop again because a new replacement has been
	282	+ * requested.
	283	+ */
	284	+again:
	285	+ scoped_guard(rcu) {
	286	+ struct futex_private_hash *fph;
	287	+
	288	+ fph = rcu_dereference(mm->futex_phash);
	289	+ if (!fph)
	290	+ return NULL;
	291	+
	292	+ if (fph->immutable)
	293	+ return fph;
	294	+ if (rcuref_get(&fph->users))
	295	+ return fph;
	296	+ }
	297	+ futex_pivot_hash(mm);
	298	+ goto again;
	299	+}
	300	+
117	301	struct futex_hash_bucket futex_hash(union futex_key key)
118	302	{
119		− u32 hash = jhash2((u32 )key, offsetof(typeof(key), both.offset) / 4,
120		− key->both.offset);
	303	+ struct futex_private_hash *fph;
	304	+ struct futex_hash_bucket *hb;
	305	+
	306	+again:
	307	+ scoped_guard(rcu) {
	308	+ hb = __futex_hash(key, NULL);
	309	+ fph = hb->priv;
	310	+
	311	+ if (!fph \|\| futex_private_hash_get(fph))
	312	+ return hb;
	313	+ }
	314	+ futex_pivot_hash(key->private.mm);
	315	+ goto again;
	316	+}
	317	+
	318	+#else /* !CONFIG_FUTEX_PRIVATE_HASH */
	319	+
	320	+static struct futex_hash_bucket *
	321	+__futex_hash_private(union futex_key key, struct futex_private_hash fph)
	322	+{
	323	+ return NULL;
	324	+}
	325	+
	326	+struct futex_hash_bucket futex_hash(union futex_key key)
	327	+{
	328	+ return __futex_hash(key, NULL);
	329	+}
	330	+
	331	+#endif /* CONFIG_FUTEX_PRIVATE_HASH */
	332	+
	333	+#ifdef CONFIG_FUTEX_MPOL
	334	+
	335	+static int __futex_key_to_node(struct mm_struct *mm, unsigned long addr)
	336	+{
	337	+ struct vm_area_struct *vma = vma_lookup(mm, addr);
	338	+ struct mempolicy *mpol;
	339	+ int node = FUTEX_NO_NODE;
	340	+
	341	+ if (!vma)
	342	+ return FUTEX_NO_NODE;
	343	+
	344	+ mpol = vma_policy(vma);
	345	+ if (!mpol)
	346	+ return FUTEX_NO_NODE;
	347	+
	348	+ switch (mpol->mode) {
	349	+ case MPOL_PREFERRED:
	350	+ node = first_node(mpol->nodes);
	351	+ break;
	352	+ case MPOL_PREFERRED_MANY:
	353	+ case MPOL_BIND:
	354	+ if (mpol->home_node != NUMA_NO_NODE)
	355	+ node = mpol->home_node;
	356	+ break;
	357	+ default:
	358	+ break;
	359	+ }
	360	+
	361	+ return node;
	362	+}
	363	+
	364	+static int futex_key_to_node_opt(struct mm_struct *mm, unsigned long addr)
	365	+{
	366	+ int seq, node;
	367	+
	368	+ guard(rcu)();
	369	+
	370	+ if (!mmap_lock_speculate_try_begin(mm, &seq))
	371	+ return -EBUSY;
	372	+
	373	+ node = __futex_key_to_node(mm, addr);
	374	+
	375	+ if (mmap_lock_speculate_retry(mm, seq))
	376	+ return -EAGAIN;
	377	+
	378	+ return node;
	379	+}
	380	+
	381	+static int futex_mpol(struct mm_struct *mm, unsigned long addr)
	382	+{
	383	+ int node;
	384	+
	385	+ node = futex_key_to_node_opt(mm, addr);
	386	+ if (node >= FUTEX_NO_NODE)
	387	+ return node;
121	388
122		− return &futex_queues[hash & futex_hashmask];
	389	+ guard(mmap_read_lock)(mm);
	390	+ return __futex_key_to_node(mm, addr);
123	391	}
124	392
	393	+#else /* !CONFIG_FUTEX_MPOL */
	394	+
	395	+static int futex_mpol(struct mm_struct *mm, unsigned long addr)
	396	+{
	397	+ return FUTEX_NO_NODE;
	398	+}
	399	+
	400	+#endif /* CONFIG_FUTEX_MPOL */
	401	+
	402	+/**
	403	+ * __futex_hash - Return the hash bucket
	404	+ * @key: Pointer to the futex key for which the hash is calculated
	405	+ * @fph: Pointer to private hash if known
	406	+ *
	407	+ * We hash on the keys returned from get_futex_key (see below) and return the
	408	+ * corresponding hash bucket.
	409	+ * If the FUTEX is PROCESS_PRIVATE then a per-process hash bucket (from the
	410	+ * private hash) is returned if existing. Otherwise a hash bucket from the
	411	+ * global hash is returned.
	412	+ */
	413	+static struct futex_hash_bucket *
	414	+__futex_hash(union futex_key key, struct futex_private_hash fph)
	415	+{
	416	+ int node = key->both.node;
	417	+ u32 hash;
	418	+
	419	+ if (node == FUTEX_NO_NODE) {
	420	+ struct futex_hash_bucket *hb;
	421	+
	422	+ hb = __futex_hash_private(key, fph);
	423	+ if (hb)
	424	+ return hb;
	425	+ }
	426	+
	427	+ hash = jhash2((u32 *)key,
	428	+ offsetof(typeof(*key), both.offset) / sizeof(u32),
	429	+ key->both.offset);
	430	+
	431	+ if (node == FUTEX_NO_NODE) {
	432	+ /*
	433	+ * In case of !FLAGS_NUMA, use some unused hash bits to pick a
	434	+ * node -- this ensures regular futexes are interleaved across
	435	+ * the nodes and avoids having to allocate multiple
	436	+ * hash-tables.
	437	+ *
	438	+ * NOTE: this isn't perfectly uniform, but it is fast and
	439	+ * handles sparse node masks.
	440	+ */
	441	+ node = (hash >> futex_hashshift) % nr_node_ids;
	442	+ if (!node_possible(node)) {
	443	+ node = find_next_bit_wrap(node_possible_map.bits,
	444	+ nr_node_ids, node);
	445	+ }
	446	+ }
	447	+
	448	+ return &futex_queues[node][hash & futex_hashmask];
	449	+}
125	450
126	451	/**
127	452	* futex_setup_timer - set up the sleeping hrtimer.
@@ -227,25 +552,60 @@ int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key, · 99 unmodified lines
227	552	struct page *page;
228	553	struct folio *folio;
229	554	struct address_space *mapping;
230		− int err, ro = 0;
	555	+ int node, err, size, ro = 0;
	556	+ bool node_updated = false;
231	557	bool fshared;
232	558
233	559	fshared = flags & FLAGS_SHARED;
	560	+ size = futex_size(flags);
	561	+ if (flags & FLAGS_NUMA)
	562	+ size *= 2;
234	563
235	564	/*
236	565	* The futex address must be "naturally" aligned.
237	566	*/
238	567	key->both.offset = address % PAGE_SIZE;
239		− if (unlikely((address % sizeof(u32)) != 0))
	568	+ if (unlikely((address % size) != 0))
240	569	return -EINVAL;
241	570	address -= key->both.offset;
242	571
243		− if (unlikely(!access_ok(uaddr, sizeof(u32))))
	572	+ if (unlikely(!access_ok(uaddr, size)))
244	573	return -EFAULT;
245	574
246	575	if (unlikely(should_fail_futex(fshared)))
247	576	return -EFAULT;
248	577
	578	+ node = FUTEX_NO_NODE;
	579	+
	580	+ if (flags & FLAGS_NUMA) {
	581	+ u32 __user naddr = (void )uaddr + size / 2;
	582	+
	583	+ if (futex_get_value(&node, naddr))
	584	+ return -EFAULT;
	585	+
	586	+ if (node != FUTEX_NO_NODE &&
	587	+ (node >= MAX_NUMNODES \|\| !node_possible(node)))
	588	+ return -EINVAL;
	589	+ }
	590	+
	591	+ if (node == FUTEX_NO_NODE && (flags & FLAGS_MPOL)) {
	592	+ node = futex_mpol(mm, address);
	593	+ node_updated = true;
	594	+ }
	595	+
	596	+ if (flags & FLAGS_NUMA) {
	597	+ u32 __user naddr = (void )uaddr + size / 2;
	598	+
	599	+ if (node == FUTEX_NO_NODE) {
	600	+ node = numa_node_id();
	601	+ node_updated = true;
	602	+ }
	603	+ if (node_updated && futex_put_value(node, naddr))
	604	+ return -EFAULT;
	605	+ }
	606	+
	607	+ key->both.node = node;
	608	+
249	609	/*
250	610	* PROCESS_PRIVATE futexes are fast.
251	611	* As the mm cannot disappear under us and the 'key' only needs
@@ -502,13 +862,9 @@ void __futex_unqueue(struct futex_q *q) · 250 unmodified lines
502	862	}
503	863
504	864	/* The key must be already stored in q->key. */
505		−struct futex_hash_bucket futex_q_lock(struct futex_q q)
	865	+void futex_q_lock(struct futex_q q, struct futex_hash_bucket hb)
506	866	__acquires(&hb->lock)
507	867	{
508		− struct futex_hash_bucket *hb;
509		−
510		− hb = futex_hash(&q->key);
511		−
512	868	/*
513	869	* Increment the counter before taking the lock so that
514	870	* a potential waker won't miss a to-be-slept task that is
@@ -522,14 +878,13 @@ struct futex_hash_bucket futex_q_lock(struct futex_q q) · 7 unmodified lines
522	878	q->lock_ptr = &hb->lock;
523	879
524	880	spin_lock(&hb->lock);
525		− return hb;
526	881	}
527	882
528	883	void futex_q_unlock(struct futex_hash_bucket *hb)
529	884	__releases(&hb->lock)
530	885	{
531		− spin_unlock(&hb->lock);
532	886	futex_hb_waiters_dec(hb);
	887	+ spin_unlock(&hb->lock);
533	888	}
534	889
535	890	void __futex_queue(struct futex_q q, struct futex_hash_bucket hb,
@@ -568,6 +923,8 @@ int futex_unqueue(struct futex_q *q) · 32 unmodified lines
568	923	spinlock_t *lock_ptr;
569	924	int ret = 0;
570	925
	926	+ /* RCU so lock_ptr is not going away during locking. */
	927	+ guard(rcu)();
571	928	/* In the common case we don't take the spinlock, which is nice. */
572	929	retry:
573	930	/*
@@ -606,6 +963,24 @@ retry: · 32 unmodified lines
606	963	return ret;
607	964	}
608	965
	966	+void futex_q_lockptr_lock(struct futex_q *q)
	967	+{
	968	+ spinlock_t *lock_ptr;
	969	+
	970	+ /*
	971	+ * See futex_unqueue() why lock_ptr can change.
	972	+ */
	973	+ guard(rcu)();
	974	+retry:
	975	+ lock_ptr = READ_ONCE(q->lock_ptr);
	976	+ spin_lock(lock_ptr);
	977	+
	978	+ if (unlikely(lock_ptr != q->lock_ptr)) {
	979	+ spin_unlock(lock_ptr);
	980	+ goto retry;
	981	+ }
	982	+}
	983	+
609	984	/*
610	985	* PI futexes can not be requeued and must remove themselves from the hash
611	986	* bucket. The hash bucket lock (i.e. lock_ptr) is held.
@@ -949,9 +1324,19 @@ static void exit_pi_state_list(struct task_struct *curr) · 337 unmodified lines
949	1324	{
950	1325	struct list_head next, head = &curr->pi_state_list;
951	1326	struct futex_pi_state *pi_state;
952		− struct futex_hash_bucket *hb;
953	1327	union futex_key key = FUTEX_KEY_INIT;
954	1328
	1329	+ /*
	1330	+ * The mutex mm_struct::futex_hash_lock might be acquired.
	1331	+ */
	1332	+ might_sleep();
	1333	+ /*
	1334	+ * Ensure the hash remains stable (no resize) during the while loop
	1335	+ * below. The hb pointer is acquired under the pi_lock so we can't block
	1336	+ * on the mutex.
	1337	+ */
	1338	+ WARN_ON(curr != current);
	1339	+ guard(private_hash)();
955	1340	/*
956	1341	* We are a ZOMBIE and nobody can enqueue itself on
957	1342	* pi_state_list anymore, but we have to be careful
@@ -962,50 +1347,52 @@ static void exit_pi_state_list(struct task_struct *curr) · 4 unmodified lines
962	1347	next = head->next;
963	1348	pi_state = list_entry(next, struct futex_pi_state, list);
964	1349	key = pi_state->key;
965		− hb = futex_hash(&key);
966		−
967		− /*
968		− * We can race against put_pi_state() removing itself from the
969		− * list (a waiter going away). put_pi_state() will first
970		− * decrement the reference count and then modify the list, so
971		− * its possible to see the list entry but fail this reference
972		− * acquire.
973		− *
974		− * In that case; drop the locks to let put_pi_state() make
975		− * progress and retry the loop.
976		− */
977		− if (!refcount_inc_not_zero(&pi_state->refcount)) {
	1350	+ if (1) {
	1351	+ CLASS(hb, hb)(&key);
	1352	+
	1353	+ /*
	1354	+ * We can race against put_pi_state() removing itself from the
	1355	+ * list (a waiter going away). put_pi_state() will first
	1356	+ * decrement the reference count and then modify the list, so
	1357	+ * its possible to see the list entry but fail this reference
	1358	+ * acquire.
	1359	+ *
	1360	+ * In that case; drop the locks to let put_pi_state() make
	1361	+ * progress and retry the loop.
	1362	+ */
	1363	+ if (!refcount_inc_not_zero(&pi_state->refcount)) {
	1364	+ raw_spin_unlock_irq(&curr->pi_lock);
	1365	+ cpu_relax();
	1366	+ raw_spin_lock_irq(&curr->pi_lock);
	1367	+ continue;
	1368	+ }
978	1369	raw_spin_unlock_irq(&curr->pi_lock);
979		− cpu_relax();
980		− raw_spin_lock_irq(&curr->pi_lock);
981		− continue;
982		− }
983		− raw_spin_unlock_irq(&curr->pi_lock);
984	1370
985		− spin_lock(&hb->lock);
986		− raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
987		− raw_spin_lock(&curr->pi_lock);
988		− /*
989		− * We dropped the pi-lock, so re-check whether this
990		− * task still owns the PI-state:
991		− */
992		− if (head->next != next) {
993		− /* retain curr->pi_lock for the loop invariant */
994		− raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
	1371	+ spin_lock(&hb->lock);
	1372	+ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
	1373	+ raw_spin_lock(&curr->pi_lock);
	1374	+ /*
	1375	+ * We dropped the pi-lock, so re-check whether this
	1376	+ * task still owns the PI-state:
	1377	+ */
	1378	+ if (head->next != next) {
	1379	+ /* retain curr->pi_lock for the loop invariant */
	1380	+ raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
	1381	+ spin_unlock(&hb->lock);
	1382	+ put_pi_state(pi_state);
	1383	+ continue;
	1384	+ }
	1385	+
	1386	+ WARN_ON(pi_state->owner != curr);
	1387	+ WARN_ON(list_empty(&pi_state->list));
	1388	+ list_del_init(&pi_state->list);
	1389	+ pi_state->owner = NULL;
	1390	+
	1391	+ raw_spin_unlock(&curr->pi_lock);
	1392	+ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
995	1393	spin_unlock(&hb->lock);
996		− put_pi_state(pi_state);
997		− continue;
998	1394	}
999	1395
1000		− WARN_ON(pi_state->owner != curr);
1001		− WARN_ON(list_empty(&pi_state->list));
1002		− list_del_init(&pi_state->list);
1003		− pi_state->owner = NULL;
1004		−
1005		− raw_spin_unlock(&curr->pi_lock);
1006		− raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1007		− spin_unlock(&hb->lock);
1008		−
1009	1396	rt_mutex_futex_unlock(&pi_state->pi_mutex);
1010	1397	put_pi_state(pi_state);
1011	1398
@@ -1125,30 +1512,304 @@ void futex_exit_release(struct task_struct *tsk) · 113 unmodified lines
1125	1512	futex_cleanup_end(tsk, FUTEX_STATE_DEAD);
1126	1513	}
1127	1514
	1515	+static void futex_hash_bucket_init(struct futex_hash_bucket *fhb,
	1516	+ struct futex_private_hash *fph)
	1517	+{
	1518	+#ifdef CONFIG_FUTEX_PRIVATE_HASH
	1519	+ fhb->priv = fph;
	1520	+#endif
	1521	+ atomic_set(&fhb->waiters, 0);
	1522	+ plist_head_init(&fhb->chain);
	1523	+ spin_lock_init(&fhb->lock);
	1524	+}
	1525	+
	1526	+#define FH_CUSTOM 0x01
	1527	+#define FH_IMMUTABLE 0x02
	1528	+
	1529	+#ifdef CONFIG_FUTEX_PRIVATE_HASH
	1530	+void futex_hash_free(struct mm_struct *mm)
	1531	+{
	1532	+ struct futex_private_hash *fph;
	1533	+
	1534	+ kvfree(mm->futex_phash_new);
	1535	+ fph = rcu_dereference_raw(mm->futex_phash);
	1536	+ if (fph) {
	1537	+ WARN_ON_ONCE(rcuref_read(&fph->users) > 1);
	1538	+ kvfree(fph);
	1539	+ }
	1540	+}
	1541	+
	1542	+static bool futex_pivot_pending(struct mm_struct *mm)
	1543	+{
	1544	+ struct futex_private_hash *fph;
	1545	+
	1546	+ guard(rcu)();
	1547	+
	1548	+ if (!mm->futex_phash_new)
	1549	+ return true;
	1550	+
	1551	+ fph = rcu_dereference(mm->futex_phash);
	1552	+ return rcuref_is_dead(&fph->users);
	1553	+}
	1554	+
	1555	+static bool futex_hash_less(struct futex_private_hash *a,
	1556	+ struct futex_private_hash *b)
	1557	+{
	1558	+ /* user provided always wins */
	1559	+ if (!a->custom && b->custom)
	1560	+ return true;
	1561	+ if (a->custom && !b->custom)
	1562	+ return false;
	1563	+
	1564	+ /* zero-sized hash wins */
	1565	+ if (!b->hash_mask)
	1566	+ return true;
	1567	+ if (!a->hash_mask)
	1568	+ return false;
	1569	+
	1570	+ /* keep the biggest */
	1571	+ if (a->hash_mask < b->hash_mask)
	1572	+ return true;
	1573	+ if (a->hash_mask > b->hash_mask)
	1574	+ return false;
	1575	+
	1576	+ return false; /* equal */
	1577	+}
	1578	+
	1579	+static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
	1580	+{
	1581	+ struct mm_struct *mm = current->mm;
	1582	+ struct futex_private_hash *fph;
	1583	+ bool custom = flags & FH_CUSTOM;
	1584	+ int i;
	1585	+
	1586	+ if (hash_slots && (hash_slots == 1 \|\| !is_power_of_2(hash_slots)))
	1587	+ return -EINVAL;
	1588	+
	1589	+ /*
	1590	+ * Once we've disabled the global hash there is no way back.
	1591	+ */
	1592	+ scoped_guard(rcu) {
	1593	+ fph = rcu_dereference(mm->futex_phash);
	1594	+ if (fph && (!fph->hash_mask \|\| fph->immutable)) {
	1595	+ if (custom)
	1596	+ return -EBUSY;
	1597	+ return 0;
	1598	+ }
	1599	+ }
	1600	+
	1601	+ fph = kvzalloc(struct_size(fph, queues, hash_slots), GFP_KERNEL_ACCOUNT \| __GFP_NOWARN);
	1602	+ if (!fph)
	1603	+ return -ENOMEM;
	1604	+
	1605	+ rcuref_init(&fph->users, 1);
	1606	+ fph->hash_mask = hash_slots ? hash_slots - 1 : 0;
	1607	+ fph->custom = custom;
	1608	+ fph->immutable = !!(flags & FH_IMMUTABLE);
	1609	+ fph->mm = mm;
	1610	+
	1611	+ for (i = 0; i < hash_slots; i++)
	1612	+ futex_hash_bucket_init(&fph->queues[i], fph);
	1613	+
	1614	+ if (custom) {
	1615	+ /*
	1616	+ * Only let prctl() wait / retry; don't unduly delay clone().
	1617	+ */
	1618	+again:
	1619	+ wait_var_event(mm, futex_pivot_pending(mm));
	1620	+ }
	1621	+
	1622	+ scoped_guard(mutex, &mm->futex_hash_lock) {
	1623	+ struct futex_private_hash *free __free(kvfree) = NULL;
	1624	+ struct futex_private_hash cur, new;
	1625	+
	1626	+ cur = rcu_dereference_protected(mm->futex_phash,
	1627	+ lockdep_is_held(&mm->futex_hash_lock));
	1628	+ new = mm->futex_phash_new;
	1629	+ mm->futex_phash_new = NULL;
	1630	+
	1631	+ if (fph) {
	1632	+ if (cur && !new) {
	1633	+ /*
	1634	+ * If we have an existing hash, but do not yet have
	1635	+ * allocated a replacement hash, drop the initial
	1636	+ * reference on the existing hash.
	1637	+ */
	1638	+ futex_private_hash_put(cur);
	1639	+ }
	1640	+
	1641	+ if (new) {
	1642	+ /*
	1643	+ * Two updates raced; throw out the lesser one.
	1644	+ */
	1645	+ if (futex_hash_less(new, fph)) {
	1646	+ free = new;
	1647	+ new = fph;
	1648	+ } else {
	1649	+ free = fph;
	1650	+ }
	1651	+ } else {
	1652	+ new = fph;
	1653	+ }
	1654	+ fph = NULL;
	1655	+ }
	1656	+
	1657	+ if (new) {
	1658	+ /*
	1659	+ * Will set mm->futex_phash_new on failure;
	1660	+ * futex_private_hash_get() will try again.
	1661	+ */
	1662	+ if (!__futex_pivot_hash(mm, new) && custom)
	1663	+ goto again;
	1664	+ }
	1665	+ }
	1666	+ return 0;
	1667	+}
	1668	+
	1669	+int futex_hash_allocate_default(void)
	1670	+{
	1671	+ unsigned int threads, buckets, current_buckets = 0;
	1672	+ struct futex_private_hash *fph;
	1673	+
	1674	+ if (!current->mm)
	1675	+ return 0;
	1676	+
	1677	+ scoped_guard(rcu) {
	1678	+ threads = min_t(unsigned int,
	1679	+ get_nr_threads(current),
	1680	+ num_online_cpus());
	1681	+
	1682	+ fph = rcu_dereference(current->mm->futex_phash);
	1683	+ if (fph) {
	1684	+ if (fph->custom)
	1685	+ return 0;
	1686	+
	1687	+ current_buckets = fph->hash_mask + 1;
	1688	+ }
	1689	+ }
	1690	+
	1691	+ /*
	1692	+ * The default allocation will remain within
	1693	+ * 16 <= threads * 4 <= global hash size
	1694	+ */
	1695	+ buckets = roundup_pow_of_two(4 * threads);
	1696	+ buckets = clamp(buckets, 16, futex_hashmask + 1);
	1697	+
	1698	+ if (current_buckets >= buckets)
	1699	+ return 0;
	1700	+
	1701	+ return futex_hash_allocate(buckets, 0);
	1702	+}
	1703	+
	1704	+static int futex_hash_get_slots(void)
	1705	+{
	1706	+ struct futex_private_hash *fph;
	1707	+
	1708	+ guard(rcu)();
	1709	+ fph = rcu_dereference(current->mm->futex_phash);
	1710	+ if (fph && fph->hash_mask)
	1711	+ return fph->hash_mask + 1;
	1712	+ return 0;
	1713	+}
	1714	+
	1715	+static int futex_hash_get_immutable(void)
	1716	+{
	1717	+ struct futex_private_hash *fph;
	1718	+
	1719	+ guard(rcu)();
	1720	+ fph = rcu_dereference(current->mm->futex_phash);
	1721	+ if (fph && fph->immutable)
	1722	+ return 1;
	1723	+ if (fph && !fph->hash_mask)
	1724	+ return 1;
	1725	+ return 0;
	1726	+}
	1727	+
	1728	+#else
	1729	+
	1730	+static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
	1731	+{
	1732	+ return -EINVAL;
	1733	+}
	1734	+
	1735	+static int futex_hash_get_slots(void)
	1736	+{
	1737	+ return 0;
	1738	+}
	1739	+
	1740	+static int futex_hash_get_immutable(void)
	1741	+{
	1742	+ return 0;
	1743	+}
	1744	+#endif
	1745	+
	1746	+int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
	1747	+{
	1748	+ unsigned int flags = FH_CUSTOM;
	1749	+ int ret;
	1750	+
	1751	+ switch (arg2) {
	1752	+ case PR_FUTEX_HASH_SET_SLOTS:
	1753	+ if (arg4 & ~FH_FLAG_IMMUTABLE)
	1754	+ return -EINVAL;
	1755	+ if (arg4 & FH_FLAG_IMMUTABLE)
	1756	+ flags \|= FH_IMMUTABLE;
	1757	+ ret = futex_hash_allocate(arg3, flags);
	1758	+ break;
	1759	+
	1760	+ case PR_FUTEX_HASH_GET_SLOTS:
	1761	+ ret = futex_hash_get_slots();
	1762	+ break;
	1763	+
	1764	+ case PR_FUTEX_HASH_GET_IMMUTABLE:
	1765	+ ret = futex_hash_get_immutable();
	1766	+ break;
	1767	+
	1768	+ default:
	1769	+ ret = -EINVAL;
	1770	+ break;
	1771	+ }
	1772	+ return ret;
	1773	+}
	1774	+
1128	1775	static int __init futex_init(void)
1129	1776	{
1130	1777	unsigned long hashsize, i;
1131		− unsigned int futex_shift;
	1778	+ unsigned int order, n;
	1779	+ unsigned long size;
1132	1780
1133	1781	#ifdef CONFIG_BASE_SMALL
1134	1782	hashsize = 16;
1135	1783	#else
1136		− hashsize = roundup_pow_of_two(256 * num_possible_cpus());
	1784	+ hashsize = 256 * num_possible_cpus();
	1785	+ hashsize /= num_possible_nodes();
	1786	+ hashsize = max(4, hashsize);
	1787	+ hashsize = roundup_pow_of_two(hashsize);
1137	1788	#endif
	1789	+ futex_hashshift = ilog2(hashsize);
	1790	+ size = sizeof(struct futex_hash_bucket) * hashsize;
	1791	+ order = get_order(size);
	1792	+
	1793	+ for_each_node(n) {
	1794	+ struct futex_hash_bucket *table;
	1795	+
	1796	+ if (order > MAX_PAGE_ORDER)
	1797	+ table = vmalloc_huge_node(size, GFP_KERNEL, n);
	1798	+ else
	1799	+ table = alloc_pages_exact_nid(n, size, GFP_KERNEL);
	1800	+
	1801	+ BUG_ON(!table);
1138	1802
1139		− futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),
1140		− hashsize, 0, 0,
1141		− &futex_shift, NULL,
1142		− hashsize, hashsize);
1143		− hashsize = 1UL << futex_shift;
	1803	+ for (i = 0; i < hashsize; i++)
	1804	+ futex_hash_bucket_init(&table[i], NULL);
1144	1805
1145		− for (i = 0; i < hashsize; i++) {
1146		− atomic_set(&futex_queues[i].waiters, 0);
1147		− plist_head_init(&futex_queues[i].chain);
1148		− spin_lock_init(&futex_queues[i].lock);
	1806	+ futex_queues[n] = table;
1149	1807	}
1150	1808
1151	1809	futex_hashmask = hashsize - 1;
	1810	+ pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n",
	1811	+ hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024,
	1812	+ order > MAX_PAGE_ORDER ? "vmalloc" : "linear");
1152	1813	return 0;
1153	1814	}
1154	1815	core_initcall(futex_init);

kernel/futex/futex.h+66−8

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/sched/wake_q.h>
8	8	#include <linux/compat.h>
9	9	#include <linux/uaccess.h>
	10	+#include <linux/cleanup.h>
10	11
11	12	#ifdef CONFIG_PREEMPT_RT
12	13	#include <linux/rcuwait.h>
@@ -38,6 +39,7 @@ · 25 unmodified lines
38	39	#define FLAGS_HAS_TIMEOUT 0x0040
39	40	#define FLAGS_NUMA 0x0080
40	41	#define FLAGS_STRICT 0x0100
	42	+#define FLAGS_MPOL 0x0200
41	43
42	44	/* FUTEX_ to FLAGS_ */
43	45	static inline unsigned int futex_to_flags(unsigned int op)
@@ -53,7 +55,7 @@ static inline unsigned int futex_to_flags(unsigned int op) · 9 unmodified lines
53	55	return flags;
54	56	}
55	57
56		−#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_PRIVATE)
	58	+#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK \| FUTEX2_NUMA \| FUTEX2_MPOL \| FUTEX2_PRIVATE)
57	59
58	60	/* FUTEX2_ to FLAGS_ */
59	61	static inline unsigned int futex2_to_flags(unsigned int flags2)
@@ -66,6 +68,9 @@ static inline unsigned int futex2_to_flags(unsigned int flags2) · 6 unmodified lines
66	68	if (flags2 & FUTEX2_NUMA)
67	69	flags \|= FLAGS_NUMA;
68	70
	71	+ if (flags2 & FUTEX2_MPOL)
	72	+ flags \|= FLAGS_MPOL;
	73	+
69	74	return flags;
70	75	}
71	76
@@ -86,6 +91,19 @@ static inline bool futex_flags_valid(unsigned int flags) · 14 unmodified lines
86	91	if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
87	92	return false;
88	93
	94	+ /*
	95	+ * Must be able to represent both FUTEX_NO_NODE and every valid nodeid
	96	+ * in a futex word.
	97	+ */
	98	+ if (flags & FLAGS_NUMA) {
	99	+ int bits = 8 * futex_size(flags);
	100	+ u64 max = ~0ULL;
	101	+
	102	+ max >>= 64 - bits;
	103	+ if (nr_node_ids >= max)
	104	+ return false;
	105	+ }
	106	+
89	107	return true;
90	108	}
91	109
@@ -117,6 +135,7 @@ struct futex_hash_bucket { · 25 unmodified lines
117	135	atomic_t waiters;
118	136	spinlock_t lock;
119	137	struct plist_head chain;
	138	+ struct futex_private_hash *priv;
120	139	} ____cacheline_aligned_in_smp;
121	140
122	141	/*
@@ -156,6 +175,7 @@ typedef void (futex_wake_fn)(struct wake_q_head wake_q, struct futex_q q); · 33 unmodified lines
156	175	* @requeue_pi_key: the requeue_pi target futex key
157	176	* @bitset: bitset for the optional bitmasked wakeup
158	177	* @requeue_state: State field for futex_requeue_pi()
	178	+ * @drop_hb_ref: Waiter should drop the extra hash bucket reference if true
159	179	* @requeue_wait: RCU wait for futex_requeue_pi() (RT only)
160	180	*
161	181	* We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
@@ -182,6 +202,7 @@ struct futex_q { · 20 unmodified lines
182	202	union futex_key *requeue_pi_key;
183	203	u32 bitset;
184	204	atomic_t requeue_state;
	205	+ bool drop_hb_ref;
185	206	#ifdef CONFIG_PREEMPT_RT
186	207	struct rcuwait requeue_wait;
187	208	#endif
@@ -196,12 +217,35 @@ enum futex_access { · 8 unmodified lines
196	217
197	218	extern int get_futex_key(u32 __user uaddr, unsigned int flags, union futex_key key,
198	219	enum futex_access rw);
199		−
	220	+extern void futex_q_lockptr_lock(struct futex_q *q);
200	221	extern struct hrtimer_sleeper *
201	222	futex_setup_timer(ktime_t time, struct hrtimer_sleeper timeout,
202	223	int flags, u64 range_ns);
203	224
204	225	extern struct futex_hash_bucket futex_hash(union futex_key key);
	226	+#ifdef CONFIG_FUTEX_PRIVATE_HASH
	227	+extern void futex_hash_get(struct futex_hash_bucket *hb);
	228	+extern void futex_hash_put(struct futex_hash_bucket *hb);
	229	+
	230	+extern struct futex_private_hash *futex_private_hash(void);
	231	+extern bool futex_private_hash_get(struct futex_private_hash *fph);
	232	+extern void futex_private_hash_put(struct futex_private_hash *fph);
	233	+
	234	+#else /* !CONFIG_FUTEX_PRIVATE_HASH */
	235	+static inline void futex_hash_get(struct futex_hash_bucket *hb) { }
	236	+static inline void futex_hash_put(struct futex_hash_bucket *hb) { }
	237	+static inline struct futex_private_hash *futex_private_hash(void) { return NULL; }
	238	+static inline bool futex_private_hash_get(void) { return false; }
	239	+static inline void futex_private_hash_put(struct futex_private_hash *fph) { }
	240	+#endif
	241	+
	242	+DEFINE_CLASS(hb, struct futex_hash_bucket *,
	243	+ if (_T) futex_hash_put(_T),
	244	+ futex_hash(key), union futex_key *key);
	245	+
	246	+DEFINE_CLASS(private_hash, struct futex_private_hash *,
	247	+ if (_T) futex_private_hash_put(_T),
	248	+ futex_private_hash(), void);
205	249
206	250	/**
207	251	* futex_match - Check whether two futex keys are equal
@@ -219,9 +263,9 @@ static inline int futex_match(union futex_key key1, union futex_key key2) · 11 unmodified lines
219	263	}
220	264
221	265	extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
222		− struct futex_q q, struct futex_hash_bucket *hb);
223		−extern void futex_wait_queue(struct futex_hash_bucket hb, struct futex_q q,
224		− struct hrtimer_sleeper *timeout);
	266	+ struct futex_q q, union futex_key key2,
	267	+ struct task_struct *task);
	268	+extern void futex_do_wait(struct futex_q q, struct hrtimer_sleeper timeout);
225	269	extern bool __futex_wake_mark(struct futex_q *q);
226	270	extern void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q);
227	271
@@ -256,7 +300,7 @@ static inline int futex_cmpxchg_value_locked(u32 curval, u32 __user uaddr, u32 · 28 unmodified lines
256	300	* This looks a bit overkill, but generally just results in a couple
257	301	* of instructions.
258	302	*/
259		−static __always_inline int futex_read_inatomic(u32 dest, u32 __user from)
	303	+static __always_inline int futex_get_value(u32 dest, u32 __user from)
260	304	{
261	305	u32 val;
262	306
@@ -273,12 +317,26 @@ Efault: · 10 unmodified lines
273	317	return -EFAULT;
274	318	}
275	319
	320	+static __always_inline int futex_put_value(u32 val, u32 __user *to)
	321	+{
	322	+ if (can_do_masked_user_access())
	323	+ to = masked_user_access_begin(to);
	324	+ else if (!user_read_access_begin(to, sizeof(*to)))
	325	+ return -EFAULT;
	326	+ unsafe_put_user(val, to, Efault);
	327	+ user_read_access_end();
	328	+ return 0;
	329	+Efault:
	330	+ user_read_access_end();
	331	+ return -EFAULT;
	332	+}
	333	+
276	334	static inline int futex_get_value_locked(u32 dest, u32 __user from)
277	335	{
278	336	int ret;
279	337
280	338	pagefault_disable();
281		− ret = futex_read_inatomic(dest, from);
	339	+ ret = futex_get_value(dest, from);
282	340	pagefault_enable();
283	341
284	342	return ret;
@@ -354,7 +412,7 @@ static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb) · 69 unmodified lines
354	412	#endif
355	413	}
356	414
357		−extern struct futex_hash_bucket futex_q_lock(struct futex_q q);
	415	+extern void futex_q_lock(struct futex_q q, struct futex_hash_bucket hb);
358	416	extern void futex_q_unlock(struct futex_hash_bucket *hb);
359	417
360	418

kernel/futex/pi.c+166−142

@@ -806,7 +806,7 @@ handle_err: · 805 unmodified lines
806	806	break;
807	807	}
808	808
809		− spin_lock(q->lock_ptr);
	809	+ futex_q_lockptr_lock(q);
810	810	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
811	811
812	812	/*
@@ -920,7 +920,6 @@ int futex_lock_pi(u32 __user uaddr, unsigned int flags, ktime_t time, int tryl · 107 unmodified lines
920	920	struct hrtimer_sleeper timeout, *to;
921	921	struct task_struct *exiting = NULL;
922	922	struct rt_mutex_waiter rt_waiter;
923		− struct futex_hash_bucket *hb;
924	923	struct futex_q q = futex_q_init;
925	924	DEFINE_WAKE_Q(wake_q);
926	925	int res, ret;
@@ -939,151 +938,183 @@ retry: · 12 unmodified lines
939	938	goto out;
940	939
941	940	retry_private:
942		− hb = futex_q_lock(&q);
	941	+ if (1) {
	942	+ CLASS(hb, hb)(&q.key);
943	943
944		− ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,
945		− &exiting, 0);
946		− if (unlikely(ret)) {
947		− /*
948		− * Atomic work succeeded and we got the lock,
949		− * or failed. Either way, we do _not_ block.
950		− */
951		− switch (ret) {
952		− case 1:
953		− /* We got the lock. */
954		− ret = 0;
955		− goto out_unlock_put_key;
956		− case -EFAULT:
957		− goto uaddr_faulted;
958		− case -EBUSY:
959		− case -EAGAIN:
960		− /*
961		− * Two reasons for this:
962		− * - EBUSY: Task is exiting and we just wait for the
963		− * exit to complete.
964		− * - EAGAIN: The user space value changed.
965		− */
966		− futex_q_unlock(hb);
	944	+ futex_q_lock(&q, hb);
	945	+
	946	+ ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,
	947	+ &exiting, 0);
	948	+ if (unlikely(ret)) {
967	949	/*
968		− * Handle the case where the owner is in the middle of
969		− * exiting. Wait for the exit to complete otherwise
970		− * this task might loop forever, aka. live lock.
	950	+ * Atomic work succeeded and we got the lock,
	951	+ * or failed. Either way, we do _not_ block.
971	952	*/
972		− wait_for_owner_exiting(ret, exiting);
973		− cond_resched();
974		− goto retry;
975		− default:
976		− goto out_unlock_put_key;
	953	+ switch (ret) {
	954	+ case 1:
	955	+ /* We got the lock. */
	956	+ ret = 0;
	957	+ goto out_unlock_put_key;
	958	+ case -EFAULT:
	959	+ goto uaddr_faulted;
	960	+ case -EBUSY:
	961	+ case -EAGAIN:
	962	+ /*
	963	+ * Two reasons for this:
	964	+ * - EBUSY: Task is exiting and we just wait for the
	965	+ * exit to complete.
	966	+ * - EAGAIN: The user space value changed.
	967	+ */
	968	+ futex_q_unlock(hb);
	969	+ /*
	970	+ * Handle the case where the owner is in the middle of
	971	+ * exiting. Wait for the exit to complete otherwise
	972	+ * this task might loop forever, aka. live lock.
	973	+ */
	974	+ wait_for_owner_exiting(ret, exiting);
	975	+ cond_resched();
	976	+ goto retry;
	977	+ default:
	978	+ goto out_unlock_put_key;
	979	+ }
977	980	}
978		− }
979	981
980		− WARN_ON(!q.pi_state);
	982	+ WARN_ON(!q.pi_state);
981	983
982		− /*
983		− * Only actually queue now that the atomic ops are done:
984		− */
985		− __futex_queue(&q, hb, current);
	984	+ /*
	985	+ * Only actually queue now that the atomic ops are done:
	986	+ */
	987	+ __futex_queue(&q, hb, current);
986	988
987		− if (trylock) {
988		− ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
989		− /* Fixup the trylock return value: */
990		− ret = ret ? 0 : -EWOULDBLOCK;
991		− goto no_block;
992		− }
	989	+ if (trylock) {
	990	+ ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
	991	+ /* Fixup the trylock return value: */
	992	+ ret = ret ? 0 : -EWOULDBLOCK;
	993	+ goto no_block;
	994	+ }
993	995
994		− /*
995		− * Must be done before we enqueue the waiter, here is unfortunately
996		− * under the hb lock, but that should work because it does nothing.
997		− */
998		− rt_mutex_pre_schedule();
	996	+ /*
	997	+ * Caution; releasing @hb in-scope. The hb->lock is still locked
	998	+ * while the reference is dropped. The reference can not be dropped
	999	+ * after the unlock because if a user initiated resize is in progress
	1000	+ * then we might need to wake him. This can not be done after the
	1001	+ * rt_mutex_pre_schedule() invocation. The hb will remain valid because
	1002	+ * the thread, performing resize, will block on hb->lock during
	1003	+ * the requeue.
	1004	+ */
	1005	+ futex_hash_put(no_free_ptr(hb));
	1006	+ /*
	1007	+ * Must be done before we enqueue the waiter, here is unfortunately
	1008	+ * under the hb lock, but that should work because it does nothing.
	1009	+ */
	1010	+ rt_mutex_pre_schedule();
999	1011
1000		− rt_mutex_init_waiter(&rt_waiter);
	1012	+ rt_mutex_init_waiter(&rt_waiter);
1001	1013
1002		− /*
1003		− * On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not
1004		− * hold it while doing rt_mutex_start_proxy(), because then it will
1005		− * include hb->lock in the blocking chain, even through we'll not in
1006		− * fact hold it while blocking. This will lead it to report -EDEADLK
1007		− * and BUG when futex_unlock_pi() interleaves with this.
1008		− *
1009		− * Therefore acquire wait_lock while holding hb->lock, but drop the
1010		− * latter before calling __rt_mutex_start_proxy_lock(). This
1011		− * interleaves with futex_unlock_pi() -- which does a similar lock
1012		− * handoff -- such that the latter can observe the futex_q::pi_state
1013		− * before __rt_mutex_start_proxy_lock() is done.
1014		− */
1015		− raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
1016		− spin_unlock(q.lock_ptr);
1017		− /*
1018		− * __rt_mutex_start_proxy_lock() unconditionally enqueues the @rt_waiter
1019		− * such that futex_unlock_pi() is guaranteed to observe the waiter when
1020		− * it sees the futex_q::pi_state.
1021		− */
1022		− ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);
1023		− raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);
	1014	+ /*
	1015	+ * On PREEMPT_RT, when hb->lock becomes an rt_mutex, we must not
	1016	+ * hold it while doing rt_mutex_start_proxy(), because then it will
	1017	+ * include hb->lock in the blocking chain, even through we'll not in
	1018	+ * fact hold it while blocking. This will lead it to report -EDEADLK
	1019	+ * and BUG when futex_unlock_pi() interleaves with this.
	1020	+ *
	1021	+ * Therefore acquire wait_lock while holding hb->lock, but drop the
	1022	+ * latter before calling __rt_mutex_start_proxy_lock(). This
	1023	+ * interleaves with futex_unlock_pi() -- which does a similar lock
	1024	+ * handoff -- such that the latter can observe the futex_q::pi_state
	1025	+ * before __rt_mutex_start_proxy_lock() is done.
	1026	+ */
	1027	+ raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
	1028	+ spin_unlock(q.lock_ptr);
	1029	+ /*
	1030	+ * __rt_mutex_start_proxy_lock() unconditionally enqueues the @rt_waiter
	1031	+ * such that futex_unlock_pi() is guaranteed to observe the waiter when
	1032	+ * it sees the futex_q::pi_state.
	1033	+ */
	1034	+ ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current, &wake_q);
	1035	+ raw_spin_unlock_irq_wake(&q.pi_state->pi_mutex.wait_lock, &wake_q);
1024	1036
1025		− if (ret) {
1026		− if (ret == 1)
1027		− ret = 0;
1028		− goto cleanup;
1029		− }
	1037	+ if (ret) {
	1038	+ if (ret == 1)
	1039	+ ret = 0;
	1040	+ goto cleanup;
	1041	+ }
1030	1042
1031		− if (unlikely(to))
1032		− hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
	1043	+ if (unlikely(to))
	1044	+ hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
1033	1045
1034		− ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
	1046	+ ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
1035	1047
1036	1048	cleanup:
1037		− /*
1038		− * If we failed to acquire the lock (deadlock/signal/timeout), we must
1039		− * must unwind the above, however we canont lock hb->lock because
1040		− * rt_mutex already has a waiter enqueued and hb->lock can itself try
1041		− * and enqueue an rt_waiter through rtlock.
1042		− *
1043		− * Doing the cleanup without holding hb->lock can cause inconsistent
1044		− * state between hb and pi_state, but only in the direction of not
1045		− * seeing a waiter that is leaving.
1046		− *
1047		− * See futex_unlock_pi(), it deals with this inconsistency.
1048		− *
1049		− * There be dragons here, since we must deal with the inconsistency on
1050		− * the way out (here), it is impossible to detect/warn about the race
1051		− * the other way around (missing an incoming waiter).
1052		− *
1053		− * What could possibly go wrong...
1054		− */
1055		− if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
1056		− ret = 0;
	1049	+ /*
	1050	+ * If we failed to acquire the lock (deadlock/signal/timeout), we must
	1051	+ * unwind the above, however we canont lock hb->lock because
	1052	+ * rt_mutex already has a waiter enqueued and hb->lock can itself try
	1053	+ * and enqueue an rt_waiter through rtlock.
	1054	+ *
	1055	+ * Doing the cleanup without holding hb->lock can cause inconsistent
	1056	+ * state between hb and pi_state, but only in the direction of not
	1057	+ * seeing a waiter that is leaving.
	1058	+ *
	1059	+ * See futex_unlock_pi(), it deals with this inconsistency.
	1060	+ *
	1061	+ * There be dragons here, since we must deal with the inconsistency on
	1062	+ * the way out (here), it is impossible to detect/warn about the race
	1063	+ * the other way around (missing an incoming waiter).
	1064	+ *
	1065	+ * What could possibly go wrong...
	1066	+ */
	1067	+ if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
	1068	+ ret = 0;
1057	1069
1058		− /*
1059		− * Now that the rt_waiter has been dequeued, it is safe to use
1060		− * spinlock/rtlock (which might enqueue its own rt_waiter) and fix up
1061		− * the
1062		− */
1063		− spin_lock(q.lock_ptr);
1064		− /*
1065		− * Waiter is unqueued.
1066		− */
1067		− rt_mutex_post_schedule();
	1070	+ /*
	1071	+ * Now that the rt_waiter has been dequeued, it is safe to use
	1072	+ * spinlock/rtlock (which might enqueue its own rt_waiter) and fix up
	1073	+ * the
	1074	+ */
	1075	+ futex_q_lockptr_lock(&q);
	1076	+ /*
	1077	+ * Waiter is unqueued.
	1078	+ */
	1079	+ rt_mutex_post_schedule();
1068	1080	no_block:
1069		− /*
1070		− * Fixup the pi_state owner and possibly acquire the lock if we
1071		− * haven't already.
1072		− */
1073		− res = fixup_pi_owner(uaddr, &q, !ret);
1074		− /*
1075		− * If fixup_pi_owner() returned an error, propagate that. If it acquired
1076		− * the lock, clear our -ETIMEDOUT or -EINTR.
1077		− */
1078		− if (res)
1079		− ret = (res < 0) ? res : 0;
1080		−
1081		− futex_unqueue_pi(&q);
1082		− spin_unlock(q.lock_ptr);
1083		− goto out;
	1081	+ /*
	1082	+ * Fixup the pi_state owner and possibly acquire the lock if we
	1083	+ * haven't already.
	1084	+ */
	1085	+ res = fixup_pi_owner(uaddr, &q, !ret);
	1086	+ /*
	1087	+ * If fixup_pi_owner() returned an error, propagate that. If it acquired
	1088	+ * the lock, clear our -ETIMEDOUT or -EINTR.
	1089	+ */
	1090	+ if (res)
	1091	+ ret = (res < 0) ? res : 0;
	1092	+
	1093	+ futex_unqueue_pi(&q);
	1094	+ spin_unlock(q.lock_ptr);
	1095	+ if (q.drop_hb_ref) {
	1096	+ CLASS(hb, hb)(&q.key);
	1097	+ /* Additional reference from futex_unlock_pi() */
	1098	+ futex_hash_put(hb);
	1099	+ }
	1100	+ goto out;
1084	1101
1085	1102	out_unlock_put_key:
1086		− futex_q_unlock(hb);
	1103	+ futex_q_unlock(hb);
	1104	+ goto out;
	1105	+
	1106	+uaddr_faulted:
	1107	+ futex_q_unlock(hb);
	1108	+
	1109	+ ret = fault_in_user_writeable(uaddr);
	1110	+ if (ret)
	1111	+ goto out;
	1112	+
	1113	+ if (!(flags & FLAGS_SHARED))
	1114	+ goto retry_private;
	1115	+
	1116	+ goto retry;
	1117	+ }
1087	1118
1088	1119	out:
1089	1120	if (to) {
@@ -1091,18 +1122,6 @@ out: · 1 unmodified line
1091	1122	destroy_hrtimer_on_stack(&to->timer);
1092	1123	}
1093	1124	return ret != -EINTR ? ret : -ERESTARTNOINTR;
1094		−
1095		−uaddr_faulted:
1096		− futex_q_unlock(hb);
1097		−
1098		− ret = fault_in_user_writeable(uaddr);
1099		− if (ret)
1100		− goto out;
1101		−
1102		− if (!(flags & FLAGS_SHARED))
1103		− goto retry_private;
1104		−
1105		− goto retry;
1106	1125	}
1107	1126
1108	1127	/*
@@ -1114,7 +1133,6 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) · 5 unmodified lines
1114	1133	{
1115	1134	u32 curval, uval, vpid = task_pid_vnr(current);
1116	1135	union futex_key key = FUTEX_KEY_INIT;
1117		− struct futex_hash_bucket *hb;
1118	1136	struct futex_q *top_waiter;
1119	1137	int ret;
1120	1138
@@ -1134,7 +1152,7 @@ retry: · 13 unmodified lines
1134	1152	if (ret)
1135	1153	return ret;
1136	1154
1137		− hb = futex_hash(&key);
	1155	+ CLASS(hb, hb)(&key);
1138	1156	spin_lock(&hb->lock);
1139	1157	retry_hb:
1140	1158
@@ -1187,6 +1205,12 @@ retry_hb: · 46 unmodified lines
1187	1205	*/
1188	1206	rt_waiter = rt_mutex_top_waiter(&pi_state->pi_mutex);
1189	1207	if (!rt_waiter) {
	1208	+ /*
	1209	+ * Acquire a reference for the leaving waiter to ensure
	1210	+ * valid futex_q::lock_ptr.
	1211	+ */
	1212	+ futex_hash_get(hb);
	1213	+ top_waiter->drop_hb_ref = true;
1190	1214	__futex_unqueue(top_waiter);
1191	1215	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1192	1216	goto retry_hb;

kernel/futex/requeue.c+234−226

@@ -87,6 +87,11 @@ void requeue_futex(struct futex_q q, struct futex_hash_bucket hb1, · 86 unmodified lines
87	87	futex_hb_waiters_inc(hb2);
88	88	plist_add(&q->list, &hb2->chain);
89	89	q->lock_ptr = &hb2->lock;
	90	+ /*
	91	+ * hb1 and hb2 belong to the same futex_hash_bucket_private
	92	+ * because if we managed get a reference on hb1 then it can't be
	93	+ * replaced. Therefore we avoid put(hb1)+get(hb2) here.
	94	+ */
90	95	}
91	96	q->key = *key2;
92	97	}
@@ -231,7 +236,12 @@ void requeue_pi_wake_futex(struct futex_q q, union futex_key key, · 138 unmodified lines
231	236
232	237	WARN_ON(!q->rt_waiter);
233	238	q->rt_waiter = NULL;
234		−
	239	+ /*
	240	+ * Acquire a reference for the waiter to ensure valid
	241	+ * futex_q::lock_ptr.
	242	+ */
	243	+ futex_hash_get(hb);
	244	+ q->drop_hb_ref = true;
235	245	q->lock_ptr = &hb->lock;
236	246
237	247	/* Signal locked state to the waiter */
@@ -371,7 +381,6 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1, · 133 unmodified lines
371	381	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
372	382	int task_count = 0, ret;
373	383	struct futex_pi_state *pi_state = NULL;
374		− struct futex_hash_bucket hb1, hb2;
375	384	struct futex_q this, next;
376	385	DEFINE_WAKE_Q(wake_q);
377	386
@@ -443,240 +452,242 @@ retry: · 65 unmodified lines
443	452	if (requeue_pi && futex_match(&key1, &key2))
444	453	return -EINVAL;
445	454
446		− hb1 = futex_hash(&key1);
447		− hb2 = futex_hash(&key2);
448		−
449	455	retry_private:
450		− futex_hb_waiters_inc(hb2);
451		− double_lock_hb(hb1, hb2);
	456	+ if (1) {
	457	+ CLASS(hb, hb1)(&key1);
	458	+ CLASS(hb, hb2)(&key2);
452	459
453		− if (likely(cmpval != NULL)) {
454		− u32 curval;
	460	+ futex_hb_waiters_inc(hb2);
	461	+ double_lock_hb(hb1, hb2);
455	462
456		− ret = futex_get_value_locked(&curval, uaddr1);
	463	+ if (likely(cmpval != NULL)) {
	464	+ u32 curval;
457	465
458		− if (unlikely(ret)) {
459		− double_unlock_hb(hb1, hb2);
460		− futex_hb_waiters_dec(hb2);
	466	+ ret = futex_get_value_locked(&curval, uaddr1);
461	467
462		− ret = get_user(curval, uaddr1);
463		− if (ret)
464		− return ret;
	468	+ if (unlikely(ret)) {
	469	+ futex_hb_waiters_dec(hb2);
	470	+ double_unlock_hb(hb1, hb2);
465	471
466		− if (!(flags1 & FLAGS_SHARED))
467		− goto retry_private;
	472	+ ret = get_user(curval, uaddr1);
	473	+ if (ret)
	474	+ return ret;
468	475
469		− goto retry;
470		− }
471		− if (curval != *cmpval) {
472		− ret = -EAGAIN;
473		− goto out_unlock;
474		− }
475		− }
	476	+ if (!(flags1 & FLAGS_SHARED))
	477	+ goto retry_private;
476	478
477		− if (requeue_pi) {
478		− struct task_struct *exiting = NULL;
	479	+ goto retry;
	480	+ }
	481	+ if (curval != *cmpval) {
	482	+ ret = -EAGAIN;
	483	+ goto out_unlock;
	484	+ }
	485	+ }
479	486
480		− /*
481		− * Attempt to acquire uaddr2 and wake the top waiter. If we
482		− * intend to requeue waiters, force setting the FUTEX_WAITERS
483		− * bit. We force this here where we are able to easily handle
484		− * faults rather in the requeue loop below.
485		− *
486		− * Updates topwaiter::requeue_state if a top waiter exists.
487		− */
488		− ret = futex_proxy_trylock_atomic(uaddr2, hb1, hb2, &key1,
489		− &key2, &pi_state,
490		− &exiting, nr_requeue);
	487	+ if (requeue_pi) {
	488	+ struct task_struct *exiting = NULL;
491	489
492		− /*
493		− * At this point the top_waiter has either taken uaddr2 or
494		− * is waiting on it. In both cases pi_state has been
495		− * established and an initial refcount on it. In case of an
496		− * error there's nothing.
497		− *
498		− * The top waiter's requeue_state is up to date:
499		− *
500		− * - If the lock was acquired atomically (ret == 1), then
501		− * the state is Q_REQUEUE_PI_LOCKED.
502		− *
503		− * The top waiter has been dequeued and woken up and can
504		− * return to user space immediately. The kernel/user
505		− * space state is consistent. In case that there must be
506		− * more waiters requeued the WAITERS bit in the user
507		− * space futex is set so the top waiter task has to go
508		− * into the syscall slowpath to unlock the futex. This
509		− * will block until this requeue operation has been
510		− * completed and the hash bucket locks have been
511		− * dropped.
512		− *
513		− * - If the trylock failed with an error (ret < 0) then
514		− * the state is either Q_REQUEUE_PI_NONE, i.e. "nothing
515		− * happened", or Q_REQUEUE_PI_IGNORE when there was an
516		− * interleaved early wakeup.
517		− *
518		− * - If the trylock did not succeed (ret == 0) then the
519		− * state is either Q_REQUEUE_PI_IN_PROGRESS or
520		− * Q_REQUEUE_PI_WAIT if an early wakeup interleaved.
521		− * This will be cleaned up in the loop below, which
522		− * cannot fail because futex_proxy_trylock_atomic() did
523		− * the same sanity checks for requeue_pi as the loop
524		− * below does.
525		− */
526		− switch (ret) {
527		− case 0:
528		− /* We hold a reference on the pi state. */
529		− break;
530		−
531		− case 1:
532	490	/*
533		− * futex_proxy_trylock_atomic() acquired the user space
534		− * futex. Adjust task_count.
	491	+ * Attempt to acquire uaddr2 and wake the top waiter. If we
	492	+ * intend to requeue waiters, force setting the FUTEX_WAITERS
	493	+ * bit. We force this here where we are able to easily handle
	494	+ * faults rather in the requeue loop below.
	495	+ *
	496	+ * Updates topwaiter::requeue_state if a top waiter exists.
535	497	*/
536		− task_count++;
537		− ret = 0;
538		− break;
	498	+ ret = futex_proxy_trylock_atomic(uaddr2, hb1, hb2, &key1,
	499	+ &key2, &pi_state,
	500	+ &exiting, nr_requeue);
539	501
540		− /*
541		− * If the above failed, then pi_state is NULL and
542		− * waiter::requeue_state is correct.
543		− */
544		− case -EFAULT:
545		− double_unlock_hb(hb1, hb2);
546		− futex_hb_waiters_dec(hb2);
547		− ret = fault_in_user_writeable(uaddr2);
548		− if (!ret)
549		− goto retry;
550		− return ret;
551		− case -EBUSY:
552		− case -EAGAIN:
553		− /*
554		− * Two reasons for this:
555		− * - EBUSY: Owner is exiting and we just wait for the
556		− * exit to complete.
557		− * - EAGAIN: The user space value changed.
558		− */
559		− double_unlock_hb(hb1, hb2);
560		− futex_hb_waiters_dec(hb2);
561	502	/*
562		− * Handle the case where the owner is in the middle of
563		− * exiting. Wait for the exit to complete otherwise
564		− * this task might loop forever, aka. live lock.
	503	+ * At this point the top_waiter has either taken uaddr2 or
	504	+ * is waiting on it. In both cases pi_state has been
	505	+ * established and an initial refcount on it. In case of an
	506	+ * error there's nothing.
	507	+ *
	508	+ * The top waiter's requeue_state is up to date:
	509	+ *
	510	+ * - If the lock was acquired atomically (ret == 1), then
	511	+ * the state is Q_REQUEUE_PI_LOCKED.
	512	+ *
	513	+ * The top waiter has been dequeued and woken up and can
	514	+ * return to user space immediately. The kernel/user
	515	+ * space state is consistent. In case that there must be
	516	+ * more waiters requeued the WAITERS bit in the user
	517	+ * space futex is set so the top waiter task has to go
	518	+ * into the syscall slowpath to unlock the futex. This
	519	+ * will block until this requeue operation has been
	520	+ * completed and the hash bucket locks have been
	521	+ * dropped.
	522	+ *
	523	+ * - If the trylock failed with an error (ret < 0) then
	524	+ * the state is either Q_REQUEUE_PI_NONE, i.e. "nothing
	525	+ * happened", or Q_REQUEUE_PI_IGNORE when there was an
	526	+ * interleaved early wakeup.
	527	+ *
	528	+ * - If the trylock did not succeed (ret == 0) then the
	529	+ * state is either Q_REQUEUE_PI_IN_PROGRESS or
	530	+ * Q_REQUEUE_PI_WAIT if an early wakeup interleaved.
	531	+ * This will be cleaned up in the loop below, which
	532	+ * cannot fail because futex_proxy_trylock_atomic() did
	533	+ * the same sanity checks for requeue_pi as the loop
	534	+ * below does.
565	535	*/
566		− wait_for_owner_exiting(ret, exiting);
567		− cond_resched();
568		− goto retry;
569		− default:
570		− goto out_unlock;
571		− }
572		− }
573		−
574		− plist_for_each_entry_safe(this, next, &hb1->chain, list) {
575		− if (task_count - nr_wake >= nr_requeue)
576		− break;
577		−
578		− if (!futex_match(&this->key, &key1))
579		− continue;
580		−
581		− /*
582		− * FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI should always
583		− * be paired with each other and no other futex ops.
584		− *
585		− * We should never be requeueing a futex_q with a pi_state,
586		− * which is awaiting a futex_unlock_pi().
587		− */
588		− if ((requeue_pi && !this->rt_waiter) \|\|
589		− (!requeue_pi && this->rt_waiter) \|\|
590		− this->pi_state) {
591		− ret = -EINVAL;
592		− break;
	536	+ switch (ret) {
	537	+ case 0:
	538	+ /* We hold a reference on the pi state. */
	539	+ break;
	540	+
	541	+ case 1:
	542	+ /*
	543	+ * futex_proxy_trylock_atomic() acquired the user space
	544	+ * futex. Adjust task_count.
	545	+ */
	546	+ task_count++;
	547	+ ret = 0;
	548	+ break;
	549	+
	550	+ /*
	551	+ * If the above failed, then pi_state is NULL and
	552	+ * waiter::requeue_state is correct.
	553	+ */
	554	+ case -EFAULT:
	555	+ futex_hb_waiters_dec(hb2);
	556	+ double_unlock_hb(hb1, hb2);
	557	+ ret = fault_in_user_writeable(uaddr2);
	558	+ if (!ret)
	559	+ goto retry;
	560	+ return ret;
	561	+ case -EBUSY:
	562	+ case -EAGAIN:
	563	+ /*
	564	+ * Two reasons for this:
	565	+ * - EBUSY: Owner is exiting and we just wait for the
	566	+ * exit to complete.
	567	+ * - EAGAIN: The user space value changed.
	568	+ */
	569	+ futex_hb_waiters_dec(hb2);
	570	+ double_unlock_hb(hb1, hb2);
	571	+ /*
	572	+ * Handle the case where the owner is in the middle of
	573	+ * exiting. Wait for the exit to complete otherwise
	574	+ * this task might loop forever, aka. live lock.
	575	+ */
	576	+ wait_for_owner_exiting(ret, exiting);
	577	+ cond_resched();
	578	+ goto retry;
	579	+ default:
	580	+ goto out_unlock;
	581	+ }
593	582	}
594	583
595		− /* Plain futexes just wake or requeue and are done */
596		− if (!requeue_pi) {
597		− if (++task_count <= nr_wake)
598		− this->wake(&wake_q, this);
599		− else
600		− requeue_futex(this, hb1, hb2, &key2);
601		− continue;
602		− }
	584	+ plist_for_each_entry_safe(this, next, &hb1->chain, list) {
	585	+ if (task_count - nr_wake >= nr_requeue)
	586	+ break;
603	587
604		− /* Ensure we requeue to the expected futex for requeue_pi. */
605		− if (!futex_match(this->requeue_pi_key, &key2)) {
606		− ret = -EINVAL;
607		− break;
608		− }
	588	+ if (!futex_match(&this->key, &key1))
	589	+ continue;
609	590
610		− /*
611		− * Requeue nr_requeue waiters and possibly one more in the case
612		− * of requeue_pi if we couldn't acquire the lock atomically.
613		− *
614		− * Prepare the waiter to take the rt_mutex. Take a refcount
615		− * on the pi_state and store the pointer in the futex_q
616		− * object of the waiter.
617		− */
618		− get_pi_state(pi_state);
619		−
620		− /* Don't requeue when the waiter is already on the way out. */
621		− if (!futex_requeue_pi_prepare(this, pi_state)) {
622	591	/*
623		− * Early woken waiter signaled that it is on the
624		− * way out. Drop the pi_state reference and try the
625		− * next waiter. @this->pi_state is still NULL.
	592	+ * FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI should always
	593	+ * be paired with each other and no other futex ops.
	594	+ *
	595	+ * We should never be requeueing a futex_q with a pi_state,
	596	+ * which is awaiting a futex_unlock_pi().
626	597	*/
627		− put_pi_state(pi_state);
628		− continue;
629		− }
	598	+ if ((requeue_pi && !this->rt_waiter) \|\|
	599	+ (!requeue_pi && this->rt_waiter) \|\|
	600	+ this->pi_state) {
	601	+ ret = -EINVAL;
	602	+ break;
	603	+ }
	604	+
	605	+ /* Plain futexes just wake or requeue and are done */
	606	+ if (!requeue_pi) {
	607	+ if (++task_count <= nr_wake)
	608	+ this->wake(&wake_q, this);
	609	+ else
	610	+ requeue_futex(this, hb1, hb2, &key2);
	611	+ continue;
	612	+ }
	613	+
	614	+ /* Ensure we requeue to the expected futex for requeue_pi. */
	615	+ if (!futex_match(this->requeue_pi_key, &key2)) {
	616	+ ret = -EINVAL;
	617	+ break;
	618	+ }
630	619
631		− ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
632		− this->rt_waiter,
633		− this->task);
634		−
635		− if (ret == 1) {
636		− /*
637		− * We got the lock. We do neither drop the refcount
638		− * on pi_state nor clear this->pi_state because the
639		− * waiter needs the pi_state for cleaning up the
640		− * user space value. It will drop the refcount
641		− * after doing so. this::requeue_state is updated
642		− * in the wakeup as well.
643		− */
644		− requeue_pi_wake_futex(this, &key2, hb2);
645		− task_count++;
646		− } else if (!ret) {
647		− /* Waiter is queued, move it to hb2 */
648		− requeue_futex(this, hb1, hb2, &key2);
649		− futex_requeue_pi_complete(this, 0);
650		− task_count++;
651		− } else {
652	620	/*
653		− * rt_mutex_start_proxy_lock() detected a potential
654		− * deadlock when we tried to queue that waiter.
655		− * Drop the pi_state reference which we took above
656		− * and remove the pointer to the state from the
657		− * waiters futex_q object.
	621	+ * Requeue nr_requeue waiters and possibly one more in the case
	622	+ * of requeue_pi if we couldn't acquire the lock atomically.
	623	+ *
	624	+ * Prepare the waiter to take the rt_mutex. Take a refcount
	625	+ * on the pi_state and store the pointer in the futex_q
	626	+ * object of the waiter.
658	627	*/
659		− this->pi_state = NULL;
660		− put_pi_state(pi_state);
661		− futex_requeue_pi_complete(this, ret);
662		− /*
663		− * We stop queueing more waiters and let user space
664		− * deal with the mess.
665		− */
666		− break;
	628	+ get_pi_state(pi_state);
	629	+
	630	+ /* Don't requeue when the waiter is already on the way out. */
	631	+ if (!futex_requeue_pi_prepare(this, pi_state)) {
	632	+ /*
	633	+ * Early woken waiter signaled that it is on the
	634	+ * way out. Drop the pi_state reference and try the
	635	+ * next waiter. @this->pi_state is still NULL.
	636	+ */
	637	+ put_pi_state(pi_state);
	638	+ continue;
	639	+ }
	640	+
	641	+ ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
	642	+ this->rt_waiter,
	643	+ this->task);
	644	+
	645	+ if (ret == 1) {
	646	+ /*
	647	+ * We got the lock. We do neither drop the refcount
	648	+ * on pi_state nor clear this->pi_state because the
	649	+ * waiter needs the pi_state for cleaning up the
	650	+ * user space value. It will drop the refcount
	651	+ * after doing so. this::requeue_state is updated
	652	+ * in the wakeup as well.
	653	+ */
	654	+ requeue_pi_wake_futex(this, &key2, hb2);
	655	+ task_count++;
	656	+ } else if (!ret) {
	657	+ /* Waiter is queued, move it to hb2 */
	658	+ requeue_futex(this, hb1, hb2, &key2);
	659	+ futex_requeue_pi_complete(this, 0);
	660	+ task_count++;
	661	+ } else {
	662	+ /*
	663	+ * rt_mutex_start_proxy_lock() detected a potential
	664	+ * deadlock when we tried to queue that waiter.
	665	+ * Drop the pi_state reference which we took above
	666	+ * and remove the pointer to the state from the
	667	+ * waiters futex_q object.
	668	+ */
	669	+ this->pi_state = NULL;
	670	+ put_pi_state(pi_state);
	671	+ futex_requeue_pi_complete(this, ret);
	672	+ /*
	673	+ * We stop queueing more waiters and let user space
	674	+ * deal with the mess.
	675	+ */
	676	+ break;
	677	+ }
667	678	}
668		− }
669	679
670		− /*
671		− * We took an extra initial reference to the pi_state in
672		− * futex_proxy_trylock_atomic(). We need to drop it here again.
673		− */
674		− put_pi_state(pi_state);
	680	+ /*
	681	+ * We took an extra initial reference to the pi_state in
	682	+ * futex_proxy_trylock_atomic(). We need to drop it here again.
	683	+ */
	684	+ put_pi_state(pi_state);
675	685
676	686	out_unlock:
677		− double_unlock_hb(hb1, hb2);
	687	+ futex_hb_waiters_dec(hb2);
	688	+ double_unlock_hb(hb1, hb2);
	689	+ }
678	690	wake_up_q(&wake_q);
679		− futex_hb_waiters_dec(hb2);
680	691	return ret ? ret : task_count;
681	692	}
682	693
@@ -769,7 +780,6 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 86 unmodified lines
769	780	{
770	781	struct hrtimer_sleeper timeout, *to;
771	782	struct rt_mutex_waiter rt_waiter;
772		− struct futex_hash_bucket *hb;
773	783	union futex_key key2 = FUTEX_KEY_INIT;
774	784	struct futex_q q = futex_q_init;
775	785	struct rt_mutex_base *pi_mutex;
@@ -805,35 +815,28 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 29 unmodified lines
805	815	* Prepare to wait on uaddr. On success, it holds hb->lock and q
806	816	* is initialized.
807	817	*/
808		− ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
	818	+ ret = futex_wait_setup(uaddr, val, flags, &q, &key2, current);
809	819	if (ret)
810	820	goto out;
811	821
812		− /*
813		− * The check above which compares uaddrs is not sufficient for
814		− * shared futexes. We need to compare the keys:
815		− */
816		− if (futex_match(&q.key, &key2)) {
817		− futex_q_unlock(hb);
818		− ret = -EINVAL;
819		− goto out;
820		− }
821		−
822	822	/* Queue the futex_q, drop the hb lock, wait for wakeup. */
823		− futex_wait_queue(hb, &q, to);
	823	+ futex_do_wait(&q, to);
824	824
825	825	switch (futex_requeue_pi_wakeup_sync(&q)) {
826	826	case Q_REQUEUE_PI_IGNORE:
827		− /* The waiter is still on uaddr1 */
828		− spin_lock(&hb->lock);
829		− ret = handle_early_requeue_pi_wakeup(hb, &q, to);
830		− spin_unlock(&hb->lock);
	827	+ {
	828	+ CLASS(hb, hb)(&q.key);
	829	+ /* The waiter is still on uaddr1 */
	830	+ spin_lock(&hb->lock);
	831	+ ret = handle_early_requeue_pi_wakeup(hb, &q, to);
	832	+ spin_unlock(&hb->lock);
	833	+ }
831	834	break;
832	835
833	836	case Q_REQUEUE_PI_LOCKED:
834	837	/* The requeue acquired the lock */
835	838	if (q.pi_state && (q.pi_state->owner != current)) {
836		− spin_lock(q.lock_ptr);
	839	+ futex_q_lockptr_lock(&q);
837	840	ret = fixup_pi_owner(uaddr2, &q, true);
838	841	/*
839	842	* Drop the reference to the pi state which the
@@ -860,7 +863,7 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 20 unmodified lines
860	863	if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
861	864	ret = 0;
862	865
863		− spin_lock(q.lock_ptr);
	866	+ futex_q_lockptr_lock(&q);
864	867	debug_rt_mutex_free_waiter(&rt_waiter);
865	868	/*
866	869	* Fixup the pi_state owner and possibly acquire the lock if we
@@ -892,6 +895,11 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, · 25 unmodified lines
892	895	default:
893	896	BUG();
894	897	}
	898	+ if (q.drop_hb_ref) {
	899	+ CLASS(hb, hb)(&q.key);
	900	+ /* Additional reference from requeue_pi_wake_futex() */
	901	+ futex_hash_put(hb);
	902	+ }
895	903
896	904	out:
897	905	if (to) {

kernel/futex/waitwake.c+113−94

@@ -154,7 +154,6 @@ void futex_wake_mark(struct wake_q_head wake_q, struct futex_q q) · 153 unmodified lines
154	154	*/
155	155	int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
156	156	{
157		− struct futex_hash_bucket *hb;
158	157	struct futex_q this, next;
159	158	union futex_key key = FUTEX_KEY_INIT;
160	159	DEFINE_WAKE_Q(wake_q);
@@ -170,7 +169,7 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) · 9 unmodified lines
170	169	if ((flags & FLAGS_STRICT) && !nr_wake)
171	170	return 0;
172	171
173		− hb = futex_hash(&key);
	172	+ CLASS(hb, hb)(&key);
174	173
175	174	/* Make sure we really have tasks to wakeup */
176	175	if (!futex_hb_waiters_pending(hb))
@@ -253,7 +252,6 @@ int futex_wake_op(u32 __user uaddr1, unsigned int flags, u32 __user uaddr2, · 76 unmodified lines
253	252	int nr_wake, int nr_wake2, int op)
254	253	{
255	254	union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
256		− struct futex_hash_bucket hb1, hb2;
257	255	struct futex_q this, next;
258	256	int ret, op_ret;
259	257	DEFINE_WAKE_Q(wake_q);
@@ -266,67 +264,69 @@ retry: · 6 unmodified lines
266	264	if (unlikely(ret != 0))
267	265	return ret;
268	266
269		− hb1 = futex_hash(&key1);
270		− hb2 = futex_hash(&key2);
271		−
272	267	retry_private:
273		− double_lock_hb(hb1, hb2);
274		− op_ret = futex_atomic_op_inuser(op, uaddr2);
275		− if (unlikely(op_ret < 0)) {
276		− double_unlock_hb(hb1, hb2);
277		−
278		− if (!IS_ENABLED(CONFIG_MMU) \|\|
279		− unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
280		− /*
281		− * we don't get EFAULT from MMU faults if we don't have
282		− * an MMU, but we might get them from range checking
283		− */
284		− ret = op_ret;
285		− return ret;
286		− }
287		−
288		− if (op_ret == -EFAULT) {
289		− ret = fault_in_user_writeable(uaddr2);
290		− if (ret)
	268	+ if (1) {
	269	+ CLASS(hb, hb1)(&key1);
	270	+ CLASS(hb, hb2)(&key2);
	271	+
	272	+ double_lock_hb(hb1, hb2);
	273	+ op_ret = futex_atomic_op_inuser(op, uaddr2);
	274	+ if (unlikely(op_ret < 0)) {
	275	+ double_unlock_hb(hb1, hb2);
	276	+
	277	+ if (!IS_ENABLED(CONFIG_MMU) \|\|
	278	+ unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
	279	+ /*
	280	+ * we don't get EFAULT from MMU faults if we don't have
	281	+ * an MMU, but we might get them from range checking
	282	+ */
	283	+ ret = op_ret;
291	284	return ret;
292		− }
293		−
294		− cond_resched();
295		− if (!(flags & FLAGS_SHARED))
296		− goto retry_private;
297		− goto retry;
298		− }
	285	+ }
299	286
300		− plist_for_each_entry_safe(this, next, &hb1->chain, list) {
301		− if (futex_match (&this->key, &key1)) {
302		− if (this->pi_state \|\| this->rt_waiter) {
303		− ret = -EINVAL;
304		− goto out_unlock;
	287	+ if (op_ret == -EFAULT) {
	288	+ ret = fault_in_user_writeable(uaddr2);
	289	+ if (ret)
	290	+ return ret;
305	291	}
306		− this->wake(&wake_q, this);
307		− if (++ret >= nr_wake)
308		− break;
	292	+
	293	+ cond_resched();
	294	+ if (!(flags & FLAGS_SHARED))
	295	+ goto retry_private;
	296	+ goto retry;
309	297	}
310		− }
311	298
312		− if (op_ret > 0) {
313		− op_ret = 0;
314		− plist_for_each_entry_safe(this, next, &hb2->chain, list) {
315		− if (futex_match (&this->key, &key2)) {
	299	+ plist_for_each_entry_safe(this, next, &hb1->chain, list) {
	300	+ if (futex_match(&this->key, &key1)) {
316	301	if (this->pi_state \|\| this->rt_waiter) {
317	302	ret = -EINVAL;
318	303	goto out_unlock;
319	304	}
320	305	this->wake(&wake_q, this);
321		− if (++op_ret >= nr_wake2)
	306	+ if (++ret >= nr_wake)
322	307	break;
323	308	}
324	309	}
325		− ret += op_ret;
326		− }
	310	+
	311	+ if (op_ret > 0) {
	312	+ op_ret = 0;
	313	+ plist_for_each_entry_safe(this, next, &hb2->chain, list) {
	314	+ if (futex_match(&this->key, &key2)) {
	315	+ if (this->pi_state \|\| this->rt_waiter) {
	316	+ ret = -EINVAL;
	317	+ goto out_unlock;
	318	+ }
	319	+ this->wake(&wake_q, this);
	320	+ if (++op_ret >= nr_wake2)
	321	+ break;
	322	+ }
	323	+ }
	324	+ ret += op_ret;
	325	+ }
327	326
328	327	out_unlock:
329		− double_unlock_hb(hb1, hb2);
	328	+ double_unlock_hb(hb1, hb2);
	329	+ }
330	330	wake_up_q(&wake_q);
331	331	return ret;
332	332	}
@@ -334,23 +334,12 @@ out_unlock: · 1 unmodified line
334	334	static long futex_wait_restart(struct restart_block *restart);
335	335
336	336	/**
337		− * futex_wait_queue() - futex_queue() and wait for wakeup, timeout, or signal
338		− * @hb: the futex hash bucket, must be locked by the caller
	337	+ * futex_do_wait() - wait for wakeup, timeout, or signal
339	338	* @q: the futex_q to queue up on
340	339	* @timeout: the prepared hrtimer_sleeper, or null for no timeout
341	340	*/
342		−void futex_wait_queue(struct futex_hash_bucket hb, struct futex_q q,
343		− struct hrtimer_sleeper *timeout)
	341	+void futex_do_wait(struct futex_q q, struct hrtimer_sleeper timeout)
344	342	{
345		− /*
346		− * The task state is guaranteed to be set before another task can
347		− * wake it. set_current_state() is implemented using smp_store_mb() and
348		− * futex_queue() calls spin_unlock() upon completion, both serializing
349		− * access to the hash list and forcing another memory barrier.
350		− */
351		− set_current_state(TASK_INTERRUPTIBLE\|TASK_FREEZABLE);
352		− futex_queue(q, hb, current);
353		−
354	343	/* Arm the timer */
355	344	if (timeout)
356	345	hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
@@ -412,11 +401,16 @@ int futex_unqueue_multiple(struct futex_vector *v, int count) · 55 unmodified lines
412	401	*/
413	402	int futex_wait_multiple_setup(struct futex_vector vs, int count, int woken)
414	403	{
415		− struct futex_hash_bucket *hb;
416	404	bool retry = false;
417	405	int ret, i;
418	406	u32 uval;
419	407
	408	+ /*
	409	+ * Make sure to have a reference on the private_hash such that we
	410	+ * don't block on rehash after changing the task state below.
	411	+ */
	412	+ guard(private_hash)();
	413	+
420	414	/*
421	415	* Enqueuing multiple futexes is tricky, because we need to enqueue
422	416	* each futex on the list before dealing with the next one to avoid
@@ -451,20 +445,24 @@ retry: · 28 unmodified lines
451	445	struct futex_q *q = &vs[i].q;
452	446	u32 val = vs[i].w.val;
453	447
454		− hb = futex_q_lock(q);
455		− ret = futex_get_value_locked(&uval, uaddr);
	448	+ if (1) {
	449	+ CLASS(hb, hb)(&q->key);
456	450
457		− if (!ret && uval == val) {
458		− /*
459		− * The bucket lock can't be held while dealing with the
460		− * next futex. Queue each futex at this moment so hb can
461		− * be unlocked.
462		− */
463		− futex_queue(q, hb, current);
464		− continue;
465		− }
	451	+ futex_q_lock(q, hb);
	452	+ ret = futex_get_value_locked(&uval, uaddr);
	453	+
	454	+ if (!ret && uval == val) {
	455	+ /*
	456	+ * The bucket lock can't be held while dealing with the
	457	+ * next futex. Queue each futex at this moment so hb can
	458	+ * be unlocked.
	459	+ */
	460	+ futex_queue(q, hb, current);
	461	+ continue;
	462	+ }
466	463
467		− futex_q_unlock(hb);
	464	+ futex_q_unlock(hb);
	465	+ }
468	466	__set_current_state(TASK_RUNNING);
469	467
470	468	/*
@@ -578,7 +576,8 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, · 107 unmodified lines
578	576	* @val: the expected value
579	577	* @flags: futex flags (FLAGS_SHARED, etc.)
580	578	* @q: the associated futex_q
581		− * @hb: storage for hash_bucket pointer to be returned to caller
	579	+ * @key2: the second futex_key if used for requeue PI
	580	+ * @task: Task queueing this futex
582	581	*
583	582	* Setup the futex_q and locate the hash_bucket. Get the futex value and
584	583	* compare it with the expected value. Handle atomic faults internally.
@@ -586,10 +585,12 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, · 1 unmodified line
586	585	*
587	586	* Return:
588	587	* - 0 - uaddr contains val and hb has been locked;
589		− * - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
	588	+ * - <0 - On error and the hb is unlocked. A possible reason: the uaddr can not
	589	+ * be read, does not contain the expected value or is not properly aligned.
590	590	*/
591	591	int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
592		− struct futex_q q, struct futex_hash_bucket *hb)
	592	+ struct futex_q q, union futex_key key2,
	593	+ struct task_struct *task)
593	594	{
594	595	u32 uval;
595	596	int ret;
@@ -618,26 +619,45 @@ retry: · 22 unmodified lines
618	619	return ret;
619	620
620	621	retry_private:
621		− *hb = futex_q_lock(q);
	622	+ if (1) {
	623	+ CLASS(hb, hb)(&q->key);
622	624
623		− ret = futex_get_value_locked(&uval, uaddr);
	625	+ futex_q_lock(q, hb);
624	626
625		− if (ret) {
626		− futex_q_unlock(*hb);
	627	+ ret = futex_get_value_locked(&uval, uaddr);
627	628
628		− ret = get_user(uval, uaddr);
629		− if (ret)
630		− return ret;
	629	+ if (ret) {
	630	+ futex_q_unlock(hb);
631	631
632		− if (!(flags & FLAGS_SHARED))
633		− goto retry_private;
	632	+ ret = get_user(uval, uaddr);
	633	+ if (ret)
	634	+ return ret;
634	635
635		− goto retry;
636		− }
	636	+ if (!(flags & FLAGS_SHARED))
	637	+ goto retry_private;
	638	+
	639	+ goto retry;
	640	+ }
637	641
638		− if (uval != val) {
639		− futex_q_unlock(*hb);
640		− ret = -EWOULDBLOCK;
	642	+ if (uval != val) {
	643	+ futex_q_unlock(hb);
	644	+ return -EWOULDBLOCK;
	645	+ }
	646	+
	647	+ if (key2 && futex_match(&q->key, key2)) {
	648	+ futex_q_unlock(hb);
	649	+ return -EINVAL;
	650	+ }
	651	+
	652	+ /*
	653	+ * The task state is guaranteed to be set before another task can
	654	+ * wake it. set_current_state() is implemented using smp_store_mb() and
	655	+ * futex_queue() calls spin_unlock() upon completion, both serializing
	656	+ * access to the hash list and forcing another memory barrier.
	657	+ */
	658	+ if (task == current)
	659	+ set_current_state(TASK_INTERRUPTIBLE\|TASK_FREEZABLE);
	660	+ futex_queue(q, hb, task);
641	661	}
642	662
643	663	return ret;
@@ -647,7 +667,6 @@ int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, · 3 unmodified lines
647	667	struct hrtimer_sleeper *to, u32 bitset)
648	668	{
649	669	struct futex_q q = futex_q_init;
650		− struct futex_hash_bucket *hb;
651	670	int ret;
652	671
653	672	if (!bitset)
@@ -660,12 +679,12 @@ retry: · 6 unmodified lines
660	679	* Prepare to wait on uaddr. On success, it holds hb->lock and q
661	680	* is initialized.
662	681	*/
663		− ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
	682	+ ret = futex_wait_setup(uaddr, val, flags, &q, NULL, current);
664	683	if (ret)
665	684	return ret;
666	685
667	686	/* futex_queue and wait for wakeup, timeout, or a signal. */
668		− futex_wait_queue(hb, &q, to);
	687	+ futex_do_wait(&q, to);
669	688
670	689	/* If we were woken (and unqueued), we succeeded, whatever. */
671	690	if (!futex_unqueue(&q))

kernel/sys.c+4−0

@@ -52,6 +52,7 @@ · 51 unmodified lines
52	52	#include <linux/user_namespace.h>
53	53	#include <linux/time_namespace.h>
54	54	#include <linux/binfmts.h>
	55	+#include <linux/futex.h>
55	56
56	57	#include <linux/sched.h>
57	58	#include <linux/sched/autogroup.h>
@@ -2820,6 +2821,9 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, · 2762 unmodified lines
2820	2821	return -EINVAL;
2821	2822	error = posixtimer_create_prctl(arg2);
2822	2823	break;
	2824	+ case PR_FUTEX_HASH:
	2825	+ error = futex_hash_prctl(arg2, arg3, arg4);
	2826	+ break;
2823	2827	default:
2824	2828	trace_task_prctl_unknown(option, arg2, arg3, arg4, arg5);
2825	2829	error = -EINVAL;

mm/nommu.c+17−1

@@ -200,7 +200,23 @@ void *vmalloc_noprof(unsigned long size) · 199 unmodified lines
200	200	}
201	201	EXPORT_SYMBOL(vmalloc_noprof);
202	202
203		−void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask) __weak __alias(__vmalloc_noprof);
	203	+/*
	204	+ * vmalloc_huge_node - allocate virtually contiguous memory, on a node
	205	+ *
	206	+ * @size: allocation size
	207	+ * @gfp_mask: flags for the page level allocator
	208	+ * @node: node to use for allocation or NUMA_NO_NODE
	209	+ *
	210	+ * Allocate enough pages to cover @size from the page level
	211	+ * allocator and map them into contiguous kernel virtual space.
	212	+ *
	213	+ * Due to NOMMU implications the node argument and HUGE page attribute is
	214	+ * ignored.
	215	+ */
	216	+void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node)
	217	+{
	218	+ return __vmalloc_noprof(size, gfp_mask);
	219	+}
204	220
205	221	/*
206	222	* vzalloc - allocate virtually contiguous memory with zero fill

mm/vmalloc.c+6−5

@@ -3943,9 +3943,10 @@ void *vmalloc_noprof(unsigned long size) · 3942 unmodified lines
3943	3943	EXPORT_SYMBOL(vmalloc_noprof);
3944	3944
3945	3945	/**
3946		− * vmalloc_huge - allocate virtually contiguous memory, allow huge pages
	3946	+ * vmalloc_huge_node - allocate virtually contiguous memory, allow huge pages
3947	3947	* @size: allocation size
3948	3948	* @gfp_mask: flags for the page level allocator
	3949	+ * @node: node to use for allocation or NUMA_NO_NODE
3949	3950	*
3950	3951	* Allocate enough pages to cover @size from the page level
3951	3952	* allocator and map them into contiguous kernel virtual space.
@@ -3954,13 +3955,13 @@ EXPORT_SYMBOL(vmalloc_noprof); · 2 unmodified lines
3954	3955	*
3955	3956	* Return: pointer to the allocated memory or %NULL on error
3956	3957	*/
3957		−void *vmalloc_huge_noprof(unsigned long size, gfp_t gfp_mask)
	3958	+void *vmalloc_huge_node_noprof(unsigned long size, gfp_t gfp_mask, int node)
3958	3959	{
3959	3960	return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END,
3960		− gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
3961		− NUMA_NO_NODE, __builtin_return_address(0));
	3961	+ gfp_mask, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
	3962	+ node, __builtin_return_address(0));
3962	3963	}
3963		−EXPORT_SYMBOL_GPL(vmalloc_huge_noprof);
	3964	+EXPORT_SYMBOL_GPL(vmalloc_huge_node_noprof);
3964	3965
3965	3966	/**
3966	3967	* vzalloc - allocate virtually contiguous memory with zero fill

tools/include/uapi/linux/prctl.h+44−1

@@ -230,7 +230,7 @@ struct prctl_mm_map { · 229 unmodified lines
230	230	# define PR_PAC_APDBKEY (1UL << 3)
231	231	# define PR_PAC_APGAKEY (1UL << 4)
232	232
233		−/* Tagged user address controls for arm64 */
	233	+/* Tagged user address controls for arm64 and RISC-V */
234	234	#define PR_SET_TAGGED_ADDR_CTRL 55
235	235	#define PR_GET_TAGGED_ADDR_CTRL 56
236	236	# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
@@ -244,6 +244,9 @@ struct prctl_mm_map { · 7 unmodified lines
244	244	# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
245	245	/* Unused; kept only for source compatibility */
246	246	# define PR_MTE_TCF_SHIFT 1
	247	+/* RISC-V pointer masking tag length */
	248	+# define PR_PMLEN_SHIFT 24
	249	+# define PR_PMLEN_MASK (0x7fUL << PR_PMLEN_SHIFT)
247	250
248	251	/* Control reclaim behavior when allocating memory */
249	252	#define PR_SET_IO_FLUSHER 57
@@ -328,4 +331,44 @@ struct prctl_mm_map { · 78 unmodified lines
328	331	# define PR_PPC_DEXCR_CTRL_CLEAR_ONEXEC 0x10 /* Clear the aspect on exec */
329	332	# define PR_PPC_DEXCR_CTRL_MASK 0x1f
330	333
	334	+/*
	335	+ * Get the current shadow stack configuration for the current thread,
	336	+ * this will be the value configured via PR_SET_SHADOW_STACK_STATUS.
	337	+ */
	338	+#define PR_GET_SHADOW_STACK_STATUS 74
	339	+
	340	+/*
	341	+ * Set the current shadow stack configuration. Enabling the shadow
	342	+ * stack will cause a shadow stack to be allocated for the thread.
	343	+ */
	344	+#define PR_SET_SHADOW_STACK_STATUS 75
	345	+# define PR_SHADOW_STACK_ENABLE (1UL << 0)
	346	+# define PR_SHADOW_STACK_WRITE (1UL << 1)
	347	+# define PR_SHADOW_STACK_PUSH (1UL << 2)
	348	+
	349	+/*
	350	+ * Prevent further changes to the specified shadow stack
	351	+ * configuration. All bits may be locked via this call, including
	352	+ * undefined bits.
	353	+ */
	354	+#define PR_LOCK_SHADOW_STACK_STATUS 76
	355	+
	356	+/*
	357	+ * Controls the mode of timer_create() for CRIU restore operations.
	358	+ * Enabling this allows CRIU to restore timers with explicit IDs.
	359	+ *
	360	+ * Don't use for normal operations as the result might be undefined.
	361	+ */
	362	+#define PR_TIMER_CREATE_RESTORE_IDS 77
	363	+# define PR_TIMER_CREATE_RESTORE_IDS_OFF 0
	364	+# define PR_TIMER_CREATE_RESTORE_IDS_ON 1
	365	+# define PR_TIMER_CREATE_RESTORE_IDS_GET 2
	366	+
	367	+/* FUTEX hash management */
	368	+#define PR_FUTEX_HASH 78
	369	+# define PR_FUTEX_HASH_SET_SLOTS 1
	370	+# define FH_FLAG_IMMUTABLE (1ULL << 0)
	371	+# define PR_FUTEX_HASH_GET_SLOTS 2
	372	+# define PR_FUTEX_HASH_GET_IMMUTABLE 3
	373	+
331	374	#endif /* _LINUX_PRCTL_H */

tools/perf/bench/Build+1−0

@@ -3,6 +3,7 @@ perf-bench-y += sched-pipe.o · 2 unmodified lines
3	3	perf-bench-y += sched-seccomp-notify.o
4	4	perf-bench-y += syscall.o
5	5	perf-bench-y += mem-functions.o
	6	+perf-bench-y += futex.o
6	7	perf-bench-y += futex-hash.o
7	8	perf-bench-y += futex-wake.o
8	9	perf-bench-y += futex-wake-parallel.o

tools/perf/bench/futex-hash.c+7−0

@@ -18,9 +18,11 @@ · 17 unmodified lines
18	18	#include <stdlib.h>
19	19	#include <linux/compiler.h>
20	20	#include <linux/kernel.h>
	21	+#include <linux/prctl.h>
21	22	#include <linux/zalloc.h>
22	23	#include <sys/time.h>
23	24	#include <sys/mman.h>
	25	+#include <sys/prctl.h>
24	26	#include <perf/cpumap.h>
25	27
26	28	#include "../util/mutex.h"
@@ -50,9 +52,12 @@ struct worker { · 23 unmodified lines
50	52	static struct bench_futex_parameters params = {
51	53	.nfutexes = 1024,
52	54	.runtime = 10,
	55	+ .nbuckets = -1,
53	56	};
54	57
55	58	static const struct option options[] = {
	59	+ OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
	60	+ OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
56	61	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
57	62	OPT_UINTEGER('r', "runtime", &params.runtime, "Specify runtime (in seconds)"),
58	63	OPT_UINTEGER('f', "futexes", &params.nfutexes, "Specify amount of futexes per threads"),
@@ -118,6 +123,7 @@ static void print_summary(void) · 59 unmodified lines
118	123	printf("%sAveraged %ld operations/sec (+- %.2f%%), total secs = %d\n",
119	124	!params.silent ? "\n" : "", avg, rel_stddev_stats(stddev, avg),
120	125	(int)bench__runtime.tv_sec);
	126	+ futex_print_nbuckets(&params);
121	127	}
122	128
123	129	int bench_futex_hash(int argc, const char **argv)
@@ -161,6 +167,7 @@ int bench_futex_hash(int argc, const char **argv) · 37 unmodified lines
161	167
162	168	if (!params.fshared)
163	169	futex_flag = FUTEX_PRIVATE_FLAG;
	170	+ futex_set_nbuckets_param(&params);
164	171
165	172	printf("Run summary [PID %d]: %d threads, each operating on %d [%s] futexes for %d secs.\n\n",
166	173	getpid(), params.nthreads, params.nfutexes, params.fshared ? "shared":"private", params.runtime);

tools/perf/bench/futex-lock-pi.c+5−0

@@ -41,10 +41,13 @@ static struct stats throughput_stats; · 40 unmodified lines
41	41	static struct cond thread_parent, thread_worker;
42	42
43	43	static struct bench_futex_parameters params = {
	44	+ .nbuckets = -1,
44	45	.runtime = 10,
45	46	};
46	47
47	48	static const struct option options[] = {
	49	+ OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
	50	+ OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
48	51	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
49	52	OPT_UINTEGER('r', "runtime", &params.runtime, "Specify runtime (in seconds)"),
50	53	OPT_BOOLEAN( 'M', "multi", &params.multi, "Use multiple futexes"),
@@ -67,6 +70,7 @@ static void print_summary(void) · 16 unmodified lines
67	70	printf("%sAveraged %ld operations/sec (+- %.2f%%), total secs = %d\n",
68	71	!params.silent ? "\n" : "", avg, rel_stddev_stats(stddev, avg),
69	72	(int)bench__runtime.tv_sec);
	73	+ futex_print_nbuckets(&params);
70	74	}
71	75
72	76	static void toggle_done(int sig __maybe_unused,
@@ -203,6 +207,7 @@ int bench_futex_lock_pi(int argc, const char **argv) · 130 unmodified lines
203	207	mutex_init(&thread_lock);
204	208	cond_init(&thread_parent);
205	209	cond_init(&thread_worker);
	210	+ futex_set_nbuckets_param(&params);
206	211
207	212	threads_starting = params.nthreads;
208	213	gettimeofday(&bench__start, NULL);

tools/perf/bench/futex-requeue.c+6−0

@@ -42,6 +42,7 @@ static unsigned int threads_starting; · 41 unmodified lines
42	42	static int futex_flag = 0;
43	43
44	44	static struct bench_futex_parameters params = {
	45	+ .nbuckets = -1,
45	46	/*
46	47	* How many tasks to requeue at a time.
47	48	* Default to 1 in order to make the kernel work more.
@@ -50,6 +51,8 @@ static struct bench_futex_parameters params = { · 2 unmodified lines
50	51	};
51	52
52	53	static const struct option options[] = {
	54	+ OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
	55	+ OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
53	56	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
54	57	OPT_UINTEGER('q', "nrequeue", &params.nrequeue, "Specify amount of threads to requeue at once"),
55	58	OPT_BOOLEAN( 's', "silent", &params.silent, "Silent mode: do not display data/details"),
@@ -77,6 +80,7 @@ static void print_summary(void) · 21 unmodified lines
77	80	params.nthreads,
78	81	requeuetime_avg / USEC_PER_MSEC,
79	82	rel_stddev_stats(requeuetime_stddev, requeuetime_avg));
	83	+ futex_print_nbuckets(&params);
80	84	}
81	85
82	86	static void workerfn(void arg __maybe_unused)
@@ -204,6 +208,8 @@ int bench_futex_requeue(int argc, const char **argv) · 121 unmodified lines
204	208	if (params.broadcast)
205	209	params.nrequeue = params.nthreads;
206	210
	211	+ futex_set_nbuckets_param(&params);
	212	+
207	213	printf("Run summary [PID %d]: Requeuing %d threads (from [%s] %p to %s%p), "
208	214	"%d at a time.\n\n", getpid(), params.nthreads,
209	215	params.fshared ? "shared":"private", &futex1,

tools/perf/bench/futex-wake-parallel.c+8−1

@@ -57,9 +57,13 @@ static struct stats waketime_stats, wakeup_stats; · 56 unmodified lines
57	57	static unsigned int threads_starting;
58	58	static int futex_flag = 0;
59	59
60		−static struct bench_futex_parameters params;
	60	+static struct bench_futex_parameters params = {
	61	+ .nbuckets = -1,
	62	+};
61	63
62	64	static const struct option options[] = {
	65	+ OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
	66	+ OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
63	67	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
64	68	OPT_UINTEGER('w', "nwakers", &params.nwakes, "Specify amount of waking threads"),
65	69	OPT_BOOLEAN( 's', "silent", &params.silent, "Silent mode: do not display data/details"),
@@ -218,6 +222,7 @@ static void print_summary(void) · 152 unmodified lines
218	222	params.nthreads,
219	223	waketime_avg / USEC_PER_MSEC,
220	224	rel_stddev_stats(waketime_stddev, waketime_avg));
	225	+ futex_print_nbuckets(&params);
221	226	}
222	227
223	228
@@ -291,6 +296,8 @@ int bench_futex_wake_parallel(int argc, const char **argv) · 67 unmodified lines
291	296	if (!params.fshared)
292	297	futex_flag = FUTEX_PRIVATE_FLAG;
293	298
	299	+ futex_set_nbuckets_param(&params);
	300	+
294	301	printf("Run summary [PID %d]: blocking on %d threads (at [%s] "
295	302	"futex %p), %d threads waking up %d at a time.\n\n",
296	303	getpid(), params.nthreads, params.fshared ? "shared":"private",

tools/perf/bench/futex-wake.c+4−0

@@ -42,6 +42,7 @@ static unsigned int threads_starting; · 41 unmodified lines
42	42	static int futex_flag = 0;
43	43
44	44	static struct bench_futex_parameters params = {
	45	+ .nbuckets = -1,
45	46	/*
46	47	* How many wakeups to do at a time.
47	48	* Default to 1 in order to make the kernel work more.
@@ -50,6 +51,8 @@ static struct bench_futex_parameters params = { · 2 unmodified lines
50	51	};
51	52
52	53	static const struct option options[] = {
	54	+ OPT_INTEGER( 'b', "buckets", &params.nbuckets, "Specify amount of hash buckets"),
	55	+ OPT_BOOLEAN( 'I', "immutable", &params.buckets_immutable, "Make the hash buckets immutable"),
53	56	OPT_UINTEGER('t', "threads", &params.nthreads, "Specify amount of threads"),
54	57	OPT_UINTEGER('w', "nwakes", &params.nwakes, "Specify amount of threads to wake at once"),
55	58	OPT_BOOLEAN( 's', "silent", &params.silent, "Silent mode: do not display data/details"),
@@ -93,6 +96,7 @@ static void print_summary(void) · 37 unmodified lines
93	96	params.nthreads,
94	97	waketime_avg / USEC_PER_MSEC,
95	98	rel_stddev_stats(waketime_stddev, waketime_avg));
	99	+ futex_print_nbuckets(&params);
96	100	}
97	101
98	102	static void block_threads(pthread_t w, struct perf_cpu_map cpu)

tools/perf/bench/futex.c+67−0

@@ -0,0 +1,67 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <err.h>
	3	+#include <stdio.h>
	4	+#include <stdlib.h>
	5	+#include <linux/prctl.h>
	6	+#include <sys/prctl.h>
	7	+
	8	+#include "futex.h"
	9	+
	10	+void futex_set_nbuckets_param(struct bench_futex_parameters *params)
	11	+{
	12	+ unsigned long flags;
	13	+ int ret;
	14	+
	15	+ if (params->nbuckets < 0)
	16	+ return;
	17	+
	18	+ flags = params->buckets_immutable ? FH_FLAG_IMMUTABLE : 0;
	19	+ ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, params->nbuckets, flags);
	20	+ if (ret) {
	21	+ printf("Requesting %d hash buckets failed: %d/%m\n",
	22	+ params->nbuckets, ret);
	23	+ err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	24	+ }
	25	+}
	26	+
	27	+void futex_print_nbuckets(struct bench_futex_parameters *params)
	28	+{
	29	+ char *futex_hash_mode;
	30	+ int ret;
	31	+
	32	+ ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_SLOTS);
	33	+ if (params->nbuckets >= 0) {
	34	+ if (ret != params->nbuckets) {
	35	+ if (ret < 0) {
	36	+ printf("Can't query number of buckets: %m\n");
	37	+ err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	38	+ }
	39	+ printf("Requested number of hash buckets does not currently used.\n");
	40	+ printf("Requested: %d in usage: %d\n", params->nbuckets, ret);
	41	+ err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	42	+ }
	43	+ if (params->nbuckets == 0) {
	44	+ ret = asprintf(&futex_hash_mode, "Futex hashing: global hash");
	45	+ } else {
	46	+ ret = prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_IMMUTABLE);
	47	+ if (ret < 0) {
	48	+ printf("Can't check if the hash is immutable: %m\n");
	49	+ err(EXIT_FAILURE, "prctl(PR_FUTEX_HASH)");
	50	+ }
	51	+ ret = asprintf(&futex_hash_mode, "Futex hashing: %d hash buckets %s",
	52	+ params->nbuckets,
	53	+ ret == 1 ? "(immutable)" : "");
	54	+ }
	55	+ } else {
	56	+ if (ret <= 0) {
	57	+ ret = asprintf(&futex_hash_mode, "Futex hashing: global hash");
	58	+ } else {
	59	+ ret = asprintf(&futex_hash_mode, "Futex hashing: auto resized to %d buckets",
	60	+ ret);
	61	+ }
	62	+ }
	63	+ if (ret < 0)
	64	+ err(EXIT_FAILURE, "ENOMEM, futex_hash_mode");
	65	+ printf("%s\n", futex_hash_mode);
	66	+ free(futex_hash_mode);
	67	+}

tools/perf/bench/futex.h+5−0

@@ -25,6 +25,8 @@ struct bench_futex_parameters { · 24 unmodified lines
25	25	unsigned int nfutexes;
26	26	unsigned int nwakes;
27	27	unsigned int nrequeue;
	28	+ int nbuckets;
	29	+ bool buckets_immutable;
28	30	};
29	31
30	32	/**
@@ -143,4 +145,7 @@ futex_cmp_requeue_pi(u_int32_t uaddr, u_int32_t val, u_int32_t uaddr2, · 112 unmodified lines
143	145	val, opflags);
144	146	}
145	147
	148	+void futex_set_nbuckets_param(struct bench_futex_parameters *params);
	149	+void futex_print_nbuckets(struct bench_futex_parameters *params);
	150	+
146	151	#endif /* _FUTEX_H */

tools/testing/selftests/futex/functional/.gitignore+4−2

@@ -1,11 +1,13 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
	2	+futex_numa_mpol
	3	+futex_priv_hash
	4	+futex_requeue
2	5	futex_requeue_pi
3	6	futex_requeue_pi_mismatched_ops
4	7	futex_requeue_pi_signal_restart
	8	+futex_wait
5	9	futex_wait_private_mapped_file
6	10	futex_wait_timeout
7	11	futex_wait_uninitialized_heap
8	12	futex_wait_wouldblock
9		−futex_wait
10		−futex_requeue
11	13	futex_waitv

tools/testing/selftests/futex/functional/Makefile+5−2

@@ -1,7 +1,7 @@
1	1	# SPDX-License-Identifier: GPL-2.0
2	2	INCLUDES := -I../include -I../../ $(KHDR_INCLUDES)
3	3	CFLAGS := $(CFLAGS) -g -O2 -Wall -pthread $(INCLUDES) $(KHDR_INCLUDES)
4		−LDLIBS := -lpthread -lrt
	4	+LDLIBS := -lpthread -lrt -lnuma
5	5
6	6	LOCAL_HDRS := \
7	7	../include/futextest.h \
@@ -17,7 +17,10 @@ TEST_GEN_PROGS := \ · 9 unmodified lines
17	17	futex_wait_private_mapped_file \
18	18	futex_wait \
19	19	futex_requeue \
20		− futex_waitv
	20	+ futex_priv_hash \
	21	+ futex_numa_mpol \
	22	+ futex_waitv \
	23	+ futex_numa
21	24
22	25	TEST_PROGS := run.sh
23	26

tools/testing/selftests/futex/functional/futex_numa.c+262−0

@@ -0,0 +1,262 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <pthread.h>
	4	+#include <sys/shm.h>
	5	+#include <sys/mman.h>
	6	+#include <fcntl.h>
	7	+#include <stdbool.h>
	8	+#include <time.h>
	9	+#include <assert.h>
	10	+#include "logging.h"
	11	+#include "futextest.h"
	12	+#include "futex2test.h"
	13	+
	14	+typedef u_int32_t u32;
	15	+typedef int32_t s32;
	16	+typedef u_int64_t u64;
	17	+
	18	+static unsigned int fflags = (FUTEX2_SIZE_U32 \| FUTEX2_PRIVATE);
	19	+static int fnode = FUTEX_NO_NODE;
	20	+
	21	+/* fairly stupid test-and-set lock with a waiter flag */
	22	+
	23	+#define N_LOCK 0x0000001
	24	+#define N_WAITERS 0x0001000
	25	+
	26	+struct futex_numa_32 {
	27	+ union {
	28	+ u64 full;
	29	+ struct {
	30	+ u32 val;
	31	+ u32 node;
	32	+ };
	33	+ };
	34	+};
	35	+
	36	+void futex_numa_32_lock(struct futex_numa_32 *lock)
	37	+{
	38	+ for (;;) {
	39	+ struct futex_numa_32 new, old = {
	40	+ .full = __atomic_load_n(&lock->full, __ATOMIC_RELAXED),
	41	+ };
	42	+
	43	+ for (;;) {
	44	+ new = old;
	45	+ if (old.val == 0) {
	46	+ /* no waiter, no lock -> first lock, set no-node */
	47	+ new.node = fnode;
	48	+ }
	49	+ if (old.val & N_LOCK) {
	50	+ /* contention, set waiter */
	51	+ new.val \|= N_WAITERS;
	52	+ }
	53	+ new.val \|= N_LOCK;
	54	+
	55	+ /* nothing changed, ready to block */
	56	+ if (old.full == new.full)
	57	+ break;
	58	+
	59	+ /*
	60	+ * Use u64 cmpxchg to set the futex value and node in a
	61	+ * consistent manner.
	62	+ */
	63	+ if (__atomic_compare_exchange_n(&lock->full,
	64	+ &old.full, new.full,
	65	+ /* .weak */ false,
	66	+ __ATOMIC_ACQUIRE,
	67	+ __ATOMIC_RELAXED)) {
	68	+
	69	+ /* if we just set N_LOCK, we own it */
	70	+ if (!(old.val & N_LOCK))
	71	+ return;
	72	+
	73	+ /* go block */
	74	+ break;
	75	+ }
	76	+ }
	77	+
	78	+ futex2_wait(lock, new.val, fflags, NULL, 0);
	79	+ }
	80	+}
	81	+
	82	+void futex_numa_32_unlock(struct futex_numa_32 *lock)
	83	+{
	84	+ u32 val = __atomic_sub_fetch(&lock->val, N_LOCK, __ATOMIC_RELEASE);
	85	+ assert((s32)val >= 0);
	86	+ if (val & N_WAITERS) {
	87	+ int woken = futex2_wake(lock, 1, fflags);
	88	+ assert(val == N_WAITERS);
	89	+ if (!woken) {
	90	+ __atomic_compare_exchange_n(&lock->val, &val, 0U,
	91	+ false, __ATOMIC_RELAXED,
	92	+ __ATOMIC_RELAXED);
	93	+ }
	94	+ }
	95	+}
	96	+
	97	+static long nanos = 50000;
	98	+
	99	+struct thread_args {
	100	+ pthread_t tid;
	101	+ volatile int * done;
	102	+ struct futex_numa_32 *lock;
	103	+ int val;
	104	+ int val1, val2;
	105	+ int node;
	106	+};
	107	+
	108	+static void threadfn(void _arg)
	109	+{
	110	+ struct thread_args *args = _arg;
	111	+ struct timespec ts = {
	112	+ .tv_nsec = nanos,
	113	+ };
	114	+ int node;
	115	+
	116	+ while (!*args->done) {
	117	+
	118	+ futex_numa_32_lock(args->lock);
	119	+ args->val++;
	120	+
	121	+ assert(args->val1 == args->val2);
	122	+ (*args->val1)++;
	123	+ nanosleep(&ts, NULL);
	124	+ (*args->val2)++;
	125	+
	126	+ node = args->lock->node;
	127	+ futex_numa_32_unlock(args->lock);
	128	+
	129	+ if (node != args->node) {
	130	+ args->node = node;
	131	+ printf("node: %d\n", node);
	132	+ }
	133	+
	134	+ nanosleep(&ts, NULL);
	135	+ }
	136	+
	137	+ return NULL;
	138	+}
	139	+
	140	+static void contendfn(void _arg)
	141	+{
	142	+ struct thread_args *args = _arg;
	143	+
	144	+ while (!*args->done) {
	145	+ /*
	146	+ * futex2_wait() will take hb-lock, verify *var == val and
	147	+ * queue/abort. By knowingly setting val 'wrong' this will
	148	+ * abort and thereby generate hb-lock contention.
	149	+ */
	150	+ futex2_wait(&args->lock->val, ~0U, fflags, NULL, 0);
	151	+ args->val++;
	152	+ }
	153	+
	154	+ return NULL;
	155	+}
	156	+
	157	+static volatile int done = 0;
	158	+static struct futex_numa_32 lock = { .val = 0, };
	159	+static int val1, val2;
	160	+
	161	+int main(int argc, char *argv[])
	162	+{
	163	+ struct thread_args tas[512], cas[512];
	164	+ int c, t, threads = 2, contenders = 0;
	165	+ int sleeps = 10;
	166	+ int total = 0;
	167	+
	168	+ while ((c = getopt(argc, argv, "c:t:s:n:N::")) != -1) {
	169	+ switch (c) {
	170	+ case 'c':
	171	+ contenders = atoi(optarg);
	172	+ break;
	173	+ case 't':
	174	+ threads = atoi(optarg);
	175	+ break;
	176	+ case 's':
	177	+ sleeps = atoi(optarg);
	178	+ break;
	179	+ case 'n':
	180	+ nanos = atoi(optarg);
	181	+ break;
	182	+ case 'N':
	183	+ fflags \|= FUTEX2_NUMA;
	184	+ if (optarg)
	185	+ fnode = atoi(optarg);
	186	+ break;
	187	+ default:
	188	+ exit(1);
	189	+ break;
	190	+ }
	191	+ }
	192	+
	193	+ for (t = 0; t < contenders; t++) {
	194	+ struct thread_args args = calloc(1, sizeof(args));
	195	+ if (!args) {
	196	+ perror("thread_args");
	197	+ exit(-1);
	198	+ }
	199	+
	200	+ args->done = &done;
	201	+ args->lock = &lock;
	202	+ args->val1 = &val1;
	203	+ args->val2 = &val2;
	204	+ args->node = -1;
	205	+
	206	+ if (pthread_create(&args->tid, NULL, contendfn, args)) {
	207	+ perror("pthread_create");
	208	+ exit(-1);
	209	+ }
	210	+
	211	+ cas[t] = args;
	212	+ }
	213	+
	214	+ for (t = 0; t < threads; t++) {
	215	+ struct thread_args args = calloc(1, sizeof(args));
	216	+ if (!args) {
	217	+ perror("thread_args");
	218	+ exit(-1);
	219	+ }
	220	+
	221	+ args->done = &done;
	222	+ args->lock = &lock;
	223	+ args->val1 = &val1;
	224	+ args->val2 = &val2;
	225	+ args->node = -1;
	226	+
	227	+ if (pthread_create(&args->tid, NULL, threadfn, args)) {
	228	+ perror("pthread_create");
	229	+ exit(-1);
	230	+ }
	231	+
	232	+ tas[t] = args;
	233	+ }
	234	+
	235	+ sleep(sleeps);
	236	+
	237	+ done = true;
	238	+
	239	+ for (t = 0; t < threads; t++) {
	240	+ struct thread_args *args = tas[t];
	241	+
	242	+ pthread_join(args->tid, NULL);
	243	+ total += args->val;
	244	+// printf("tval: %d\n", args->val);
	245	+ }
	246	+ printf("total: %d\n", total);
	247	+
	248	+ if (contenders) {
	249	+ total = 0;
	250	+ for (t = 0; t < contenders; t++) {
	251	+ struct thread_args *args = cas[t];
	252	+
	253	+ pthread_join(args->tid, NULL);
	254	+ total += args->val;
	255	+ // printf("tval: %d\n", args->val);
	256	+ }
	257	+ printf("contenders: %d\n", total);
	258	+ }
	259	+
	260	+ return 0;
	261	+}
	262	+

tools/testing/selftests/futex/functional/futex_numa_mpol.c+231−0

@@ -0,0 +1,231 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
	2	+/*
	3	+ * Copyright (C) 2025 Sebastian Andrzej Siewior <bigeasy@linutronix.de>
	4	+ */
	5	+
	6	+#define _GNU_SOURCE
	7	+
	8	+#include <errno.h>
	9	+#include <pthread.h>
	10	+#include <stdio.h>
	11	+#include <stdlib.h>
	12	+#include <unistd.h>
	13	+#include <numa.h>
	14	+#include <numaif.h>
	15	+
	16	+#include <linux/futex.h>
	17	+#include <sys/mman.h>
	18	+
	19	+#include "logging.h"
	20	+#include "futextest.h"
	21	+#include "futex2test.h"
	22	+
	23	+#define MAX_THREADS 64
	24	+
	25	+static pthread_barrier_t barrier_main;
	26	+static pthread_t threads[MAX_THREADS];
	27	+
	28	+struct thread_args {
	29	+ void *futex_ptr;
	30	+ unsigned int flags;
	31	+ int result;
	32	+};
	33	+
	34	+static struct thread_args thread_args[MAX_THREADS];
	35	+
	36	+#ifndef FUTEX_NO_NODE
	37	+#define FUTEX_NO_NODE (-1)
	38	+#endif
	39	+
	40	+#ifndef FUTEX2_MPOL
	41	+#define FUTEX2_MPOL 0x08
	42	+#endif
	43	+
	44	+static void thread_lock_fn(void arg)
	45	+{
	46	+ struct thread_args *args = arg;
	47	+ int ret;
	48	+
	49	+ pthread_barrier_wait(&barrier_main);
	50	+ ret = futex2_wait(args->futex_ptr, 0, args->flags, NULL, 0);
	51	+ args->result = ret;
	52	+ return NULL;
	53	+}
	54	+
	55	+static void create_max_threads(void *futex_ptr)
	56	+{
	57	+ int i, ret;
	58	+
	59	+ for (i = 0; i < MAX_THREADS; i++) {
	60	+ thread_args[i].futex_ptr = futex_ptr;
	61	+ thread_args[i].flags = FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA;
	62	+ thread_args[i].result = 0;
	63	+ ret = pthread_create(&threads[i], NULL, thread_lock_fn, &thread_args[i]);
	64	+ if (ret)
	65	+ ksft_exit_fail_msg("pthread_create failed\n");
	66	+ }
	67	+}
	68	+
	69	+static void join_max_threads(void)
	70	+{
	71	+ int i, ret;
	72	+
	73	+ for (i = 0; i < MAX_THREADS; i++) {
	74	+ ret = pthread_join(threads[i], NULL);
	75	+ if (ret)
	76	+ ksft_exit_fail_msg("pthread_join failed for thread %d\n", i);
	77	+ }
	78	+}
	79	+
	80	+static void __test_futex(void *futex_ptr, int must_fail, unsigned int futex_flags)
	81	+{
	82	+ int to_wake, ret, i, need_exit = 0;
	83	+
	84	+ pthread_barrier_init(&barrier_main, NULL, MAX_THREADS + 1);
	85	+ create_max_threads(futex_ptr);
	86	+ pthread_barrier_wait(&barrier_main);
	87	+ to_wake = MAX_THREADS;
	88	+
	89	+ do {
	90	+ ret = futex2_wake(futex_ptr, to_wake, futex_flags);
	91	+ if (must_fail) {
	92	+ if (ret < 0)
	93	+ break;
	94	+ ksft_exit_fail_msg("futex2_wake(%d, 0x%x) should fail, but didn't\n",
	95	+ to_wake, futex_flags);
	96	+ }
	97	+ if (ret < 0) {
	98	+ ksft_exit_fail_msg("Failed futex2_wake(%d, 0x%x): %m\n",
	99	+ to_wake, futex_flags);
	100	+ }
	101	+ if (!ret)
	102	+ usleep(50);
	103	+ to_wake -= ret;
	104	+
	105	+ } while (to_wake);
	106	+ join_max_threads();
	107	+
	108	+ for (i = 0; i < MAX_THREADS; i++) {
	109	+ if (must_fail && thread_args[i].result != -1) {
	110	+ ksft_print_msg("Thread %d should fail but succeeded (%d)\n",
	111	+ i, thread_args[i].result);
	112	+ need_exit = 1;
	113	+ }
	114	+ if (!must_fail && thread_args[i].result != 0) {
	115	+ ksft_print_msg("Thread %d failed (%d)\n", i, thread_args[i].result);
	116	+ need_exit = 1;
	117	+ }
	118	+ }
	119	+ if (need_exit)
	120	+ ksft_exit_fail_msg("Aborting due to earlier errors.\n");
	121	+}
	122	+
	123	+static void test_futex(void *futex_ptr, int must_fail)
	124	+{
	125	+ __test_futex(futex_ptr, must_fail, FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA);
	126	+}
	127	+
	128	+static void test_futex_mpol(void *futex_ptr, int must_fail)
	129	+{
	130	+ __test_futex(futex_ptr, must_fail, FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA \| FUTEX2_MPOL);
	131	+}
	132	+
	133	+static void usage(char *prog)
	134	+{
	135	+ printf("Usage: %s\n", prog);
	136	+ printf(" -c Use color\n");
	137	+ printf(" -h Display this help message\n");
	138	+ printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n",
	139	+ VQUIET, VCRITICAL, VINFO);
	140	+}
	141	+
	142	+int main(int argc, char *argv[])
	143	+{
	144	+ struct futex32_numa *futex_numa;
	145	+ int mem_size, i;
	146	+ void *futex_ptr;
	147	+ char c;
	148	+
	149	+ while ((c = getopt(argc, argv, "chv:")) != -1) {
	150	+ switch (c) {
	151	+ case 'c':
	152	+ log_color(1);
	153	+ break;
	154	+ case 'h':
	155	+ usage(basename(argv[0]));
	156	+ exit(0);
	157	+ break;
	158	+ case 'v':
	159	+ log_verbosity(atoi(optarg));
	160	+ break;
	161	+ default:
	162	+ usage(basename(argv[0]));
	163	+ exit(1);
	164	+ }
	165	+ }
	166	+
	167	+ ksft_print_header();
	168	+ ksft_set_plan(1);
	169	+
	170	+ mem_size = sysconf(_SC_PAGE_SIZE);
	171	+ futex_ptr = mmap(NULL, mem_size, PROT_READ \| PROT_WRITE, MAP_PRIVATE \| MAP_ANONYMOUS, 0, 0);
	172	+ if (futex_ptr == MAP_FAILED)
	173	+ ksft_exit_fail_msg("mmap() for %d bytes failed\n", mem_size);
	174	+
	175	+ futex_numa = futex_ptr;
	176	+
	177	+ ksft_print_msg("Regular test\n");
	178	+ futex_numa->futex = 0;
	179	+ futex_numa->numa = FUTEX_NO_NODE;
	180	+ test_futex(futex_ptr, 0);
	181	+
	182	+ if (futex_numa->numa == FUTEX_NO_NODE)
	183	+ ksft_exit_fail_msg("NUMA node is left uninitialized\n");
	184	+
	185	+ ksft_print_msg("Memory too small\n");
	186	+ test_futex(futex_ptr + mem_size - 4, 1);
	187	+
	188	+ ksft_print_msg("Memory out of range\n");
	189	+ test_futex(futex_ptr + mem_size, 1);
	190	+
	191	+ futex_numa->numa = FUTEX_NO_NODE;
	192	+ mprotect(futex_ptr, mem_size, PROT_READ);
	193	+ ksft_print_msg("Memory, RO\n");
	194	+ test_futex(futex_ptr, 1);
	195	+
	196	+ mprotect(futex_ptr, mem_size, PROT_NONE);
	197	+ ksft_print_msg("Memory, no access\n");
	198	+ test_futex(futex_ptr, 1);
	199	+
	200	+ mprotect(futex_ptr, mem_size, PROT_READ \| PROT_WRITE);
	201	+ ksft_print_msg("Memory back to RW\n");
	202	+ test_futex(futex_ptr, 0);
	203	+
	204	+ /* MPOL test. Does not work as expected */
	205	+ for (i = 0; i < 4; i++) {
	206	+ unsigned long nodemask;
	207	+ int ret;
	208	+
	209	+ nodemask = 1 << i;
	210	+ ret = mbind(futex_ptr, mem_size, MPOL_BIND, &nodemask,
	211	+ sizeof(nodemask) * 8, 0);
	212	+ if (ret == 0) {
	213	+ ksft_print_msg("Node %d test\n", i);
	214	+ futex_numa->futex = 0;
	215	+ futex_numa->numa = FUTEX_NO_NODE;
	216	+
	217	+ ret = futex2_wake(futex_ptr, 0, FUTEX2_SIZE_U32 \| FUTEX_PRIVATE_FLAG \| FUTEX2_NUMA \| FUTEX2_MPOL);
	218	+ if (ret < 0)
	219	+ ksft_test_result_fail("Failed to wake 0 with MPOL: %m\n");
	220	+ if (0)
	221	+ test_futex_mpol(futex_numa, 0);
	222	+ if (futex_numa->numa != i) {
	223	+ ksft_test_result_fail("Returned NUMA node is %d expected %d\n",
	224	+ futex_numa->numa, i);
	225	+ }
	226	+ }
	227	+ }
	228	+ ksft_test_result_pass("NUMA MPOL tests passed\n");
	229	+ ksft_finished();
	230	+ return 0;
	231	+}

tools/testing/selftests/futex/functional/futex_priv_hash.c+292−0

@@ -0,0 +1,292 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
	2	+/*
	3	+ * Copyright (C) 2025 Sebastian Andrzej Siewior <bigeasy@linutronix.de>
	4	+ */
	5	+
	6	+#define _GNU_SOURCE
	7	+
	8	+#include <errno.h>
	9	+#include <pthread.h>
	10	+#include <stdio.h>
	11	+#include <stdlib.h>
	12	+#include <unistd.h>
	13	+
	14	+#include <linux/prctl.h>
	15	+#include <sys/prctl.h>
	16	+
	17	+#include "logging.h"
	18	+
	19	+#define MAX_THREADS 64
	20	+
	21	+static pthread_barrier_t barrier_main;
	22	+static pthread_mutex_t global_lock;
	23	+static pthread_t threads[MAX_THREADS];
	24	+static int counter;
	25	+
	26	+#ifndef PR_FUTEX_HASH
	27	+#define PR_FUTEX_HASH 78
	28	+# define PR_FUTEX_HASH_SET_SLOTS 1
	29	+# define FH_FLAG_IMMUTABLE (1ULL << 0)
	30	+# define PR_FUTEX_HASH_GET_SLOTS 2
	31	+# define PR_FUTEX_HASH_GET_IMMUTABLE 3
	32	+#endif
	33	+
	34	+static int futex_hash_slots_set(unsigned int slots, int flags)
	35	+{
	36	+ return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_SET_SLOTS, slots, flags);
	37	+}
	38	+
	39	+static int futex_hash_slots_get(void)
	40	+{
	41	+ return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_SLOTS);
	42	+}
	43	+
	44	+static int futex_hash_immutable_get(void)
	45	+{
	46	+ return prctl(PR_FUTEX_HASH, PR_FUTEX_HASH_GET_IMMUTABLE);
	47	+}
	48	+
	49	+static void futex_hash_slots_set_verify(int slots)
	50	+{
	51	+ int ret;
	52	+
	53	+ ret = futex_hash_slots_set(slots, 0);
	54	+ if (ret != 0) {
	55	+ ksft_test_result_fail("Failed to set slots to %d: %m\n", slots);
	56	+ ksft_finished();
	57	+ }
	58	+ ret = futex_hash_slots_get();
	59	+ if (ret != slots) {
	60	+ ksft_test_result_fail("Set %d slots but PR_FUTEX_HASH_GET_SLOTS returns: %d, %m\n",
	61	+ slots, ret);
	62	+ ksft_finished();
	63	+ }
	64	+ ksft_test_result_pass("SET and GET slots %d passed\n", slots);
	65	+}
	66	+
	67	+static void futex_hash_slots_set_must_fail(int slots, int flags)
	68	+{
	69	+ int ret;
	70	+
	71	+ ret = futex_hash_slots_set(slots, flags);
	72	+ ksft_test_result(ret < 0, "futex_hash_slots_set(%d, %d)\n",
	73	+ slots, flags);
	74	+}
	75	+
	76	+static void thread_return_fn(void arg)
	77	+{
	78	+ return NULL;
	79	+}
	80	+
	81	+static void thread_lock_fn(void arg)
	82	+{
	83	+ pthread_barrier_wait(&barrier_main);
	84	+
	85	+ pthread_mutex_lock(&global_lock);
	86	+ counter++;
	87	+ usleep(20);
	88	+ pthread_mutex_unlock(&global_lock);
	89	+ return NULL;
	90	+}
	91	+
	92	+static void create_max_threads(void (thread_fn)(void *))
	93	+{
	94	+ int i, ret;
	95	+
	96	+ for (i = 0; i < MAX_THREADS; i++) {
	97	+ ret = pthread_create(&threads[i], NULL, thread_fn, NULL);
	98	+ if (ret)
	99	+ ksft_exit_fail_msg("pthread_create failed: %m\n");
	100	+ }
	101	+}
	102	+
	103	+static void join_max_threads(void)
	104	+{
	105	+ int i, ret;
	106	+
	107	+ for (i = 0; i < MAX_THREADS; i++) {
	108	+ ret = pthread_join(threads[i], NULL);
	109	+ if (ret)
	110	+ ksft_exit_fail_msg("pthread_join failed for thread %d\n", i);
	111	+ }
	112	+}
	113	+
	114	+static void usage(char *prog)
	115	+{
	116	+ printf("Usage: %s\n", prog);
	117	+ printf(" -c Use color\n");
	118	+ printf(" -g Test global hash instead intead local immutable \n");
	119	+ printf(" -h Display this help message\n");
	120	+ printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n",
	121	+ VQUIET, VCRITICAL, VINFO);
	122	+}
	123	+
	124	+static const char *test_msg_auto_create = "Automatic hash bucket init on thread creation.\n";
	125	+static const char *test_msg_auto_inc = "Automatic increase with more than 16 CPUs\n";
	126	+
	127	+int main(int argc, char *argv[])
	128	+{
	129	+ int futex_slots1, futex_slotsn, online_cpus;
	130	+ pthread_mutexattr_t mutex_attr_pi;
	131	+ int use_global_hash = 0;
	132	+ int ret;
	133	+ char c;
	134	+
	135	+ while ((c = getopt(argc, argv, "cghv:")) != -1) {
	136	+ switch (c) {
	137	+ case 'c':
	138	+ log_color(1);
	139	+ break;
	140	+ case 'g':
	141	+ use_global_hash = 1;
	142	+ break;
	143	+ case 'h':
	144	+ usage(basename(argv[0]));
	145	+ exit(0);
	146	+ break;
	147	+ case 'v':
	148	+ log_verbosity(atoi(optarg));
	149	+ break;
	150	+ default:
	151	+ usage(basename(argv[0]));
	152	+ exit(1);
	153	+ }
	154	+ }
	155	+
	156	+ ksft_print_header();
	157	+ ksft_set_plan(22);
	158	+
	159	+ ret = pthread_mutexattr_init(&mutex_attr_pi);
	160	+ ret \|= pthread_mutexattr_setprotocol(&mutex_attr_pi, PTHREAD_PRIO_INHERIT);
	161	+ ret \|= pthread_mutex_init(&global_lock, &mutex_attr_pi);
	162	+ if (ret != 0) {
	163	+ ksft_exit_fail_msg("Failed to initialize pthread mutex.\n");
	164	+ }
	165	+ /* First thread, expect to be 0, not yet initialized */
	166	+ ret = futex_hash_slots_get();
	167	+ if (ret != 0)
	168	+ ksft_exit_fail_msg("futex_hash_slots_get() failed: %d, %m\n", ret);
	169	+
	170	+ ret = futex_hash_immutable_get();
	171	+ if (ret != 0)
	172	+ ksft_exit_fail_msg("futex_hash_immutable_get() failed: %d, %m\n", ret);
	173	+
	174	+ ksft_test_result_pass("Basic get slots and immutable status.\n");
	175	+ ret = pthread_create(&threads[0], NULL, thread_return_fn, NULL);
	176	+ if (ret != 0)
	177	+ ksft_exit_fail_msg("pthread_create() failed: %d, %m\n", ret);
	178	+
	179	+ ret = pthread_join(threads[0], NULL);
	180	+ if (ret != 0)
	181	+ ksft_exit_fail_msg("pthread_join() failed: %d, %m\n", ret);
	182	+
	183	+ /* First thread, has to initialiaze private hash */
	184	+ futex_slots1 = futex_hash_slots_get();
	185	+ if (futex_slots1 <= 0) {
	186	+ ksft_print_msg("Current hash buckets: %d\n", futex_slots1);
	187	+ ksft_exit_fail_msg(test_msg_auto_create);
	188	+ }
	189	+
	190	+ ksft_test_result_pass(test_msg_auto_create);
	191	+
	192	+ online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
	193	+ ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS + 1);
	194	+ if (ret != 0)
	195	+ ksft_exit_fail_msg("pthread_barrier_init failed: %m.\n");
	196	+
	197	+ ret = pthread_mutex_lock(&global_lock);
	198	+ if (ret != 0)
	199	+ ksft_exit_fail_msg("pthread_mutex_lock failed: %m.\n");
	200	+
	201	+ counter = 0;
	202	+ create_max_threads(thread_lock_fn);
	203	+ pthread_barrier_wait(&barrier_main);
	204	+
	205	+ /*
	206	+ * The current default size of hash buckets is 16. The auto increase
	207	+ * works only if more than 16 CPUs are available.
	208	+ */
	209	+ ksft_print_msg("Online CPUs: %d\n", online_cpus);
	210	+ if (online_cpus > 16) {
	211	+ futex_slotsn = futex_hash_slots_get();
	212	+ if (futex_slotsn < 0 \|\| futex_slots1 == futex_slotsn) {
	213	+ ksft_print_msg("Expected increase of hash buckets but got: %d -> %d\n",
	214	+ futex_slots1, futex_slotsn);
	215	+ ksft_exit_fail_msg(test_msg_auto_inc);
	216	+ }
	217	+ ksft_test_result_pass(test_msg_auto_inc);
	218	+ } else {
	219	+ ksft_test_result_skip(test_msg_auto_inc);
	220	+ }
	221	+ ret = pthread_mutex_unlock(&global_lock);
	222	+
	223	+ /* Once the user changes it, it has to be what is set */
	224	+ futex_hash_slots_set_verify(2);
	225	+ futex_hash_slots_set_verify(4);
	226	+ futex_hash_slots_set_verify(8);
	227	+ futex_hash_slots_set_verify(32);
	228	+ futex_hash_slots_set_verify(16);
	229	+
	230	+ ret = futex_hash_slots_set(15, 0);
	231	+ ksft_test_result(ret < 0, "Use 15 slots\n");
	232	+
	233	+ futex_hash_slots_set_verify(2);
	234	+ join_max_threads();
	235	+ ksft_test_result(counter == MAX_THREADS, "Created of waited for %d of %d threads\n",
	236	+ counter, MAX_THREADS);
	237	+ counter = 0;
	238	+ /* Once the user set something, auto reisze must be disabled */
	239	+ ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS);
	240	+
	241	+ create_max_threads(thread_lock_fn);
	242	+ join_max_threads();
	243	+
	244	+ ret = futex_hash_slots_get();
	245	+ ksft_test_result(ret == 2, "No more auto-resize after manaul setting, got %d\n",
	246	+ ret);
	247	+
	248	+ futex_hash_slots_set_must_fail(1 << 29, 0);
	249	+
	250	+ /*
	251	+ * Once the private hash has been made immutable or global hash has been requested,
	252	+ * then this requested can not be undone.
	253	+ */
	254	+ if (use_global_hash) {
	255	+ ret = futex_hash_slots_set(0, 0);
	256	+ ksft_test_result(ret == 0, "Global hash request\n");
	257	+ } else {
	258	+ ret = futex_hash_slots_set(4, FH_FLAG_IMMUTABLE);
	259	+ ksft_test_result(ret == 0, "Immutable resize to 4\n");
	260	+ }
	261	+ if (ret != 0)
	262	+ goto out;
	263	+
	264	+ futex_hash_slots_set_must_fail(4, 0);
	265	+ futex_hash_slots_set_must_fail(4, FH_FLAG_IMMUTABLE);
	266	+ futex_hash_slots_set_must_fail(8, 0);
	267	+ futex_hash_slots_set_must_fail(8, FH_FLAG_IMMUTABLE);
	268	+ futex_hash_slots_set_must_fail(0, FH_FLAG_IMMUTABLE);
	269	+ futex_hash_slots_set_must_fail(6, FH_FLAG_IMMUTABLE);
	270	+
	271	+ ret = pthread_barrier_init(&barrier_main, NULL, MAX_THREADS);
	272	+ if (ret != 0) {
	273	+ ksft_exit_fail_msg("pthread_barrier_init failed: %m\n");
	274	+ return 1;
	275	+ }
	276	+ create_max_threads(thread_lock_fn);
	277	+ join_max_threads();
	278	+
	279	+ ret = futex_hash_slots_get();
	280	+ if (use_global_hash) {
	281	+ ksft_test_result(ret == 0, "Continue to use global hash\n");
	282	+ } else {
	283	+ ksft_test_result(ret == 4, "Continue to use the 4 hash buckets\n");
	284	+ }
	285	+
	286	+ ret = futex_hash_immutable_get();
	287	+ ksft_test_result(ret == 1, "Hash reports to be immutable\n");
	288	+
	289	+out:
	290	+ ksft_finished();
	291	+ return 0;
	292	+}

tools/testing/selftests/futex/functional/run.sh+7−0

@@ -82,3 +82,10 @@ echo · 81 unmodified lines
82	82
83	83	echo
84	84	./futex_waitv $COLOR
	85	+
	86	+echo
	87	+./futex_priv_hash $COLOR
	88	+./futex_priv_hash -g $COLOR
	89	+
	90	+echo
	91	+./futex_numa_mpol $COLOR

tools/testing/selftests/futex/include/futex2test.h+70−0

@@ -8,6 +8,53 @@ · 7 unmodified lines
8	8
9	9	#define u64_to_ptr(x) ((void *)(uintptr_t)(x))
10	10
	11	+#ifndef __NR_futex_waitv
	12	+#define __NR_futex_waitv 449
	13	+struct futex_waitv {
	14	+ __u64 val;
	15	+ __u64 uaddr;
	16	+ __u32 flags;
	17	+ __u32 __reserved;
	18	+};
	19	+#endif
	20	+
	21	+#ifndef __NR_futex_wake
	22	+#define __NR_futex_wake 454
	23	+#endif
	24	+
	25	+#ifndef __NR_futex_wait
	26	+#define __NR_futex_wait 455
	27	+#endif
	28	+
	29	+#ifndef FUTEX2_SIZE_U32
	30	+#define FUTEX2_SIZE_U32 0x02
	31	+#endif
	32	+
	33	+#ifndef FUTEX2_NUMA
	34	+#define FUTEX2_NUMA 0x04
	35	+#endif
	36	+
	37	+#ifndef FUTEX2_MPOL
	38	+#define FUTEX2_MPOL 0x08
	39	+#endif
	40	+
	41	+#ifndef FUTEX2_PRIVATE
	42	+#define FUTEX2_PRIVATE FUTEX_PRIVATE_FLAG
	43	+#endif
	44	+
	45	+#ifndef FUTEX2_NO_NODE
	46	+#define FUTEX_NO_NODE (-1)
	47	+#endif
	48	+
	49	+#ifndef FUTEX_32
	50	+#define FUTEX_32 FUTEX2_SIZE_U32
	51	+#endif
	52	+
	53	+struct futex32_numa {
	54	+ futex_t futex;
	55	+ futex_t numa;
	56	+};
	57	+
11	58	/**
12	59	* futex_waitv - Wait at multiple futexes, wake on any
13	60	* @waiters: Array of waiters
@@ -20,3 +67,26 @@ static inline int futex_waitv(volatile struct futex_waitv *waiters, unsigned lon · 6 unmodified lines
20	67	{
21	68	return syscall(__NR_futex_waitv, waiters, nr_waiters, flags, timo, clockid);
22	69	}
	70	+
	71	+/*
	72	+ * futex_wait() - block on uaddr with optional timeout
	73	+ * @val: Expected value
	74	+ * @flags: FUTEX2 flags
	75	+ * @timeout: Relative timeout
	76	+ * @clockid: Clock id for the timeout
	77	+ */
	78	+static inline int futex2_wait(void *uaddr, long val, unsigned int flags,
	79	+ struct timespec *timeout, clockid_t clockid)
	80	+{
	81	+ return syscall(__NR_futex_wait, uaddr, val, ~0U, flags, timeout, clockid);
	82	+}
	83	+
	84	+/*
	85	+ * futex2_wake() - Wake a number of futexes
	86	+ * @nr: Number of threads to wake at most
	87	+ * @flags: FUTEX2 flags
	88	+ */
	89	+static inline int futex2_wake(void *uaddr, int nr, unsigned int flags)
	90	+{
	91	+ return syscall(__NR_futex_wake, uaddr, ~0U, nr, flags);
	92	+}