History

futex: Implement FUTEX2_NUMA

Extend the futex2 interface to be numa aware. When FUTEX2_NUMA is specified for a futex, the user value is extended to two words (of the same size). The first is the user value we all know, the second one will be the node to place this futex on. struct futex_numa_32 { u32 val; u32 node; }; When node is set to ~0, WAIT will set it to the current node_id such that WAKE knows where to find it. If userspace corrupts the node value between WAIT and WAKE, the futex will not be found and no wakeup will happen. When FUTEX2_NUMA is not set, the node is simply an extension of the hash, such that traditional futexes are still interleaved over the nodes. This is done to avoid having to have a separate !numa hash-table. [bigeasy: ensure to have at least hashsize of 4 in futex_init(), add pr_info() for size and allocation information. Cast the naddr math to void*]

Peter Zijlstra committed 1 year ago

Commit cec199c1 parent 63e85954 files · +123 −20Browse files

Unified · Split

4 files
  1. include/linux/futex.h +3 −0
  2. include/uapi/linux/futex.h +7 −0
  3. kernel/futex/core.c +83 −17
  4. kernel/futex/futex.h +30 −3
include/linux/futex.h+3−0
@@ -34,6 +34,7 @@ union futex_key { · 33 unmodified lines
34 u64 i_seq;34 u64 i_seq;
35 unsigned long pgoff;35 unsigned long pgoff;
36 unsigned int offset;36 unsigned int offset;
37 /* unsigned int node; */
37 } shared;38 } shared;
38 struct {39 struct {
39 union {40 union {
@@ -42,11 +43,13 @@ union futex_key { · 2 unmodified lines
42 };43 };
43 unsigned long address;44 unsigned long address;
44 unsigned int offset;45 unsigned int offset;
46 /* unsigned int node; */
45 } private;47 } private;
46 struct {48 struct {
47 u64 ptr;49 u64 ptr;
48 unsigned long word;50 unsigned long word;
49 unsigned int offset;51 unsigned int offset;
52 unsigned int node; /* NOT hashed! */
50 } both;53 } both;
51};54};
52 55
include/uapi/linux/futex.h+7−0
@@ -74,6 +74,13 @@ · 73 unmodified lines
74/* do not use */74/* do not use */
75#define FUTEX_32 FUTEX2_SIZE_U32 /* historical accident :-( */75#define FUTEX_32 FUTEX2_SIZE_U32 /* historical accident :-( */
76 76
77/*
78 * When FUTEX2_NUMA doubles the futex word, the second word is a node value.
79 * The special value -1 indicates no-node. This is the same value as
80 * NUMA_NO_NODE, except that value is not ABI, this is.
81 */
82#define FUTEX_NO_NODE (-1)
83
77/*84/*
78 * Max numbers of elements in a futex_waitv array85 * Max numbers of elements in a futex_waitv array
79 */86 */
kernel/futex/core.c+83−17
@@ -36,6 +36,8 @@ · 35 unmodified lines
36#include <linux/pagemap.h>36#include <linux/pagemap.h>
37#include <linux/debugfs.h>37#include <linux/debugfs.h>
38#include <linux/plist.h>38#include <linux/plist.h>
39#include <linux/gfp.h>
40#include <linux/vmalloc.h>
39#include <linux/memblock.h>41#include <linux/memblock.h>
40#include <linux/fault-inject.h>42#include <linux/fault-inject.h>
41#include <linux/slab.h>43#include <linux/slab.h>
@@ -51,11 +53,14 @@ · 9 unmodified lines
51 * reside in the same cacheline.53 * reside in the same cacheline.
52 */54 */
53static struct {55static struct {
54 struct futex_hash_bucket *queues;
55 unsigned long hashmask;56 unsigned long hashmask;
57 unsigned int hashshift;
58 struct futex_hash_bucket *queues[MAX_NUMNODES];
56} __futex_data __read_mostly __aligned(2*sizeof(long));59} __futex_data __read_mostly __aligned(2*sizeof(long));
57#define futex_queues (__futex_data.queues)60
58#define futex_hashmask (__futex_data.hashmask)61#define futex_hashmask (__futex_data.hashmask)
62#define futex_hashshift (__futex_data.hashshift)
63#define futex_queues (__futex_data.queues)
59 64
60struct futex_private_hash {65struct futex_private_hash {
61 rcuref_t users;66 rcuref_t users;
@@ -339,15 +344,35 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph) · 277 unmodified lines
339{344{
340 struct futex_hash_bucket *hb;345 struct futex_hash_bucket *hb;
341 u32 hash;346 u32 hash;
347 int node;
342 348
343 hb = __futex_hash_private(key, fph);349 hb = __futex_hash_private(key, fph);
344 if (hb)350 if (hb)
345 return hb;351 return hb;
346 352
347 hash = jhash2((u32 *)key,353 hash = jhash2((u32 *)key,
348 offsetof(typeof(*key), both.offset) / 4,354 offsetof(typeof(*key), both.offset) / sizeof(u32),
349 key->both.offset);355 key->both.offset);
350 return &futex_queues[hash & futex_hashmask];356 node = key->both.node;
357
358 if (node == FUTEX_NO_NODE) {
359 /*
360 * In case of !FLAGS_NUMA, use some unused hash bits to pick a
361 * node -- this ensures regular futexes are interleaved across
362 * the nodes and avoids having to allocate multiple
363 * hash-tables.
364 *
365 * NOTE: this isn't perfectly uniform, but it is fast and
366 * handles sparse node masks.
367 */
368 node = (hash >> futex_hashshift) % nr_node_ids;
369 if (!node_possible(node)) {
370 node = find_next_bit_wrap(node_possible_map.bits,
371 nr_node_ids, node);
372 }
373 }
374
375 return &futex_queues[node][hash & futex_hashmask];
351}376}
352 377
353/**378/**
@@ -454,25 +479,49 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, · 100 unmodified lines
454 struct page *page;479 struct page *page;
455 struct folio *folio;480 struct folio *folio;
456 struct address_space *mapping;481 struct address_space *mapping;
457 int err, ro = 0;482 int node, err, size, ro = 0;
458 bool fshared;483 bool fshared;
459 484
460 fshared = flags & FLAGS_SHARED;485 fshared = flags & FLAGS_SHARED;
486 size = futex_size(flags);
487 if (flags & FLAGS_NUMA)
488 size *= 2;
461 489
462 /*490 /*
463 * The futex address must be "naturally" aligned.491 * The futex address must be "naturally" aligned.
464 */492 */
465 key->both.offset = address % PAGE_SIZE;493 key->both.offset = address % PAGE_SIZE;
466 if (unlikely((address % sizeof(u32)) != 0))494 if (unlikely((address % size) != 0))
467 return -EINVAL;495 return -EINVAL;
468 address -= key->both.offset;496 address -= key->both.offset;
469 497
470 if (unlikely(!access_ok(uaddr, sizeof(u32))))498 if (unlikely(!access_ok(uaddr, size)))
471 return -EFAULT;499 return -EFAULT;
472 500
473 if (unlikely(should_fail_futex(fshared)))501 if (unlikely(should_fail_futex(fshared)))
474 return -EFAULT;502 return -EFAULT;
475 503
504 if (flags & FLAGS_NUMA) {
505 u32 __user *naddr = (void *)uaddr + size / 2;
506
507 if (futex_get_value(&node, naddr))
508 return -EFAULT;
509
510 if (node == FUTEX_NO_NODE) {
511 node = numa_node_id();
512 if (futex_put_value(node, naddr))
513 return -EFAULT;
514
515 } else if (node >= MAX_NUMNODES || !node_possible(node)) {
516 return -EINVAL;
517 }
518
519 key->both.node = node;
520
521 } else {
522 key->both.node = FUTEX_NO_NODE;
523 }
524
476 /*525 /*
477 * PROCESS_PRIVATE futexes are fast.526 * PROCESS_PRIVATE futexes are fast.
478 * As the mm cannot disappear under us and the 'key' only needs527 * As the mm cannot disappear under us and the 'key' only needs
@@ -1642,24 +1691,41 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) · 1163 unmodified lines
1642static int __init futex_init(void)1691static int __init futex_init(void)
1643{1692{
1644 unsigned long hashsize, i;1693 unsigned long hashsize, i;
1645 unsigned int futex_shift;1694 unsigned int order, n;
1695 unsigned long size;
1646 1696
1647#ifdef CONFIG_BASE_SMALL1697#ifdef CONFIG_BASE_SMALL
1648 hashsize = 16;1698 hashsize = 16;
1649#else1699#else
1650 hashsize = roundup_pow_of_two(256 * num_possible_cpus());1700 hashsize = 256 * num_possible_cpus();
1701 hashsize /= num_possible_nodes();
1702 hashsize = max(4, hashsize);
1703 hashsize = roundup_pow_of_two(hashsize);
1651#endif1704#endif
1705 futex_hashshift = ilog2(hashsize);
1706 size = sizeof(struct futex_hash_bucket) * hashsize;
1707 order = get_order(size);
1652 1708
1653 futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),1709 for_each_node(n) {
1654 hashsize, 0, 0,1710 struct futex_hash_bucket *table;
1655 &futex_shift, NULL,
1656 hashsize, hashsize);
1657 hashsize = 1UL << futex_shift;
1658 1711
1659 for (i = 0; i < hashsize; i++)1712 if (order > MAX_PAGE_ORDER)
1660 futex_hash_bucket_init(&futex_queues[i], NULL);1713 table = vmalloc_huge_node(size, GFP_KERNEL, n);
1714 else
1715 table = alloc_pages_exact_nid(n, size, GFP_KERNEL);
1716
1717 BUG_ON(!table);
1718
1719 for (i = 0; i < hashsize; i++)
1720 futex_hash_bucket_init(&table[i], NULL);
1721
1722 futex_queues[n] = table;
1723 }
1661 1724
1662 futex_hashmask = hashsize - 1;1725 futex_hashmask = hashsize - 1;
1726 pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n",
1727 hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024,
1728 order > MAX_PAGE_ORDER ? "vmalloc" : "linear");
1663 return 0;1729 return 0;
1664}1730}
1665core_initcall(futex_init);1731core_initcall(futex_init);
kernel/futex/futex.h+30−3
@@ -54,7 +54,7 @@ static inline unsigned int futex_to_flags(unsigned int op) · 53 unmodified lines
54 return flags;54 return flags;
55}55}
56 56
57#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE)57#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_NUMA | FUTEX2_PRIVATE)
58 58
59/* FUTEX2_ to FLAGS_ */59/* FUTEX2_ to FLAGS_ */
60static inline unsigned int futex2_to_flags(unsigned int flags2)60static inline unsigned int futex2_to_flags(unsigned int flags2)
@@ -87,6 +87,19 @@ static inline bool futex_flags_valid(unsigned int flags) · 26 unmodified lines
87 if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)87 if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
88 return false;88 return false;
89 89
90 /*
91 * Must be able to represent both FUTEX_NO_NODE and every valid nodeid
92 * in a futex word.
93 */
94 if (flags & FLAGS_NUMA) {
95 int bits = 8 * futex_size(flags);
96 u64 max = ~0ULL;
97
98 max >>= 64 - bits;
99 if (nr_node_ids >= max)
100 return false;
101 }
102
90 return true;103 return true;
91}104}
92 105
@@ -282,7 +295,7 @@ static inline int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 · 189 unmodified lines
282 * This looks a bit overkill, but generally just results in a couple295 * This looks a bit overkill, but generally just results in a couple
283 * of instructions.296 * of instructions.
284 */297 */
285static __always_inline int futex_read_inatomic(u32 *dest, u32 __user *from)298static __always_inline int futex_get_value(u32 *dest, u32 __user *from)
286{299{
287 u32 val;300 u32 val;
288 301
@@ -299,12 +312,26 @@ Efault: · 10 unmodified lines
299 return -EFAULT;312 return -EFAULT;
300}313}
301 314
315static __always_inline int futex_put_value(u32 val, u32 __user *to)
316{
317 if (can_do_masked_user_access())
318 to = masked_user_access_begin(to);
319 else if (!user_read_access_begin(to, sizeof(*to)))
320 return -EFAULT;
321 unsafe_put_user(val, to, Efault);
322 user_read_access_end();
323 return 0;
324Efault:
325 user_read_access_end();
326 return -EFAULT;
327}
328
302static inline int futex_get_value_locked(u32 *dest, u32 __user *from)329static inline int futex_get_value_locked(u32 *dest, u32 __user *from)
303{330{
304 int ret;331 int ret;
305 332
306 pagefault_disable();333 pagefault_disable();
307 ret = futex_read_inatomic(dest, from);334 ret = futex_get_value(dest, from);
308 pagefault_enable();335 pagefault_enable();
309 336
310 return ret;337 return ret;