History

Merge tag 'x86-urgent-2026-03-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Ingo Molnar: - Fix SEV guest boot failures in certain circumstances, due to very early code relying on a BSS-zeroed variable that isn't actually zeroed yet an may contain non-zero bootup values Move the variable into the .data section go gain even earlier zeroing - Expose & allow the IBPB-on-Entry feature on SNP guests, which was not properly exposed to guests due to initial implementational caution - Fix O= build failure when CONFIG_EFI_SBAT_FILE is using relative file paths - Fix the various SNC (Sub-NUMA Clustering) topology enumeration bugs/artifacts (sched-domain build errors mostly). SNC enumeration data got more complicated with Granite Rapids X (GNR) and Clearwater Forest X (CWF), which exposed these bugs and made their effects more serious - Also use the now sane(r) SNC code to fix resctrl SNC detection bugs - Work around a historic libgcc unwinder bug in the vdso32 sigreturn code (again), which regressed during an overly aggressive recent cleanup of DWARF annotations * tag 'x86-urgent-2026-03-08' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/entry/vdso32: Work around libgcc unwinder bug x86/resctrl: Fix SNC detection x86/topo: Fix SNC topology mess x86/topo: Replace x86_has_numa_in_package x86/topo: Add topology_num_nodes_per_package() x86/numa: Store extra copy of numa_nodes_parsed x86/boot: Handle relative CONFIG_EFI_SBAT_FILE file paths x86/sev: Allow IBPB-on-Entry feature for SNP guests x86/boot/sev: Move SEV decompressor variables into the .data section

Linus Torvalds committed 8 months ago

Commit c23719a2 parents 6ff1020 b5ef09a14 files · +228 −95Browse files

Unified · Split

14 files
  1. arch/x86/boot/compressed/Makefile +1 −0
  2. arch/x86/boot/compressed/sev.c +5 −4
  3. arch/x86/boot/startup/sev-shared.c +1 −1
  4. arch/x86/coco/sev/core.c +1 −0
  5. arch/x86/entry/vdso/vdso32/sigreturn.S +30 −0
  6. arch/x86/include/asm/msr-index.h +4 −1
  7. arch/x86/include/asm/numa.h +6 −0
  8. arch/x86/include/asm/topology.h +6 −0
  9. arch/x86/kernel/cpu/common.c +3 −0
  10. arch/x86/kernel/cpu/resctrl/monitor.c +5 −31
  11. arch/x86/kernel/cpu/topology.c +11 −2
  12. arch/x86/kernel/smpboot.c +145 −56
  13. arch/x86/mm/numa.c +8 −0
  14. arch/x86/mm/srat.c +2 −0
arch/x86/boot/compressed/Makefile+1−0
@@ -113,6 +113,7 @@ vmlinux-objs-$(CONFIG_EFI_SBAT) += $(obj)/sbat.o · 112 unmodified lines
113 113
114ifdef CONFIG_EFI_SBAT114ifdef CONFIG_EFI_SBAT
115$(obj)/sbat.o: $(CONFIG_EFI_SBAT_FILE)115$(obj)/sbat.o: $(CONFIG_EFI_SBAT_FILE)
116AFLAGS_sbat.o += -I $(srctree)
116endif117endif
117 118
118$(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE119$(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE
arch/x86/boot/compressed/sev.c+5−4
@@ -28,17 +28,17 @@ · 27 unmodified lines
28#include "sev.h"28#include "sev.h"
29 29
30static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);30static struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
31struct ghcb *boot_ghcb;31struct ghcb *boot_ghcb __section(".data");
32 32
33#undef __init33#undef __init
34#define __init34#define __init
35 35
36#define __BOOT_COMPRESSED36#define __BOOT_COMPRESSED
37 37
38u8 snp_vmpl;38u8 snp_vmpl __section(".data");
39u16 ghcb_version;39u16 ghcb_version __section(".data");
40 40
41u64 boot_svsm_caa_pa;41u64 boot_svsm_caa_pa __section(".data");
42 42
43/* Include code for early handlers */43/* Include code for early handlers */
44#include "../../boot/startup/sev-shared.c"44#include "../../boot/startup/sev-shared.c"
@@ -188,6 +188,7 @@ bool sev_es_check_ghcb_fault(unsigned long address) · 143 unmodified lines
188 MSR_AMD64_SNP_RESERVED_BIT13 | \188 MSR_AMD64_SNP_RESERVED_BIT13 | \
189 MSR_AMD64_SNP_RESERVED_BIT15 | \189 MSR_AMD64_SNP_RESERVED_BIT15 | \
190 MSR_AMD64_SNP_SECURE_AVIC | \190 MSR_AMD64_SNP_SECURE_AVIC | \
191 MSR_AMD64_SNP_RESERVED_BITS19_22 | \
191 MSR_AMD64_SNP_RESERVED_MASK)192 MSR_AMD64_SNP_RESERVED_MASK)
192 193
193#ifdef CONFIG_AMD_SECURE_AVIC194#ifdef CONFIG_AMD_SECURE_AVIC
arch/x86/boot/startup/sev-shared.c+1−1
@@ -31,7 +31,7 @@ static u32 cpuid_std_range_max __ro_after_init; · 30 unmodified lines
31static u32 cpuid_hyp_range_max __ro_after_init;31static u32 cpuid_hyp_range_max __ro_after_init;
32static u32 cpuid_ext_range_max __ro_after_init;32static u32 cpuid_ext_range_max __ro_after_init;
33 33
34bool sev_snp_needs_sfw;34bool sev_snp_needs_sfw __section(".data");
35 35
36void __noreturn36void __noreturn
37sev_es_terminate(unsigned int set, unsigned int reason)37sev_es_terminate(unsigned int set, unsigned int reason)
arch/x86/coco/sev/core.c+1−0
@@ -89,6 +89,7 @@ static const char * const sev_status_feat_names[] = { · 88 unmodified lines
89 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt",89 [MSR_AMD64_SNP_VMSA_REG_PROT_BIT] = "VMSARegProt",
90 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt",90 [MSR_AMD64_SNP_SMT_PROT_BIT] = "SMTProt",
91 [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC",91 [MSR_AMD64_SNP_SECURE_AVIC_BIT] = "SecureAVIC",
92 [MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT] = "IBPBOnEntry",
92};93};
93 94
94/*95/*
arch/x86/entry/vdso/vdso32/sigreturn.S+30−0
@@ -35,9 +35,38 @@ · 34 unmodified lines
35#endif35#endif
36.endm36.endm
37 37
38/*
39 * WARNING:
40 *
41 * A bug in the libgcc unwinder as of at least gcc 15.2 (2026) means that
42 * the unwinder fails to recognize the signal frame flag.
43 *
44 * There is a hacky legacy fallback path in libgcc which ends up
45 * getting invoked instead. It happens to work as long as BOTH of the
46 * following conditions are true:
47 *
48 * 1. There is at least one byte before the each of the sigreturn
49 * functions which falls outside any function. This is enforced by
50 * an explicit nop instruction before the ALIGN.
51 * 2. The code sequences between the entry point up to and including
52 * the int $0x80 below need to match EXACTLY. Do not change them
53 * in any way. The exact byte sequences are:
54 *
55 * __kernel_sigreturn:
56 * 0: 58 pop %eax
57 * 1: b8 77 00 00 00 mov $0x77,%eax
58 * 6: cd 80 int $0x80
59 *
60 * __kernel_rt_sigreturn:
61 * 0: b8 ad 00 00 00 mov $0xad,%eax
62 * 5: cd 80 int $0x80
63 *
64 * For details, see: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=124050
65 */
38 .text66 .text
39 .globl __kernel_sigreturn67 .globl __kernel_sigreturn
40 .type __kernel_sigreturn,@function68 .type __kernel_sigreturn,@function
69 nop /* libgcc hack: see comment above */
41 ALIGN70 ALIGN
42__kernel_sigreturn:71__kernel_sigreturn:
43 STARTPROC_SIGNAL_FRAME IA32_SIGFRAME_sigcontext72 STARTPROC_SIGNAL_FRAME IA32_SIGFRAME_sigcontext
@@ -52,6 +81,7 @@ SYM_INNER_LABEL(vdso32_sigreturn_landing_pad, SYM_L_GLOBAL) · 8 unmodified lines
52 81
53 .globl __kernel_rt_sigreturn82 .globl __kernel_rt_sigreturn
54 .type __kernel_rt_sigreturn,@function83 .type __kernel_rt_sigreturn,@function
84 nop /* libgcc hack: see comment above */
55 ALIGN85 ALIGN
56__kernel_rt_sigreturn:86__kernel_rt_sigreturn:
57 STARTPROC_SIGNAL_FRAME IA32_RT_SIGFRAME_sigcontext87 STARTPROC_SIGNAL_FRAME IA32_RT_SIGFRAME_sigcontext
arch/x86/include/asm/msr-index.h+4−1
@@ -740,7 +740,10 @@ · 739 unmodified lines
740#define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT)740#define MSR_AMD64_SNP_SMT_PROT BIT_ULL(MSR_AMD64_SNP_SMT_PROT_BIT)
741#define MSR_AMD64_SNP_SECURE_AVIC_BIT 18741#define MSR_AMD64_SNP_SECURE_AVIC_BIT 18
742#define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT)742#define MSR_AMD64_SNP_SECURE_AVIC BIT_ULL(MSR_AMD64_SNP_SECURE_AVIC_BIT)
743#define MSR_AMD64_SNP_RESV_BIT 19743#define MSR_AMD64_SNP_RESERVED_BITS19_22 GENMASK_ULL(22, 19)
744#define MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT 23
745#define MSR_AMD64_SNP_IBPB_ON_ENTRY BIT_ULL(MSR_AMD64_SNP_IBPB_ON_ENTRY_BIT)
746#define MSR_AMD64_SNP_RESV_BIT 24
744#define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT)747#define MSR_AMD64_SNP_RESERVED_MASK GENMASK_ULL(63, MSR_AMD64_SNP_RESV_BIT)
745#define MSR_AMD64_SAVIC_CONTROL 0xc0010138748#define MSR_AMD64_SAVIC_CONTROL 0xc0010138
746#define MSR_AMD64_SAVIC_EN_BIT 0749#define MSR_AMD64_SAVIC_EN_BIT 0
arch/x86/include/asm/numa.h+6−0
@@ -22,6 +22,7 @@ extern int numa_off; · 21 unmodified lines
22 */22 */
23extern s16 __apicid_to_node[MAX_LOCAL_APIC];23extern s16 __apicid_to_node[MAX_LOCAL_APIC];
24extern nodemask_t numa_nodes_parsed __initdata;24extern nodemask_t numa_nodes_parsed __initdata;
25extern nodemask_t numa_phys_nodes_parsed __initdata;
25 26
26static inline void set_apicid_to_node(int apicid, s16 node)27static inline void set_apicid_to_node(int apicid, s16 node)
27{28{
@@ -48,6 +49,7 @@ extern void __init init_cpu_to_node(void); · 20 unmodified lines
48extern void numa_add_cpu(unsigned int cpu);49extern void numa_add_cpu(unsigned int cpu);
49extern void numa_remove_cpu(unsigned int cpu);50extern void numa_remove_cpu(unsigned int cpu);
50extern void init_gi_nodes(void);51extern void init_gi_nodes(void);
52extern int num_phys_nodes(void);
51#else /* CONFIG_NUMA */53#else /* CONFIG_NUMA */
52static inline void numa_set_node(int cpu, int node) { }54static inline void numa_set_node(int cpu, int node) { }
53static inline void numa_clear_node(int cpu) { }55static inline void numa_clear_node(int cpu) { }
@@ -55,6 +57,10 @@ static inline void init_cpu_to_node(void) { } · 1 unmodified line
55static inline void numa_add_cpu(unsigned int cpu) { }57static inline void numa_add_cpu(unsigned int cpu) { }
56static inline void numa_remove_cpu(unsigned int cpu) { }58static inline void numa_remove_cpu(unsigned int cpu) { }
57static inline void init_gi_nodes(void) { }59static inline void init_gi_nodes(void) { }
60static inline int num_phys_nodes(void)
61{
62 return 1;
63}
58#endif /* CONFIG_NUMA */64#endif /* CONFIG_NUMA */
59 65
60#ifdef CONFIG_DEBUG_PER_CPU_MAPS66#ifdef CONFIG_DEBUG_PER_CPU_MAPS
arch/x86/include/asm/topology.h+6−0
@@ -155,6 +155,7 @@ extern unsigned int __max_logical_packages; · 154 unmodified lines
155extern unsigned int __max_threads_per_core;155extern unsigned int __max_threads_per_core;
156extern unsigned int __num_threads_per_package;156extern unsigned int __num_threads_per_package;
157extern unsigned int __num_cores_per_package;157extern unsigned int __num_cores_per_package;
158extern unsigned int __num_nodes_per_package;
158 159
159const char *get_topology_cpu_type_name(struct cpuinfo_x86 *c);160const char *get_topology_cpu_type_name(struct cpuinfo_x86 *c);
160enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c);161enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c);
@@ -179,6 +180,11 @@ static inline unsigned int topology_num_threads_per_package(void) · 18 unmodified lines
179 return __num_threads_per_package;180 return __num_threads_per_package;
180}181}
181 182
183static inline unsigned int topology_num_nodes_per_package(void)
184{
185 return __num_nodes_per_package;
186}
187
182#ifdef CONFIG_X86_LOCAL_APIC188#ifdef CONFIG_X86_LOCAL_APIC
183int topology_get_logical_id(u32 apicid, enum x86_topology_domains at_level);189int topology_get_logical_id(u32 apicid, enum x86_topology_domains at_level);
184#else190#else
arch/x86/kernel/cpu/common.c+3−0
@@ -95,6 +95,9 @@ EXPORT_SYMBOL(__max_dies_per_package); · 94 unmodified lines
95unsigned int __max_logical_packages __ro_after_init = 1;95unsigned int __max_logical_packages __ro_after_init = 1;
96EXPORT_SYMBOL(__max_logical_packages);96EXPORT_SYMBOL(__max_logical_packages);
97 97
98unsigned int __num_nodes_per_package __ro_after_init = 1;
99EXPORT_SYMBOL(__num_nodes_per_package);
100
98unsigned int __num_cores_per_package __ro_after_init = 1;101unsigned int __num_cores_per_package __ro_after_init = 1;
99EXPORT_SYMBOL(__num_cores_per_package);102EXPORT_SYMBOL(__num_cores_per_package);
100 103
arch/x86/kernel/cpu/resctrl/monitor.c+5−31
@@ -364,7 +364,7 @@ void arch_mon_domain_online(struct rdt_resource *r, struct rdt_l3_mon_domain *d) · 363 unmodified lines
364 msr_clear_bit(MSR_RMID_SNC_CONFIG, 0);364 msr_clear_bit(MSR_RMID_SNC_CONFIG, 0);
365}365}
366 366
367/* CPU models that support MSR_RMID_SNC_CONFIG */367/* CPU models that support SNC and MSR_RMID_SNC_CONFIG */
368static const struct x86_cpu_id snc_cpu_ids[] __initconst = {368static const struct x86_cpu_id snc_cpu_ids[] __initconst = {
369 X86_MATCH_VFM(INTEL_ICELAKE_X, 0),369 X86_MATCH_VFM(INTEL_ICELAKE_X, 0),
370 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, 0),370 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, 0),
@@ -375,40 +375,14 @@ static const struct x86_cpu_id snc_cpu_ids[] __initconst = { · 4 unmodified lines
375 {}375 {}
376};376};
377 377
378/*
379 * There isn't a simple hardware bit that indicates whether a CPU is running
380 * in Sub-NUMA Cluster (SNC) mode. Infer the state by comparing the
381 * number of CPUs sharing the L3 cache with CPU0 to the number of CPUs in
382 * the same NUMA node as CPU0.
383 * It is not possible to accurately determine SNC state if the system is
384 * booted with a maxcpus=N parameter. That distorts the ratio of SNC nodes
385 * to L3 caches. It will be OK if system is booted with hyperthreading
386 * disabled (since this doesn't affect the ratio).
387 */
388static __init int snc_get_config(void)378static __init int snc_get_config(void)
389{379{
390 struct cacheinfo *ci = get_cpu_cacheinfo_level(0, RESCTRL_L3_CACHE);380 int ret = topology_num_nodes_per_package();
391 const cpumask_t *node0_cpumask;
392 int cpus_per_node, cpus_per_l3;
393 int ret;
394
395 if (!x86_match_cpu(snc_cpu_ids) || !ci)
396 return 1;
397 381
398 cpus_read_lock();382 if (ret > 1 && !x86_match_cpu(snc_cpu_ids)) {
399 if (num_online_cpus() != num_present_cpus())383 pr_warn("CoD enabled system? Resctrl not supported\n");
400 pr_warn("Some CPUs offline, SNC detection may be incorrect\n");
401 cpus_read_unlock();
402
403 node0_cpumask = cpumask_of_node(cpu_to_node(0));
404
405 cpus_per_node = cpumask_weight(node0_cpumask);
406 cpus_per_l3 = cpumask_weight(&ci->shared_cpu_map);
407
408 if (!cpus_per_node || !cpus_per_l3)
409 return 1;384 return 1;
410 385 }
411 ret = cpus_per_l3 / cpus_per_node;
412 386
413 /* sanity check: Only valid results are 1, 2, 3, 4, 6 */387 /* sanity check: Only valid results are 1, 2, 3, 4, 6 */
414 switch (ret) {388 switch (ret) {
arch/x86/kernel/cpu/topology.c+11−2
@@ -31,6 +31,7 @@ · 30 unmodified lines
31#include <asm/mpspec.h>31#include <asm/mpspec.h>
32#include <asm/msr.h>32#include <asm/msr.h>
33#include <asm/smp.h>33#include <asm/smp.h>
34#include <asm/numa.h>
34 35
35#include "cpu.h"36#include "cpu.h"
36 37
@@ -492,11 +493,19 @@ void __init topology_init_possible_cpus(void) · 455 unmodified lines
492 set_nr_cpu_ids(allowed);493 set_nr_cpu_ids(allowed);
493 494
494 cnta = domain_weight(TOPO_PKG_DOMAIN);495 cnta = domain_weight(TOPO_PKG_DOMAIN);
495 cntb = domain_weight(TOPO_DIE_DOMAIN);
496 __max_logical_packages = cnta;496 __max_logical_packages = cnta;
497
498 pr_info("Max. logical packages: %3u\n", __max_logical_packages);
499
500 cntb = num_phys_nodes();
501 __num_nodes_per_package = DIV_ROUND_UP(cntb, cnta);
502
503 pr_info("Max. logical nodes: %3u\n", cntb);
504 pr_info("Num. nodes per package:%3u\n", __num_nodes_per_package);
505
506 cntb = domain_weight(TOPO_DIE_DOMAIN);
497 __max_dies_per_package = 1U << (get_count_order(cntb) - get_count_order(cnta));507 __max_dies_per_package = 1U << (get_count_order(cntb) - get_count_order(cnta));
498 508
499 pr_info("Max. logical packages: %3u\n", cnta);
500 pr_info("Max. logical dies: %3u\n", cntb);509 pr_info("Max. logical dies: %3u\n", cntb);
501 pr_info("Max. dies per package: %3u\n", __max_dies_per_package);510 pr_info("Max. dies per package: %3u\n", __max_dies_per_package);
502 511
arch/x86/kernel/smpboot.c+145−56
@@ -468,13 +468,6 @@ static int x86_cluster_flags(void) · 467 unmodified lines
468}468}
469#endif469#endif
470 470
471/*
472 * Set if a package/die has multiple NUMA nodes inside.
473 * AMD Magny-Cours, Intel Cluster-on-Die, and Intel
474 * Sub-NUMA Clustering have this.
475 */
476static bool x86_has_numa_in_package;
477
478static struct sched_domain_topology_level x86_topology[] = {471static struct sched_domain_topology_level x86_topology[] = {
479 SDTL_INIT(tl_smt_mask, cpu_smt_flags, SMT),472 SDTL_INIT(tl_smt_mask, cpu_smt_flags, SMT),
480#ifdef CONFIG_SCHED_CLUSTER473#ifdef CONFIG_SCHED_CLUSTER
@@ -496,7 +489,7 @@ static void __init build_sched_topology(void) · 15 unmodified lines
496 * PKG domain since the NUMA domains will auto-magically create the489 * PKG domain since the NUMA domains will auto-magically create the
497 * right spanning domains based on the SLIT.490 * right spanning domains based on the SLIT.
498 */491 */
499 if (x86_has_numa_in_package) {492 if (topology_num_nodes_per_package() > 1) {
500 unsigned int pkgdom = ARRAY_SIZE(x86_topology) - 2;493 unsigned int pkgdom = ARRAY_SIZE(x86_topology) - 2;
501 494
502 memset(&x86_topology[pkgdom], 0, sizeof(x86_topology[pkgdom]));495 memset(&x86_topology[pkgdom], 0, sizeof(x86_topology[pkgdom]));
@@ -513,33 +506,149 @@ static void __init build_sched_topology(void) · 10 unmodified lines
513}506}
514 507
515#ifdef CONFIG_NUMA508#ifdef CONFIG_NUMA
516static int sched_avg_remote_distance;509/*
517static int avg_remote_numa_distance(void)510 * Test if the on-trace cluster at (N,N) is symmetric.
511 * Uses upper triangle iteration to avoid obvious duplicates.
512 */
513static bool slit_cluster_symmetric(int N)
518{514{
519 int i, j;515 int u = topology_num_nodes_per_package();
520 int distance, nr_remote, total_distance;516
521 517 for (int k = 0; k < u; k++) {
522 if (sched_avg_remote_distance > 0)518 for (int l = k; l < u; l++) {
523 return sched_avg_remote_distance;519 if (node_distance(N + k, N + l) !=
524 520 node_distance(N + l, N + k))
525 nr_remote = 0;521 return false;
526 total_distance = 0;
527 for_each_node_state(i, N_CPU) {
528 for_each_node_state(j, N_CPU) {
529 distance = node_distance(i, j);
530
531 if (distance >= REMOTE_DISTANCE) {
532 nr_remote++;
533 total_distance += distance;
534 }
535 }522 }
536 }523 }
537 if (nr_remote)
538 sched_avg_remote_distance = total_distance / nr_remote;
539 else
540 sched_avg_remote_distance = REMOTE_DISTANCE;
541 524
542 return sched_avg_remote_distance;525 return true;
526}
527
528/*
529 * Return the package-id of the cluster, or ~0 if indeterminate.
530 * Each node in the on-trace cluster should have the same package-id.
531 */
532static u32 slit_cluster_package(int N)
533{
534 int u = topology_num_nodes_per_package();
535 u32 pkg_id = ~0;
536
537 for (int n = 0; n < u; n++) {
538 const struct cpumask *cpus = cpumask_of_node(N + n);
539 int cpu;
540
541 for_each_cpu(cpu, cpus) {
542 u32 id = topology_logical_package_id(cpu);
543
544 if (pkg_id == ~0)
545 pkg_id = id;
546 if (pkg_id != id)
547 return ~0;
548 }
549 }
550
551 return pkg_id;
552}
553
554/*
555 * Validate the SLIT table is of the form expected for SNC, specifically:
556 *
557 * - each on-trace cluster should be symmetric,
558 * - each on-trace cluster should have a unique package-id.
559 *
560 * If you NUMA_EMU on top of SNC, you get to keep the pieces.
561 */
562static bool slit_validate(void)
563{
564 int u = topology_num_nodes_per_package();
565 u32 pkg_id, prev_pkg_id = ~0;
566
567 for (int pkg = 0; pkg < topology_max_packages(); pkg++) {
568 int n = pkg * u;
569
570 /*
571 * Ensure the on-trace cluster is symmetric and each cluster
572 * has a different package id.
573 */
574 if (!slit_cluster_symmetric(n))
575 return false;
576 pkg_id = slit_cluster_package(n);
577 if (pkg_id == ~0)
578 return false;
579 if (pkg && pkg_id == prev_pkg_id)
580 return false;
581
582 prev_pkg_id = pkg_id;
583 }
584
585 return true;
586}
587
588/*
589 * Compute a sanitized SLIT table for SNC; notably SNC-3 can end up with
590 * asymmetric off-trace clusters, reflecting physical assymmetries. However
591 * this leads to 'unfortunate' sched_domain configurations.
592 *
593 * For example dual socket GNR with SNC-3:
594 *
595 * node distances:
596 * node 0 1 2 3 4 5
597 * 0: 10 15 17 21 28 26
598 * 1: 15 10 15 23 26 23
599 * 2: 17 15 10 26 23 21
600 * 3: 21 28 26 10 15 17
601 * 4: 23 26 23 15 10 15
602 * 5: 26 23 21 17 15 10
603 *
604 * Fix things up by averaging out the off-trace clusters; resulting in:
605 *
606 * node 0 1 2 3 4 5
607 * 0: 10 15 17 24 24 24
608 * 1: 15 10 15 24 24 24
609 * 2: 17 15 10 24 24 24
610 * 3: 24 24 24 10 15 17
611 * 4: 24 24 24 15 10 15
612 * 5: 24 24 24 17 15 10
613 */
614static int slit_cluster_distance(int i, int j)
615{
616 static int slit_valid = -1;
617 int u = topology_num_nodes_per_package();
618 long d = 0;
619 int x, y;
620
621 if (slit_valid < 0) {
622 slit_valid = slit_validate();
623 if (!slit_valid)
624 pr_err(FW_BUG "SLIT table doesn't have the expected form for SNC -- fixup disabled!\n");
625 else
626 pr_info("Fixing up SNC SLIT table.\n");
627 }
628
629 /*
630 * Is this a unit cluster on the trace?
631 */
632 if ((i / u) == (j / u) || !slit_valid)
633 return node_distance(i, j);
634
635 /*
636 * Off-trace cluster.
637 *
638 * Notably average out the symmetric pair of off-trace clusters to
639 * ensure the resulting SLIT table is symmetric.
640 */
641 x = i - (i % u);
642 y = j - (j % u);
643
644 for (i = x; i < x + u; i++) {
645 for (j = y; j < y + u; j++) {
646 d += node_distance(i, j);
647 d += node_distance(j, i);
648 }
649 }
650
651 return d / (2*u*u);
543}652}
544 653
545int arch_sched_node_distance(int from, int to)654int arch_sched_node_distance(int from, int to)
@@ -549,34 +658,14 @@ int arch_sched_node_distance(int from, int to) · 3 unmodified lines
549 switch (boot_cpu_data.x86_vfm) {658 switch (boot_cpu_data.x86_vfm) {
550 case INTEL_GRANITERAPIDS_X:659 case INTEL_GRANITERAPIDS_X:
551 case INTEL_ATOM_DARKMONT_X:660 case INTEL_ATOM_DARKMONT_X:
552 661 if (topology_max_packages() == 1 ||
553 if (!x86_has_numa_in_package || topology_max_packages() == 1 ||662 topology_num_nodes_per_package() < 3)
554 d < REMOTE_DISTANCE)
555 return d;663 return d;
556 664
557 /*665 /*
558 * With SNC enabled, there could be too many levels of remote666 * Handle SNC-3 asymmetries.
559 * NUMA node distances, creating NUMA domain levels
560 * including local nodes and partial remote nodes.
561 *
562 * Trim finer distance tuning for NUMA nodes in remote package
563 * for the purpose of building sched domains. Group NUMA nodes
564 * in the remote package in the same sched group.
565 * Simplify NUMA domains and avoid extra NUMA levels including
566 * different remote NUMA nodes and local nodes.
567 *
568 * GNR and CWF don't expect systems with more than 2 packages
569 * and more than 2 hops between packages. Single average remote
570 * distance won't be appropriate if there are more than 2
571 * packages as average distance to different remote packages
572 * could be different.
573 */667 */
574 WARN_ONCE(topology_max_packages() > 2,668 return slit_cluster_distance(from, to);
575 "sched: Expect only up to 2 packages for GNR or CWF, "
576 "but saw %d packages when building sched domains.",
577 topology_max_packages());
578
579 d = avg_remote_numa_distance();
580 }669 }
581 return d;670 return d;
582}671}
@@ -606,7 +695,7 @@ void set_cpu_sibling_map(int cpu) · 23 unmodified lines
606 o = &cpu_data(i);695 o = &cpu_data(i);
607 696
608 if (match_pkg(c, o) && !topology_same_node(c, o))697 if (match_pkg(c, o) && !topology_same_node(c, o))
609 x86_has_numa_in_package = true;698 WARN_ON_ONCE(topology_num_nodes_per_package() == 1);
610 699
611 if ((i == cpu) || (has_smt && match_smt(c, o)))700 if ((i == cpu) || (has_smt && match_smt(c, o)))
612 link_mask(topology_sibling_cpumask, cpu, i);701 link_mask(topology_sibling_cpumask, cpu, i);
arch/x86/mm/numa.c+8−0
@@ -48,6 +48,8 @@ s16 __apicid_to_node[MAX_LOCAL_APIC] = { · 47 unmodified lines
48 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE48 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
49};49};
50 50
51nodemask_t numa_phys_nodes_parsed __initdata;
52
51int numa_cpu_node(int cpu)53int numa_cpu_node(int cpu)
52{54{
53 u32 apicid = early_per_cpu(x86_cpu_to_apicid, cpu);55 u32 apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
@@ -57,6 +59,11 @@ int numa_cpu_node(int cpu) · 3 unmodified lines
57 return NUMA_NO_NODE;59 return NUMA_NO_NODE;
58}60}
59 61
62int __init num_phys_nodes(void)
63{
64 return bitmap_weight(numa_phys_nodes_parsed.bits, MAX_NUMNODES);
65}
66
60cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];67cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
61EXPORT_SYMBOL(node_to_cpumask_map);68EXPORT_SYMBOL(node_to_cpumask_map);
62 69
@@ -210,6 +217,7 @@ static int __init dummy_numa_init(void) · 147 unmodified lines
210 0LLU, PFN_PHYS(max_pfn) - 1);217 0LLU, PFN_PHYS(max_pfn) - 1);
211 218
212 node_set(0, numa_nodes_parsed);219 node_set(0, numa_nodes_parsed);
220 node_set(0, numa_phys_nodes_parsed);
213 numa_add_memblk(0, 0, PFN_PHYS(max_pfn));221 numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
214 222
215 return 0;223 return 0;
arch/x86/mm/srat.c+2−0
@@ -57,6 +57,7 @@ acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa) · 56 unmodified lines
57 }57 }
58 set_apicid_to_node(apic_id, node);58 set_apicid_to_node(apic_id, node);
59 node_set(node, numa_nodes_parsed);59 node_set(node, numa_nodes_parsed);
60 node_set(node, numa_phys_nodes_parsed);
60 pr_debug("SRAT: PXM %u -> APIC 0x%04x -> Node %u\n", pxm, apic_id, node);61 pr_debug("SRAT: PXM %u -> APIC 0x%04x -> Node %u\n", pxm, apic_id, node);
61}62}
62 63
@@ -97,6 +98,7 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) · 34 unmodified lines
97 98
98 set_apicid_to_node(apic_id, node);99 set_apicid_to_node(apic_id, node);
99 node_set(node, numa_nodes_parsed);100 node_set(node, numa_nodes_parsed);
101 node_set(node, numa_phys_nodes_parsed);
100 pr_debug("SRAT: PXM %u -> APIC 0x%02x -> Node %u\n", pxm, apic_id, node);102 pr_debug("SRAT: PXM %u -> APIC 0x%02x -> Node %u\n", pxm, apic_id, node);
101}103}
102 104