NUMA topologies where the shortest path between some two nodes requires three or more hops (i.e. diameter > 2) end up being misrepresented in the scheduler topology structures. This is currently detected when booting a kernel with CONFIG_SCHED_DEBUG=y + sched_debug on the cmdline, although this will only yield a warning about sched_group spans not matching sched_domain spans: ERROR: groups don't span domain->span Add an explicit warning for that case, triggered regardless of CONFIG_SCHED_DEBUG, and decorate it with an appropriate comment. The topology described in the comment can be booted up on QEMU by appending the following to your usual QEMU incantation: -smp cores=4 \ -numa node,cpus=0,nodeid=0 -numa node,cpus=1,nodeid=1, \ -numa node,cpus=2,nodeid=2, -numa node,cpus=3,nodeid=3, \ -numa dist,src=0,dst=1,val=20, -numa dist,src=0,dst=2,val=30, \ -numa dist,src=0,dst=3,val=40, -numa dist,src=1,dst=2,val=20, \ -numa dist,src=1,dst=3,val=30, -numa dist,src=2,dst=3,val=20 A somewhat more realistic topology (6-node mesh) with the same affliction can be conjured with: -smp cores=6 \ -numa node,cpus=0,nodeid=0 -numa node,cpus=1,nodeid=1, \ -numa node,cpus=2,nodeid=2, -numa node,cpus=3,nodeid=3, \ -numa node,cpus=4,nodeid=4, -numa node,cpus=5,nodeid=5, \ -numa dist,src=0,dst=1,val=20, -numa dist,src=0,dst=2,val=30, \ -numa dist,src=0,dst=3,val=40, -numa dist,src=0,dst=4,val=30, \ -numa dist,src=0,dst=5,val=20, \ -numa dist,src=1,dst=2,val=20, -numa dist,src=1,dst=3,val=30, \ -numa dist,src=1,dst=4,val=20, -numa dist,src=1,dst=5,val=30, \ -numa dist,src=2,dst=3,val=20, -numa dist,src=2,dst=4,val=30, \ -numa dist,src=2,dst=5,val=40, \ -numa dist,src=3,dst=4,val=20, -numa dist,src=3,dst=5,val=30, \ -numa dist,src=4,dst=5,val=20
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -675,6 +675,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) · 674 unmodified lines | |||
| 675 | { | 675 | { |
| 676 | struct rq *rq = cpu_rq(cpu); | 676 | struct rq *rq = cpu_rq(cpu); |
| 677 | struct sched_domain *tmp; | 677 | struct sched_domain *tmp; |
| 678 | int numa_distance = 0; | ||
| 678 | 679 | ||
| 679 | /* Remove the sched domains which do not contribute to scheduling. */ | 680 | /* Remove the sched domains which do not contribute to scheduling. */ |
| 680 | for (tmp = sd; tmp; ) { | 681 | for (tmp = sd; tmp; ) { |
| @@ -706,6 +707,38 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) · 25 unmodified lines | |||
| 706 | sd->child = NULL; | 707 | sd->child = NULL; |
| 707 | } | 708 | } |
| 708 | 709 | ||
| 710 | for (tmp = sd; tmp; tmp = tmp->parent) | ||
| 711 | numa_distance += !!(tmp->flags & SD_NUMA); | ||
| 712 | |||
| 713 | /* | ||
| 714 | * FIXME: Diameter >=3 is misrepresented. | ||
| 715 | * | ||
| 716 | * Smallest diameter=3 topology is: | ||
| 717 | * | ||
| 718 | * node 0 1 2 3 | ||
| 719 | * 0: 10 20 30 40 | ||
| 720 | * 1: 20 10 20 30 | ||
| 721 | * 2: 30 20 10 20 | ||
| 722 | * 3: 40 30 20 10 | ||
| 723 | * | ||
| 724 | * 0 --- 1 --- 2 --- 3 | ||
| 725 | * | ||
| 726 | * NUMA-3 0-3 N/A N/A 0-3 | ||
| 727 | * groups: {0-2},{1-3} {1-3},{0-2} | ||
| 728 | * | ||
| 729 | * NUMA-2 0-2 0-3 0-3 1-3 | ||
| 730 | * groups: {0-1},{1-3} {0-2},{2-3} {1-3},{0-1} {2-3},{0-2} | ||
| 731 | * | ||
| 732 | * NUMA-1 0-1 0-2 1-3 2-3 | ||
| 733 | * groups: {0},{1} {1},{2},{0} {2},{3},{1} {3},{2} | ||
| 734 | * | ||
| 735 | * NUMA-0 0 1 2 3 | ||
| 736 | * | ||
| 737 | * The NUMA-2 groups for nodes 0 and 3 are obviously buggered, as the | ||
| 738 | * group span isn't a subset of the domain span. | ||
| 739 | */ | ||
| 740 | WARN_ONCE(numa_distance > 2, "Shortest NUMA path spans too many nodes\n"); | ||
| 741 | |||
| 709 | sched_domain_debug(sd, cpu); | 742 | sched_domain_debug(sd, cpu); |
| 710 | 743 | ||
| 711 | rq_attach_root(rq, rd); | 744 | rq_attach_root(rq, rd); |