History

sched/numa: Prefer using an idle CPU as a migration target instead of comparing tasks

task_numa_find_cpu() can scan a node multiple times. Minimally it scans to gather statistics and later to find a suitable target. In some cases, the second scan will simply pick an idle CPU if the load is not imbalanced. This patch caches information on an idle core while gathering statistics and uses it immediately if load is not imbalanced to avoid a second scan of the node runqueues. Preference is given to an idle core rather than an idle SMT sibling to avoid packing HT siblings due to linearly scanning the node cpumask. As a side-effect, even when the second scan is necessary, the importance of using select_idle_sibling is much reduced because information on idle CPUs is cached and can be reused. Note that this patch actually makes is harder to move to an idle CPU as multiple tasks can race for the same idle CPU due to a race checking numa_migrate_on. This is addressed in the next patch.

Mel Gorman committed 6 years ago

Commit ff7db0b1 parent 070f5e81 file · +102 −17Browse files

Showing the changes to kernel (1 of 1 file). Show all

Unified · Split

kernel/sched/fair.c+102−17
@@ -1500,8 +1500,29 @@ struct numa_stats { · 1499 unmodified lines
15001500 unsigned int nr_running;
15011501 unsigned int weight;
15021502 enum numa_type node_type;
1503+ int idle_cpu;
15031504 };
15041505
1506+static inline bool is_core_idle(int cpu)
1507+{
1508+#ifdef CONFIG_SCHED_SMT
1509+ int sibling;
1510+
1511+ for_each_cpu(sibling, cpu_smt_mask(cpu)) {
1512+ if (cpu == sibling)
1513+ continue;
1514+
1515+ if (!idle_cpu(cpu))
1516+ return false;
1517+ }
1518+#endif
1519+
1520+ return true;
1521+}
1522+
1523+/* Forward declarations of select_idle_sibling helpers */
1524+static inline bool test_idle_cores(int cpu, bool def);
1525+
15051526 struct task_numa_env {
15061527 struct task_struct *p;
15071528
@@ -1537,15 +1558,39 @@ numa_type numa_classify(unsigned int imbalance_pct, · 29 unmodified lines
15371558 return node_fully_busy;
15381559 }
15391560
1561+static inline int numa_idle_core(int idle_core, int cpu)
1562+{
1563+#ifdef CONFIG_SCHED_SMT
1564+ if (!static_branch_likely(&sched_smt_present) ||
1565+ idle_core >= 0 || !test_idle_cores(cpu, false))
1566+ return idle_core;
1567+
1568+ /*
1569+ * Prefer cores instead of packing HT siblings
1570+ * and triggering future load balancing.
1571+ */
1572+ if (is_core_idle(cpu))
1573+ idle_core = cpu;
1574+#endif
1575+
1576+ return idle_core;
1577+}
1578+
15401579 /*
1541− * XXX borrowed from update_sg_lb_stats
1580+ * Gather all necessary information to make NUMA balancing placement
1581+ * decisions that are compatible with standard load balancer. This
1582+ * borrows code and logic from update_sg_lb_stats but sharing a
1583+ * common implementation is impractical.
15421584 */
15431585 static void update_numa_stats(struct task_numa_env *env,
1544− struct numa_stats *ns, int nid)
1586+ struct numa_stats *ns, int nid,
1587+ bool find_idle)
15451588 {
1546− int cpu;
1589+ int cpu, idle_core = -1;
15471590
15481591 memset(ns, 0, sizeof(*ns));
1592+ ns->idle_cpu = -1;
1593+
15491594 for_each_cpu(cpu, cpumask_of_node(nid)) {
15501595 struct rq *rq = cpu_rq(cpu);
15511596
@@ -1553,11 +1598,25 @@ static void update_numa_stats(struct task_numa_env *env, · 1 unmodified line
15531598 ns->util += cpu_util(cpu);
15541599 ns->nr_running += rq->cfs.h_nr_running;
15551600 ns->compute_capacity += capacity_of(cpu);
1601+
1602+ if (find_idle && !rq->nr_running && idle_cpu(cpu)) {
1603+ if (READ_ONCE(rq->numa_migrate_on) ||
1604+ !cpumask_test_cpu(cpu, env->p->cpus_ptr))
1605+ continue;
1606+
1607+ if (ns->idle_cpu == -1)
1608+ ns->idle_cpu = cpu;
1609+
1610+ idle_core = numa_idle_core(idle_core, cpu);
1611+ }
15561612 }
15571613
15581614 ns->weight = cpumask_weight(cpumask_of_node(nid));
15591615
15601616 ns->node_type = numa_classify(env->imbalance_pct, ns);
1617+
1618+ if (idle_core >= 0)
1619+ ns->idle_cpu = idle_core;
15611620 }
15621621
15631622 static void task_numa_assign(struct task_numa_env *env,
@@ -1566,7 +1625,7 @@ static void task_numa_assign(struct task_numa_env *env, · 2 unmodified lines
15661625 struct rq *rq = cpu_rq(env->dst_cpu);
15671626
15681627 /* Bail out if run-queue part of active NUMA balance. */
1569− if (xchg(&rq->numa_migrate_on, 1))
1628+ if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1))
15701629 return;
15711630
15721631 /*
@@ -1730,19 +1789,39 @@ static void task_numa_compare(struct task_numa_env *env, · 157 unmodified lines
17301789 goto unlock;
17311790
17321791 assign:
1733− /*
1734− * One idle CPU per node is evaluated for a task numa move.
1735− * Call select_idle_sibling to maybe find a better one.
1736− */
1792+ /* Evaluate an idle CPU for a task numa move. */
17371793 if (!cur) {
1794+ int cpu = env->dst_stats.idle_cpu;
1795+
1796+ /* Nothing cached so current CPU went idle since the search. */
1797+ if (cpu < 0)
1798+ cpu = env->dst_cpu;
1799+
17381800 /*
1739− * select_idle_siblings() uses an per-CPU cpumask that
1740− * can be used from IRQ context.
1801+ * If the CPU is no longer truly idle and the previous best CPU
1802+ * is, keep using it.
17411803 */
1742− local_irq_disable();
1743− env->dst_cpu = select_idle_sibling(env->p, env->src_cpu,
1804+ if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
1805+ idle_cpu(env->best_cpu)) {
1806+ cpu = env->best_cpu;
1807+ }
1808+
1809+ /*
1810+ * Use select_idle_sibling if the previously found idle CPU is
1811+ * not idle any more.
1812+ */
1813+ if (!idle_cpu(cpu)) {
1814+ /*
1815+ * select_idle_siblings() uses an per-CPU cpumask that
1816+ * can be used from IRQ context.
1817+ */
1818+ local_irq_disable();
1819+ cpu = select_idle_sibling(env->p, env->src_cpu,
17441820 env->dst_cpu);
1745− local_irq_enable();
1821+ local_irq_enable();
1822+ }
1823+
1824+ env->dst_cpu = cpu;
17461825 }
17471826
17481827 task_numa_assign(env, cur, imp);
@@ -1776,8 +1855,14 @@ static void task_numa_find_cpu(struct task_numa_env *env, · 27 unmodified lines
17761855 imbalance = adjust_numa_imbalance(imbalance, src_running);
17771856
17781857 /* Use idle CPU if there is no imbalance */
1779− if (!imbalance)
1858+ if (!imbalance) {
17801859 maymove = true;
1860+ if (env->dst_stats.idle_cpu >= 0) {
1861+ env->dst_cpu = env->dst_stats.idle_cpu;
1862+ task_numa_assign(env, NULL, 0);
1863+ return;
1864+ }
1865+ }
17811866 } else {
17821867 long src_load, dst_load, load;
17831868 /*
@@ -1850,10 +1935,10 @@ static int task_numa_migrate(struct task_struct *p) · 66 unmodified lines
18501935 dist = env.dist = node_distance(env.src_nid, env.dst_nid);
18511936 taskweight = task_weight(p, env.src_nid, dist);
18521937 groupweight = group_weight(p, env.src_nid, dist);
1853− update_numa_stats(&env, &env.src_stats, env.src_nid);
1938+ update_numa_stats(&env, &env.src_stats, env.src_nid, false);
18541939 taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
18551940 groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
1856− update_numa_stats(&env, &env.dst_stats, env.dst_nid);
1941+ update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
18571942
18581943 /* Try to find a spot on the preferred nid. */
18591944 task_numa_find_cpu(&env, taskimp, groupimp);
@@ -1886,7 +1971,7 @@ static int task_numa_migrate(struct task_struct *p) · 26 unmodified lines
18861971
18871972 env.dist = dist;
18881973 env.dst_nid = nid;
1889− update_numa_stats(&env, &env.dst_stats, env.dst_nid);
1974+ update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
18901975 task_numa_find_cpu(&env, taskimp, groupimp);
18911976 }
18921977 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t