History

sched/numa: Prefer using an idle CPU as a migration target instead of comparing tasks

task_numa_find_cpu() can scan a node multiple times. Minimally it scans to gather statistics and later to find a suitable target. In some cases, the second scan will simply pick an idle CPU if the load is not imbalanced. This patch caches information on an idle core while gathering statistics and uses it immediately if load is not imbalanced to avoid a second scan of the node runqueues. Preference is given to an idle core rather than an idle SMT sibling to avoid packing HT siblings due to linearly scanning the node cpumask. As a side-effect, even when the second scan is necessary, the importance of using select_idle_sibling is much reduced because information on idle CPUs is cached and can be reused. Note that this patch actually makes is harder to move to an idle CPU as multiple tasks can race for the same idle CPU due to a race checking numa_migrate_on. This is addressed in the next patch.

Mel Gorman committed 6 years ago

Commit ff7db0b1 parent 070f5e81 file · +102 −17Browse files

Showing the changes to kernel (1 of 1 file). Show all

Unified · Split

kernel/sched/fair.c+102−17
@@ -1500,8 +1500,29 @@ struct numa_stats { · 1499 unmodified lines
1500 unsigned int nr_running;1500 unsigned int nr_running;
1501 unsigned int weight;1501 unsigned int weight;
1502 enum numa_type node_type;1502 enum numa_type node_type;
1503 int idle_cpu;
1503};1504};
1504 1505
1506static inline bool is_core_idle(int cpu)
1507{
1508#ifdef CONFIG_SCHED_SMT
1509 int sibling;
1510
1511 for_each_cpu(sibling, cpu_smt_mask(cpu)) {
1512 if (cpu == sibling)
1513 continue;
1514
1515 if (!idle_cpu(cpu))
1516 return false;
1517 }
1518#endif
1519
1520 return true;
1521}
1522
1523/* Forward declarations of select_idle_sibling helpers */
1524static inline bool test_idle_cores(int cpu, bool def);
1525
1505struct task_numa_env {1526struct task_numa_env {
1506 struct task_struct *p;1527 struct task_struct *p;
1507 1528
@@ -1537,15 +1558,39 @@ numa_type numa_classify(unsigned int imbalance_pct, · 29 unmodified lines
1537 return node_fully_busy;1558 return node_fully_busy;
1538}1559}
1539 1560
1561static inline int numa_idle_core(int idle_core, int cpu)
1562{
1563#ifdef CONFIG_SCHED_SMT
1564 if (!static_branch_likely(&sched_smt_present) ||
1565 idle_core >= 0 || !test_idle_cores(cpu, false))
1566 return idle_core;
1567
1568 /*
1569 * Prefer cores instead of packing HT siblings
1570 * and triggering future load balancing.
1571 */
1572 if (is_core_idle(cpu))
1573 idle_core = cpu;
1574#endif
1575
1576 return idle_core;
1577}
1578
1540/*1579/*
1541 * XXX borrowed from update_sg_lb_stats1580 * Gather all necessary information to make NUMA balancing placement
1581 * decisions that are compatible with standard load balancer. This
1582 * borrows code and logic from update_sg_lb_stats but sharing a
1583 * common implementation is impractical.
1542 */1584 */
1543static void update_numa_stats(struct task_numa_env *env,1585static void update_numa_stats(struct task_numa_env *env,
1544 struct numa_stats *ns, int nid)1586 struct numa_stats *ns, int nid,
1587 bool find_idle)
1545{1588{
1546 int cpu;1589 int cpu, idle_core = -1;
1547 1590
1548 memset(ns, 0, sizeof(*ns));1591 memset(ns, 0, sizeof(*ns));
1592 ns->idle_cpu = -1;
1593
1549 for_each_cpu(cpu, cpumask_of_node(nid)) {1594 for_each_cpu(cpu, cpumask_of_node(nid)) {
1550 struct rq *rq = cpu_rq(cpu);1595 struct rq *rq = cpu_rq(cpu);
1551 1596
@@ -1553,11 +1598,25 @@ static void update_numa_stats(struct task_numa_env *env, · 1 unmodified line
1553 ns->util += cpu_util(cpu);1598 ns->util += cpu_util(cpu);
1554 ns->nr_running += rq->cfs.h_nr_running;1599 ns->nr_running += rq->cfs.h_nr_running;
1555 ns->compute_capacity += capacity_of(cpu);1600 ns->compute_capacity += capacity_of(cpu);
1601
1602 if (find_idle && !rq->nr_running && idle_cpu(cpu)) {
1603 if (READ_ONCE(rq->numa_migrate_on) ||
1604 !cpumask_test_cpu(cpu, env->p->cpus_ptr))
1605 continue;
1606
1607 if (ns->idle_cpu == -1)
1608 ns->idle_cpu = cpu;
1609
1610 idle_core = numa_idle_core(idle_core, cpu);
1611 }
1556 }1612 }
1557 1613
1558 ns->weight = cpumask_weight(cpumask_of_node(nid));1614 ns->weight = cpumask_weight(cpumask_of_node(nid));
1559 1615
1560 ns->node_type = numa_classify(env->imbalance_pct, ns);1616 ns->node_type = numa_classify(env->imbalance_pct, ns);
1617
1618 if (idle_core >= 0)
1619 ns->idle_cpu = idle_core;
1561}1620}
1562 1621
1563static void task_numa_assign(struct task_numa_env *env,1622static void task_numa_assign(struct task_numa_env *env,
@@ -1566,7 +1625,7 @@ static void task_numa_assign(struct task_numa_env *env, · 2 unmodified lines
1566 struct rq *rq = cpu_rq(env->dst_cpu);1625 struct rq *rq = cpu_rq(env->dst_cpu);
1567 1626
1568 /* Bail out if run-queue part of active NUMA balance. */1627 /* Bail out if run-queue part of active NUMA balance. */
1569 if (xchg(&rq->numa_migrate_on, 1))1628 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1))
1570 return;1629 return;
1571 1630
1572 /*1631 /*
@@ -1730,19 +1789,39 @@ static void task_numa_compare(struct task_numa_env *env, · 157 unmodified lines
1730 goto unlock;1789 goto unlock;
1731 1790
1732assign:1791assign:
1733 /*1792 /* Evaluate an idle CPU for a task numa move. */
1734 * One idle CPU per node is evaluated for a task numa move.
1735 * Call select_idle_sibling to maybe find a better one.
1736 */
1737 if (!cur) {1793 if (!cur) {
1794 int cpu = env->dst_stats.idle_cpu;
1795
1796 /* Nothing cached so current CPU went idle since the search. */
1797 if (cpu < 0)
1798 cpu = env->dst_cpu;
1799
1738 /*1800 /*
1739 * select_idle_siblings() uses an per-CPU cpumask that1801 * If the CPU is no longer truly idle and the previous best CPU
1740 * can be used from IRQ context.1802 * is, keep using it.
1741 */1803 */
1742 local_irq_disable();1804 if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
1743 env->dst_cpu = select_idle_sibling(env->p, env->src_cpu,1805 idle_cpu(env->best_cpu)) {
1806 cpu = env->best_cpu;
1807 }
1808
1809 /*
1810 * Use select_idle_sibling if the previously found idle CPU is
1811 * not idle any more.
1812 */
1813 if (!idle_cpu(cpu)) {
1814 /*
1815 * select_idle_siblings() uses an per-CPU cpumask that
1816 * can be used from IRQ context.
1817 */
1818 local_irq_disable();
1819 cpu = select_idle_sibling(env->p, env->src_cpu,
1744 env->dst_cpu);1820 env->dst_cpu);
1745 local_irq_enable();1821 local_irq_enable();
1822 }
1823
1824 env->dst_cpu = cpu;
1746 }1825 }
1747 1826
1748 task_numa_assign(env, cur, imp);1827 task_numa_assign(env, cur, imp);
@@ -1776,8 +1855,14 @@ static void task_numa_find_cpu(struct task_numa_env *env, · 27 unmodified lines
1776 imbalance = adjust_numa_imbalance(imbalance, src_running);1855 imbalance = adjust_numa_imbalance(imbalance, src_running);
1777 1856
1778 /* Use idle CPU if there is no imbalance */1857 /* Use idle CPU if there is no imbalance */
1779 if (!imbalance)1858 if (!imbalance) {
1780 maymove = true;1859 maymove = true;
1860 if (env->dst_stats.idle_cpu >= 0) {
1861 env->dst_cpu = env->dst_stats.idle_cpu;
1862 task_numa_assign(env, NULL, 0);
1863 return;
1864 }
1865 }
1781 } else {1866 } else {
1782 long src_load, dst_load, load;1867 long src_load, dst_load, load;
1783 /*1868 /*
@@ -1850,10 +1935,10 @@ static int task_numa_migrate(struct task_struct *p) · 66 unmodified lines
1850 dist = env.dist = node_distance(env.src_nid, env.dst_nid);1935 dist = env.dist = node_distance(env.src_nid, env.dst_nid);
1851 taskweight = task_weight(p, env.src_nid, dist);1936 taskweight = task_weight(p, env.src_nid, dist);
1852 groupweight = group_weight(p, env.src_nid, dist);1937 groupweight = group_weight(p, env.src_nid, dist);
1853 update_numa_stats(&env, &env.src_stats, env.src_nid);1938 update_numa_stats(&env, &env.src_stats, env.src_nid, false);
1854 taskimp = task_weight(p, env.dst_nid, dist) - taskweight;1939 taskimp = task_weight(p, env.dst_nid, dist) - taskweight;
1855 groupimp = group_weight(p, env.dst_nid, dist) - groupweight;1940 groupimp = group_weight(p, env.dst_nid, dist) - groupweight;
1856 update_numa_stats(&env, &env.dst_stats, env.dst_nid);1941 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
1857 1942
1858 /* Try to find a spot on the preferred nid. */1943 /* Try to find a spot on the preferred nid. */
1859 task_numa_find_cpu(&env, taskimp, groupimp);1944 task_numa_find_cpu(&env, taskimp, groupimp);
@@ -1886,7 +1971,7 @@ static int task_numa_migrate(struct task_struct *p) · 26 unmodified lines
1886 1971
1887 env.dist = dist;1972 env.dist = dist;
1888 env.dst_nid = nid;1973 env.dst_nid = nid;
1889 update_numa_stats(&env, &env.dst_stats, env.dst_nid);1974 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true);
1890 task_numa_find_cpu(&env, taskimp, groupimp);1975 task_numa_find_cpu(&env, taskimp, groupimp);
1891 }1976 }
1892 }1977 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t