| @@ -10,6 +10,7 @@ · 9 unmodified lines |
| 10 | #include <linux/spinlock.h> | 10 | #include <linux/spinlock.h> |
| 11 | #include <linux/timerqueue.h> | 11 | #include <linux/timerqueue.h> |
| 12 | #include <trace/events/ipi.h> | 12 | #include <trace/events/ipi.h> |
| | 13 | #include <linux/sched/isolation.h> |
| 13 | | 14 | |
| 14 | #include "timer_migration.h" | 15 | #include "timer_migration.h" |
| 15 | #include "tick-internal.h" | 16 | #include "tick-internal.h" |
| @@ -420,14 +421,54 @@ static struct list_head *tmigr_level_list __read_mostly; · 404 unmodified lines |
| 420 | static unsigned int tmigr_hierarchy_levels __read_mostly; | 421 | static unsigned int tmigr_hierarchy_levels __read_mostly; |
| 421 | static unsigned int tmigr_crossnode_level __read_mostly; | 422 | static unsigned int tmigr_crossnode_level __read_mostly; |
| 422 | | 423 | |
| | 424 | static struct tmigr_group *tmigr_root; |
| | 425 | |
| 423 | static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu); | 426 | static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu); |
| 424 | | 427 | |
| | 428 | /* |
| | 429 | * CPUs available for timer migration. |
| | 430 | * Protected by cpuset_mutex (with cpus_read_lock held) or cpus_write_lock. |
| | 431 | * Additionally tmigr_available_mutex serializes set/clear operations with each other. |
| | 432 | */ |
| | 433 | static cpumask_var_t tmigr_available_cpumask; |
| | 434 | static DEFINE_MUTEX(tmigr_available_mutex); |
| | 435 | |
| | 436 | /* Enabled during late initcall */ |
| | 437 | static DEFINE_STATIC_KEY_FALSE(tmigr_exclude_isolated); |
| | 438 | |
| 425 | #define TMIGR_NONE 0xFF | 439 | #define TMIGR_NONE 0xFF |
| 426 | #define BIT_CNT 8 | 440 | #define BIT_CNT 8 |
| 427 | | 441 | |
| 428 | static inline bool tmigr_is_not_available(struct tmigr_cpu *tmc) | 442 | static inline bool tmigr_is_not_available(struct tmigr_cpu *tmc) |
| 429 | { | 443 | { |
| 430 | return !(tmc->tmgroup && tmc->online); | 444 | return !(tmc->tmgroup && tmc->available); |
| | 445 | } |
| | 446 | |
| | 447 | /* |
| | 448 | * Returns true if @cpu should be excluded from the hierarchy as isolated. |
| | 449 | * Domain isolated CPUs don't participate in timer migration, nohz_full CPUs |
| | 450 | * are still part of the hierarchy but become idle (from a tick and timer |
| | 451 | * migration perspective) when they stop their tick. This lets the timekeeping |
| | 452 | * CPU handle their global timers. Marking also isolated CPUs as idle would be |
| | 453 | * too costly, hence they are completely excluded from the hierarchy. |
| | 454 | * This check is necessary, for instance, to prevent offline isolated CPUs from |
| | 455 | * being incorrectly marked as available once getting back online. |
| | 456 | * |
| | 457 | * This function returns false during early boot and the isolation logic is |
| | 458 | * enabled only after isolated CPUs are marked as unavailable at late boot. |
| | 459 | * The tick CPU can be isolated at boot, however we cannot mark it as |
| | 460 | * unavailable to avoid having no global migrator for the nohz_full CPUs. This |
| | 461 | * should be ensured by the callers of this function: implicitly from hotplug |
| | 462 | * callbacks and explicitly in tmigr_init_isolation() and |
| | 463 | * tmigr_isolated_exclude_cpumask(). |
| | 464 | */ |
| | 465 | static inline bool tmigr_is_isolated(int cpu) |
| | 466 | { |
| | 467 | if (!static_branch_unlikely(&tmigr_exclude_isolated)) |
| | 468 | return false; |
| | 469 | return (!housekeeping_cpu(cpu, HK_TYPE_DOMAIN) || |
| | 470 | cpuset_cpu_is_isolated(cpu)) && |
| | 471 | housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE); |
| 431 | } | 472 | } |
| 432 | | 473 | |
| 433 | /* | 474 | /* |
| @@ -502,11 +543,6 @@ static bool tmigr_check_lonely(struct tmigr_group *group) · 68 unmodified lines |
| 502 | * @now: timer base monotonic | 543 | * @now: timer base monotonic |
| 503 | * @check: is set if there is the need to handle remote timers; | 544 | * @check: is set if there is the need to handle remote timers; |
| 504 | * required in tmigr_requires_handle_remote() only | 545 | * required in tmigr_requires_handle_remote() only |
| 505 | * @tmc_active: this flag indicates, whether the CPU which triggers | | |
| 506 | * the hierarchy walk is !idle in the timer migration | | |
| 507 | * hierarchy. When the CPU is idle and the whole hierarchy is | | |
| 508 | * idle, only the first event of the top level has to be | | |
| 509 | * considered. | | |
| 510 | */ | 546 | */ |
| 511 | struct tmigr_walk { | 547 | struct tmigr_walk { |
| 512 | u64 nextexp; | 548 | u64 nextexp; |
| @@ -517,16 +553,13 @@ struct tmigr_walk { · 4 unmodified lines |
| 517 | unsigned long basej; | 553 | unsigned long basej; |
| 518 | u64 now; | 554 | u64 now; |
| 519 | bool check; | 555 | bool check; |
| 520 | bool tmc_active; | | |
| 521 | }; | 556 | }; |
| 522 | | 557 | |
| 523 | typedef bool (*up_f)(struct tmigr_group *, struct tmigr_group *, struct tmigr_walk *); | 558 | typedef bool (*up_f)(struct tmigr_group *, struct tmigr_group *, struct tmigr_walk *); |
| 524 | | 559 | |
| 525 | static void __walk_groups(up_f up, struct tmigr_walk *data, | 560 | static void __walk_groups_from(up_f up, struct tmigr_walk *data, |
| 526 | struct tmigr_cpu *tmc) | 561 | struct tmigr_group *child, struct tmigr_group *group) |
| 527 | { | 562 | { |
| 528 | struct tmigr_group *child = NULL, *group = tmc->tmgroup; | | |
| 529 | | | |
| 530 | do { | 563 | do { |
| 531 | WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels); | 564 | WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels); |
| 532 | | 565 | |
| @@ -544,6 +577,12 @@ static void __walk_groups(up_f up, struct tmigr_walk *data, · 11 unmodified lines |
| 544 | } while (group); | 577 | } while (group); |
| 545 | } | 578 | } |
| 546 | | 579 | |
| | 580 | static void __walk_groups(up_f up, struct tmigr_walk *data, |
| | 581 | struct tmigr_cpu *tmc) |
| | 582 | { |
| | 583 | __walk_groups_from(up, data, NULL, tmc->tmgroup); |
| | 584 | } |
| | 585 | |
| 547 | static void walk_groups(up_f up, struct tmigr_walk *data, struct tmigr_cpu *tmc) | 586 | static void walk_groups(up_f up, struct tmigr_walk *data, struct tmigr_cpu *tmc) |
| 548 | { | 587 | { |
| 549 | lockdep_assert_held(&tmc->lock); | 588 | lockdep_assert_held(&tmc->lock); |
| @@ -708,7 +747,7 @@ void tmigr_cpu_activate(void) · 158 unmodified lines |
| 708 | /* | 747 | /* |
| 709 | * Returns true, if there is nothing to be propagated to the next level | 748 | * Returns true, if there is nothing to be propagated to the next level |
| 710 | * | 749 | * |
| 711 | * @data->firstexp is set to expiry of first gobal event of the (top level of | 750 | * @data->firstexp is set to expiry of first global event of the (top level of |
| 712 | * the) hierarchy, but only when hierarchy is completely idle. | 751 | * the) hierarchy, but only when hierarchy is completely idle. |
| 713 | * | 752 | * |
| 714 | * The child and group states need to be read under the lock, to prevent a race | 753 | * The child and group states need to be read under the lock, to prevent a race |
| @@ -926,7 +965,7 @@ static void tmigr_handle_remote_cpu(unsigned int cpu, u64 now, · 211 unmodified lines |
| 926 | * updated the event takes care when hierarchy is completely | 965 | * updated the event takes care when hierarchy is completely |
| 927 | * idle. Otherwise the migrator does it as the event is enqueued. | 966 | * idle. Otherwise the migrator does it as the event is enqueued. |
| 928 | */ | 967 | */ |
| 929 | if (!tmc->online || tmc->remote || tmc->cpuevt.ignore || | 968 | if (!tmc->available || tmc->remote || tmc->cpuevt.ignore || |
| 930 | now < tmc->cpuevt.nextevt.expires) { | 969 | now < tmc->cpuevt.nextevt.expires) { |
| 931 | raw_spin_unlock_irq(&tmc->lock); | 970 | raw_spin_unlock_irq(&tmc->lock); |
| 932 | return; | 971 | return; |
| @@ -973,7 +1012,7 @@ static void tmigr_handle_remote_cpu(unsigned int cpu, u64 now, · 40 unmodified lines |
| 973 | * (See also section "Required event and timerqueue update after a | 1012 | * (See also section "Required event and timerqueue update after a |
| 974 | * remote expiry" in the documentation at the top) | 1013 | * remote expiry" in the documentation at the top) |
| 975 | */ | 1014 | */ |
| 976 | if (!tmc->online || !tmc->idle) { | 1015 | if (!tmc->available || !tmc->idle) { |
| 977 | timer_unlock_remote_bases(cpu); | 1016 | timer_unlock_remote_bases(cpu); |
| 978 | goto unlock; | 1017 | goto unlock; |
| 979 | } | 1018 | } |
| @@ -1113,15 +1152,6 @@ static bool tmigr_requires_handle_remote_up(struct tmigr_group *group, · 133 unmodified lines |
| 1113 | */ | 1152 | */ |
| 1114 | if (!tmigr_check_migrator(group, childmask)) | 1153 | if (!tmigr_check_migrator(group, childmask)) |
| 1115 | return true; | 1154 | return true; |
| 1116 | | | |
| 1117 | /* | | |
| 1118 | * When there is a parent group and the CPU which triggered the | | |
| 1119 | * hierarchy walk is not active, proceed the walk to reach the top level | | |
| 1120 | * group before reading the next_expiry value. | | |
| 1121 | */ | | |
| 1122 | if (group->parent && !data->tmc_active) | | |
| 1123 | return false; | | |
| 1124 | | | |
| 1125 | /* | 1155 | /* |
| 1126 | * The lock is required on 32bit architectures to read the variable | 1156 | * The lock is required on 32bit architectures to read the variable |
| 1127 | * consistently with a concurrent writer. On 64bit the lock is not | 1157 | * consistently with a concurrent writer. On 64bit the lock is not |
| @@ -1166,7 +1196,6 @@ bool tmigr_requires_handle_remote(void) · 38 unmodified lines |
| 1166 | data.now = get_jiffies_update(&jif); | 1196 | data.now = get_jiffies_update(&jif); |
| 1167 | data.childmask = tmc->groupmask; | 1197 | data.childmask = tmc->groupmask; |
| 1168 | data.firstexp = KTIME_MAX; | 1198 | data.firstexp = KTIME_MAX; |
| 1169 | data.tmc_active = !tmc->idle; | | |
| 1170 | data.check = false; | 1199 | data.check = false; |
| 1171 | | 1200 | |
| 1172 | /* | 1201 | /* |
| @@ -1432,38 +1461,43 @@ static long tmigr_trigger_active(void *unused) · 259 unmodified lines |
| 1432 | { | 1461 | { |
| 1433 | struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); | 1462 | struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); |
| 1434 | | 1463 | |
| 1435 | WARN_ON_ONCE(!tmc->online || tmc->idle); | 1464 | WARN_ON_ONCE(!tmc->available || tmc->idle); |
| 1436 | | 1465 | |
| 1437 | return 0; | 1466 | return 0; |
| 1438 | } | 1467 | } |
| 1439 | | 1468 | |
| 1440 | static int tmigr_cpu_offline(unsigned int cpu) | 1469 | static int tmigr_clear_cpu_available(unsigned int cpu) |
| 1441 | { | 1470 | { |
| 1442 | struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); | 1471 | struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); |
| 1443 | int migrator; | 1472 | int migrator; |
| 1444 | u64 firstexp; | 1473 | u64 firstexp; |
| 1445 | | 1474 | |
| 1446 | raw_spin_lock_irq(&tmc->lock); | 1475 | guard(mutex)(&tmigr_available_mutex); |
| 1447 | tmc->online = false; | | |
| 1448 | WRITE_ONCE(tmc->wakeup, KTIME_MAX); | | |
| 1449 | | 1476 | |
| 1450 | /* | 1477 | cpumask_clear_cpu(cpu, tmigr_available_cpumask); |
| 1451 | * CPU has to handle the local events on his own, when on the way to | 1478 | scoped_guard(raw_spinlock_irq, &tmc->lock) { |
| 1452 | * offline; Therefore nextevt value is set to KTIME_MAX | 1479 | if (!tmc->available) |
| 1453 | */ | 1480 | return 0; |
| 1454 | firstexp = __tmigr_cpu_deactivate(tmc, KTIME_MAX); | 1481 | tmc->available = false; |
| 1455 | trace_tmigr_cpu_offline(tmc); | 1482 | WRITE_ONCE(tmc->wakeup, KTIME_MAX); |
| 1456 | raw_spin_unlock_irq(&tmc->lock); | 1483 | |
| | 1484 | /* |
| | 1485 | * CPU has to handle the local events on his own, when on the way to |
| | 1486 | * offline; Therefore nextevt value is set to KTIME_MAX |
| | 1487 | */ |
| | 1488 | firstexp = __tmigr_cpu_deactivate(tmc, KTIME_MAX); |
| | 1489 | trace_tmigr_cpu_unavailable(tmc); |
| | 1490 | } |
| 1457 | | 1491 | |
| 1458 | if (firstexp != KTIME_MAX) { | 1492 | if (firstexp != KTIME_MAX) { |
| 1459 | migrator = cpumask_any_but(cpu_online_mask, cpu); | 1493 | migrator = cpumask_any(tmigr_available_cpumask); |
| 1460 | work_on_cpu(migrator, tmigr_trigger_active, NULL); | 1494 | work_on_cpu(migrator, tmigr_trigger_active, NULL); |
| 1461 | } | 1495 | } |
| 1462 | | 1496 | |
| 1463 | return 0; | 1497 | return 0; |
| 1464 | } | 1498 | } |
| 1465 | | 1499 | |
| 1466 | static int tmigr_cpu_online(unsigned int cpu) | 1500 | static int tmigr_set_cpu_available(unsigned int cpu) |
| 1467 | { | 1501 | { |
| 1468 | struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); | 1502 | struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu); |
| 1469 | | 1503 | |
| @@ -1471,16 +1505,123 @@ static int tmigr_cpu_online(unsigned int cpu) · 1 unmodified line |
| 1471 | if (WARN_ON_ONCE(!tmc->tmgroup)) | 1505 | if (WARN_ON_ONCE(!tmc->tmgroup)) |
| 1472 | return -EINVAL; | 1506 | return -EINVAL; |
| 1473 | | 1507 | |
| 1474 | raw_spin_lock_irq(&tmc->lock); | 1508 | if (tmigr_is_isolated(cpu)) |
| 1475 | trace_tmigr_cpu_online(tmc); | 1509 | return 0; |
| 1476 | tmc->idle = timer_base_is_idle(); | 1510 | |
| 1477 | if (!tmc->idle) | 1511 | guard(mutex)(&tmigr_available_mutex); |
| 1478 | __tmigr_cpu_activate(tmc); | 1512 | |
| 1479 | tmc->online = true; | 1513 | cpumask_set_cpu(cpu, tmigr_available_cpumask); |
| 1480 | raw_spin_unlock_irq(&tmc->lock); | 1514 | scoped_guard(raw_spinlock_irq, &tmc->lock) { |
| | 1515 | if (tmc->available) |
| | 1516 | return 0; |
| | 1517 | trace_tmigr_cpu_available(tmc); |
| | 1518 | tmc->idle = timer_base_is_idle(); |
| | 1519 | if (!tmc->idle) |
| | 1520 | __tmigr_cpu_activate(tmc); |
| | 1521 | tmc->available = true; |
| | 1522 | } |
| 1481 | return 0; | 1523 | return 0; |
| 1482 | } | 1524 | } |
| 1483 | | 1525 | |
| | 1526 | static void tmigr_cpu_isolate(struct work_struct *ignored) |
| | 1527 | { |
| | 1528 | tmigr_clear_cpu_available(smp_processor_id()); |
| | 1529 | } |
| | 1530 | |
| | 1531 | static void tmigr_cpu_unisolate(struct work_struct *ignored) |
| | 1532 | { |
| | 1533 | tmigr_set_cpu_available(smp_processor_id()); |
| | 1534 | } |
| | 1535 | |
| | 1536 | /** |
| | 1537 | * tmigr_isolated_exclude_cpumask - Exclude given CPUs from hierarchy |
| | 1538 | * @exclude_cpumask: the cpumask to be excluded from timer migration hierarchy |
| | 1539 | * |
| | 1540 | * This function can be called from cpuset code to provide the new set of |
| | 1541 | * isolated CPUs that should be excluded from the hierarchy. |
| | 1542 | * Online CPUs not present in exclude_cpumask but already excluded are brought |
| | 1543 | * back to the hierarchy. |
| | 1544 | * Functions to isolate/unisolate need to be called locally and can sleep. |
| | 1545 | */ |
| | 1546 | int tmigr_isolated_exclude_cpumask(struct cpumask *exclude_cpumask) |
| | 1547 | { |
| | 1548 | struct work_struct __percpu *works __free(free_percpu) = |
| | 1549 | alloc_percpu(struct work_struct); |
| | 1550 | cpumask_var_t cpumask __free(free_cpumask_var) = CPUMASK_VAR_NULL; |
| | 1551 | int cpu; |
| | 1552 | |
| | 1553 | lockdep_assert_cpus_held(); |
| | 1554 | |
| | 1555 | if (!works) |
| | 1556 | return -ENOMEM; |
| | 1557 | if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) |
| | 1558 | return -ENOMEM; |
| | 1559 | |
| | 1560 | /* |
| | 1561 | * First set previously isolated CPUs as available (unisolate). |
| | 1562 | * This cpumask contains only CPUs that switched to available now. |
| | 1563 | */ |
| | 1564 | cpumask_andnot(cpumask, cpu_online_mask, exclude_cpumask); |
| | 1565 | cpumask_andnot(cpumask, cpumask, tmigr_available_cpumask); |
| | 1566 | |
| | 1567 | for_each_cpu(cpu, cpumask) { |
| | 1568 | struct work_struct *work = per_cpu_ptr(works, cpu); |
| | 1569 | |
| | 1570 | INIT_WORK(work, tmigr_cpu_unisolate); |
| | 1571 | schedule_work_on(cpu, work); |
| | 1572 | } |
| | 1573 | for_each_cpu(cpu, cpumask) |
| | 1574 | flush_work(per_cpu_ptr(works, cpu)); |
| | 1575 | |
| | 1576 | /* |
| | 1577 | * Then clear previously available CPUs (isolate). |
| | 1578 | * This cpumask contains only CPUs that switched to not available now. |
| | 1579 | * There cannot be overlap with the newly available ones. |
| | 1580 | */ |
| | 1581 | cpumask_and(cpumask, exclude_cpumask, tmigr_available_cpumask); |
| | 1582 | cpumask_and(cpumask, cpumask, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE)); |
| | 1583 | /* |
| | 1584 | * Handle this here and not in the cpuset code because exclude_cpumask |
| | 1585 | * might include also the tick CPU if included in isolcpus. |
| | 1586 | */ |
| | 1587 | for_each_cpu(cpu, cpumask) { |
| | 1588 | if (!tick_nohz_cpu_hotpluggable(cpu)) { |
| | 1589 | cpumask_clear_cpu(cpu, cpumask); |
| | 1590 | break; |
| | 1591 | } |
| | 1592 | } |
| | 1593 | |
| | 1594 | for_each_cpu(cpu, cpumask) { |
| | 1595 | struct work_struct *work = per_cpu_ptr(works, cpu); |
| | 1596 | |
| | 1597 | INIT_WORK(work, tmigr_cpu_isolate); |
| | 1598 | schedule_work_on(cpu, work); |
| | 1599 | } |
| | 1600 | for_each_cpu(cpu, cpumask) |
| | 1601 | flush_work(per_cpu_ptr(works, cpu)); |
| | 1602 | |
| | 1603 | return 0; |
| | 1604 | } |
| | 1605 | |
| | 1606 | static int __init tmigr_init_isolation(void) |
| | 1607 | { |
| | 1608 | cpumask_var_t cpumask __free(free_cpumask_var) = CPUMASK_VAR_NULL; |
| | 1609 | |
| | 1610 | static_branch_enable(&tmigr_exclude_isolated); |
| | 1611 | |
| | 1612 | if (!housekeeping_enabled(HK_TYPE_DOMAIN)) |
| | 1613 | return 0; |
| | 1614 | if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) |
| | 1615 | return -ENOMEM; |
| | 1616 | |
| | 1617 | cpumask_andnot(cpumask, cpu_possible_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)); |
| | 1618 | |
| | 1619 | /* Protect against RCU torture hotplug testing */ |
| | 1620 | guard(cpus_read_lock)(); |
| | 1621 | return tmigr_isolated_exclude_cpumask(cpumask); |
| | 1622 | } |
| | 1623 | late_initcall(tmigr_init_isolation); |
| | 1624 | |
| 1484 | static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, | 1625 | static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, |
| 1485 | int node) | 1626 | int node) |
| 1486 | { | 1627 | { |
| @@ -1498,21 +1639,6 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, · 11 unmodified lines |
| 1498 | s.seq = 0; | 1639 | s.seq = 0; |
| 1499 | atomic_set(&group->migr_state, s.state); | 1640 | atomic_set(&group->migr_state, s.state); |
| 1500 | | 1641 | |
| 1501 | /* | | |
| 1502 | * If this is a new top-level, prepare its groupmask in advance. | | |
| 1503 | * This avoids accidents where yet another new top-level is | | |
| 1504 | * created in the future and made visible before the current groupmask. | | |
| 1505 | */ | | |
| 1506 | if (list_empty(&tmigr_level_list[lvl])) { | | |
| 1507 | group->groupmask = BIT(0); | | |
| 1508 | /* | | |
| 1509 | * The previous top level has prepared its groupmask already, | | |
| 1510 | * simply account it as the first child. | | |
| 1511 | */ | | |
| 1512 | if (lvl > 0) | | |
| 1513 | group->num_children = 1; | | |
| 1514 | } | | |
| 1515 | | | |
| 1516 | timerqueue_init_head(&group->events); | 1642 | timerqueue_init_head(&group->events); |
| 1517 | timerqueue_init(&group->groupevt.nextevt); | 1643 | timerqueue_init(&group->groupevt.nextevt); |
| 1518 | group->groupevt.nextevt.expires = KTIME_MAX; | 1644 | group->groupevt.nextevt.expires = KTIME_MAX; |
| @@ -1520,8 +1646,7 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, · 1 unmodified line |
| 1520 | group->groupevt.ignore = true; | 1646 | group->groupevt.ignore = true; |
| 1521 | } | 1647 | } |
| 1522 | | 1648 | |
| 1523 | static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node, | 1649 | static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl) |
| 1524 | unsigned int lvl) | | |
| 1525 | { | 1650 | { |
| 1526 | struct tmigr_group *tmp, *group = NULL; | 1651 | struct tmigr_group *tmp, *group = NULL; |
| 1527 | | 1652 | |
| @@ -1567,25 +1692,51 @@ static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node, · 39 unmodified lines |
| 1567 | return group; | 1692 | return group; |
| 1568 | } | 1693 | } |
| 1569 | | 1694 | |
| | 1695 | static bool tmigr_init_root(struct tmigr_group *group, bool activate) |
| | 1696 | { |
| | 1697 | if (!group->parent && group != tmigr_root) { |
| | 1698 | /* |
| | 1699 | * This is the new top-level, prepare its groupmask in advance |
| | 1700 | * to avoid accidents where yet another new top-level is |
| | 1701 | * created in the future and made visible before this groupmask. |
| | 1702 | */ |
| | 1703 | group->groupmask = BIT(0); |
| | 1704 | WARN_ON_ONCE(activate); |
| | 1705 | |
| | 1706 | return true; |
| | 1707 | } |
| | 1708 | |
| | 1709 | return false; |
| | 1710 | |
| | 1711 | } |
| | 1712 | |
| 1570 | static void tmigr_connect_child_parent(struct tmigr_group *child, | 1713 | static void tmigr_connect_child_parent(struct tmigr_group *child, |
| 1571 | struct tmigr_group *parent, | 1714 | struct tmigr_group *parent, |
| 1572 | bool activate) | 1715 | bool activate) |
| 1573 | { | 1716 | { |
| 1574 | struct tmigr_walk data; | 1717 | if (tmigr_init_root(parent, activate)) { |
| 1575 | | 1718 | /* |
| 1576 | raw_spin_lock_irq(&child->lock); | 1719 | * The previous top level had prepared its groupmask already, |
| 1577 | raw_spin_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING); | 1720 | * simply account it in advance as the first child. If some groups |
| | 1721 | * have been created between the old and new root due to node |
| | 1722 | * mismatch, the new root's child will be intialized accordingly. |
| | 1723 | */ |
| | 1724 | parent->num_children = 1; |
| | 1725 | } |
| 1578 | | 1726 | |
| 1579 | if (activate) { | 1727 | /* Connecting old root to new root ? */ |
| | 1728 | if (!parent->parent && activate) { |
| 1580 | /* | 1729 | /* |
| 1581 | * @child is the old top and @parent the new one. In this | 1730 | * @child is the old top, or in case of node mismatch, some |
| 1582 | * case groupmask is pre-initialized and @child already | 1731 | * intermediate group between the old top and the new one in |
| 1583 | * accounted, along with its new sibling corresponding to the | 1732 | * @parent. In this case the @child must be pre-accounted above |
| 1584 | * CPU going up. | 1733 | * as the first child. Its new inactive sibling corresponding |
| | 1734 | * to the CPU going up has been accounted as the second child. |
| 1585 | */ | 1735 | */ |
| 1586 | WARN_ON_ONCE(child->groupmask != BIT(0) || parent->num_children != 2); | 1736 | WARN_ON_ONCE(parent->num_children != 2); |
| | 1737 | child->groupmask = BIT(0); |
| 1587 | } else { | 1738 | } else { |
| 1588 | /* Adding @child for the CPU going up to @parent. */ | 1739 | /* Common case adding @child for the CPU going up to @parent. */ |
| 1589 | child->groupmask = BIT(parent->num_children++); | 1740 | child->groupmask = BIT(parent->num_children++); |
| 1590 | } | 1741 | } |
| 1591 | | 1742 | |
| @@ -1596,87 +1747,61 @@ static void tmigr_connect_child_parent(struct tmigr_group *child, · 4 unmodified lines |
| 1596 | */ | 1747 | */ |
| 1597 | smp_store_release(&child->parent, parent); | 1748 | smp_store_release(&child->parent, parent); |
| 1598 | | 1749 | |
| 1599 | raw_spin_unlock(&parent->lock); | | |
| 1600 | raw_spin_unlock_irq(&child->lock); | | |
| 1601 | | | |
| 1602 | trace_tmigr_connect_child_parent(child); | 1750 | trace_tmigr_connect_child_parent(child); |
| 1603 | | | |
| 1604 | if (!activate) | | |
| 1605 | return; | | |
| 1606 | | | |
| 1607 | /* | | |
| 1608 | * To prevent inconsistent states, active children need to be active in | | |
| 1609 | * the new parent as well. Inactive children are already marked inactive | | |
| 1610 | * in the parent group: | | |
| 1611 | * | | |
| 1612 | * * When new groups were created by tmigr_setup_groups() starting from | | |
| 1613 | * the lowest level (and not higher then one level below the current | | |
| 1614 | * top level), then they are not active. They will be set active when | | |
| 1615 | * the new online CPU comes active. | | |
| 1616 | * | | |
| 1617 | * * But if a new group above the current top level is required, it is | | |
| 1618 | * mandatory to propagate the active state of the already existing | | |
| 1619 | * child to the new parent. So tmigr_connect_child_parent() is | | |
| 1620 | * executed with the formerly top level group (child) and the newly | | |
| 1621 | * created group (parent). | | |
| 1622 | * | | |
| 1623 | * * It is ensured that the child is active, as this setup path is | | |
| 1624 | * executed in hotplug prepare callback. This is exectued by an | | |
| 1625 | * already connected and !idle CPU. Even if all other CPUs go idle, | | |
| 1626 | * the CPU executing the setup will be responsible up to current top | | |
| 1627 | * level group. And the next time it goes inactive, it will release | | |
| 1628 | * the new childmask and parent to subsequent walkers through this | | |
| 1629 | * @child. Therefore propagate active state unconditionally. | | |
| 1630 | */ | | |
| 1631 | data.childmask = child->groupmask; | | |
| 1632 | | | |
| 1633 | /* | | |
| 1634 | * There is only one new level per time (which is protected by | | |
| 1635 | * tmigr_mutex). When connecting the child and the parent and set the | | |
| 1636 | * child active when the parent is inactive, the parent needs to be the | | |
| 1637 | * uppermost level. Otherwise there went something wrong! | | |
| 1638 | */ | | |
| 1639 | WARN_ON(!tmigr_active_up(parent, child, &data) && parent->parent); | | |
| 1640 | } | 1751 | } |
| 1641 | | 1752 | |
| 1642 | static int tmigr_setup_groups(unsigned int cpu, unsigned int node) | 1753 | static int tmigr_setup_groups(unsigned int cpu, unsigned int node, |
| | 1754 | struct tmigr_group *start, bool activate) |
| 1643 | { | 1755 | { |
| 1644 | struct tmigr_group *group, *child, **stack; | 1756 | struct tmigr_group *group, *child, **stack; |
| 1645 | int top = 0, err = 0, i = 0; | 1757 | int i, top = 0, err = 0, start_lvl = 0; |
| 1646 | struct list_head *lvllist; | 1758 | bool root_mismatch = false; |
| 1647 | | 1759 | |
| 1648 | stack = kcalloc(tmigr_hierarchy_levels, sizeof(*stack), GFP_KERNEL); | 1760 | stack = kcalloc(tmigr_hierarchy_levels, sizeof(*stack), GFP_KERNEL); |
| 1649 | if (!stack) | 1761 | if (!stack) |
| 1650 | return -ENOMEM; | 1762 | return -ENOMEM; |
| 1651 | | 1763 | |
| 1652 | do { | 1764 | if (start) { |
| 1653 | group = tmigr_get_group(cpu, node, i); | 1765 | stack[start->level] = start; |
| | 1766 | start_lvl = start->level + 1; |
| | 1767 | } |
| | 1768 | |
| | 1769 | if (tmigr_root) |
| | 1770 | root_mismatch = tmigr_root->numa_node != node; |
| | 1771 | |
| | 1772 | for (i = start_lvl; i < tmigr_hierarchy_levels; i++) { |
| | 1773 | group = tmigr_get_group(node, i); |
| 1654 | if (IS_ERR(group)) { | 1774 | if (IS_ERR(group)) { |
| 1655 | err = PTR_ERR(group); | 1775 | err = PTR_ERR(group); |
| | 1776 | i--; |
| 1656 | break; | 1777 | break; |
| 1657 | } | 1778 | } |
| 1658 | | 1779 | |
| 1659 | top = i; | 1780 | top = i; |
| 1660 | stack[i++] = group; | 1781 | stack[i] = group; |
| 1661 | | 1782 | |
| 1662 | /* | 1783 | /* |
| 1663 | * When booting only less CPUs of a system than CPUs are | 1784 | * When booting only less CPUs of a system than CPUs are |
| 1664 | * available, not all calculated hierarchy levels are required. | 1785 | * available, not all calculated hierarchy levels are required, |
| | 1786 | * unless a node mismatch is detected. |
| 1665 | * | 1787 | * |
| 1666 | * The loop is aborted as soon as the highest level, which might | 1788 | * The loop is aborted as soon as the highest level, which might |
| 1667 | * be different from tmigr_hierarchy_levels, contains only a | 1789 | * be different from tmigr_hierarchy_levels, contains only a |
| 1668 | * single group. | 1790 | * single group, unless the nodes mismatch below tmigr_crossnode_level |
| 1669 | */ | 1791 | */ |
| 1670 | if (group->parent || list_is_singular(&tmigr_level_list[i - 1])) | 1792 | if (group->parent) |
| 1671 | break; | 1793 | break; |
| | 1794 | if ((!root_mismatch || i >= tmigr_crossnode_level) && |
| | 1795 | list_is_singular(&tmigr_level_list[i])) |
| | 1796 | break; |
| | 1797 | } |
| 1672 | | 1798 | |
| 1673 | } while (i < tmigr_hierarchy_levels); | 1799 | /* Assert single root without parent */ |
| 1674 | | 1800 | if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) |
| 1675 | /* Assert single root */ | 1801 | return -EINVAL; |
| 1676 | WARN_ON_ONCE(!err && !group->parent && !list_is_singular(&tmigr_level_list[top])); | | |
| 1677 | | 1802 | |
| 1678 | while (i > 0) { | 1803 | for (; i >= start_lvl; i--) { |
| 1679 | group = stack[--i]; | 1804 | group = stack[i]; |
| 1680 | | 1805 | |
| 1681 | if (err < 0) { | 1806 | if (err < 0) { |
| 1682 | list_del(&group->list); | 1807 | list_del(&group->list); |
| @@ -1692,12 +1817,10 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) · 9 unmodified lines |
| 1692 | if (i == 0) { | 1817 | if (i == 0) { |
| 1693 | struct tmigr_cpu *tmc = per_cpu_ptr(&tmigr_cpu, cpu); | 1818 | struct tmigr_cpu *tmc = per_cpu_ptr(&tmigr_cpu, cpu); |
| 1694 | | 1819 | |
| 1695 | raw_spin_lock_irq(&group->lock); | | |
| 1696 | | | |
| 1697 | tmc->tmgroup = group; | 1820 | tmc->tmgroup = group; |
| 1698 | tmc->groupmask = BIT(group->num_children++); | 1821 | tmc->groupmask = BIT(group->num_children++); |
| 1699 | | 1822 | |
| 1700 | raw_spin_unlock_irq(&group->lock); | 1823 | tmigr_init_root(group, activate); |
| 1701 | | 1824 | |
| 1702 | trace_tmigr_connect_cpu_parent(tmc); | 1825 | trace_tmigr_connect_cpu_parent(tmc); |
| 1703 | | 1826 | |
| @@ -1705,42 +1828,58 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) · 1 unmodified line |
| 1705 | continue; | 1828 | continue; |
| 1706 | } else { | 1829 | } else { |
| 1707 | child = stack[i - 1]; | 1830 | child = stack[i - 1]; |
| 1708 | /* Will be activated at online time */ | 1831 | tmigr_connect_child_parent(child, group, activate); |
| 1709 | tmigr_connect_child_parent(child, group, false); | | |
| 1710 | } | 1832 | } |
| | 1833 | } |
| 1711 | | 1834 | |
| 1712 | /* check if uppermost level was newly created */ | 1835 | if (err < 0) |
| 1713 | if (top != i) | 1836 | goto out; |
| 1714 | continue; | | |
| 1715 | | | |
| 1716 | WARN_ON_ONCE(top == 0); | | |
| 1717 | | 1837 | |
| 1718 | lvllist = &tmigr_level_list[top]; | 1838 | if (activate) { |
| | 1839 | struct tmigr_walk data; |
| | 1840 | union tmigr_state state; |
| 1719 | | 1841 | |
| 1720 | /* | 1842 | /* |
| 1721 | * Newly created root level should have accounted the upcoming | 1843 | * To prevent inconsistent states, active children need to be active in |
| 1722 | * CPU's child group and pre-accounted the old root. | 1844 | * the new parent as well. Inactive children are already marked inactive |
| | 1845 | * in the parent group: |
| | 1846 | * |
| | 1847 | * * When new groups were created by tmigr_setup_groups() starting from |
| | 1848 | * the lowest level, then they are not active. They will be set active |
| | 1849 | * when the new online CPU comes active. |
| | 1850 | * |
| | 1851 | * * But if new groups above the current top level are required, it is |
| | 1852 | * mandatory to propagate the active state of the already existing |
| | 1853 | * child to the new parents. So tmigr_active_up() activates the |
| | 1854 | * new parents while walking up from the old root to the new. |
| | 1855 | * |
| | 1856 | * * It is ensured that @start is active, as this setup path is |
| | 1857 | * executed in hotplug prepare callback. This is executed by an |
| | 1858 | * already connected and !idle CPU. Even if all other CPUs go idle, |
| | 1859 | * the CPU executing the setup will be responsible up to current top |
| | 1860 | * level group. And the next time it goes inactive, it will release |
| | 1861 | * the new childmask and parent to subsequent walkers through this |
| | 1862 | * @child. Therefore propagate active state unconditionally. |
| 1723 | */ | 1863 | */ |
| 1724 | if (group->num_children == 2 && list_is_singular(lvllist)) { | 1864 | state.state = atomic_read(&start->migr_state); |
| 1725 | /* | 1865 | WARN_ON_ONCE(!state.active); |
| 1726 | * The target CPU must never do the prepare work, except | 1866 | WARN_ON_ONCE(!start->parent); |
| 1727 | * on early boot when the boot CPU is the target. Otherwise | 1867 | data.childmask = start->groupmask; |
| 1728 | * it may spuriously activate the old top level group inside | 1868 | __walk_groups_from(tmigr_active_up, &data, start, start->parent); |
| 1729 | * the new one (nevertheless whether old top level group is | 1869 | } |
| 1730 | * active or not) and/or release an uninitialized childmask. | | |
| 1731 | */ | | |
| 1732 | WARN_ON_ONCE(cpu == raw_smp_processor_id()); | | |
| 1733 | | | |
| 1734 | lvllist = &tmigr_level_list[top - 1]; | | |
| 1735 | list_for_each_entry(child, lvllist, list) { | | |
| 1736 | if (child->parent) | | |
| 1737 | continue; | | |
| 1738 | | 1870 | |
| 1739 | tmigr_connect_child_parent(child, group, true); | 1871 | /* Root update */ |
| 1740 | } | 1872 | if (list_is_singular(&tmigr_level_list[top])) { |
| | 1873 | group = list_first_entry(&tmigr_level_list[top], |
| | 1874 | typeof(*group), list); |
| | 1875 | WARN_ON_ONCE(group->parent); |
| | 1876 | if (tmigr_root) { |
| | 1877 | /* Old root should be the same or below */ |
| | 1878 | WARN_ON_ONCE(tmigr_root->level > top); |
| 1741 | } | 1879 | } |
| | 1880 | tmigr_root = group; |
| 1742 | } | 1881 | } |
| 1743 | | 1882 | out: |
| 1744 | kfree(stack); | 1883 | kfree(stack); |
| 1745 | | 1884 | |
| 1746 | return err; | 1885 | return err; |
| @@ -1748,12 +1887,31 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) · 1 unmodified line |
| 1748 | | 1887 | |
| 1749 | static int tmigr_add_cpu(unsigned int cpu) | 1888 | static int tmigr_add_cpu(unsigned int cpu) |
| 1750 | { | 1889 | { |
| | 1890 | struct tmigr_group *old_root = tmigr_root; |
| 1751 | int node = cpu_to_node(cpu); | 1891 | int node = cpu_to_node(cpu); |
| 1752 | int ret; | 1892 | int ret; |
| 1753 | | 1893 | |
| 1754 | mutex_lock(&tmigr_mutex); | 1894 | guard(mutex)(&tmigr_mutex); |
| 1755 | ret = tmigr_setup_groups(cpu, node); | 1895 | |
| 1756 | mutex_unlock(&tmigr_mutex); | 1896 | ret = tmigr_setup_groups(cpu, node, NULL, false); |
| | 1897 | |
| | 1898 | /* Root has changed? Connect the old one to the new */ |
| | 1899 | if (ret >= 0 && old_root && old_root != tmigr_root) { |
| | 1900 | /* |
| | 1901 | * The target CPU must never do the prepare work, except |
| | 1902 | * on early boot when the boot CPU is the target. Otherwise |
| | 1903 | * it may spuriously activate the old top level group inside |
| | 1904 | * the new one (nevertheless whether old top level group is |
| | 1905 | * active or not) and/or release an uninitialized childmask. |
| | 1906 | */ |
| | 1907 | WARN_ON_ONCE(cpu == raw_smp_processor_id()); |
| | 1908 | /* |
| | 1909 | * The (likely) current CPU is expected to be online in the hierarchy, |
| | 1910 | * otherwise the old root may not be active as expected. |
| | 1911 | */ |
| | 1912 | WARN_ON_ONCE(!per_cpu_ptr(&tmigr_cpu, raw_smp_processor_id())->available); |
| | 1913 | ret = tmigr_setup_groups(-1, old_root->numa_node, old_root, true); |
| | 1914 | } |
| 1757 | | 1915 | |
| 1758 | return ret; | 1916 | return ret; |
| 1759 | } | 1917 | } |
| @@ -1798,6 +1956,11 @@ static int __init tmigr_init(void) · 38 unmodified lines |
| 1798 | if (ncpus == 1) | 1956 | if (ncpus == 1) |
| 1799 | return 0; | 1957 | return 0; |
| 1800 | | 1958 | |
| | 1959 | if (!zalloc_cpumask_var(&tmigr_available_cpumask, GFP_KERNEL)) { |
| | 1960 | ret = -ENOMEM; |
| | 1961 | goto err; |
| | 1962 | } |
| | 1963 | |
| 1801 | /* | 1964 | /* |
| 1802 | * Calculate the required hierarchy levels. Unfortunately there is no | 1965 | * Calculate the required hierarchy levels. Unfortunately there is no |
| 1803 | * reliable information available, unless all possible CPUs have been | 1966 | * reliable information available, unless all possible CPUs have been |
| @@ -1847,7 +2010,7 @@ static int __init tmigr_init(void) · 43 unmodified lines |
| 1847 | goto err; | 2010 | goto err; |
| 1848 | | 2011 | |
| 1849 | ret = cpuhp_setup_state(CPUHP_AP_TMIGR_ONLINE, "tmigr:online", | 2012 | ret = cpuhp_setup_state(CPUHP_AP_TMIGR_ONLINE, "tmigr:online", |
| 1850 | tmigr_cpu_online, tmigr_cpu_offline); | 2013 | tmigr_set_cpu_available, tmigr_clear_cpu_available); |
| 1851 | if (ret) | 2014 | if (ret) |
| 1852 | goto err; | 2015 | goto err; |
| 1853 | | 2016 | |