Merge tag 'timers-core-2025-11-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull timer core updates from Thomas Gleixner: - Prevent a thundering herd problem when the timekeeper CPU is delayed and a large number of CPUs compete to acquire jiffies_lock to do the update. Limit it to one CPU with a separate "uncontended" atomic variable. - A set of improvements for the timer migration mechanism: - Support imbalanced NUMA trees correctly - Support dynamic exclusion of CPUs from the migrator duty to allow the cpuset/isolation mechanism to exclude them from handling timers of remote idle CPUs - The usual small updates, cleanups and enhancements * tag 'timers-core-2025-11-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: timers/migration: Exclude isolated cpus from hierarchy cpumask: Add initialiser to use cleanup helpers sched/isolation: Force housekeeping if isolcpus and nohz_full don't leave any cgroup/cpuset: Rename update_unbound_workqueue_cpumask() to update_isolation_cpumasks() timers/migration: Use scoped_guard on available flag set/clear timers/migration: Add mask for CPUs available in the hierarchy timers/migration: Rename 'online' bit to 'available' selftests/timers/nanosleep: Add tests for return of remaining time selftests/timers: Clean up kernel version check in posix_timers time: Fix a few typos in time[r] related code comments time: tick-oneshot: Add missing Return and parameter descriptions to kernel-doc hrtimer: Store time as ktime_t in restart block timers/migration: Remove dead code handling idle CPU checking for remote timers timers/migration: Remove unused "cpu" parameter from tmigr_get_group() timers/migration: Assert that hotplug preparing CPU is part of stable active hierarchy timers/migration: Fix imbalanced NUMA trees timers/migration: Remove locking on group connection timers/migration: Convert "while" loops to use "for" tick/sched: Limit non-timekeeper CPUs calling jiffies update

Linus Torvalds committed 11 months ago

Commit d42e5042 parents 5028f42 7dec06216 files · +503 −196Browse files

Unified · Split

16 files

include/linux/cpumask.h +2 −0
include/linux/delay.h +4 −4
include/linux/restart_block.h +1 −1
include/linux/timer.h +9 −0
include/trace/events/timer_migration.h +2 −2
kernel/cgroup/cpuset.c +9 −6
kernel/sched/isolation.c +23 −0
kernel/time/hrtimer.c +2 −2
kernel/time/posix-cpu-timers.c +2 −2
kernel/time/posix-timers.c +1 −1
kernel/time/tick-oneshot.c +19 −1
kernel/time/tick-sched.c +26 −4
kernel/time/timer_migration.c +325 −162
kernel/time/timer_migration.h +1 −1
tools/testing/selftests/timers/nanosleep.c +55 −0
tools/testing/selftests/timers/posix_timers.c +22 −10

include/linux/cpumask.h+2−0

@@ -1022,6 +1022,7 @@ static __always_inline unsigned int cpumask_size(void) · 1021 unmodified lines
1022	1022
1023	1023	#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
1024	1024	#define __cpumask_var_read_mostly __read_mostly
	1025	+#define CPUMASK_VAR_NULL NULL
1025	1026
1026	1027	bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
1027	1028
@@ -1068,6 +1069,7 @@ static __always_inline bool cpumask_available(cpumask_var_t mask) · 40 unmodified lines
1068	1069
1069	1070	#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
1070	1071	#define __cpumask_var_read_mostly
	1072	+#define CPUMASK_VAR_NULL {}
1071	1073
1072	1074	static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
1073	1075	{

include/linux/delay.h+4−4

@@ -68,7 +68,7 @@ void usleep_range_state(unsigned long min, unsigned long max, · 67 unmodified lines
68	68	* @min: Minimum time in microseconds to sleep
69	69	* @max: Maximum time in microseconds to sleep
70	70	*
71		− * For basic information please refere to usleep_range_state().
	71	+ * For basic information please refer to usleep_range_state().
72	72	*
73	73	* The task will be in the state TASK_UNINTERRUPTIBLE during the sleep.
74	74	*/
@@ -82,10 +82,10 @@ static inline void usleep_range(unsigned long min, unsigned long max) · 7 unmodified lines
82	82	* @min: Minimum time in microseconds to sleep
83	83	* @max: Maximum time in microseconds to sleep
84	84	*
85		− * For basic information please refere to usleep_range_state().
	85	+ * For basic information please refer to usleep_range_state().
86	86	*
87	87	* The sleeping task has the state TASK_IDLE during the sleep to prevent
88		− * contribution to the load avarage.
	88	+ * contribution to the load average.
89	89	*/
90	90	static inline void usleep_range_idle(unsigned long min, unsigned long max)
91	91	{
@@ -96,7 +96,7 @@ static inline void usleep_range_idle(unsigned long min, unsigned long max) · 4 unmodified lines
96	96	* ssleep - wrapper for seconds around msleep
97	97	* @seconds: Requested sleep duration in seconds
98	98	*
99		− * Please refere to msleep() for detailed information.
	99	+ * Please refer to msleep() for detailed information.
100	100	*/
101	101	static inline void ssleep(unsigned int seconds)
102	102	{

include/linux/restart_block.h+1−1

@@ -43,7 +43,7 @@ struct restart_block { · 42 unmodified lines
43	43	struct __kernel_timespec __user *rmtp;
44	44	struct old_timespec32 __user *compat_rmtp;
45	45	};
46		− u64 expires;
	46	+ ktime_t expires;
47	47	} nanosleep;
48	48	/* For poll */
49	49	struct {

include/linux/timer.h+9−0

@@ -188,4 +188,13 @@ int timers_dead_cpu(unsigned int cpu); · 187 unmodified lines
188	188	#define timers_dead_cpu NULL
189	189	#endif
190	190
	191	+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
	192	+extern int tmigr_isolated_exclude_cpumask(struct cpumask *exclude_cpumask);
	193	+#else
	194	+static inline int tmigr_isolated_exclude_cpumask(struct cpumask *exclude_cpumask)
	195	+{
	196	+ return 0;
	197	+}
	198	+#endif
	199	+
191	200	#endif

include/trace/events/timer_migration.h+2−2

@@ -173,14 +173,14 @@ DEFINE_EVENT(tmigr_cpugroup, tmigr_cpu_active, · 172 unmodified lines
173	173	TP_ARGS(tmc)
174	174	);
175	175
176		−DEFINE_EVENT(tmigr_cpugroup, tmigr_cpu_online,
	176	+DEFINE_EVENT(tmigr_cpugroup, tmigr_cpu_available,
177	177
178	178	TP_PROTO(struct tmigr_cpu *tmc),
179	179
180	180	TP_ARGS(tmc)
181	181	);
182	182
183		−DEFINE_EVENT(tmigr_cpugroup, tmigr_cpu_offline,
	183	+DEFINE_EVENT(tmigr_cpugroup, tmigr_cpu_unavailable,
184	184
185	185	TP_PROTO(struct tmigr_cpu *tmc),
186	186

kernel/cgroup/cpuset.c+9−6

@@ -1391,7 +1391,7 @@ static bool partition_xcpus_del(int old_prs, struct cpuset *parent, · 1390 unmodified lines
1391	1391	return isolcpus_updated;
1392	1392	}
1393	1393
1394		−static void update_unbound_workqueue_cpumask(bool isolcpus_updated)
	1394	+static void update_isolation_cpumasks(bool isolcpus_updated)
1395	1395	{
1396	1396	int ret;
1397	1397
@@ -1402,6 +1402,9 @@ static void update_unbound_workqueue_cpumask(bool isolcpus_updated) · 4 unmodified lines
1402	1402
1403	1403	ret = workqueue_unbound_exclude_cpumask(isolated_cpus);
1404	1404	WARN_ON_ONCE(ret < 0);
	1405	+
	1406	+ ret = tmigr_isolated_exclude_cpumask(isolated_cpus);
	1407	+ WARN_ON_ONCE(ret < 0);
1405	1408	}
1406	1409
1407	1410	/**
@@ -1555,7 +1558,7 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, · 147 unmodified lines
1555	1558	list_add(&cs->remote_sibling, &remote_children);
1556	1559	cpumask_copy(cs->effective_xcpus, tmp->new_cpus);
1557	1560	spin_unlock_irq(&callback_lock);
1558		− update_unbound_workqueue_cpumask(isolcpus_updated);
	1561	+ update_isolation_cpumasks(isolcpus_updated);
1559	1562	cpuset_force_rebuild();
1560	1563	cs->prs_err = 0;
1561	1564
@@ -1596,7 +1599,7 @@ static void remote_partition_disable(struct cpuset cs, struct tmpmasks tmp) · 34 unmodified lines
1596	1599	compute_excpus(cs, cs->effective_xcpus);
1597	1600	reset_partition_data(cs);
1598	1601	spin_unlock_irq(&callback_lock);
1599		− update_unbound_workqueue_cpumask(isolcpus_updated);
	1602	+ update_isolation_cpumasks(isolcpus_updated);
1600	1603	cpuset_force_rebuild();
1601	1604
1602	1605	/*
@@ -1665,7 +1668,7 @@ static void remote_cpus_update(struct cpuset cs, struct cpumask xcpus, · 62 unmodified lines
1665	1668	if (xcpus)
1666	1669	cpumask_copy(cs->exclusive_cpus, xcpus);
1667	1670	spin_unlock_irq(&callback_lock);
1668		− update_unbound_workqueue_cpumask(isolcpus_updated);
	1671	+ update_isolation_cpumasks(isolcpus_updated);
1669	1672	if (adding \|\| deleting)
1670	1673	cpuset_force_rebuild();
1671	1674
@@ -2023,7 +2026,7 @@ write_error: · 351 unmodified lines
2023	2026	WARN_ON_ONCE(parent->nr_subparts < 0);
2024	2027	}
2025	2028	spin_unlock_irq(&callback_lock);
2026		− update_unbound_workqueue_cpumask(isolcpus_updated);
	2029	+ update_isolation_cpumasks(isolcpus_updated);
2027	2030
2028	2031	if ((old_prs != new_prs) && (cmd == partcmd_update))
2029	2032	update_partition_exclusive_flag(cs, new_prs);
@@ -3043,7 +3046,7 @@ out: · 1013 unmodified lines
3043	3046	else if (isolcpus_updated)
3044	3047	isolated_cpus_update(old_prs, new_prs, cs->effective_xcpus);
3045	3048	spin_unlock_irq(&callback_lock);
3046		− update_unbound_workqueue_cpumask(isolcpus_updated);
	3049	+ update_isolation_cpumasks(isolcpus_updated);
3047	3050
3048	3051	/* Force update if switching back to member & update effective_xcpus */
3049	3052	update_cpumasks_hier(cs, &tmpmask, !new_prs);

kernel/sched/isolation.c+23−0

@@ -167,6 +167,29 @@ static int __init housekeeping_setup(char *str, unsigned long flags) · 166 unmodified lines
167	167	}
168	168	}
169	169
	170	+ /*
	171	+ * Check the combination of nohz_full and isolcpus=domain,
	172	+ * necessary to avoid problems with the timer migration
	173	+ * hierarchy. managed_irq is ignored by this check since it
	174	+ * isn't considered in the timer migration logic.
	175	+ */
	176	+ iter_flags = housekeeping.flags & (HK_FLAG_KERNEL_NOISE \| HK_FLAG_DOMAIN);
	177	+ type = find_first_bit(&iter_flags, HK_TYPE_MAX);
	178	+ /*
	179	+ * Pass the check if none of these flags were previously set or
	180	+ * are not in the current selection.
	181	+ */
	182	+ iter_flags = flags & (HK_FLAG_KERNEL_NOISE \| HK_FLAG_DOMAIN);
	183	+ first_cpu = (type == HK_TYPE_MAX \|\| !iter_flags) ? 0 :
	184	+ cpumask_first_and_and(cpu_present_mask,
	185	+ housekeeping_staging, housekeeping.cpumasks[type]);
	186	+ if (first_cpu >= min(nr_cpu_ids, setup_max_cpus)) {
	187	+ pr_warn("Housekeeping: must include one present CPU "
	188	+ "neither in nohz_full= nor in isolcpus=domain, "
	189	+ "ignoring setting %s\n", str);
	190	+ goto free_housekeeping_staging;
	191	+ }
	192	+
170	193	iter_flags = flags & ~housekeeping.flags;
171	194
172	195	for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)

kernel/time/hrtimer.c+2−2

@@ -2145,7 +2145,7 @@ static long __sched hrtimer_nanosleep_restart(struct restart_block *restart) · 2144 unmodified lines
2145	2145	int ret;
2146	2146
2147	2147	hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS);
2148		− hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
	2148	+ hrtimer_set_expires(&t.timer, restart->nanosleep.expires);
2149	2149	ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
2150	2150	destroy_hrtimer_on_stack(&t.timer);
2151	2151	return ret;
@@ -2172,7 +2172,7 @@ long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, · 20 unmodified lines
2172	2172
2173	2173	restart = &current->restart_block;
2174	2174	restart->nanosleep.clockid = t.timer.base->clockid;
2175		− restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
	2175	+ restart->nanosleep.expires = hrtimer_get_expires(&t.timer);
2176	2176	set_restart_fn(restart, hrtimer_nanosleep_restart);
2177	2177	out:
2178	2178	destroy_hrtimer_on_stack(&t.timer);

kernel/time/posix-cpu-timers.c+2−2

@@ -1557,7 +1557,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, · 1556 unmodified lines
1557	1557	* Report back to the user the time still remaining.
1558	1558	*/
1559	1559	restart = &current->restart_block;
1560		− restart->nanosleep.expires = expires;
	1560	+ restart->nanosleep.expires = ns_to_ktime(expires);
1561	1561	if (restart->nanosleep.type != TT_NONE)
1562	1562	error = nanosleep_copyout(restart, &it.it_value);
1563	1563	}
@@ -1599,7 +1599,7 @@ static long posix_cpu_nsleep_restart(struct restart_block *restart_block) · 35 unmodified lines
1599	1599	clockid_t which_clock = restart_block->nanosleep.clockid;
1600	1600	struct timespec64 t;
1601	1601
1602		− t = ns_to_timespec64(restart_block->nanosleep.expires);
	1602	+ t = ktime_to_timespec64(restart_block->nanosleep.expires);
1603	1603
1604	1604	return do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t);
1605	1605	}

kernel/time/posix-timers.c+1−1

@@ -1242,7 +1242,7 @@ SYSCALL_DEFINE2(clock_adjtime, const clockid_t, which_clock, · 1241 unmodified lines
1242	1242	* sys_clock_settime(). The kernel internal timekeeping is always using
1243	1243	* nanoseconds precision independent of the clocksource device which is
1244	1244	* used to read the time from. The resolution of that device only
1245		− * affects the presicion of the time returned by sys_clock_gettime().
	1245	+ * affects the precision of the time returned by sys_clock_gettime().
1246	1246	*
1247	1247	* Returns:
1248	1248	* 0 Success. @tp contains the resolution

kernel/time/tick-oneshot.c+19−1

@@ -19,6 +19,10 @@ · 18 unmodified lines
19	19
20	20	/**
21	21	* tick_program_event - program the CPU local timer device for the next event
	22	+ * @expires: the time at which the next timer event should occur
	23	+ * @force: flag to force reprograming even if the event time hasn't changed
	24	+ *
	25	+ * Return: 0 on success, negative error code on failure
22	26	*/
23	27	int tick_program_event(ktime_t expires, int force)
24	28	{
@@ -57,6 +61,13 @@ void tick_resume_oneshot(void) · 32 unmodified lines
57	61
58	62	/**
59	63	* tick_setup_oneshot - setup the event device for oneshot mode (hres or nohz)
	64	+ * @newdev: Pointer to the clock event device to configure
	65	+ * @handler: Function to be called when the event device triggers an interrupt
	66	+ * @next_event: Initial expiry time for the next event (in ktime)
	67	+ *
	68	+ * Configures the specified clock event device for onshot mode,
	69	+ * assigns the given handler as its event callback, and programs
	70	+ * the device to trigger at the specified next event time.
60	71	*/
61	72	void tick_setup_oneshot(struct clock_event_device *newdev,
62	73	void (handler)(struct clock_event_device ),
@@ -69,6 +80,10 @@ void tick_setup_oneshot(struct clock_event_device *newdev, · 6 unmodified lines
69	80
70	81	/**
71	82	* tick_switch_to_oneshot - switch to oneshot mode
	83	+ * @handler: function to call when an event occurs on the tick device
	84	+ *
	85	+ * Return: 0 on success, -EINVAL if the tick device is not present,
	86	+ * not functional, or does not support oneshot mode.
72	87	*/
73	88	int tick_switch_to_oneshot(void (handler)(struct clock_event_device ))
74	89	{
@@ -101,7 +116,7 @@ int tick_switch_to_oneshot(void (handler)(struct clock_event_device )) · 26 unmodified lines
101	116	/**
102	117	* tick_oneshot_mode_active - check whether the system is in oneshot mode
103	118	*
104		− * returns 1 when either nohz or highres are enabled. otherwise 0.
	119	+ * Return: 1 when either nohz or highres are enabled, otherwise 0.
105	120	*/
106	121	int tick_oneshot_mode_active(void)
107	122	{
@@ -120,6 +135,9 @@ int tick_oneshot_mode_active(void) · 12 unmodified lines
120	135	* tick_init_highres - switch to high resolution mode
121	136	*
122	137	* Called with interrupts disabled.
	138	+ *
	139	+ * Return: 0 on success, -EINVAL if the tick device cannot switch
	140	+ * to oneshot/high-resolution mode.
123	141	*/
124	142	int tick_init_highres(void)
125	143	{

kernel/time/tick-sched.c+26−4

@@ -201,6 +201,27 @@ static inline void tick_sched_flag_clear(struct tick_sched *ts, · 200 unmodified lines
201	201	ts->flags &= ~flag;
202	202	}
203	203
	204	+/*
	205	+ * Allow only one non-timekeeper CPU at a time update jiffies from
	206	+ * the timer tick.
	207	+ *
	208	+ * Returns true if update was run.
	209	+ */
	210	+static bool tick_limited_update_jiffies64(struct tick_sched *ts, ktime_t now)
	211	+{
	212	+ static atomic_t in_progress;
	213	+ int inp;
	214	+
	215	+ inp = atomic_read(&in_progress);
	216	+ if (inp \|\| !atomic_try_cmpxchg(&in_progress, &inp, 1))
	217	+ return false;
	218	+
	219	+ if (ts->last_tick_jiffies == jiffies)
	220	+ tick_do_update_jiffies64(now);
	221	+ atomic_set(&in_progress, 0);
	222	+ return true;
	223	+}
	224	+
204	225	#define MAX_STALLED_JIFFIES 5
205	226
206	227	static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
@@ -239,10 +260,11 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) · 32 unmodified lines
239	260	ts->stalled_jiffies = 0;
240	261	ts->last_tick_jiffies = READ_ONCE(jiffies);
241	262	} else {
242		− if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) {
243		− tick_do_update_jiffies64(now);
244		− ts->stalled_jiffies = 0;
245		− ts->last_tick_jiffies = READ_ONCE(jiffies);
	263	+ if (++ts->stalled_jiffies >= MAX_STALLED_JIFFIES) {
	264	+ if (tick_limited_update_jiffies64(ts, now)) {
	265	+ ts->stalled_jiffies = 0;
	266	+ ts->last_tick_jiffies = READ_ONCE(jiffies);
	267	+ }
246	268	}
247	269	}
248	270

kernel/time/timer_migration.c+325−162

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/spinlock.h>
11	11	#include <linux/timerqueue.h>
12	12	#include <trace/events/ipi.h>
	13	+#include <linux/sched/isolation.h>
13	14
14	15	#include "timer_migration.h"
15	16	#include "tick-internal.h"
@@ -420,14 +421,54 @@ static struct list_head *tmigr_level_list __read_mostly; · 404 unmodified lines
420	421	static unsigned int tmigr_hierarchy_levels __read_mostly;
421	422	static unsigned int tmigr_crossnode_level __read_mostly;
422	423
	424	+static struct tmigr_group *tmigr_root;
	425	+
423	426	static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu);
424	427
	428	+/*
	429	+ * CPUs available for timer migration.
	430	+ * Protected by cpuset_mutex (with cpus_read_lock held) or cpus_write_lock.
	431	+ * Additionally tmigr_available_mutex serializes set/clear operations with each other.
	432	+ */
	433	+static cpumask_var_t tmigr_available_cpumask;
	434	+static DEFINE_MUTEX(tmigr_available_mutex);
	435	+
	436	+/* Enabled during late initcall */
	437	+static DEFINE_STATIC_KEY_FALSE(tmigr_exclude_isolated);
	438	+
425	439	#define TMIGR_NONE 0xFF
426	440	#define BIT_CNT 8
427	441
428	442	static inline bool tmigr_is_not_available(struct tmigr_cpu *tmc)
429	443	{
430		− return !(tmc->tmgroup && tmc->online);
	444	+ return !(tmc->tmgroup && tmc->available);
	445	+}
	446	+
	447	+/*
	448	+ * Returns true if @cpu should be excluded from the hierarchy as isolated.
	449	+ * Domain isolated CPUs don't participate in timer migration, nohz_full CPUs
	450	+ * are still part of the hierarchy but become idle (from a tick and timer
	451	+ * migration perspective) when they stop their tick. This lets the timekeeping
	452	+ * CPU handle their global timers. Marking also isolated CPUs as idle would be
	453	+ * too costly, hence they are completely excluded from the hierarchy.
	454	+ * This check is necessary, for instance, to prevent offline isolated CPUs from
	455	+ * being incorrectly marked as available once getting back online.
	456	+ *
	457	+ * This function returns false during early boot and the isolation logic is
	458	+ * enabled only after isolated CPUs are marked as unavailable at late boot.
	459	+ * The tick CPU can be isolated at boot, however we cannot mark it as
	460	+ * unavailable to avoid having no global migrator for the nohz_full CPUs. This
	461	+ * should be ensured by the callers of this function: implicitly from hotplug
	462	+ * callbacks and explicitly in tmigr_init_isolation() and
	463	+ * tmigr_isolated_exclude_cpumask().
	464	+ */
	465	+static inline bool tmigr_is_isolated(int cpu)
	466	+{
	467	+ if (!static_branch_unlikely(&tmigr_exclude_isolated))
	468	+ return false;
	469	+ return (!housekeeping_cpu(cpu, HK_TYPE_DOMAIN) \|\|
	470	+ cpuset_cpu_is_isolated(cpu)) &&
	471	+ housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE);
431	472	}
432	473
433	474	/*
@@ -502,11 +543,6 @@ static bool tmigr_check_lonely(struct tmigr_group *group) · 68 unmodified lines
502	543	* @now: timer base monotonic
503	544	* @check: is set if there is the need to handle remote timers;
504	545	* required in tmigr_requires_handle_remote() only
505		− * @tmc_active: this flag indicates, whether the CPU which triggers
506		− * the hierarchy walk is !idle in the timer migration
507		− * hierarchy. When the CPU is idle and the whole hierarchy is
508		− * idle, only the first event of the top level has to be
509		− * considered.
510	546	*/
511	547	struct tmigr_walk {
512	548	u64 nextexp;
@@ -517,16 +553,13 @@ struct tmigr_walk { · 4 unmodified lines
517	553	unsigned long basej;
518	554	u64 now;
519	555	bool check;
520		− bool tmc_active;
521	556	};
522	557
523	558	typedef bool (up_f)(struct tmigr_group , struct tmigr_group , struct tmigr_walk );
524	559
525		−static void __walk_groups(up_f up, struct tmigr_walk *data,
526		− struct tmigr_cpu *tmc)
	560	+static void __walk_groups_from(up_f up, struct tmigr_walk *data,
	561	+ struct tmigr_group child, struct tmigr_group group)
527	562	{
528		− struct tmigr_group child = NULL, group = tmc->tmgroup;
529		−
530	563	do {
531	564	WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels);
532	565
@@ -544,6 +577,12 @@ static void __walk_groups(up_f up, struct tmigr_walk *data, · 11 unmodified lines
544	577	} while (group);
545	578	}
546	579
	580	+static void __walk_groups(up_f up, struct tmigr_walk *data,
	581	+ struct tmigr_cpu *tmc)
	582	+{
	583	+ __walk_groups_from(up, data, NULL, tmc->tmgroup);
	584	+}
	585	+
547	586	static void walk_groups(up_f up, struct tmigr_walk data, struct tmigr_cpu tmc)
548	587	{
549	588	lockdep_assert_held(&tmc->lock);
@@ -708,7 +747,7 @@ void tmigr_cpu_activate(void) · 158 unmodified lines
708	747	/*
709	748	* Returns true, if there is nothing to be propagated to the next level
710	749	*
711		− * @data->firstexp is set to expiry of first gobal event of the (top level of
	750	+ * @data->firstexp is set to expiry of first global event of the (top level of
712	751	* the) hierarchy, but only when hierarchy is completely idle.
713	752	*
714	753	* The child and group states need to be read under the lock, to prevent a race
@@ -926,7 +965,7 @@ static void tmigr_handle_remote_cpu(unsigned int cpu, u64 now, · 211 unmodified lines
926	965	* updated the event takes care when hierarchy is completely
927	966	* idle. Otherwise the migrator does it as the event is enqueued.
928	967	*/
929		− if (!tmc->online \|\| tmc->remote \|\| tmc->cpuevt.ignore \|\|
	968	+ if (!tmc->available \|\| tmc->remote \|\| tmc->cpuevt.ignore \|\|
930	969	now < tmc->cpuevt.nextevt.expires) {
931	970	raw_spin_unlock_irq(&tmc->lock);
932	971	return;
@@ -973,7 +1012,7 @@ static void tmigr_handle_remote_cpu(unsigned int cpu, u64 now, · 40 unmodified lines
973	1012	* (See also section "Required event and timerqueue update after a
974	1013	* remote expiry" in the documentation at the top)
975	1014	*/
976		− if (!tmc->online \|\| !tmc->idle) {
	1015	+ if (!tmc->available \|\| !tmc->idle) {
977	1016	timer_unlock_remote_bases(cpu);
978	1017	goto unlock;
979	1018	}
@@ -1113,15 +1152,6 @@ static bool tmigr_requires_handle_remote_up(struct tmigr_group *group, · 133 unmodified lines
1113	1152	*/
1114	1153	if (!tmigr_check_migrator(group, childmask))
1115	1154	return true;
1116		−
1117		− /*
1118		− * When there is a parent group and the CPU which triggered the
1119		− * hierarchy walk is not active, proceed the walk to reach the top level
1120		− * group before reading the next_expiry value.
1121		− */
1122		− if (group->parent && !data->tmc_active)
1123		− return false;
1124		−
1125	1155	/*
1126	1156	* The lock is required on 32bit architectures to read the variable
1127	1157	* consistently with a concurrent writer. On 64bit the lock is not
@@ -1166,7 +1196,6 @@ bool tmigr_requires_handle_remote(void) · 38 unmodified lines
1166	1196	data.now = get_jiffies_update(&jif);
1167	1197	data.childmask = tmc->groupmask;
1168	1198	data.firstexp = KTIME_MAX;
1169		− data.tmc_active = !tmc->idle;
1170	1199	data.check = false;
1171	1200
1172	1201	/*
@@ -1432,38 +1461,43 @@ static long tmigr_trigger_active(void *unused) · 259 unmodified lines
1432	1461	{
1433	1462	struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
1434	1463
1435		− WARN_ON_ONCE(!tmc->online \|\| tmc->idle);
	1464	+ WARN_ON_ONCE(!tmc->available \|\| tmc->idle);
1436	1465
1437	1466	return 0;
1438	1467	}
1439	1468
1440		−static int tmigr_cpu_offline(unsigned int cpu)
	1469	+static int tmigr_clear_cpu_available(unsigned int cpu)
1441	1470	{
1442	1471	struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
1443	1472	int migrator;
1444	1473	u64 firstexp;
1445	1474
1446		− raw_spin_lock_irq(&tmc->lock);
1447		− tmc->online = false;
1448		− WRITE_ONCE(tmc->wakeup, KTIME_MAX);
	1475	+ guard(mutex)(&tmigr_available_mutex);
1449	1476
1450		− /*
1451		− * CPU has to handle the local events on his own, when on the way to
1452		− * offline; Therefore nextevt value is set to KTIME_MAX
1453		− */
1454		− firstexp = __tmigr_cpu_deactivate(tmc, KTIME_MAX);
1455		− trace_tmigr_cpu_offline(tmc);
1456		− raw_spin_unlock_irq(&tmc->lock);
	1477	+ cpumask_clear_cpu(cpu, tmigr_available_cpumask);
	1478	+ scoped_guard(raw_spinlock_irq, &tmc->lock) {
	1479	+ if (!tmc->available)
	1480	+ return 0;
	1481	+ tmc->available = false;
	1482	+ WRITE_ONCE(tmc->wakeup, KTIME_MAX);
	1483	+
	1484	+ /*
	1485	+ * CPU has to handle the local events on his own, when on the way to
	1486	+ * offline; Therefore nextevt value is set to KTIME_MAX
	1487	+ */
	1488	+ firstexp = __tmigr_cpu_deactivate(tmc, KTIME_MAX);
	1489	+ trace_tmigr_cpu_unavailable(tmc);
	1490	+ }
1457	1491
1458	1492	if (firstexp != KTIME_MAX) {
1459		− migrator = cpumask_any_but(cpu_online_mask, cpu);
	1493	+ migrator = cpumask_any(tmigr_available_cpumask);
1460	1494	work_on_cpu(migrator, tmigr_trigger_active, NULL);
1461	1495	}
1462	1496
1463	1497	return 0;
1464	1498	}
1465	1499
1466		−static int tmigr_cpu_online(unsigned int cpu)
	1500	+static int tmigr_set_cpu_available(unsigned int cpu)
1467	1501	{
1468	1502	struct tmigr_cpu *tmc = this_cpu_ptr(&tmigr_cpu);
1469	1503
@@ -1471,16 +1505,123 @@ static int tmigr_cpu_online(unsigned int cpu) · 1 unmodified line
1471	1505	if (WARN_ON_ONCE(!tmc->tmgroup))
1472	1506	return -EINVAL;
1473	1507
1474		− raw_spin_lock_irq(&tmc->lock);
1475		− trace_tmigr_cpu_online(tmc);
1476		− tmc->idle = timer_base_is_idle();
1477		− if (!tmc->idle)
1478		− __tmigr_cpu_activate(tmc);
1479		− tmc->online = true;
1480		− raw_spin_unlock_irq(&tmc->lock);
	1508	+ if (tmigr_is_isolated(cpu))
	1509	+ return 0;
	1510	+
	1511	+ guard(mutex)(&tmigr_available_mutex);
	1512	+
	1513	+ cpumask_set_cpu(cpu, tmigr_available_cpumask);
	1514	+ scoped_guard(raw_spinlock_irq, &tmc->lock) {
	1515	+ if (tmc->available)
	1516	+ return 0;
	1517	+ trace_tmigr_cpu_available(tmc);
	1518	+ tmc->idle = timer_base_is_idle();
	1519	+ if (!tmc->idle)
	1520	+ __tmigr_cpu_activate(tmc);
	1521	+ tmc->available = true;
	1522	+ }
1481	1523	return 0;
1482	1524	}
1483	1525
	1526	+static void tmigr_cpu_isolate(struct work_struct *ignored)
	1527	+{
	1528	+ tmigr_clear_cpu_available(smp_processor_id());
	1529	+}
	1530	+
	1531	+static void tmigr_cpu_unisolate(struct work_struct *ignored)
	1532	+{
	1533	+ tmigr_set_cpu_available(smp_processor_id());
	1534	+}
	1535	+
	1536	+/**
	1537	+ * tmigr_isolated_exclude_cpumask - Exclude given CPUs from hierarchy
	1538	+ * @exclude_cpumask: the cpumask to be excluded from timer migration hierarchy
	1539	+ *
	1540	+ * This function can be called from cpuset code to provide the new set of
	1541	+ * isolated CPUs that should be excluded from the hierarchy.
	1542	+ * Online CPUs not present in exclude_cpumask but already excluded are brought
	1543	+ * back to the hierarchy.
	1544	+ * Functions to isolate/unisolate need to be called locally and can sleep.
	1545	+ */
	1546	+int tmigr_isolated_exclude_cpumask(struct cpumask *exclude_cpumask)
	1547	+{
	1548	+ struct work_struct __percpu *works __free(free_percpu) =
	1549	+ alloc_percpu(struct work_struct);
	1550	+ cpumask_var_t cpumask __free(free_cpumask_var) = CPUMASK_VAR_NULL;
	1551	+ int cpu;
	1552	+
	1553	+ lockdep_assert_cpus_held();
	1554	+
	1555	+ if (!works)
	1556	+ return -ENOMEM;
	1557	+ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL))
	1558	+ return -ENOMEM;
	1559	+
	1560	+ /*
	1561	+ * First set previously isolated CPUs as available (unisolate).
	1562	+ * This cpumask contains only CPUs that switched to available now.
	1563	+ */
	1564	+ cpumask_andnot(cpumask, cpu_online_mask, exclude_cpumask);
	1565	+ cpumask_andnot(cpumask, cpumask, tmigr_available_cpumask);
	1566	+
	1567	+ for_each_cpu(cpu, cpumask) {
	1568	+ struct work_struct *work = per_cpu_ptr(works, cpu);
	1569	+
	1570	+ INIT_WORK(work, tmigr_cpu_unisolate);
	1571	+ schedule_work_on(cpu, work);
	1572	+ }
	1573	+ for_each_cpu(cpu, cpumask)
	1574	+ flush_work(per_cpu_ptr(works, cpu));
	1575	+
	1576	+ /*
	1577	+ * Then clear previously available CPUs (isolate).
	1578	+ * This cpumask contains only CPUs that switched to not available now.
	1579	+ * There cannot be overlap with the newly available ones.
	1580	+ */
	1581	+ cpumask_and(cpumask, exclude_cpumask, tmigr_available_cpumask);
	1582	+ cpumask_and(cpumask, cpumask, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE));
	1583	+ /*
	1584	+ * Handle this here and not in the cpuset code because exclude_cpumask
	1585	+ * might include also the tick CPU if included in isolcpus.
	1586	+ */
	1587	+ for_each_cpu(cpu, cpumask) {
	1588	+ if (!tick_nohz_cpu_hotpluggable(cpu)) {
	1589	+ cpumask_clear_cpu(cpu, cpumask);
	1590	+ break;
	1591	+ }
	1592	+ }
	1593	+
	1594	+ for_each_cpu(cpu, cpumask) {
	1595	+ struct work_struct *work = per_cpu_ptr(works, cpu);
	1596	+
	1597	+ INIT_WORK(work, tmigr_cpu_isolate);
	1598	+ schedule_work_on(cpu, work);
	1599	+ }
	1600	+ for_each_cpu(cpu, cpumask)
	1601	+ flush_work(per_cpu_ptr(works, cpu));
	1602	+
	1603	+ return 0;
	1604	+}
	1605	+
	1606	+static int __init tmigr_init_isolation(void)
	1607	+{
	1608	+ cpumask_var_t cpumask __free(free_cpumask_var) = CPUMASK_VAR_NULL;
	1609	+
	1610	+ static_branch_enable(&tmigr_exclude_isolated);
	1611	+
	1612	+ if (!housekeeping_enabled(HK_TYPE_DOMAIN))
	1613	+ return 0;
	1614	+ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL))
	1615	+ return -ENOMEM;
	1616	+
	1617	+ cpumask_andnot(cpumask, cpu_possible_mask, housekeeping_cpumask(HK_TYPE_DOMAIN));
	1618	+
	1619	+ /* Protect against RCU torture hotplug testing */
	1620	+ guard(cpus_read_lock)();
	1621	+ return tmigr_isolated_exclude_cpumask(cpumask);
	1622	+}
	1623	+late_initcall(tmigr_init_isolation);
	1624	+
1484	1625	static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl,
1485	1626	int node)
1486	1627	{
@@ -1498,21 +1639,6 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, · 11 unmodified lines
1498	1639	s.seq = 0;
1499	1640	atomic_set(&group->migr_state, s.state);
1500	1641
1501		− /*
1502		− * If this is a new top-level, prepare its groupmask in advance.
1503		− * This avoids accidents where yet another new top-level is
1504		− * created in the future and made visible before the current groupmask.
1505		− */
1506		− if (list_empty(&tmigr_level_list[lvl])) {
1507		− group->groupmask = BIT(0);
1508		− /*
1509		− * The previous top level has prepared its groupmask already,
1510		− * simply account it as the first child.
1511		− */
1512		− if (lvl > 0)
1513		− group->num_children = 1;
1514		− }
1515		−
1516	1642	timerqueue_init_head(&group->events);
1517	1643	timerqueue_init(&group->groupevt.nextevt);
1518	1644	group->groupevt.nextevt.expires = KTIME_MAX;
@@ -1520,8 +1646,7 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, · 1 unmodified line
1520	1646	group->groupevt.ignore = true;
1521	1647	}
1522	1648
1523		−static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node,
1524		− unsigned int lvl)
	1649	+static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl)
1525	1650	{
1526	1651	struct tmigr_group tmp, group = NULL;
1527	1652
@@ -1567,25 +1692,51 @@ static struct tmigr_group *tmigr_get_group(unsigned int cpu, int node, · 39 unmodified lines
1567	1692	return group;
1568	1693	}
1569	1694
	1695	+static bool tmigr_init_root(struct tmigr_group *group, bool activate)
	1696	+{
	1697	+ if (!group->parent && group != tmigr_root) {
	1698	+ /*
	1699	+ * This is the new top-level, prepare its groupmask in advance
	1700	+ * to avoid accidents where yet another new top-level is
	1701	+ * created in the future and made visible before this groupmask.
	1702	+ */
	1703	+ group->groupmask = BIT(0);
	1704	+ WARN_ON_ONCE(activate);
	1705	+
	1706	+ return true;
	1707	+ }
	1708	+
	1709	+ return false;
	1710	+
	1711	+}
	1712	+
1570	1713	static void tmigr_connect_child_parent(struct tmigr_group *child,
1571	1714	struct tmigr_group *parent,
1572	1715	bool activate)
1573	1716	{
1574		− struct tmigr_walk data;
1575		−
1576		− raw_spin_lock_irq(&child->lock);
1577		− raw_spin_lock_nested(&parent->lock, SINGLE_DEPTH_NESTING);
	1717	+ if (tmigr_init_root(parent, activate)) {
	1718	+ /*
	1719	+ * The previous top level had prepared its groupmask already,
	1720	+ * simply account it in advance as the first child. If some groups
	1721	+ * have been created between the old and new root due to node
	1722	+ * mismatch, the new root's child will be intialized accordingly.
	1723	+ */
	1724	+ parent->num_children = 1;
	1725	+ }
1578	1726
1579		− if (activate) {
	1727	+ /* Connecting old root to new root ? */
	1728	+ if (!parent->parent && activate) {
1580	1729	/*
1581		− * @child is the old top and @parent the new one. In this
1582		− * case groupmask is pre-initialized and @child already
1583		− * accounted, along with its new sibling corresponding to the
1584		− * CPU going up.
	1730	+ * @child is the old top, or in case of node mismatch, some
	1731	+ * intermediate group between the old top and the new one in
	1732	+ * @parent. In this case the @child must be pre-accounted above
	1733	+ * as the first child. Its new inactive sibling corresponding
	1734	+ * to the CPU going up has been accounted as the second child.
1585	1735	*/
1586		− WARN_ON_ONCE(child->groupmask != BIT(0) \|\| parent->num_children != 2);
	1736	+ WARN_ON_ONCE(parent->num_children != 2);
	1737	+ child->groupmask = BIT(0);
1587	1738	} else {
1588		− /* Adding @child for the CPU going up to @parent. */
	1739	+ /* Common case adding @child for the CPU going up to @parent. */
1589	1740	child->groupmask = BIT(parent->num_children++);
1590	1741	}
1591	1742
@@ -1596,87 +1747,61 @@ static void tmigr_connect_child_parent(struct tmigr_group *child, · 4 unmodified lines
1596	1747	*/
1597	1748	smp_store_release(&child->parent, parent);
1598	1749
1599		− raw_spin_unlock(&parent->lock);
1600		− raw_spin_unlock_irq(&child->lock);
1601		−
1602	1750	trace_tmigr_connect_child_parent(child);
1603		−
1604		− if (!activate)
1605		− return;
1606		−
1607		− /*
1608		− * To prevent inconsistent states, active children need to be active in
1609		− * the new parent as well. Inactive children are already marked inactive
1610		− * in the parent group:
1611		− *
1612		− * * When new groups were created by tmigr_setup_groups() starting from
1613		− * the lowest level (and not higher then one level below the current
1614		− * top level), then they are not active. They will be set active when
1615		− * the new online CPU comes active.
1616		− *
1617		− * * But if a new group above the current top level is required, it is
1618		− * mandatory to propagate the active state of the already existing
1619		− * child to the new parent. So tmigr_connect_child_parent() is
1620		− * executed with the formerly top level group (child) and the newly
1621		− * created group (parent).
1622		− *
1623		− * * It is ensured that the child is active, as this setup path is
1624		− * executed in hotplug prepare callback. This is exectued by an
1625		− * already connected and !idle CPU. Even if all other CPUs go idle,
1626		− * the CPU executing the setup will be responsible up to current top
1627		− * level group. And the next time it goes inactive, it will release
1628		− * the new childmask and parent to subsequent walkers through this
1629		− * @child. Therefore propagate active state unconditionally.
1630		− */
1631		− data.childmask = child->groupmask;
1632		−
1633		− /*
1634		− * There is only one new level per time (which is protected by
1635		− * tmigr_mutex). When connecting the child and the parent and set the
1636		− * child active when the parent is inactive, the parent needs to be the
1637		− * uppermost level. Otherwise there went something wrong!
1638		− */
1639		− WARN_ON(!tmigr_active_up(parent, child, &data) && parent->parent);
1640	1751	}
1641	1752
1642		−static int tmigr_setup_groups(unsigned int cpu, unsigned int node)
	1753	+static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
	1754	+ struct tmigr_group *start, bool activate)
1643	1755	{
1644	1756	struct tmigr_group group, child, **stack;
1645		− int top = 0, err = 0, i = 0;
1646		− struct list_head *lvllist;
	1757	+ int i, top = 0, err = 0, start_lvl = 0;
	1758	+ bool root_mismatch = false;
1647	1759
1648	1760	stack = kcalloc(tmigr_hierarchy_levels, sizeof(*stack), GFP_KERNEL);
1649	1761	if (!stack)
1650	1762	return -ENOMEM;
1651	1763
1652		− do {
1653		− group = tmigr_get_group(cpu, node, i);
	1764	+ if (start) {
	1765	+ stack[start->level] = start;
	1766	+ start_lvl = start->level + 1;
	1767	+ }
	1768	+
	1769	+ if (tmigr_root)
	1770	+ root_mismatch = tmigr_root->numa_node != node;
	1771	+
	1772	+ for (i = start_lvl; i < tmigr_hierarchy_levels; i++) {
	1773	+ group = tmigr_get_group(node, i);
1654	1774	if (IS_ERR(group)) {
1655	1775	err = PTR_ERR(group);
	1776	+ i--;
1656	1777	break;
1657	1778	}
1658	1779
1659	1780	top = i;
1660		− stack[i++] = group;
	1781	+ stack[i] = group;
1661	1782
1662	1783	/*
1663	1784	* When booting only less CPUs of a system than CPUs are
1664		− * available, not all calculated hierarchy levels are required.
	1785	+ * available, not all calculated hierarchy levels are required,
	1786	+ * unless a node mismatch is detected.
1665	1787	*
1666	1788	* The loop is aborted as soon as the highest level, which might
1667	1789	* be different from tmigr_hierarchy_levels, contains only a
1668		− * single group.
	1790	+ * single group, unless the nodes mismatch below tmigr_crossnode_level
1669	1791	*/
1670		− if (group->parent \|\| list_is_singular(&tmigr_level_list[i - 1]))
	1792	+ if (group->parent)
1671	1793	break;
	1794	+ if ((!root_mismatch \|\| i >= tmigr_crossnode_level) &&
	1795	+ list_is_singular(&tmigr_level_list[i]))
	1796	+ break;
	1797	+ }
1672	1798
1673		− } while (i < tmigr_hierarchy_levels);
1674		−
1675		− /* Assert single root */
1676		− WARN_ON_ONCE(!err && !group->parent && !list_is_singular(&tmigr_level_list[top]));
	1799	+ /* Assert single root without parent */
	1800	+ if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels))
	1801	+ return -EINVAL;
1677	1802
1678		− while (i > 0) {
1679		− group = stack[--i];
	1803	+ for (; i >= start_lvl; i--) {
	1804	+ group = stack[i];
1680	1805
1681	1806	if (err < 0) {
1682	1807	list_del(&group->list);
@@ -1692,12 +1817,10 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) · 9 unmodified lines
1692	1817	if (i == 0) {
1693	1818	struct tmigr_cpu *tmc = per_cpu_ptr(&tmigr_cpu, cpu);
1694	1819
1695		− raw_spin_lock_irq(&group->lock);
1696		−
1697	1820	tmc->tmgroup = group;
1698	1821	tmc->groupmask = BIT(group->num_children++);
1699	1822
1700		− raw_spin_unlock_irq(&group->lock);
	1823	+ tmigr_init_root(group, activate);
1701	1824
1702	1825	trace_tmigr_connect_cpu_parent(tmc);
1703	1826
@@ -1705,42 +1828,58 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) · 1 unmodified line
1705	1828	continue;
1706	1829	} else {
1707	1830	child = stack[i - 1];
1708		− /* Will be activated at online time */
1709		− tmigr_connect_child_parent(child, group, false);
	1831	+ tmigr_connect_child_parent(child, group, activate);
1710	1832	}
	1833	+ }
1711	1834
1712		− /* check if uppermost level was newly created */
1713		− if (top != i)
1714		− continue;
1715		−
1716		− WARN_ON_ONCE(top == 0);
	1835	+ if (err < 0)
	1836	+ goto out;
1717	1837
1718		− lvllist = &tmigr_level_list[top];
	1838	+ if (activate) {
	1839	+ struct tmigr_walk data;
	1840	+ union tmigr_state state;
1719	1841
1720	1842	/*
1721		− * Newly created root level should have accounted the upcoming
1722		− * CPU's child group and pre-accounted the old root.
	1843	+ * To prevent inconsistent states, active children need to be active in
	1844	+ * the new parent as well. Inactive children are already marked inactive
	1845	+ * in the parent group:
	1846	+ *
	1847	+ * * When new groups were created by tmigr_setup_groups() starting from
	1848	+ * the lowest level, then they are not active. They will be set active
	1849	+ * when the new online CPU comes active.
	1850	+ *
	1851	+ * * But if new groups above the current top level are required, it is
	1852	+ * mandatory to propagate the active state of the already existing
	1853	+ * child to the new parents. So tmigr_active_up() activates the
	1854	+ * new parents while walking up from the old root to the new.
	1855	+ *
	1856	+ * * It is ensured that @start is active, as this setup path is
	1857	+ * executed in hotplug prepare callback. This is executed by an
	1858	+ * already connected and !idle CPU. Even if all other CPUs go idle,
	1859	+ * the CPU executing the setup will be responsible up to current top
	1860	+ * level group. And the next time it goes inactive, it will release
	1861	+ * the new childmask and parent to subsequent walkers through this
	1862	+ * @child. Therefore propagate active state unconditionally.
1723	1863	*/
1724		− if (group->num_children == 2 && list_is_singular(lvllist)) {
1725		− /*
1726		− * The target CPU must never do the prepare work, except
1727		− * on early boot when the boot CPU is the target. Otherwise
1728		− * it may spuriously activate the old top level group inside
1729		− * the new one (nevertheless whether old top level group is
1730		− * active or not) and/or release an uninitialized childmask.
1731		− */
1732		− WARN_ON_ONCE(cpu == raw_smp_processor_id());
1733		−
1734		− lvllist = &tmigr_level_list[top - 1];
1735		− list_for_each_entry(child, lvllist, list) {
1736		− if (child->parent)
1737		− continue;
	1864	+ state.state = atomic_read(&start->migr_state);
	1865	+ WARN_ON_ONCE(!state.active);
	1866	+ WARN_ON_ONCE(!start->parent);
	1867	+ data.childmask = start->groupmask;
	1868	+ __walk_groups_from(tmigr_active_up, &data, start, start->parent);
	1869	+ }
1738	1870
1739		− tmigr_connect_child_parent(child, group, true);
1740		− }
	1871	+ /* Root update */
	1872	+ if (list_is_singular(&tmigr_level_list[top])) {
	1873	+ group = list_first_entry(&tmigr_level_list[top],
	1874	+ typeof(*group), list);
	1875	+ WARN_ON_ONCE(group->parent);
	1876	+ if (tmigr_root) {
	1877	+ /* Old root should be the same or below */
	1878	+ WARN_ON_ONCE(tmigr_root->level > top);
1741	1879	}
	1880	+ tmigr_root = group;
1742	1881	}
1743		−
	1882	+out:
1744	1883	kfree(stack);
1745	1884
1746	1885	return err;
@@ -1748,12 +1887,31 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node) · 1 unmodified line
1748	1887
1749	1888	static int tmigr_add_cpu(unsigned int cpu)
1750	1889	{
	1890	+ struct tmigr_group *old_root = tmigr_root;
1751	1891	int node = cpu_to_node(cpu);
1752	1892	int ret;
1753	1893
1754		− mutex_lock(&tmigr_mutex);
1755		− ret = tmigr_setup_groups(cpu, node);
1756		− mutex_unlock(&tmigr_mutex);
	1894	+ guard(mutex)(&tmigr_mutex);
	1895	+
	1896	+ ret = tmigr_setup_groups(cpu, node, NULL, false);
	1897	+
	1898	+ /* Root has changed? Connect the old one to the new */
	1899	+ if (ret >= 0 && old_root && old_root != tmigr_root) {
	1900	+ /*
	1901	+ * The target CPU must never do the prepare work, except
	1902	+ * on early boot when the boot CPU is the target. Otherwise
	1903	+ * it may spuriously activate the old top level group inside
	1904	+ * the new one (nevertheless whether old top level group is
	1905	+ * active or not) and/or release an uninitialized childmask.
	1906	+ */
	1907	+ WARN_ON_ONCE(cpu == raw_smp_processor_id());
	1908	+ /*
	1909	+ * The (likely) current CPU is expected to be online in the hierarchy,
	1910	+ * otherwise the old root may not be active as expected.
	1911	+ */
	1912	+ WARN_ON_ONCE(!per_cpu_ptr(&tmigr_cpu, raw_smp_processor_id())->available);
	1913	+ ret = tmigr_setup_groups(-1, old_root->numa_node, old_root, true);
	1914	+ }
1757	1915
1758	1916	return ret;
1759	1917	}
@@ -1798,6 +1956,11 @@ static int __init tmigr_init(void) · 38 unmodified lines
1798	1956	if (ncpus == 1)
1799	1957	return 0;
1800	1958
	1959	+ if (!zalloc_cpumask_var(&tmigr_available_cpumask, GFP_KERNEL)) {
	1960	+ ret = -ENOMEM;
	1961	+ goto err;
	1962	+ }
	1963	+
1801	1964	/*
1802	1965	* Calculate the required hierarchy levels. Unfortunately there is no
1803	1966	* reliable information available, unless all possible CPUs have been
@@ -1847,7 +2010,7 @@ static int __init tmigr_init(void) · 43 unmodified lines
1847	2010	goto err;
1848	2011
1849	2012	ret = cpuhp_setup_state(CPUHP_AP_TMIGR_ONLINE, "tmigr:online",
1850		− tmigr_cpu_online, tmigr_cpu_offline);
	2013	+ tmigr_set_cpu_available, tmigr_clear_cpu_available);
1851	2014	if (ret)
1852	2015	goto err;
1853	2016

kernel/time/timer_migration.h+1−1

@@ -97,7 +97,7 @@ struct tmigr_group { · 96 unmodified lines
97	97	*/
98	98	struct tmigr_cpu {
99	99	raw_spinlock_t lock;
100		− bool online;
	100	+ bool available;
101	101	bool idle;
102	102	bool remote;
103	103	struct tmigr_group *tmgroup;

tools/testing/selftests/timers/nanosleep.c+55−0

@@ -116,6 +116,56 @@ int nanosleep_test(int clockid, long long ns) · 115 unmodified lines
116	116	return 0;
117	117	}
118	118
	119	+static void dummy_event_handler(int val)
	120	+{
	121	+ /* No action needed */
	122	+}
	123	+
	124	+static int nanosleep_test_remaining(int clockid)
	125	+{
	126	+ struct timespec rqtp = {}, rmtp = {};
	127	+ struct itimerspec itimer = {};
	128	+ struct sigaction sa = {};
	129	+ timer_t timer;
	130	+ int ret;
	131	+
	132	+ sa.sa_handler = dummy_event_handler;
	133	+ ret = sigaction(SIGALRM, &sa, NULL);
	134	+ if (ret)
	135	+ return -1;
	136	+
	137	+ ret = timer_create(clockid, NULL, &timer);
	138	+ if (ret)
	139	+ return -1;
	140	+
	141	+ itimer.it_value.tv_nsec = NSEC_PER_SEC / 4;
	142	+ ret = timer_settime(timer, 0, &itimer, NULL);
	143	+ if (ret)
	144	+ return -1;
	145	+
	146	+ rqtp.tv_nsec = NSEC_PER_SEC / 2;
	147	+ ret = clock_nanosleep(clockid, 0, &rqtp, &rmtp);
	148	+ if (ret != EINTR)
	149	+ return -1;
	150	+
	151	+ ret = timer_delete(timer);
	152	+ if (ret)
	153	+ return -1;
	154	+
	155	+ sa.sa_handler = SIG_DFL;
	156	+ ret = sigaction(SIGALRM, &sa, NULL);
	157	+ if (ret)
	158	+ return -1;
	159	+
	160	+ if (!in_order((struct timespec) {}, rmtp))
	161	+ return -1;
	162	+
	163	+ if (!in_order(rmtp, rqtp))
	164	+ return -1;
	165	+
	166	+ return 0;
	167	+}
	168	+
119	169	int main(int argc, char **argv)
120	170	{
121	171	long long length;
@@ -150,6 +200,11 @@ int main(int argc, char **argv) · 28 unmodified lines
150	200	}
151	201	length *= 100;
152	202	}
	203	+ ret = nanosleep_test_remaining(clockid);
	204	+ if (ret < 0) {
	205	+ ksft_test_result_fail("%-31s\n", clockstring(clockid));
	206	+ ksft_exit_fail();
	207	+ }
153	208	ksft_test_result_pass("%-31s\n", clockstring(clockid));
154	209	next:
155	210	ret = 0;

tools/testing/selftests/timers/posix_timers.c+22−10

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	#include <time.h>
19	19	#include <include/vdso/time64.h>
20	20	#include <pthread.h>
	21	+#include <stdbool.h>
21	22
22	23	#include "../kselftest.h"
23	24
@@ -670,8 +671,14 @@ static void check_timer_create_exact(void) · 646 unmodified lines
670	671
671	672	int main(int argc, char **argv)
672	673	{
	674	+ bool run_sig_ign_tests = ksft_min_kernel_version(6, 13);
	675	+
673	676	ksft_print_header();
674		− ksft_set_plan(19);
	677	+ if (run_sig_ign_tests) {
	678	+ ksft_set_plan(19);
	679	+ } else {
	680	+ ksft_set_plan(10);
	681	+ }
675	682
676	683	ksft_print_msg("Testing posix timers. False negative may happen on CPU execution \n");
677	684	ksft_print_msg("based timers if other threads run on the CPU...\n");
@@ -695,15 +702,20 @@ int main(int argc, char **argv) · 17 unmodified lines
695	702	check_timer_create(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID");
696	703	check_timer_distribution();
697	704
698		− check_sig_ign(0);
699		− check_sig_ign(1);
700		− check_rearm();
701		− check_delete();
702		− check_sigev_none(CLOCK_MONOTONIC, "CLOCK_MONOTONIC");
703		− check_sigev_none(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID");
704		− check_gettime(CLOCK_MONOTONIC, "CLOCK_MONOTONIC");
705		− check_gettime(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID");
706		− check_gettime(CLOCK_THREAD_CPUTIME_ID, "CLOCK_THREAD_CPUTIME_ID");
	705	+ if (run_sig_ign_tests) {
	706	+ check_sig_ign(0);
	707	+ check_sig_ign(1);
	708	+ check_rearm();
	709	+ check_delete();
	710	+ check_sigev_none(CLOCK_MONOTONIC, "CLOCK_MONOTONIC");
	711	+ check_sigev_none(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID");
	712	+ check_gettime(CLOCK_MONOTONIC, "CLOCK_MONOTONIC");
	713	+ check_gettime(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID");
	714	+ check_gettime(CLOCK_THREAD_CPUTIME_ID, "CLOCK_THREAD_CPUTIME_ID");
	715	+ } else {
	716	+ ksft_print_msg("Skipping SIG_IGN tests on kernel < 6.13\n");
	717	+ }
	718	+
707	719	check_overrun(CLOCK_MONOTONIC, "CLOCK_MONOTONIC");
708	720	check_overrun(CLOCK_PROCESS_CPUTIME_ID, "CLOCK_PROCESS_CPUTIME_ID");
709	721	check_overrun(CLOCK_THREAD_CPUTIME_ID, "CLOCK_THREAD_CPUTIME_ID");

Merge tag 'timers-core-2025-11-30' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code