Merge branch 'sched-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'sched-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (25 commits) sched: Fix broken assertion sched: Assert task state bits at build time sched: Update task_state_arraypwith new states sched: Add missing state chars to TASK_STATE_TO_CHAR_STR sched: Move TASK_STATE_TO_CHAR_STR near the TASK_state bits sched: Teach might_sleep() about preemptible RCU sched: Make warning less noisy sched: Simplify set_task_cpu() sched: Remove the cfs_rq dependency from set_task_cpu() sched: Add pre and post wakeup hooks sched: Move kthread_bind() back to kthread.c sched: Fix select_task_rq() vs hotplug issues sched: Fix sched_exec() balancing sched: Ensure set_task_cpu() is never called on blocked tasks sched: Use TASK_WAKING for fork wakups sched: Select_task_rq_fair() must honour SD_LOAD_BALANCE sched: Fix task_hot() test order sched: Fix set_cpu_active() in cpu_down() sched: Mark boot-cpu active before smp_init() sched: Fix cpu_clock() in NMIs, on !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK ...

Linus Torvalds committed 16 years ago

Commit aac3d392 parents 10e5453 077614e12 files · +336 −249Browse files

Unified · Split

12 files

fs/proc/array.c +12 −7
include/linux/rcutiny.h +5 −0
include/linux/rcutree.h +11 −0
include/linux/sched.h +9 −4
init/main.c +1 −6
kernel/cpu.c +3 −21
kernel/kthread.c +23 −0
kernel/sched.c +207 −194
kernel/sched_clock.c +15 −8
kernel/sched_fair.c +47 −6
kernel/sched_idletask.c +1 −1
kernel/sched_rt.c +2 −2

fs/proc/array.c+12−7

@@ -134,13 +134,16 @@ static inline void task_name(struct seq_file m, struct task_struct p) · 133 unmodified lines
134	134	* simple bit tests.
135	135	*/
136	136	static const char *task_state_array[] = {
137		− "R (running)", /* 0 */
138		− "S (sleeping)", /* 1 */
139		− "D (disk sleep)", /* 2 */
140		− "T (stopped)", /* 4 */
141		− "T (tracing stop)", /* 8 */
142		− "Z (zombie)", /* 16 */
143		− "X (dead)" /* 32 */
	137	+ "R (running)", /* 0 */
	138	+ "S (sleeping)", /* 1 */
	139	+ "D (disk sleep)", /* 2 */
	140	+ "T (stopped)", /* 4 */
	141	+ "t (tracing stop)", /* 8 */
	142	+ "Z (zombie)", /* 16 */
	143	+ "X (dead)", /* 32 */
	144	+ "x (dead)", /* 64 */
	145	+ "K (wakekill)", /* 128 */
	146	+ "W (waking)", /* 256 */
144	147	};
145	148
146	149	static inline const char get_task_state(struct task_struct tsk)
@@ -148,6 +151,8 @@ static inline const char get_task_state(struct task_struct tsk) · 1 unmodified line
148	151	unsigned int state = (tsk->state & TASK_REPORT) \| tsk->exit_state;
149	152	const char **p = &task_state_array[0];
150	153
	154	+ BUILD_BUG_ON(1 + ilog2(TASK_STATE_MAX) != ARRAY_SIZE(task_state_array));
	155	+
151	156	while (state) {
152	157	p++;
153	158	state >>= 1;

include/linux/rcutiny.h+5−0

@@ -101,4 +101,9 @@ static inline void exit_rcu(void) · 100 unmodified lines
101	101	{
102	102	}
103	103
	104	+static inline int rcu_preempt_depth(void)
	105	+{
	106	+ return 0;
	107	+}
	108	+
104	109	#endif /* __LINUX_RCUTINY_H */

include/linux/rcutree.h+11−0

@@ -45,6 +45,12 @@ extern void __rcu_read_unlock(void); · 44 unmodified lines
45	45	extern void synchronize_rcu(void);
46	46	extern void exit_rcu(void);
47	47
	48	+/*
	49	+ * Defined as macro as it is a very low level header
	50	+ * included from areas that don't even know about current
	51	+ */
	52	+#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
	53	+
48	54	#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
49	55
50	56	static inline void __rcu_read_lock(void)
@@ -63,6 +69,11 @@ static inline void exit_rcu(void) · 12 unmodified lines
63	69	{
64	70	}
65	71
	72	+static inline int rcu_preempt_depth(void)
	73	+{
	74	+ return 0;
	75	+}
	76	+
66	77	#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
67	78
68	79	static inline void __rcu_read_lock_bh(void)

include/linux/sched.h+9−4

@@ -192,6 +192,12 @@ print_cfs_rq(struct seq_file m, int cpu, struct cfs_rq cfs_rq) · 191 unmodified lines
192	192	#define TASK_DEAD 64
193	193	#define TASK_WAKEKILL 128
194	194	#define TASK_WAKING 256
	195	+#define TASK_STATE_MAX 512
	196	+
	197	+#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKW"
	198	+
	199	+extern char ___assert_task_state[1 - 2*!!(
	200	+ sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
195	201
196	202	/* Convenience macros for the sake of set_task_state */
197	203	#define TASK_KILLABLE (TASK_WAKEKILL \| TASK_UNINTERRUPTIBLE)
@@ -1091,7 +1097,8 @@ struct sched_class { · 893 unmodified lines
1091	1097	enum cpu_idle_type idle);
1092	1098	void (pre_schedule) (struct rq this_rq, struct task_struct *task);
1093	1099	void (post_schedule) (struct rq this_rq);
1094		− void (task_wake_up) (struct rq this_rq, struct task_struct *task);
	1100	+ void (task_waking) (struct rq this_rq, struct task_struct *task);
	1101	+ void (task_woken) (struct rq this_rq, struct task_struct *task);
1095	1102
1096	1103	void (set_cpus_allowed)(struct task_struct p,
1097	1104	const struct cpumask *newmask);
@@ -1115,7 +1122,7 @@ struct sched_class { · 17 unmodified lines
1115	1122	struct task_struct *task);
1116	1123
1117	1124	#ifdef CONFIG_FAIR_GROUP_SCHED
1118		− void (moved_group) (struct task_struct p);
	1125	+ void (moved_group) (struct task_struct p, int on_rq);
1119	1126	#endif
1120	1127	};
1121	1128
@@ -2594,8 +2601,6 @@ static inline void mm_init_owner(struct mm_struct mm, struct task_struct p) · 1472 unmodified lines
2594	2601	}
2595	2602	#endif /* CONFIG_MM_OWNER */
2596	2603
2597		−#define TASK_STATE_TO_CHAR_STR "RSDTtZX"
2598		−
2599	2604	#endif /* __KERNEL__ */
2600	2605
2601	2606	#endif

init/main.c+1−6

@@ -369,12 +369,6 @@ static void __init smp_init(void) · 368 unmodified lines
369	369	{
370	370	unsigned int cpu;
371	371
372		− /*
373		− * Set up the current CPU as possible to migrate to.
374		− * The other ones will be done by cpu_up/cpu_down()
375		− */
376		− set_cpu_active(smp_processor_id(), true);
377		−
378	372	/* FIXME: This should be done in userspace --RR */
379	373	for_each_present_cpu(cpu) {
380	374	if (num_online_cpus() >= setup_max_cpus)
@@ -486,6 +480,7 @@ static void __init boot_cpu_init(void) · 105 unmodified lines
486	480	int cpu = smp_processor_id();
487	481	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
488	482	set_cpu_online(cpu, true);
	483	+ set_cpu_active(cpu, true);
489	484	set_cpu_present(cpu, true);
490	485	set_cpu_possible(cpu, true);
491	486	}

kernel/cpu.c+3−21

@@ -209,6 +209,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) · 208 unmodified lines
209	209	return -ENOMEM;
210	210
211	211	cpu_hotplug_begin();
	212	+ set_cpu_active(cpu, false);
212	213	err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE \| mod,
213	214	hcpu, -1, &nr_calls);
214	215	if (err == NOTIFY_BAD) {
@@ -280,18 +281,6 @@ int __ref cpu_down(unsigned int cpu) · 65 unmodified lines
280	281	goto out;
281	282	}
282	283
283		− set_cpu_active(cpu, false);
284		−
285		− /*
286		− * Make sure the all cpus did the reschedule and are not
287		− * using stale version of the cpu_active_mask.
288		− * This is not strictly necessary becuase stop_machine()
289		− * that we run down the line already provides the required
290		− * synchronization. But it's really a side effect and we do not
291		− * want to depend on the innards of the stop_machine here.
292		− */
293		− synchronize_sched();
294		−
295	284	err = _cpu_down(cpu, 0);
296	285
297	286	out:
@@ -382,19 +371,12 @@ int disable_nonboot_cpus(void) · 84 unmodified lines
382	371	return error;
383	372	cpu_maps_update_begin();
384	373	first_cpu = cpumask_first(cpu_online_mask);
385		− /* We take down all of the non-boot CPUs in one shot to avoid races
	374	+ /*
	375	+ * We take down all of the non-boot CPUs in one shot to avoid races
386	376	* with the userspace trying to use the CPU hotplug at the same time
387	377	*/
388	378	cpumask_clear(frozen_cpus);
389	379
390		− for_each_online_cpu(cpu) {
391		− if (cpu == first_cpu)
392		− continue;
393		− set_cpu_active(cpu, false);
394		− }
395		−
396		− synchronize_sched();
397		−
398	380	printk("Disabling non-boot CPUs ...\n");
399	381	for_each_online_cpu(cpu) {
400	382	if (cpu == first_cpu)

kernel/kthread.c+23−0

@@ -149,6 +149,29 @@ struct task_struct kthread_create(int (threadfn)(void *data), · 148 unmodified lines
149	149	}
150	150	EXPORT_SYMBOL(kthread_create);
151	151
	152	+/**
	153	+ * kthread_bind - bind a just-created kthread to a cpu.
	154	+ * @p: thread created by kthread_create().
	155	+ * @cpu: cpu (might not be online, must be possible) for @k to run on.
	156	+ *
	157	+ * Description: This function is equivalent to set_cpus_allowed(),
	158	+ * except that @cpu doesn't need to be online, and the thread must be
	159	+ * stopped (i.e., just returned from kthread_create()).
	160	+ */
	161	+void kthread_bind(struct task_struct *p, unsigned int cpu)
	162	+{
	163	+ /* Must have done schedule() in kthread() before we set_task_cpu */
	164	+ if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) {
	165	+ WARN_ON(1);
	166	+ return;
	167	+ }
	168	+
	169	+ p->cpus_allowed = cpumask_of_cpu(cpu);
	170	+ p->rt.nr_cpus_allowed = 1;
	171	+ p->flags \|= PF_THREAD_BOUND;
	172	+}
	173	+EXPORT_SYMBOL(kthread_bind);
	174	+
152	175	/**
153	176	* kthread_stop - stop a thread created by kthread_create().
154	177	* @k: thread created by kthread_create().

kernel/sched.c+207−194

@@ -26,6 +26,8 @@ · 25 unmodified lines
26	26	* Thomas Gleixner, Mike Kravetz
27	27	*/
28	28
	29	+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	30	+
29	31	#include <linux/mm.h>
30	32	#include <linux/module.h>
31	33	#include <linux/nmi.h>
@@ -2002,39 +2004,6 @@ static inline void check_class_changed(struct rq rq, struct task_struct p, · 1970 unmodified lines
2002	2004	p->sched_class->prio_changed(rq, p, oldprio, running);
2003	2005	}
2004	2006
2005		−/**
2006		− * kthread_bind - bind a just-created kthread to a cpu.
2007		− * @p: thread created by kthread_create().
2008		− * @cpu: cpu (might not be online, must be possible) for @k to run on.
2009		− *
2010		− * Description: This function is equivalent to set_cpus_allowed(),
2011		− * except that @cpu doesn't need to be online, and the thread must be
2012		− * stopped (i.e., just returned from kthread_create()).
2013		− *
2014		− * Function lives here instead of kthread.c because it messes with
2015		− * scheduler internals which require locking.
2016		− */
2017		−void kthread_bind(struct task_struct *p, unsigned int cpu)
2018		−{
2019		− struct rq *rq = cpu_rq(cpu);
2020		− unsigned long flags;
2021		−
2022		− /* Must have done schedule() in kthread() before we set_task_cpu */
2023		− if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) {
2024		− WARN_ON(1);
2025		− return;
2026		− }
2027		−
2028		− raw_spin_lock_irqsave(&rq->lock, flags);
2029		− update_rq_clock(rq);
2030		− set_task_cpu(p, cpu);
2031		− p->cpus_allowed = cpumask_of_cpu(cpu);
2032		− p->rt.nr_cpus_allowed = 1;
2033		− p->flags \|= PF_THREAD_BOUND;
2034		− raw_spin_unlock_irqrestore(&rq->lock, flags);
2035		−}
2036		−EXPORT_SYMBOL(kthread_bind);
2037		−
2038	2007	#ifdef CONFIG_SMP
2039	2008	/*
2040	2009	* Is this task likely cache-hot:
@@ -2044,6 +2013,9 @@ task_hot(struct task_struct p, u64 now, struct sched_domain sd) · 3 unmodified lines
2044	2013	{
2045	2014	s64 delta;
2046	2015
	2016	+ if (p->sched_class != &fair_sched_class)
	2017	+ return 0;
	2018	+
2047	2019	/*
2048	2020	* Buddy candidates are cache hot:
2049	2021	*/
@@ -2052,9 +2024,6 @@ task_hot(struct task_struct p, u64 now, struct sched_domain sd) · 2 unmodified lines
2052	2024	&p->se == cfs_rq_of(&p->se)->last))
2053	2025	return 1;
2054	2026
2055		− if (p->sched_class != &fair_sched_class)
2056		− return 0;
2057		−
2058	2027	if (sysctl_sched_migration_cost == -1)
2059	2028	return 1;
2060	2029	if (sysctl_sched_migration_cost == 0)
@@ -2065,22 +2034,24 @@ task_hot(struct task_struct p, u64 now, struct sched_domain sd) · 4 unmodified lines
2065	2034	return delta < (s64)sysctl_sched_migration_cost;
2066	2035	}
2067	2036
2068		−
2069	2037	void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
2070	2038	{
2071		− int old_cpu = task_cpu(p);
2072		− struct cfs_rq *old_cfsrq = task_cfs_rq(p),
2073		− *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
	2039	+#ifdef CONFIG_SCHED_DEBUG
	2040	+ /*
	2041	+ * We should never call set_task_cpu() on a blocked task,
	2042	+ * ttwu() will sort out the placement.
	2043	+ */
	2044	+ WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
	2045	+ !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
	2046	+#endif
2074	2047
2075	2048	trace_sched_migrate_task(p, new_cpu);
2076	2049
2077		− if (old_cpu != new_cpu) {
2078		− p->se.nr_migrations++;
2079		− perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS,
2080		− 1, 1, NULL, 0);
2081		− }
2082		− p->se.vruntime -= old_cfsrq->min_vruntime -
2083		− new_cfsrq->min_vruntime;
	2050	+ if (task_cpu(p) == new_cpu)
	2051	+ return;
	2052	+
	2053	+ p->se.nr_migrations++;
	2054	+ perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
2084	2055
2085	2056	__set_task_cpu(p, new_cpu);
2086	2057	}
@@ -2105,13 +2076,10 @@ migrate_task(struct task_struct p, int dest_cpu, struct migration_req req) · 18 unmodified lines
2105	2076
2106	2077	/*
2107	2078	* If the task is not on a runqueue (and not running), then
2108		− * it is sufficient to simply update the task's cpu field.
	2079	+ * the next wake-up will properly place the task.
2109	2080	*/
2110		− if (!p->se.on_rq && !task_running(rq, p)) {
2111		− update_rq_clock(rq);
2112		− set_task_cpu(p, dest_cpu);
	2081	+ if (!p->se.on_rq && !task_running(rq, p))
2113	2082	return 0;
2114		− }
2115	2083
2116	2084	init_completion(&req->done);
2117	2085	req->task = p;
@@ -2317,10 +2285,73 @@ void task_oncpu_function_call(struct task_struct *p, · 199 unmodified lines
2317	2285	}
2318	2286
2319	2287	#ifdef CONFIG_SMP
	2288	+static int select_fallback_rq(int cpu, struct task_struct *p)
	2289	+{
	2290	+ int dest_cpu;
	2291	+ const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));
	2292	+
	2293	+ /* Look for allowed, online CPU in same node. */
	2294	+ for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
	2295	+ if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
	2296	+ return dest_cpu;
	2297	+
	2298	+ /* Any allowed, online CPU? */
	2299	+ dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
	2300	+ if (dest_cpu < nr_cpu_ids)
	2301	+ return dest_cpu;
	2302	+
	2303	+ /* No more Mr. Nice Guy. */
	2304	+ if (dest_cpu >= nr_cpu_ids) {
	2305	+ rcu_read_lock();
	2306	+ cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
	2307	+ rcu_read_unlock();
	2308	+ dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
	2309	+
	2310	+ /*
	2311	+ * Don't tell them about moving exiting tasks or
	2312	+ * kernel threads (both mm NULL), since they never
	2313	+ * leave kernel.
	2314	+ */
	2315	+ if (p->mm && printk_ratelimit()) {
	2316	+ printk(KERN_INFO "process %d (%s) no "
	2317	+ "longer affine to cpu%d\n",
	2318	+ task_pid_nr(p), p->comm, cpu);
	2319	+ }
	2320	+ }
	2321	+
	2322	+ return dest_cpu;
	2323	+}
	2324	+
	2325	+/*
	2326	+ * Called from:
	2327	+ *
	2328	+ * - fork, @p is stable because it isn't on the tasklist yet
	2329	+ *
	2330	+ * - exec, @p is unstable, retry loop
	2331	+ *
	2332	+ * - wake-up, we serialize ->cpus_allowed against TASK_WAKING so
	2333	+ * we should be good.
	2334	+ */
2320	2335	static inline
2321	2336	int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
2322	2337	{
2323		− return p->sched_class->select_task_rq(p, sd_flags, wake_flags);
	2338	+ int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
	2339	+
	2340	+ /*
	2341	+ * In order not to call set_task_cpu() on a blocking task we need
	2342	+ * to rely on ttwu() to place the task on a valid ->cpus_allowed
	2343	+ * cpu.
	2344	+ *
	2345	+ * Since this is common to all placement strategies, this lives here.
	2346	+ *
	2347	+ * [ this allows ->select_task() to simply return task_cpu(p) and
	2348	+ * not worry about this generic constraint ]
	2349	+ */
	2350	+ if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) \|\|
	2351	+ !cpu_active(cpu)))
	2352	+ cpu = select_fallback_rq(task_cpu(p), p);
	2353	+
	2354	+ return cpu;
2324	2355	}
2325	2356	#endif
2326	2357
@@ -2375,6 +2406,10 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, · 48 unmodified lines
2375	2406	if (task_contributes_to_load(p))
2376	2407	rq->nr_uninterruptible--;
2377	2408	p->state = TASK_WAKING;
	2409	+
	2410	+ if (p->sched_class->task_waking)
	2411	+ p->sched_class->task_waking(rq, p);
	2412	+
2378	2413	__task_rq_unlock(rq);
2379	2414
2380	2415	cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
@@ -2438,8 +2473,8 @@ out_running: · 57 unmodified lines
2438	2473
2439	2474	p->state = TASK_RUNNING;
2440	2475	#ifdef CONFIG_SMP
2441		− if (p->sched_class->task_wake_up)
2442		− p->sched_class->task_wake_up(rq, p);
	2476	+ if (p->sched_class->task_woken)
	2477	+ p->sched_class->task_woken(rq, p);
2443	2478
2444	2479	if (unlikely(rq->idle_stamp)) {
2445	2480	u64 delta = rq->clock - rq->idle_stamp;
@@ -2538,14 +2573,6 @@ static void __sched_fork(struct task_struct *p) · 92 unmodified lines
2538	2573	#ifdef CONFIG_PREEMPT_NOTIFIERS
2539	2574	INIT_HLIST_HEAD(&p->preempt_notifiers);
2540	2575	#endif
2541		−
2542		− /*
2543		− * We mark the process as running here, but have not actually
2544		− * inserted it onto the runqueue yet. This guarantees that
2545		− * nobody will actually run it, and a signal or other external
2546		− * event cannot wake it up and insert it on the runqueue either.
2547		− */
2548		− p->state = TASK_RUNNING;
2549	2576	}
2550	2577
2551	2578	/*
@@ -2556,6 +2583,12 @@ void sched_fork(struct task_struct *p, int clone_flags) · 4 unmodified lines
2556	2583	int cpu = get_cpu();
2557	2584
2558	2585	__sched_fork(p);
	2586	+ /*
	2587	+ * We mark the process as waking here. This guarantees that
	2588	+ * nobody will actually run it, and a signal or other external
	2589	+ * event cannot wake it up and insert it on the runqueue either.
	2590	+ */
	2591	+ p->state = TASK_WAKING;
2559	2592
2560	2593	/*
2561	2594	* Revert to default priority/policy on fork if requested.
@@ -2624,14 +2657,15 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags) · 62 unmodified lines
2624	2657	struct rq *rq;
2625	2658
2626	2659	rq = task_rq_lock(p, &flags);
2627		− BUG_ON(p->state != TASK_RUNNING);
	2660	+ BUG_ON(p->state != TASK_WAKING);
	2661	+ p->state = TASK_RUNNING;
2628	2662	update_rq_clock(rq);
2629	2663	activate_task(rq, p, 0);
2630	2664	trace_sched_wakeup_new(rq, p, 1);
2631	2665	check_preempt_curr(rq, p, WF_FORK);
2632	2666	#ifdef CONFIG_SMP
2633		− if (p->sched_class->task_wake_up)
2634		− p->sched_class->task_wake_up(rq, p);
	2667	+ if (p->sched_class->task_woken)
	2668	+ p->sched_class->task_woken(rq, p);
2635	2669	#endif
2636	2670	task_rq_unlock(rq, &flags);
2637	2671	}
@@ -3101,21 +3135,36 @@ static void double_rq_unlock(struct rq rq1, struct rq rq2) · 463 unmodified lines
3101	3135	}
3102	3136
3103	3137	/*
3104		− * If dest_cpu is allowed for this process, migrate the task to it.
3105		− * This is accomplished by forcing the cpu_allowed mask to only
3106		− * allow dest_cpu, which will force the cpu onto dest_cpu. Then
3107		− * the cpu_allowed mask is restored.
	3138	+ * sched_exec - execve() is a valuable balancing opportunity, because at
	3139	+ * this point the task has the smallest effective memory and cache footprint.
3108	3140	*/
3109		−static void sched_migrate_task(struct task_struct *p, int dest_cpu)
	3141	+void sched_exec(void)
3110	3142	{
	3143	+ struct task_struct *p = current;
3111	3144	struct migration_req req;
	3145	+ int dest_cpu, this_cpu;
3112	3146	unsigned long flags;
3113	3147	struct rq *rq;
3114	3148
	3149	+again:
	3150	+ this_cpu = get_cpu();
	3151	+ dest_cpu = select_task_rq(p, SD_BALANCE_EXEC, 0);
	3152	+ if (dest_cpu == this_cpu) {
	3153	+ put_cpu();
	3154	+ return;
	3155	+ }
	3156	+
3115	3157	rq = task_rq_lock(p, &flags);
	3158	+ put_cpu();
	3159	+
	3160	+ /*
	3161	+ * select_task_rq() can race against ->cpus_allowed
	3162	+ */
3116	3163	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
3117		− \|\| unlikely(!cpu_active(dest_cpu)))
3118		− goto out;
	3164	+ \|\| unlikely(!cpu_active(dest_cpu))) {
	3165	+ task_rq_unlock(rq, &flags);
	3166	+ goto again;
	3167	+ }
3119	3168
3120	3169	/* force the process onto the specified CPU */
3121	3170	if (migrate_task(p, dest_cpu, &req)) {
@@ -3130,23 +3179,9 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu) · 8 unmodified lines
3130	3179
3131	3180	return;
3132	3181	}
3133		−out:
3134	3182	task_rq_unlock(rq, &flags);
3135	3183	}
3136	3184
3137		−/*
3138		− * sched_exec - execve() is a valuable balancing opportunity, because at
3139		− * this point the task has the smallest effective memory and cache footprint.
3140		− */
3141		−void sched_exec(void)
3142		−{
3143		− int new_cpu, this_cpu = get_cpu();
3144		− new_cpu = select_task_rq(current, SD_BALANCE_EXEC, 0);
3145		− put_cpu();
3146		− if (new_cpu != this_cpu)
3147		− sched_migrate_task(current, new_cpu);
3148		−}
3149		−
3150	3185	/*
3151	3186	* pull_task - move a task from a remote runqueue to the local runqueue.
3152	3187	* Both runqueues must be locked.
@@ -5340,8 +5375,8 @@ static noinline void __schedule_bug(struct task_struct *prev) · 2187 unmodified lines
5340	5375	{
5341	5376	struct pt_regs *regs = get_irq_regs();
5342	5377
5343		− printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
5344		− prev->comm, prev->pid, preempt_count());
	5378	+ pr_err("BUG: scheduling while atomic: %s/%d/0x%08x\n",
	5379	+ prev->comm, prev->pid, preempt_count());
5345	5380
5346	5381	debug_show_held_locks(prev);
5347	5382	print_modules();
@@ -5911,14 +5946,15 @@ EXPORT_SYMBOL(wait_for_completion_killable); · 563 unmodified lines
5911	5946	*/
5912	5947	bool try_wait_for_completion(struct completion *x)
5913	5948	{
	5949	+ unsigned long flags;
5914	5950	int ret = 1;
5915	5951
5916		− spin_lock_irq(&x->wait.lock);
	5952	+ spin_lock_irqsave(&x->wait.lock, flags);
5917	5953	if (!x->done)
5918	5954	ret = 0;
5919	5955	else
5920	5956	x->done--;
5921		− spin_unlock_irq(&x->wait.lock);
	5957	+ spin_unlock_irqrestore(&x->wait.lock, flags);
5922	5958	return ret;
5923	5959	}
5924	5960	EXPORT_SYMBOL(try_wait_for_completion);
@@ -5933,12 +5969,13 @@ EXPORT_SYMBOL(try_wait_for_completion); · 8 unmodified lines
5933	5969	*/
5934	5970	bool completion_done(struct completion *x)
5935	5971	{
	5972	+ unsigned long flags;
5936	5973	int ret = 1;
5937	5974
5938		− spin_lock_irq(&x->wait.lock);
	5975	+ spin_lock_irqsave(&x->wait.lock, flags);
5939	5976	if (!x->done)
5940	5977	ret = 0;
5941		− spin_unlock_irq(&x->wait.lock);
	5978	+ spin_unlock_irqrestore(&x->wait.lock, flags);
5942	5979	return ret;
5943	5980	}
5944	5981	EXPORT_SYMBOL(completion_done);
@@ -6457,7 +6494,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) · 512 unmodified lines
6457	6494	return -EINVAL;
6458	6495
6459	6496	retval = -ESRCH;
6460		− read_lock(&tasklist_lock);
	6497	+ rcu_read_lock();
6461	6498	p = find_process_by_pid(pid);
6462	6499	if (p) {
6463	6500	retval = security_task_getscheduler(p);
@@ -6465,7 +6502,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) · 1 unmodified line
6465	6502	retval = p->policy
6466	6503	\| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
6467	6504	}
6468		− read_unlock(&tasklist_lock);
	6505	+ rcu_read_unlock();
6469	6506	return retval;
6470	6507	}
6471	6508
@@ -6483,7 +6520,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) · 11 unmodified lines
6483	6520	if (!param \|\| pid < 0)
6484	6521	return -EINVAL;
6485	6522
6486		− read_lock(&tasklist_lock);
	6523	+ rcu_read_lock();
6487	6524	p = find_process_by_pid(pid);
6488	6525	retval = -ESRCH;
6489	6526	if (!p)
@@ -6494,7 +6531,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) · 4 unmodified lines
6494	6531	goto out_unlock;
6495	6532
6496	6533	lp.sched_priority = p->rt_priority;
6497		− read_unlock(&tasklist_lock);
	6534	+ rcu_read_unlock();
6498	6535
6499	6536	/*
6500	6537	* This one might sleep, we cannot do it with a spinlock held ...
@@ -6504,7 +6541,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) · 3 unmodified lines
6504	6541	return retval;
6505	6542
6506	6543	out_unlock:
6507		− read_unlock(&tasklist_lock);
	6544	+ rcu_read_unlock();
6508	6545	return retval;
6509	6546	}
6510	6547
@@ -6515,22 +6552,18 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) · 4 unmodified lines
6515	6552	int retval;
6516	6553
6517	6554	get_online_cpus();
6518		− read_lock(&tasklist_lock);
	6555	+ rcu_read_lock();
6519	6556
6520	6557	p = find_process_by_pid(pid);
6521	6558	if (!p) {
6522		− read_unlock(&tasklist_lock);
	6559	+ rcu_read_unlock();
6523	6560	put_online_cpus();
6524	6561	return -ESRCH;
6525	6562	}
6526	6563
6527		− /*
6528		− * It is not safe to call set_cpus_allowed with the
6529		− * tasklist_lock held. We will bump the task_struct's
6530		− * usage count and then drop tasklist_lock.
6531		− */
	6564	+ /* Prevent p going away */
6532	6565	get_task_struct(p);
6533		− read_unlock(&tasklist_lock);
	6566	+ rcu_read_unlock();
6534	6567
6535	6568	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
6536	6569	retval = -ENOMEM;
@@ -6616,7 +6649,7 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask) · 79 unmodified lines
6616	6649	int retval;
6617	6650
6618	6651	get_online_cpus();
6619		− read_lock(&tasklist_lock);
	6652	+ rcu_read_lock();
6620	6653
6621	6654	retval = -ESRCH;
6622	6655	p = find_process_by_pid(pid);
@@ -6632,7 +6665,7 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask) · 9 unmodified lines
6632	6665	task_rq_unlock(rq, &flags);
6633	6666
6634	6667	out_unlock:
6635		− read_unlock(&tasklist_lock);
	6668	+ rcu_read_unlock();
6636	6669	put_online_cpus();
6637	6670
6638	6671	return retval;
@@ -6876,7 +6909,7 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, · 237 unmodified lines
6876	6909	return -EINVAL;
6877	6910
6878	6911	retval = -ESRCH;
6879		− read_lock(&tasklist_lock);
	6912	+ rcu_read_lock();
6880	6913	p = find_process_by_pid(pid);
6881	6914	if (!p)
6882	6915	goto out_unlock;
@@ -6889,13 +6922,13 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, · 6 unmodified lines
6889	6922	time_slice = p->sched_class->get_rr_interval(rq, p);
6890	6923	task_rq_unlock(rq, &flags);
6891	6924
6892		− read_unlock(&tasklist_lock);
	6925	+ rcu_read_unlock();
6893	6926	jiffies_to_timespec(time_slice, &t);
6894	6927	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
6895	6928	return retval;
6896	6929
6897	6930	out_unlock:
6898		− read_unlock(&tasklist_lock);
	6931	+ rcu_read_unlock();
6899	6932	return retval;
6900	6933	}
6901	6934
@@ -6907,23 +6940,23 @@ void sched_show_task(struct task_struct *p) · 5 unmodified lines
6907	6940	unsigned state;
6908	6941
6909	6942	state = p->state ? __ffs(p->state) + 1 : 0;
6910		− printk(KERN_INFO "%-13.13s %c", p->comm,
	6943	+ pr_info("%-13.13s %c", p->comm,
6911	6944	state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
6912	6945	#if BITS_PER_LONG == 32
6913	6946	if (state == TASK_RUNNING)
6914		− printk(KERN_CONT " running ");
	6947	+ pr_cont(" running ");
6915	6948	else
6916		− printk(KERN_CONT " %08lx ", thread_saved_pc(p));
	6949	+ pr_cont(" %08lx ", thread_saved_pc(p));
6917	6950	#else
6918	6951	if (state == TASK_RUNNING)
6919		− printk(KERN_CONT " running task ");
	6952	+ pr_cont(" running task ");
6920	6953	else
6921		− printk(KERN_CONT " %016lx ", thread_saved_pc(p));
	6954	+ pr_cont(" %016lx ", thread_saved_pc(p));
6922	6955	#endif
6923	6956	#ifdef CONFIG_DEBUG_STACK_USAGE
6924	6957	free = stack_not_used(p);
6925	6958	#endif
6926		− printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
	6959	+ pr_cont("%5lu %5d %6d 0x%08lx\n", free,
6927	6960	task_pid_nr(p), task_pid_nr(p->real_parent),
6928	6961	(unsigned long)task_thread_info(p)->flags);
6929	6962
@@ -6935,11 +6968,9 @@ void show_state_filter(unsigned long state_filter) · 5 unmodified lines
6935	6968	struct task_struct g, p;
6936	6969
6937	6970	#if BITS_PER_LONG == 32
6938		− printk(KERN_INFO
6939		− " task PC stack pid father\n");
	6971	+ pr_info(" task PC stack pid father\n");
6940	6972	#else
6941		− printk(KERN_INFO
6942		− " task PC stack pid father\n");
	6973	+ pr_info(" task PC stack pid father\n");
6943	6974	#endif
6944	6975	read_lock(&tasklist_lock);
6945	6976	do_each_thread(g, p) {
@@ -6986,6 +7017,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) · 40 unmodified lines
6986	7017	raw_spin_lock_irqsave(&rq->lock, flags);
6987	7018
6988	7019	__sched_fork(idle);
	7020	+ idle->state = TASK_RUNNING;
6989	7021	idle->se.exec_start = sched_clock();
6990	7022
6991	7023	cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
@@ -7100,7 +7132,23 @@ int set_cpus_allowed_ptr(struct task_struct p, const struct cpumask new_mask) · 108 unmodified lines
7100	7132	struct rq *rq;
7101	7133	int ret = 0;
7102	7134
	7135	+ /*
	7136	+ * Since we rely on wake-ups to migrate sleeping tasks, don't change
	7137	+ * the ->cpus_allowed mask from under waking tasks, which would be
	7138	+ * possible when we change rq->lock in ttwu(), so synchronize against
	7139	+ * TASK_WAKING to avoid that.
	7140	+ */
	7141	+again:
	7142	+ while (p->state == TASK_WAKING)
	7143	+ cpu_relax();
	7144	+
7103	7145	rq = task_rq_lock(p, &flags);
	7146	+
	7147	+ if (p->state == TASK_WAKING) {
	7148	+ task_rq_unlock(rq, &flags);
	7149	+ goto again;
	7150	+ }
	7151	+
7104	7152	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
7105	7153	ret = -EINVAL;
7106	7154	goto out;
@@ -7156,7 +7204,7 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); · 49 unmodified lines
7156	7204	static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
7157	7205	{
7158	7206	struct rq rq_dest, rq_src;
7159		− int ret = 0, on_rq;
	7207	+ int ret = 0;
7160	7208
7161	7209	if (unlikely(!cpu_active(dest_cpu)))
7162	7210	return ret;
@@ -7172,12 +7220,13 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) · 9 unmodified lines
7172	7220	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
7173	7221	goto fail;
7174	7222
7175		− on_rq = p->se.on_rq;
7176		− if (on_rq)
	7223	+ /*
	7224	+ * If we're not on a rq, the next wake-up will ensure we're
	7225	+ * placed properly.
	7226	+ */
	7227	+ if (p->se.on_rq) {
7177	7228	deactivate_task(rq_src, p, 0);
7178		−
7179		− set_task_cpu(p, dest_cpu);
7180		− if (on_rq) {
	7229	+ set_task_cpu(p, dest_cpu);
7181	7230	activate_task(rq_dest, p, 0);
7182	7231	check_preempt_curr(rq_dest, p, 0);
7183	7232	}
@@ -7273,37 +7322,10 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu) · 89 unmodified lines
7273	7322	static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
7274	7323	{
7275	7324	int dest_cpu;
7276		− const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
7277	7325
7278	7326	again:
7279		− /* Look for allowed, online CPU in same node. */
7280		− for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
7281		− if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
7282		− goto move;
7283		−
7284		− /* Any allowed, online CPU? */
7285		− dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
7286		− if (dest_cpu < nr_cpu_ids)
7287		− goto move;
7288		−
7289		− /* No more Mr. Nice Guy. */
7290		− if (dest_cpu >= nr_cpu_ids) {
7291		− cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
7292		− dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
7293		−
7294		− /*
7295		− * Don't tell them about moving exiting tasks or
7296		− * kernel threads (both mm NULL), since they never
7297		− * leave kernel.
7298		− */
7299		− if (p->mm && printk_ratelimit()) {
7300		− printk(KERN_INFO "process %d (%s) no "
7301		− "longer affine to cpu%d\n",
7302		− task_pid_nr(p), p->comm, dead_cpu);
7303		− }
7304		− }
	7327	+ dest_cpu = select_fallback_rq(dead_cpu, p);
7305	7328
7306		−move:
7307	7329	/* It can have affinity changed while we were choosing. */
7308	7330	if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
7309	7331	goto again;
@@ -7806,48 +7828,44 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, · 496 unmodified lines
7806	7828	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
7807	7829
7808	7830	if (!(sd->flags & SD_LOAD_BALANCE)) {
7809		− printk("does not load-balance\n");
	7831	+ pr_cont("does not load-balance\n");
7810	7832	if (sd->parent)
7811		− printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
7812		− " has parent");
	7833	+ pr_err("ERROR: !SD_LOAD_BALANCE domain has parent\n");
7813	7834	return -1;
7814	7835	}
7815	7836
7816		− printk(KERN_CONT "span %s level %s\n", str, sd->name);
	7837	+ pr_cont("span %s level %s\n", str, sd->name);
7817	7838
7818	7839	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
7819		− printk(KERN_ERR "ERROR: domain->span does not contain "
7820		− "CPU%d\n", cpu);
	7840	+ pr_err("ERROR: domain->span does not contain CPU%d\n", cpu);
7821	7841	}
7822	7842	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
7823		− printk(KERN_ERR "ERROR: domain->groups does not contain"
7824		− " CPU%d\n", cpu);
	7843	+ pr_err("ERROR: domain->groups does not contain CPU%d\n", cpu);
7825	7844	}
7826	7845
7827	7846	printk(KERN_DEBUG "%*s groups:", level + 1, "");
7828	7847	do {
7829	7848	if (!group) {
7830		− printk("\n");
7831		− printk(KERN_ERR "ERROR: group is NULL\n");
	7849	+ pr_cont("\n");
	7850	+ pr_err("ERROR: group is NULL\n");
7832	7851	break;
7833	7852	}
7834	7853
7835	7854	if (!group->cpu_power) {
7836		− printk(KERN_CONT "\n");
7837		− printk(KERN_ERR "ERROR: domain->cpu_power not "
7838		− "set\n");
	7855	+ pr_cont("\n");
	7856	+ pr_err("ERROR: domain->cpu_power not set\n");
7839	7857	break;
7840	7858	}
7841	7859
7842	7860	if (!cpumask_weight(sched_group_cpus(group))) {
7843		− printk(KERN_CONT "\n");
7844		− printk(KERN_ERR "ERROR: empty group\n");
	7861	+ pr_cont("\n");
	7862	+ pr_err("ERROR: empty group\n");
7845	7863	break;
7846	7864	}
7847	7865
7848	7866	if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
7849		− printk(KERN_CONT "\n");
7850		− printk(KERN_ERR "ERROR: repeated CPUs\n");
	7867	+ pr_cont("\n");
	7868	+ pr_err("ERROR: repeated CPUs\n");
7851	7869	break;
7852	7870	}
7853	7871
@@ -7855,23 +7873,21 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, · 1 unmodified line
7855	7873
7856	7874	cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
7857	7875
7858		− printk(KERN_CONT " %s", str);
	7876	+ pr_cont(" %s", str);
7859	7877	if (group->cpu_power != SCHED_LOAD_SCALE) {
7860		− printk(KERN_CONT " (cpu_power = %d)",
7861		− group->cpu_power);
	7878	+ pr_cont(" (cpu_power = %d)", group->cpu_power);
7862	7879	}
7863	7880
7864	7881	group = group->next;
7865	7882	} while (group != sd->groups);
7866		− printk(KERN_CONT "\n");
	7883	+ pr_cont("\n");
7867	7884
7868	7885	if (!cpumask_equal(sched_domain_span(sd), groupmask))
7869		− printk(KERN_ERR "ERROR: groups don't span domain->span\n");
	7886	+ pr_err("ERROR: groups don't span domain->span\n");
7870	7887
7871	7888	if (sd->parent &&
7872	7889	!cpumask_subset(groupmask, sched_domain_span(sd->parent)))
7873		− printk(KERN_ERR "ERROR: parent span is not a superset "
7874		− "of domain->span\n");
	7890	+ pr_err("ERROR: parent span is not a superset of domain->span\n");
7875	7891	return 0;
7876	7892	}
7877	7893
@@ -8427,8 +8443,7 @@ static int build_numa_sched_groups(struct s_data *d, · 549 unmodified lines
8427	8443	sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
8428	8444	GFP_KERNEL, num);
8429	8445	if (!sg) {
8430		− printk(KERN_WARNING "Can not alloc domain group for node %d\n",
8431		− num);
	8446	+ pr_warning("Can not alloc domain group for node %d\n", num);
8432	8447	return -ENOMEM;
8433	8448	}
8434	8449	d->sched_group_nodes[num] = sg;
@@ -8457,8 +8472,8 @@ static int build_numa_sched_groups(struct s_data *d, · 22 unmodified lines
8457	8472	sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
8458	8473	GFP_KERNEL, num);
8459	8474	if (!sg) {
8460		− printk(KERN_WARNING
8461		− "Can not alloc domain group for node %d\n", j);
	8475	+ pr_warning("Can not alloc domain group for node %d\n",
	8476	+ j);
8462	8477	return -ENOMEM;
8463	8478	}
8464	8479	sg->cpu_power = 0;
@@ -8686,7 +8701,7 @@ static enum s_alloc __visit_domain_allocation_hell(struct s_data *d, · 221 unmodified lines
8686	8701	d->sched_group_nodes = kcalloc(nr_node_ids,
8687	8702	sizeof(struct sched_group *), GFP_KERNEL);
8688	8703	if (!d->sched_group_nodes) {
8689		− printk(KERN_WARNING "Can not alloc sched group node list\n");
	8704	+ pr_warning("Can not alloc sched group node list\n");
8690	8705	return sa_notcovered;
8691	8706	}
8692	8707	sched_group_nodes_bycpu[cpumask_first(cpu_map)] = d->sched_group_nodes;
@@ -8703,7 +8718,7 @@ static enum s_alloc __visit_domain_allocation_hell(struct s_data *d, · 10 unmodified lines
8703	8718	return sa_send_covered;
8704	8719	d->rd = alloc_rootdomain();
8705	8720	if (!d->rd) {
8706		− printk(KERN_WARNING "Cannot alloc root domain\n");
	8721	+ pr_warning("Cannot alloc root domain\n");
8707	8722	return sa_tmpmask;
8708	8723	}
8709	8724	return sa_rootdomain;
@@ -9668,7 +9683,7 @@ void __init sched_init(void) · 958 unmodified lines
9668	9683	#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
9669	9684	static inline int preempt_count_equals(int preempt_offset)
9670	9685	{
9671		− int nested = preempt_count() & ~PREEMPT_ACTIVE;
	9686	+ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
9672	9687
9673	9688	return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
9674	9689	}
@@ -9685,13 +9700,11 @@ void __might_sleep(char *file, int line, int preempt_offset) · 10 unmodified lines
9685	9700	return;
9686	9701	prev_jiffy = jiffies;
9687	9702
9688		− printk(KERN_ERR
9689		− "BUG: sleeping function called from invalid context at %s:%d\n",
9690		− file, line);
9691		− printk(KERN_ERR
9692		− "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
9693		− in_atomic(), irqs_disabled(),
9694		− current->pid, current->comm);
	9703	+ pr_err("BUG: sleeping function called from invalid context at %s:%d\n",
	9704	+ file, line);
	9705	+ pr_err("in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
	9706	+ in_atomic(), irqs_disabled(),
	9707	+ current->pid, current->comm);
9695	9708
9696	9709	debug_show_held_locks(current);
9697	9710	if (irqs_disabled())
@@ -10083,7 +10096,7 @@ void sched_move_task(struct task_struct *tsk) · 385 unmodified lines
10083	10096
10084	10097	#ifdef CONFIG_FAIR_GROUP_SCHED
10085	10098	if (tsk->sched_class->moved_group)
10086		− tsk->sched_class->moved_group(tsk);
	10099	+ tsk->sched_class->moved_group(tsk, on_rq);
10087	10100	#endif
10088	10101
10089	10102	if (unlikely(running))

kernel/sched_clock.c+15−8

@@ -236,6 +236,18 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) · 235 unmodified lines
236	236	}
237	237	EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
238	238
	239	+unsigned long long cpu_clock(int cpu)
	240	+{
	241	+ unsigned long long clock;
	242	+ unsigned long flags;
	243	+
	244	+ local_irq_save(flags);
	245	+ clock = sched_clock_cpu(cpu);
	246	+ local_irq_restore(flags);
	247	+
	248	+ return clock;
	249	+}
	250	+
239	251	#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
240	252
241	253	void sched_clock_init(void)
@@ -251,17 +263,12 @@ u64 sched_clock_cpu(int cpu) · 9 unmodified lines
251	263	return sched_clock();
252	264	}
253	265
254		−#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
255	266
256	267	unsigned long long cpu_clock(int cpu)
257	268	{
258		− unsigned long long clock;
259		− unsigned long flags;
	269	+ return sched_clock_cpu(cpu);
	270	+}
260	271
261		− local_irq_save(flags);
262		− clock = sched_clock_cpu(cpu);
263		− local_irq_restore(flags);
	272	+#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
264	273
265		− return clock;
266		−}
267	274	EXPORT_SYMBOL_GPL(cpu_clock);

kernel/sched_fair.c+47−6

@@ -510,6 +510,7 @@ __update_curr(struct cfs_rq cfs_rq, struct sched_entity curr, · 509 unmodified lines
510	510	curr->sum_exec_runtime += delta_exec;
511	511	schedstat_add(cfs_rq, exec_clock, delta_exec);
512	512	delta_exec_weighted = calc_delta_fair(delta_exec, curr);
	513	+
513	514	curr->vruntime += delta_exec_weighted;
514	515	update_min_vruntime(cfs_rq);
515	516	}
@@ -765,16 +766,26 @@ place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int initial) · 249 unmodified lines
765	766	se->vruntime = vruntime;
766	767	}
767	768
	769	+#define ENQUEUE_WAKEUP 1
	770	+#define ENQUEUE_MIGRATE 2
	771	+
768	772	static void
769		−enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int wakeup)
	773	+enqueue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int flags)
770	774	{
	775	+ /*
	776	+ * Update the normalized vruntime before updating min_vruntime
	777	+ * through callig update_curr().
	778	+ */
	779	+ if (!(flags & ENQUEUE_WAKEUP) \|\| (flags & ENQUEUE_MIGRATE))
	780	+ se->vruntime += cfs_rq->min_vruntime;
	781	+
771	782	/*
772	783	* Update run-time statistics of the 'current'.
773	784	*/
774	785	update_curr(cfs_rq);
775	786	account_entity_enqueue(cfs_rq, se);
776	787
777		− if (wakeup) {
	788	+ if (flags & ENQUEUE_WAKEUP) {
778	789	place_entity(cfs_rq, se, 0);
779	790	enqueue_sleeper(cfs_rq, se);
780	791	}
@@ -828,6 +839,14 @@ dequeue_entity(struct cfs_rq cfs_rq, struct sched_entity se, int sleep) · 47 unmodified lines
828	839	__dequeue_entity(cfs_rq, se);
829	840	account_entity_dequeue(cfs_rq, se);
830	841	update_min_vruntime(cfs_rq);
	842	+
	843	+ /*
	844	+ * Normalize the entity after updating the min_vruntime because the
	845	+ * update can refer to the ->curr item and we need to reflect this
	846	+ * movement in our normalized position.
	847	+ */
	848	+ if (!sleep)
	849	+ se->vruntime -= cfs_rq->min_vruntime;
831	850	}
832	851
833	852	/*
@@ -1038,13 +1057,19 @@ static void enqueue_task_fair(struct rq rq, struct task_struct p, int wakeup) · 204 unmodified lines
1038	1057	{
1039	1058	struct cfs_rq *cfs_rq;
1040	1059	struct sched_entity *se = &p->se;
	1060	+ int flags = 0;
	1061	+
	1062	+ if (wakeup)
	1063	+ flags \|= ENQUEUE_WAKEUP;
	1064	+ if (p->state == TASK_WAKING)
	1065	+ flags \|= ENQUEUE_MIGRATE;
1041	1066
1042	1067	for_each_sched_entity(se) {
1043	1068	if (se->on_rq)
1044	1069	break;
1045	1070	cfs_rq = cfs_rq_of(se);
1046		− enqueue_entity(cfs_rq, se, wakeup);
1047		− wakeup = 1;
	1071	+ enqueue_entity(cfs_rq, se, flags);
	1072	+ flags = ENQUEUE_WAKEUP;
1048	1073	}
1049	1074
1050	1075	hrtick_update(rq);
@@ -1120,6 +1145,14 @@ static void yield_task_fair(struct rq *rq) · 69 unmodified lines
1120	1145
1121	1146	#ifdef CONFIG_SMP
1122	1147
	1148	+static void task_waking_fair(struct rq rq, struct task_struct p)
	1149	+{
	1150	+ struct sched_entity *se = &p->se;
	1151	+ struct cfs_rq *cfs_rq = cfs_rq_of(se);
	1152	+
	1153	+ se->vruntime -= cfs_rq->min_vruntime;
	1154	+}
	1155	+
1123	1156	#ifdef CONFIG_FAIR_GROUP_SCHED
1124	1157	/*
1125	1158	* effective_load() calculates the load change as seen from the root_task_group
@@ -1429,6 +1462,9 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag · 303 unmodified lines
1429	1462	}
1430	1463
1431	1464	for_each_domain(cpu, tmp) {
	1465	+ if (!(tmp->flags & SD_LOAD_BALANCE))
	1466	+ continue;
	1467	+
1432	1468	/*
1433	1469	* If power savings logic is enabled for a domain, see if we
1434	1470	* are not overloaded, if so, don't balance wider.
@@ -1975,6 +2011,8 @@ static void task_fork_fair(struct task_struct *p) · 540 unmodified lines
1975	2011	resched_task(rq->curr);
1976	2012	}
1977	2013
	2014	+ se->vruntime -= cfs_rq->min_vruntime;
	2015	+
1978	2016	raw_spin_unlock_irqrestore(&rq->lock, flags);
1979	2017	}
1980	2018
@@ -2028,12 +2066,13 @@ static void set_curr_task_fair(struct rq *rq) · 47 unmodified lines
2028	2066	}
2029	2067
2030	2068	#ifdef CONFIG_FAIR_GROUP_SCHED
2031		−static void moved_group_fair(struct task_struct *p)
	2069	+static void moved_group_fair(struct task_struct *p, int on_rq)
2032	2070	{
2033	2071	struct cfs_rq *cfs_rq = task_cfs_rq(p);
2034	2072
2035	2073	update_curr(cfs_rq);
2036		− place_entity(cfs_rq, &p->se, 1);
	2074	+ if (!on_rq)
	2075	+ place_entity(cfs_rq, &p->se, 1);
2037	2076	}
2038	2077	#endif
2039	2078
@@ -2073,6 +2112,8 @@ static const struct sched_class fair_sched_class = { · 33 unmodified lines
2073	2112	.move_one_task = move_one_task_fair,
2074	2113	.rq_online = rq_online_fair,
2075	2114	.rq_offline = rq_offline_fair,
	2115	+
	2116	+ .task_waking = task_waking_fair,
2076	2117	#endif
2077	2118
2078	2119	.set_curr_task = set_curr_task_fair,

kernel/sched_idletask.c+1−1

@@ -35,7 +35,7 @@ static void · 34 unmodified lines
35	35	dequeue_task_idle(struct rq rq, struct task_struct p, int sleep)
36	36	{
37	37	raw_spin_unlock_irq(&rq->lock);
38		− printk(KERN_ERR "bad: scheduling from the idle thread!\n");
	38	+ pr_err("bad: scheduling from the idle thread!\n");
39	39	dump_stack();
40	40	raw_spin_lock_irq(&rq->lock);
41	41	}

kernel/sched_rt.c+2−2

@@ -1472,7 +1472,7 @@ static void post_schedule_rt(struct rq *rq) · 1471 unmodified lines
1472	1472	* If we are not running and we are not going to reschedule soon, we should
1473	1473	* try to push tasks away now
1474	1474	*/
1475		−static void task_wake_up_rt(struct rq rq, struct task_struct p)
	1475	+static void task_woken_rt(struct rq rq, struct task_struct p)
1476	1476	{
1477	1477	if (!task_running(rq, p) &&
1478	1478	!test_tsk_need_resched(rq->curr) &&
@@ -1753,7 +1753,7 @@ static const struct sched_class rt_sched_class = { · 274 unmodified lines
1753	1753	.rq_offline = rq_offline_rt,
1754	1754	.pre_schedule = pre_schedule_rt,
1755	1755	.post_schedule = post_schedule_rt,
1756		− .task_wake_up = task_wake_up_rt,
	1756	+ .task_woken = task_woken_rt,
1757	1757	.switched_from = switched_from_rt,
1758	1758	#endif
1759	1759

Merge branch 'sched-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code