Merge tag 'sched_urgent_for_5.8_rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: "The most anticipated fix in this pull request is probably the horrible build fix for the RANDSTRUCT fail that didn't make -rc2. Also included is the cleanup that removes those BUILD_BUG_ON()s and replaces it with ugly unions. Also included is the try_to_wake_up() race fix that was first triggered by Paul's RCU-torture runs, but was independently hit by Dave Chinner's fstest runs as well" * tag 'sched_urgent_for_5.8_rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/cfs: change initial value of runnable_avg smp, irq_work: Continue smp_call_function*() and irq_work*() integration sched/core: s/WF_ON_RQ/WQ_ON_CPU/ sched/core: Fix ttwu() race sched/core: Fix PI boosting between RT and DEADLINE tasks sched/deadline: Initialize ->dl_boosted sched/core: Check cpus_mask, not cpus_ptr in __set_cpus_allowed_ptr(), to fix mask corruption sched/core: Fix CONFIG_GCC_PLUGIN_RANDSTRUCT build fail

Linus Torvalds committed 6 years ago

Commit 91a9a902 parents 098c793 e21cf439 files · +120 −65Browse files

Unified · Split

9 files

include/linux/irq_work.h +8 −18
include/linux/sched.h +1 −2
include/linux/smp.h +8 −15
include/linux/smp_types.h +66 −0
kernel/sched/core.c +34 −10
kernel/sched/deadline.c +1 −0
kernel/sched/fair.c +1 −1
kernel/sched/sched.h +1 −1
kernel/smp.c +0 −18

include/linux/irq_work.h+8−18

 #ifndef _LINUX_IRQ_WORK_H
 #define _LINUX_IRQ_WORK_H
-#include <linux/llist.h>
+#include <linux/smp_types.h>
 /*
  * An entry can be in one of four states:
  * busy      NULL, 2 -> {free, claimed} : callback in progress, can be claimed
  */
-/* flags share CSD_FLAG_ space */
-#define IRQ_WORK_PENDING	BIT(0)
-#define IRQ_WORK_BUSY		BIT(1)
-/* Doesn't want IPI, wait for tick: */
-#define IRQ_WORK_LAZY		BIT(2)
-/* Run hard IRQ context, even on RT */
-#define IRQ_WORK_HARD_IRQ	BIT(3)
-#define IRQ_WORK_CLAIMED	(IRQ_WORK_PENDING | IRQ_WORK_BUSY)
-/*
- * structure shares layout with single_call_data_t.
- */
 struct irq_work {
-	struct llist_node llnode;
+	union {
-	atomic_t flags;
+		struct __call_single_node node;
+		struct {
+			struct llist_node llnode;
+			atomic_t flags;
+		};
+	};
 	void (*func)(struct irq_work *);
 };

include/linux/sched.h+1−2

 	unsigned int			ptrace;
 #ifdef CONFIG_SMP
-	struct llist_node		wake_entry;
-	unsigned int			wake_entry_type;
 	int				on_cpu;
+	struct __call_single_node	wake_entry;
 #ifdef CONFIG_THREAD_INFO_IN_TASK
 	/* Current CPU: */
 	unsigned int			cpu;

include/linux/smp.h+8−15

 #include <linux/list.h>
 #include <linux/cpumask.h>
 #include <linux/init.h>
-#include <linux/llist.h>
+#include <linux/smp_types.h>
 typedef void (*smp_call_func_t)(void *info);
 typedef bool (*smp_cond_func_t)(int cpu, void *info);
-enum {
-	CSD_FLAG_LOCK		= 0x01,
-	/* IRQ_WORK_flags */
-	CSD_TYPE_ASYNC		= 0x00,
-	CSD_TYPE_SYNC		= 0x10,
-	CSD_TYPE_IRQ_WORK	= 0x20,
-	CSD_TYPE_TTWU		= 0x30,
-	CSD_FLAG_TYPE_MASK	= 0xF0,
-};
 /*
  * structure shares (partial) layout with struct irq_work
  */
 struct __call_single_data {
-	struct llist_node llist;
+	union {
-	unsigned int flags;
+		struct __call_single_node node;
+		struct {
+			struct llist_node llist;
+			unsigned int flags;
+		};
+	};
 	smp_call_func_t func;
 	void *info;
 };

include/linux/smp_types.h+66−0

@@ -0,0 +1,66 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __LINUX_SMP_TYPES_H
	3	#define __LINUX_SMP_TYPES_H
	4
	5	#include <linux/llist.h>
	6
	7	enum {
	8	CSD_FLAG_LOCK = 0x01,
	9
	10	IRQ_WORK_PENDING = 0x01,
	11	IRQ_WORK_BUSY = 0x02,
	12	IRQ_WORK_LAZY = 0x04, /* No IPI, wait for tick */
	13	IRQ_WORK_HARD_IRQ = 0x08, /* IRQ context on PREEMPT_RT */
	14
	15	IRQ_WORK_CLAIMED = (IRQ_WORK_PENDING \| IRQ_WORK_BUSY),
	16
	17	CSD_TYPE_ASYNC = 0x00,
	18	CSD_TYPE_SYNC = 0x10,
	19	CSD_TYPE_IRQ_WORK = 0x20,
	20	CSD_TYPE_TTWU = 0x30,
	21
	22	CSD_FLAG_TYPE_MASK = 0xF0,
	23	};
	24
	25	/*
	26	* struct __call_single_node is the primary type on
	27	* smp.c:call_single_queue.
	28	*
	29	* flush_smp_call_function_queue() only reads the type from
	30	* __call_single_node::u_flags as a regular load, the above
	31	* (anonymous) enum defines all the bits of this word.
	32	*
	33	* Other bits are not modified until the type is known.
	34	*
	35	* CSD_TYPE_SYNC/ASYNC:
	36	* struct {
	37	* struct llist_node node;
	38	* unsigned int flags;
	39	* smp_call_func_t func;
	40	* void *info;
	41	* };
	42	*
	43	* CSD_TYPE_IRQ_WORK:
	44	* struct {
	45	* struct llist_node node;
	46	* atomic_t flags;
	47	* void (func)(struct irq_work );
	48	* };
	49	*
	50	* CSD_TYPE_TTWU:
	51	* struct {
	52	* struct llist_node node;
	53	* unsigned int flags;
	54	* };
	55	*
	56	*/
	57
	58	struct __call_single_node {
	59	struct llist_node llist;
	60	union {
	61	unsigned int u_flags;
	62	atomic_t a_flags;
	63	};
	64	};
	65
	66	#endif /* __LINUX_SMP_TYPES_H */

kernel/sched/core.c+34−10

 		goto out;
 	}
-	if (cpumask_equal(p->cpus_ptr, new_mask))
+	if (cpumask_equal(&p->cpus_mask, new_mask))
 		goto out;
 	/*
 	rq_lock_irqsave(rq, &rf);
 	update_rq_clock(rq);
-	llist_for_each_entry_safe(p, t, llist, wake_entry)
+	llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
+		if (WARN_ON_ONCE(p->on_cpu))
+			smp_cond_load_acquire(&p->on_cpu, !VAL);
+		if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
+			set_task_cpu(p, cpu_of(rq));
 		ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf);
+	}
 	rq_unlock_irqrestore(rq, &rf);
 }
 	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
 	WRITE_ONCE(rq->ttwu_pending, 1);
-	__smp_call_single_queue(cpu, &p->wake_entry);
+	__smp_call_single_queue(cpu, &p->wake_entry.llist);
 }
 void wake_up_if_idle(int cpu)
 	 * the soon-to-be-idle CPU as the current CPU is likely busy.
 	 * nr_running is checked to avoid unnecessary task stacking.
 	 */
-	if ((wake_flags & WF_ON_RQ) && cpu_rq(cpu)->nr_running <= 1)
+	if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1)
 		return true;
 	return false;
 static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
 {
 	if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu, wake_flags)) {
+		if (WARN_ON_ONCE(cpu == smp_processor_id()))
+			return false;
 		sched_clock_cpu(cpu); /* Sync clocks across CPUs */
 		__ttwu_queue_wakelist(p, cpu, wake_flags);
 		return true;
 			goto out;
 		success = 1;
-		cpu = task_cpu(p);
 		trace_sched_waking(p);
 		p->state = TASK_RUNNING;
 		trace_sched_wakeup(p);
 	/* We're going to change ->state: */
 	success = 1;
-	cpu = task_cpu(p);
 	/*
 	 * Ensure we load p->on_rq _after_ p->state, otherwise it would
 	 * which potentially sends an IPI instead of spinning on p->on_cpu to
 	 * let the waker make forward progress. This is safe because IRQs are
 	 * disabled and the IPI will deliver after on_cpu is cleared.
+	 *
+	 * Ensure we load task_cpu(p) after p->on_cpu:
+	 *
+	 * set_task_cpu(p, cpu);
+	 *   STORE p->cpu = @cpu
+	 * __schedule() (switch to task 'p')
+	 *   LOCK rq->lock
+	 *   smp_mb__after_spin_lock()		smp_cond_load_acquire(&p->on_cpu)
+	 *   STORE p->on_cpu = 1		LOAD p->cpu
+	 *
+	 * to ensure we observe the correct CPU on which the task is currently
+	 * scheduling.
 	 */
-	if (READ_ONCE(p->on_cpu) && ttwu_queue_wakelist(p, cpu, wake_flags | WF_ON_RQ))
+	if (smp_load_acquire(&p->on_cpu) &&
+	    ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU))
 		goto unlock;
 	/*
 		psi_ttwu_dequeue(p);
 		set_task_cpu(p, cpu);
 	}
+#else
+	cpu = task_cpu(p);
 #endif /* CONFIG_SMP */
 	ttwu_queue(p, cpu, wake_flags);
 	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 out:
 	if (success)
-		ttwu_stat(p, cpu, wake_flags);
+		ttwu_stat(p, task_cpu(p), wake_flags);
 	preempt_enable();
 	return success;
 #endif
 	init_numa_balancing(clone_flags, p);
 #ifdef CONFIG_SMP
-	p->wake_entry_type = CSD_TYPE_TTWU;
+	p->wake_entry.u_flags = CSD_TYPE_TTWU;
 #endif
 }
 	 */
 	if (dl_prio(prio)) {
 		if (!dl_prio(p->normal_prio) ||
-		    (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
+		    (pi_task && dl_prio(pi_task->prio) &&
+		     dl_entity_preempt(&pi_task->dl, &p->dl))) {
 			p->dl.dl_boosted = 1;
 			queue_flag |= ENQUEUE_REPLENISH;
 		} else

kernel/sched/deadline.c+1−0

@@ -2692,6 +2692,7 @@ void __dl_clear_params(struct task_struct *p) · 2691 unmodified lines
2692	dl_se->dl_bw = 0;	2692	dl_se->dl_bw = 0;
2693	dl_se->dl_density = 0;	2693	dl_se->dl_density = 0;
2694		2694
		2695	dl_se->dl_boosted = 0;
2695	dl_se->dl_throttled = 0;	2696	dl_se->dl_throttled = 0;
2696	dl_se->dl_yielded = 0;	2697	dl_se->dl_yielded = 0;
2697	dl_se->dl_non_contending = 0;	2698	dl_se->dl_non_contending = 0;

kernel/sched/fair.c+1−1

@@ -806,7 +806,7 @@ void post_init_entity_util_avg(struct task_struct *p) · 805 unmodified lines
806	}	806	}
807	}	807	}
808		808
809	sa->runnable_avg = cpu_scale;	809	sa->runnable_avg = sa->util_avg;
810		810
811	if (p->sched_class != &fair_sched_class) {	811	if (p->sched_class != &fair_sched_class) {
812	/*	812	/*

kernel/sched/sched.h+1−1

@@ -1682,7 +1682,7 @@ static inline int task_on_rq_migrating(struct task_struct *p) · 1681 unmodified lines
1682	#define WF_SYNC 0x01 /* Waker goes to sleep after wakeup */	1682	#define WF_SYNC 0x01 /* Waker goes to sleep after wakeup */
1683	#define WF_FORK 0x02 /* Child wakeup after fork */	1683	#define WF_FORK 0x02 /* Child wakeup after fork */
1684	#define WF_MIGRATED 0x04 /* Internal use, task got migrated */	1684	#define WF_MIGRATED 0x04 /* Internal use, task got migrated */
1685	#define WF_ON_RQ 0x08 /* Wakee is on_rq */	1685	#define WF_ON_CPU 0x08 /* Wakee is on_cpu */
1686		1686
1687	/*	1687	/*
1688	* To aid in avoiding the subversion of "niceness" due to uneven distribution	1688	* To aid in avoiding the subversion of "niceness" due to uneven distribution

kernel/smp.c+0−18

@@ -669,24 +669,6 @@ void __init smp_init(void) · 668 unmodified lines
669	{	669	{
670	int num_nodes, num_cpus;	670	int num_nodes, num_cpus;
671		671
672	/*
673	* Ensure struct irq_work layout matches so that
674	* flush_smp_call_function_queue() can do horrible things.
675	*/
676	BUILD_BUG_ON(offsetof(struct irq_work, llnode) !=
677	offsetof(struct __call_single_data, llist));
678	BUILD_BUG_ON(offsetof(struct irq_work, func) !=
679	offsetof(struct __call_single_data, func));
680	BUILD_BUG_ON(offsetof(struct irq_work, flags) !=
681	offsetof(struct __call_single_data, flags));
682
683	/*
684	* Assert the CSD_TYPE_TTWU layout is similar enough
685	* for task_struct to be on the @call_single_queue.
686	*/
687	BUILD_BUG_ON(offsetof(struct task_struct, wake_entry_type) - offsetof(struct task_struct, wake_entry) !=
688	offsetof(struct __call_single_data, flags) - offsetof(struct __call_single_data, llist));
689
690	idle_threads_init();	672	idle_threads_init();
691	cpuhp_threads_init();	673	cpuhp_threads_init();
692		674

Merge tag 'sched_urgent_for_5.8_rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code