Pull scheduler fixes from Ingo Molnar: - Apply a number of membarrier related fixes and cleanups, which fixes a use-after-free race in the membarrier code - Introduce proper RCU protection for tasks on the runqueue - to get rid of the subtle task_rcu_dereference() interface that was easy to get wrong - Misc fixes, but also an EAS speedup * 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/fair: Avoid redundant EAS calculation sched/core: Remove double update_max_interval() call on CPU startup sched/core: Fix preempt_schedule() interrupt return comment sched/fair: Fix -Wunused-but-set-variable warnings sched/core: Fix migration to invalid CPU in __set_cpus_allowed_ptr() sched/membarrier: Return -ENOMEM to userspace on memory allocation failure sched/membarrier: Skip IPIs when mm->mm_users == 1 selftests, sched/membarrier: Add multi-threaded test sched/membarrier: Fix p->mm->membarrier_state racy load sched/membarrier: Call sync_core only before usermode for same mm sched/membarrier: Remove redundant check sched/membarrier: Fix private expedited registration check tasks, sched/core: RCUify the assignment of rq->curr tasks, sched/core: With a grace period after finish_task_switch(), remove unnecessary code tasks, sched/core: Ensure tasks are available for a grace period after leaving the runqueue tasks: Add a count of task RCU users sched/core: Convert vcpu_is_preempted() from macro to an inline function sched/fair: Remove unused cfs_rq_clock_task() function
Showing the changes to include/linux (5 of 17 files). Show all
Unified · Split
| @@ -383,6 +383,16 @@ struct mm_struct { · 382 unmodified lines | ||
| 383 | 383 | unsigned long highest_vm_end; /* highest vma end address */ |
| 384 | 384 | pgd_t * pgd; |
| 385 | 385 | |
| 386 | +#ifdef CONFIG_MEMBARRIER | |
| 387 | + /** | |
| 388 | + * @membarrier_state: Flags controlling membarrier behavior. | |
| 389 | + * | |
| 390 | + * This field is close to @pgd to hopefully fit in the same | |
| 391 | + * cache-line, which needs to be touched by switch_mm(). | |
| 392 | + */ | |
| 393 | + atomic_t membarrier_state; | |
| 394 | +#endif | |
| 395 | + | |
| 386 | 396 | /** |
| 387 | 397 | * @mm_users: The number of users including userspace. |
| 388 | 398 | * |
| @@ -452,9 +462,7 @@ struct mm_struct { · 63 unmodified lines | ||
| 452 | 462 | unsigned long flags; /* Must use atomic bitops to access */ |
| 453 | 463 | |
| 454 | 464 | struct core_state *core_state; /* coredumping support */ |
| 455 | −#ifdef CONFIG_MEMBARRIER | |
| 456 | − atomic_t membarrier_state; | |
| 457 | −#endif | |
| 465 | + | |
| 458 | 466 | #ifdef CONFIG_AIO |
| 459 | 467 | spinlock_t ioctx_lock; |
| 460 | 468 | struct kioctx_table __rcu *ioctx_table; |
| @@ -6,16 +6,11 @@ · 5 unmodified lines | ||
| 6 | 6 | |
| 7 | 7 | /* |
| 8 | 8 | * rcuwait provides a way of blocking and waking up a single |
| 9 | − * task in an rcu-safe manner; where it is forbidden to use | |
| 10 | − * after exit_notify(). task_struct is not properly rcu protected, | |
| 11 | − * unless dealing with rcu-aware lists, ie: find_task_by_*(). | |
| 9 | + * task in an rcu-safe manner. | |
| 12 | 10 | * |
| 13 | − * Alternatively we have task_rcu_dereference(), but the return | |
| 14 | − * semantics have different implications which would break the | |
| 15 | − * wakeup side. The only time @task is non-nil is when a user is | |
| 16 | − * blocked (or checking if it needs to) on a condition, and reset | |
| 17 | − * as soon as we know that the condition has succeeded and are | |
| 18 | − * awoken. | |
| 11 | + * The only time @task is non-nil is when a user is blocked (or | |
| 12 | + * checking if it needs to) on a condition, and reset as soon as we | |
| 13 | + * know that the condition has succeeded and are awoken. | |
| 19 | 14 | */ |
| 20 | 15 | struct rcuwait { |
| 21 | 16 | struct task_struct __rcu *task; |
| @@ -37,13 +32,6 @@ extern void rcuwait_wake_up(struct rcuwait *w); · 15 unmodified lines | ||
| 37 | 32 | */ |
| 38 | 33 | #define rcuwait_wait_event(w, condition) \ |
| 39 | 34 | ({ \ |
| 40 | − /* \ | |
| 41 | − * Complain if we are called after do_exit()/exit_notify(), \ | |
| 42 | − * as we cannot rely on the rcu critical region for the \ | |
| 43 | − * wakeup side. \ | |
| 44 | − */ \ | |
| 45 | − WARN_ON(current->exit_state); \ | |
| 46 | − \ | |
| 47 | 35 | rcu_assign_pointer((w)->task, current); \ |
| 48 | 36 | for (;;) { \ |
| 49 | 37 | /* \ |
| @@ -1130,7 +1130,10 @@ struct task_struct { · 1129 unmodified lines | ||
| 1130 | 1130 | |
| 1131 | 1131 | struct tlbflush_unmap_batch tlb_ubc; |
| 1132 | 1132 | |
| 1133 | − struct rcu_head rcu; | |
| 1133 | + union { | |
| 1134 | + refcount_t rcu_users; | |
| 1135 | + struct rcu_head rcu; | |
| 1136 | + }; | |
| 1134 | 1137 | |
| 1135 | 1138 | /* Cache last used pipe for splice(): */ |
| 1136 | 1139 | struct pipe_inode_info *splice_pipe; |
| @@ -1839,7 +1842,10 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) · 702 unmodified lines | ||
| 1839 | 1842 | * running or not. |
| 1840 | 1843 | */ |
| 1841 | 1844 | #ifndef vcpu_is_preempted |
| 1842 | −# define vcpu_is_preempted(cpu) false | |
| 1845 | +static inline bool vcpu_is_preempted(int cpu) | |
| 1846 | +{ | |
| 1847 | + return false; | |
| 1848 | +} | |
| 1843 | 1849 | #endif |
| 1844 | 1850 | |
| 1845 | 1851 | extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); |
| @@ -362,16 +362,16 @@ enum { · 361 unmodified lines | ||
| 362 | 362 | |
| 363 | 363 | static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) |
| 364 | 364 | { |
| 365 | + if (current->mm != mm) | |
| 366 | + return; | |
| 365 | 367 | if (likely(!(atomic_read(&mm->membarrier_state) & |
| 366 | 368 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) |
| 367 | 369 | return; |
| 368 | 370 | sync_core_before_usermode(); |
| 369 | 371 | } |
| 370 | 372 | |
| 371 | −static inline void membarrier_execve(struct task_struct *t) | |
| 372 | −{ | |
| 373 | − atomic_set(&t->mm->membarrier_state, 0); | |
| 374 | −} | |
| 373 | +extern void membarrier_exec_mmap(struct mm_struct *mm); | |
| 374 | + | |
| 375 | 375 | #else |
| 376 | 376 | #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS |
| 377 | 377 | static inline void membarrier_arch_switch_mm(struct mm_struct *prev, |
| @@ -380,7 +380,7 @@ static inline void membarrier_arch_switch_mm(struct mm_struct *prev, · 2 unmodified lines | ||
| 380 | 380 | { |
| 381 | 381 | } |
| 382 | 382 | #endif |
| 383 | −static inline void membarrier_execve(struct task_struct *t) | |
| 383 | +static inline void membarrier_exec_mmap(struct mm_struct *mm) | |
| 384 | 384 | { |
| 385 | 385 | } |
| 386 | 386 | static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) |
| @@ -119,7 +119,7 @@ static inline void put_task_struct(struct task_struct *t) · 118 unmodified lines | ||
| 119 | 119 | __put_task_struct(t); |
| 120 | 120 | } |
| 121 | 121 | |
| 122 | −struct task_struct *task_rcu_dereference(struct task_struct **ptask); | |
| 122 | +void put_task_struct_rcu_user(struct task_struct *task); | |
| 123 | 123 | |
| 124 | 124 | #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT |
| 125 | 125 | extern int arch_task_struct_size __read_mostly; |