Pull scheduler fixes from Ingo Molnar: - Apply a number of membarrier related fixes and cleanups, which fixes a use-after-free race in the membarrier code - Introduce proper RCU protection for tasks on the runqueue - to get rid of the subtle task_rcu_dereference() interface that was easy to get wrong - Misc fixes, but also an EAS speedup * 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/fair: Avoid redundant EAS calculation sched/core: Remove double update_max_interval() call on CPU startup sched/core: Fix preempt_schedule() interrupt return comment sched/fair: Fix -Wunused-but-set-variable warnings sched/core: Fix migration to invalid CPU in __set_cpus_allowed_ptr() sched/membarrier: Return -ENOMEM to userspace on memory allocation failure sched/membarrier: Skip IPIs when mm->mm_users == 1 selftests, sched/membarrier: Add multi-threaded test sched/membarrier: Fix p->mm->membarrier_state racy load sched/membarrier: Call sync_core only before usermode for same mm sched/membarrier: Remove redundant check sched/membarrier: Fix private expedited registration check tasks, sched/core: RCUify the assignment of rq->curr tasks, sched/core: With a grace period after finish_task_switch(), remove unnecessary code tasks, sched/core: Ensure tasks are available for a grace period after leaving the runqueue tasks: Add a count of task RCU users sched/core: Convert vcpu_is_preempted() from macro to an inline function sched/fair: Remove unused cfs_rq_clock_task() function
Showing the changes to include/linux (5 of 17 files). Show all
Unified · Split
| @@ -383,6 +383,16 @@ struct mm_struct { · 382 unmodified lines | |||
| 383 | unsigned long highest_vm_end; /* highest vma end address */ | 383 | unsigned long highest_vm_end; /* highest vma end address */ |
| 384 | pgd_t * pgd; | 384 | pgd_t * pgd; |
| 385 | 385 | ||
| 386 | #ifdef CONFIG_MEMBARRIER | ||
| 387 | /** | ||
| 388 | * @membarrier_state: Flags controlling membarrier behavior. | ||
| 389 | * | ||
| 390 | * This field is close to @pgd to hopefully fit in the same | ||
| 391 | * cache-line, which needs to be touched by switch_mm(). | ||
| 392 | */ | ||
| 393 | atomic_t membarrier_state; | ||
| 394 | #endif | ||
| 395 | |||
| 386 | /** | 396 | /** |
| 387 | * @mm_users: The number of users including userspace. | 397 | * @mm_users: The number of users including userspace. |
| 388 | * | 398 | * |
| @@ -452,9 +462,7 @@ struct mm_struct { · 63 unmodified lines | |||
| 452 | unsigned long flags; /* Must use atomic bitops to access */ | 462 | unsigned long flags; /* Must use atomic bitops to access */ |
| 453 | 463 | ||
| 454 | struct core_state *core_state; /* coredumping support */ | 464 | struct core_state *core_state; /* coredumping support */ |
| 455 | #ifdef CONFIG_MEMBARRIER | 465 | |
| 456 | atomic_t membarrier_state; | ||
| 457 | #endif | ||
| 458 | #ifdef CONFIG_AIO | 466 | #ifdef CONFIG_AIO |
| 459 | spinlock_t ioctx_lock; | 467 | spinlock_t ioctx_lock; |
| 460 | struct kioctx_table __rcu *ioctx_table; | 468 | struct kioctx_table __rcu *ioctx_table; |
| @@ -6,16 +6,11 @@ · 5 unmodified lines | |||
| 6 | 6 | ||
| 7 | /* | 7 | /* |
| 8 | * rcuwait provides a way of blocking and waking up a single | 8 | * rcuwait provides a way of blocking and waking up a single |
| 9 | * task in an rcu-safe manner; where it is forbidden to use | 9 | * task in an rcu-safe manner. |
| 10 | * after exit_notify(). task_struct is not properly rcu protected, | ||
| 11 | * unless dealing with rcu-aware lists, ie: find_task_by_*(). | ||
| 12 | * | 10 | * |
| 13 | * Alternatively we have task_rcu_dereference(), but the return | 11 | * The only time @task is non-nil is when a user is blocked (or |
| 14 | * semantics have different implications which would break the | 12 | * checking if it needs to) on a condition, and reset as soon as we |
| 15 | * wakeup side. The only time @task is non-nil is when a user is | 13 | * know that the condition has succeeded and are awoken. |
| 16 | * blocked (or checking if it needs to) on a condition, and reset | ||
| 17 | * as soon as we know that the condition has succeeded and are | ||
| 18 | * awoken. | ||
| 19 | */ | 14 | */ |
| 20 | struct rcuwait { | 15 | struct rcuwait { |
| 21 | struct task_struct __rcu *task; | 16 | struct task_struct __rcu *task; |
| @@ -37,13 +32,6 @@ extern void rcuwait_wake_up(struct rcuwait *w); · 15 unmodified lines | |||
| 37 | */ | 32 | */ |
| 38 | #define rcuwait_wait_event(w, condition) \ | 33 | #define rcuwait_wait_event(w, condition) \ |
| 39 | ({ \ | 34 | ({ \ |
| 40 | /* \ | ||
| 41 | * Complain if we are called after do_exit()/exit_notify(), \ | ||
| 42 | * as we cannot rely on the rcu critical region for the \ | ||
| 43 | * wakeup side. \ | ||
| 44 | */ \ | ||
| 45 | WARN_ON(current->exit_state); \ | ||
| 46 | \ | ||
| 47 | rcu_assign_pointer((w)->task, current); \ | 35 | rcu_assign_pointer((w)->task, current); \ |
| 48 | for (;;) { \ | 36 | for (;;) { \ |
| 49 | /* \ | 37 | /* \ |
| @@ -1130,7 +1130,10 @@ struct task_struct { · 1129 unmodified lines | |||
| 1130 | 1130 | ||
| 1131 | struct tlbflush_unmap_batch tlb_ubc; | 1131 | struct tlbflush_unmap_batch tlb_ubc; |
| 1132 | 1132 | ||
| 1133 | struct rcu_head rcu; | 1133 | union { |
| 1134 | refcount_t rcu_users; | ||
| 1135 | struct rcu_head rcu; | ||
| 1136 | }; | ||
| 1134 | 1137 | ||
| 1135 | /* Cache last used pipe for splice(): */ | 1138 | /* Cache last used pipe for splice(): */ |
| 1136 | struct pipe_inode_info *splice_pipe; | 1139 | struct pipe_inode_info *splice_pipe; |
| @@ -1839,7 +1842,10 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) · 702 unmodified lines | |||
| 1839 | * running or not. | 1842 | * running or not. |
| 1840 | */ | 1843 | */ |
| 1841 | #ifndef vcpu_is_preempted | 1844 | #ifndef vcpu_is_preempted |
| 1842 | # define vcpu_is_preempted(cpu) false | 1845 | static inline bool vcpu_is_preempted(int cpu) |
| 1846 | { | ||
| 1847 | return false; | ||
| 1848 | } | ||
| 1843 | #endif | 1849 | #endif |
| 1844 | 1850 | ||
| 1845 | extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); | 1851 | extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); |
| @@ -362,16 +362,16 @@ enum { · 361 unmodified lines | |||
| 362 | 362 | ||
| 363 | static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) | 363 | static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) |
| 364 | { | 364 | { |
| 365 | if (current->mm != mm) | ||
| 366 | return; | ||
| 365 | if (likely(!(atomic_read(&mm->membarrier_state) & | 367 | if (likely(!(atomic_read(&mm->membarrier_state) & |
| 366 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) | 368 | MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE))) |
| 367 | return; | 369 | return; |
| 368 | sync_core_before_usermode(); | 370 | sync_core_before_usermode(); |
| 369 | } | 371 | } |
| 370 | 372 | ||
| 371 | static inline void membarrier_execve(struct task_struct *t) | 373 | extern void membarrier_exec_mmap(struct mm_struct *mm); |
| 372 | { | 374 | |
| 373 | atomic_set(&t->mm->membarrier_state, 0); | ||
| 374 | } | ||
| 375 | #else | 375 | #else |
| 376 | #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS | 376 | #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS |
| 377 | static inline void membarrier_arch_switch_mm(struct mm_struct *prev, | 377 | static inline void membarrier_arch_switch_mm(struct mm_struct *prev, |
| @@ -380,7 +380,7 @@ static inline void membarrier_arch_switch_mm(struct mm_struct *prev, · 2 unmodified lines | |||
| 380 | { | 380 | { |
| 381 | } | 381 | } |
| 382 | #endif | 382 | #endif |
| 383 | static inline void membarrier_execve(struct task_struct *t) | 383 | static inline void membarrier_exec_mmap(struct mm_struct *mm) |
| 384 | { | 384 | { |
| 385 | } | 385 | } |
| 386 | static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) | 386 | static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm) |
| @@ -119,7 +119,7 @@ static inline void put_task_struct(struct task_struct *t) · 118 unmodified lines | |||
| 119 | __put_task_struct(t); | 119 | __put_task_struct(t); |
| 120 | } | 120 | } |
| 121 | 121 | ||
| 122 | struct task_struct *task_rcu_dereference(struct task_struct **ptask); | 122 | void put_task_struct_rcu_user(struct task_struct *task); |
| 123 | 123 | ||
| 124 | #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT | 124 | #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT |
| 125 | extern int arch_task_struct_size __read_mostly; | 125 | extern int arch_task_struct_size __read_mostly; |