History

Merge tag 'sched-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler updates from Ingo Molnar: - Updates to scheduler metrics: - PELT fixes & enhancements - PSI fixes & enhancements - Refactor cpu_util_without() - Updates to instrumentation/debugging: - Remove sched_trace_*() helper functions - can be done via debug info - Fix double update_rq_clock() warnings - Introduce & use "preemption model accessors" to simplify some of the Kconfig complexity. - Make softirq handling RT-safe. - Misc smaller fixes & cleanups. * tag 'sched-core-2022-05-23' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: topology: Remove unused cpu_cluster_mask() sched: Reverse sched_class layout sched/deadline: Remove superfluous rq clock update in push_dl_task() sched/core: Avoid obvious double update_rq_clock warning smp: Make softirq handling RT safe in flush_smp_call_function_queue() smp: Rename flush_smp_call_function_from_idle() sched: Fix missing prototype warnings sched/fair: Remove cfs_rq_tg_path() sched/fair: Remove sched_trace_*() helper functions sched/fair: Refactor cpu_util_without() sched/fair: Revise comment about lb decision matrix sched/psi: report zeroes for CPU full at the system level sched/fair: Delete useless condition in tg_unthrottle_up() sched/fair: Fix cfs_rq_clock_pelt() for throttled cfs_rq sched/fair: Move calculate of avg_load to a better location mailmap: Update my email address to @redhat.com MAINTAINERS: Add myself as scheduler topology reviewer psi: Fix trigger being fired unexpectedly at initial ftrace: Use preemption model accessors for trace header printout kcsan: Use preemption model accessors

Linus Torvalds committed 4 years ago

Commit 6f3f04c2 parents cfeb252 991d8d823 files · +213 −331Browse files

Showing the changes to kernel (16 of 23 files). Show all

Unified · Split

16 files
  1. kernel/kcsan/kcsan_test.c +3 −2
  2. kernel/sched/build_policy.c +2 −0
  3. kernel/sched/build_utility.c +1 −0
  4. kernel/sched/core.c +13 −10
  5. kernel/sched/deadline.c +4 −11
  6. kernel/sched/fair.c +67 −225
  7. kernel/sched/idle.c +1 −1
  8. kernel/sched/pelt.h +2 −2
  9. kernel/sched/psi.c +11 −7
  10. kernel/sched/rt.c +3 −2
  11. kernel/sched/sched.h +36 −19
  12. kernel/sched/smp.h +6 −0
  13. kernel/smp.c +24 −8
  14. kernel/softirq.c +13 −0
  15. kernel/stop_machine.c +0 −2
  16. kernel/trace/trace.c +4 −10
kernel/kcsan/kcsan_test.c+3−2
@@ -1380,13 +1380,14 @@ static const void *nthreads_gen_params(const void *prev, char *desc) · 1379 unmodified lines
1380 else1380 else
1381 nthreads *= 2;1381 nthreads *= 2;
1382 1382
1383 if (!IS_ENABLED(CONFIG_PREEMPT) || !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {1383 if (!preempt_model_preemptible() ||
1384 !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1384 /*1385 /*
1385 * Without any preemption, keep 2 CPUs free for other tasks, one1386 * Without any preemption, keep 2 CPUs free for other tasks, one
1386 * of which is the main test case function checking for1387 * of which is the main test case function checking for
1387 * completion or failure.1388 * completion or failure.
1388 */1389 */
1389 const long min_unused_cpus = IS_ENABLED(CONFIG_PREEMPT_NONE) ? 2 : 0;1390 const long min_unused_cpus = preempt_model_none() ? 2 : 0;
1390 const long min_required_cpus = 2 + min_unused_cpus;1391 const long min_required_cpus = 2 + min_unused_cpus;
1391 1392
1392 if (num_online_cpus() < min_required_cpus) {1393 if (num_online_cpus() < min_required_cpus) {
kernel/sched/build_policy.c+2−0
@@ -15,6 +15,7 @@ · 14 unmodified lines
15/* Headers: */15/* Headers: */
16#include <linux/sched/clock.h>16#include <linux/sched/clock.h>
17#include <linux/sched/cputime.h>17#include <linux/sched/cputime.h>
18#include <linux/sched/hotplug.h>
18#include <linux/sched/posix-timers.h>19#include <linux/sched/posix-timers.h>
19#include <linux/sched/rt.h>20#include <linux/sched/rt.h>
20 21
@@ -31,6 +32,7 @@ · 10 unmodified lines
31#include <uapi/linux/sched/types.h>32#include <uapi/linux/sched/types.h>
32 33
33#include "sched.h"34#include "sched.h"
35#include "smp.h"
34 36
35#include "autogroup.h"37#include "autogroup.h"
36#include "stats.h"38#include "stats.h"
kernel/sched/build_utility.c+1−0
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include <linux/sched/debug.h>14#include <linux/sched/debug.h>
15#include <linux/sched/isolation.h>15#include <linux/sched/isolation.h>
16#include <linux/sched/loadavg.h>16#include <linux/sched/loadavg.h>
17#include <linux/sched/nohz.h>
17#include <linux/sched/mm.h>18#include <linux/sched/mm.h>
18#include <linux/sched/rseq_api.h>19#include <linux/sched/rseq_api.h>
19#include <linux/sched/task_stack.h>20#include <linux/sched/task_stack.h>
kernel/sched/core.c+13−10
@@ -26,7 +26,10 @@ · 25 unmodified lines
26#include <linux/topology.h>26#include <linux/topology.h>
27#include <linux/sched/clock.h>27#include <linux/sched/clock.h>
28#include <linux/sched/cond_resched.h>28#include <linux/sched/cond_resched.h>
29#include <linux/sched/cputime.h>
29#include <linux/sched/debug.h>30#include <linux/sched/debug.h>
31#include <linux/sched/hotplug.h>
32#include <linux/sched/init.h>
30#include <linux/sched/isolation.h>33#include <linux/sched/isolation.h>
31#include <linux/sched/loadavg.h>34#include <linux/sched/loadavg.h>
32#include <linux/sched/mm.h>35#include <linux/sched/mm.h>
@@ -610,10 +613,10 @@ void double_rq_lock(struct rq *rq1, struct rq *rq2) · 577 unmodified lines
610 swap(rq1, rq2);613 swap(rq1, rq2);
611 614
612 raw_spin_rq_lock(rq1);615 raw_spin_rq_lock(rq1);
613 if (__rq_lockp(rq1) == __rq_lockp(rq2))616 if (__rq_lockp(rq1) != __rq_lockp(rq2))
614 return;617 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING);
615 618
616 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING);619 double_rq_clock_clear_update(rq1, rq2);
617}620}
618#endif621#endif
619 622
@@ -2190,7 +2193,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) · 1570 unmodified lines
2190{2193{
2191 if (p->sched_class == rq->curr->sched_class)2194 if (p->sched_class == rq->curr->sched_class)
2192 rq->curr->sched_class->check_preempt_curr(rq, p, flags);2195 rq->curr->sched_class->check_preempt_curr(rq, p, flags);
2193 else if (p->sched_class > rq->curr->sched_class)2196 else if (sched_class_above(p->sched_class, rq->curr->sched_class))
2194 resched_curr(rq);2197 resched_curr(rq);
2195 2198
2196 /*2199 /*
@@ -2408,7 +2411,7 @@ static int migration_cpu_stop(void *data) · 211 unmodified lines
2408 * __migrate_task() such that we will not miss enforcing cpus_ptr2411 * __migrate_task() such that we will not miss enforcing cpus_ptr
2409 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.2412 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
2410 */2413 */
2411 flush_smp_call_function_from_idle();2414 flush_smp_call_function_queue();
2412 2415
2413 raw_spin_lock(&p->pi_lock);2416 raw_spin_lock(&p->pi_lock);
2414 rq_lock(rq, &rf);2417 rq_lock(rq, &rf);
@@ -5689,7 +5692,7 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) · 3274 unmodified lines
5689 * higher scheduling class, because otherwise those lose the5692 * higher scheduling class, because otherwise those lose the
5690 * opportunity to pull in more work from other CPUs.5693 * opportunity to pull in more work from other CPUs.
5691 */5694 */
5692 if (likely(prev->sched_class <= &fair_sched_class &&5695 if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) &&
5693 rq->nr_running == rq->cfs.h_nr_running)) {5696 rq->nr_running == rq->cfs.h_nr_running)) {
5694 5697
5695 p = pick_next_task_fair(rq, prev, rf);5698 p = pick_next_task_fair(rq, prev, rf);
@@ -9469,11 +9472,11 @@ void __init sched_init(void) · 3773 unmodified lines
9469 int i;9472 int i;
9470 9473
9471 /* Make sure the linker didn't screw up */9474 /* Make sure the linker didn't screw up */
9472 BUG_ON(&idle_sched_class + 1 != &fair_sched_class ||9475 BUG_ON(&idle_sched_class != &fair_sched_class + 1 ||
9473 &fair_sched_class + 1 != &rt_sched_class ||9476 &fair_sched_class != &rt_sched_class + 1 ||
9474 &rt_sched_class + 1 != &dl_sched_class);9477 &rt_sched_class != &dl_sched_class + 1);
9475#ifdef CONFIG_SMP9478#ifdef CONFIG_SMP
9476 BUG_ON(&dl_sched_class + 1 != &stop_sched_class);9479 BUG_ON(&dl_sched_class != &stop_sched_class + 1);
9477#endif9480#endif
9478 9481
9479 wait_bit_init();9482 wait_bit_init();
kernel/sched/deadline.c+4−11
@@ -1220,8 +1220,6 @@ int dl_runtime_exceeded(struct sched_dl_entity *dl_se) · 1219 unmodified lines
1220 return (dl_se->runtime <= 0);1220 return (dl_se->runtime <= 0);
1221}1221}
1222 1222
1223extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
1224
1225/*1223/*
1226 * This function implements the GRUB accounting rule:1224 * This function implements the GRUB accounting rule:
1227 * according to the GRUB reclaiming algorithm, the runtime is1225 * according to the GRUB reclaiming algorithm, the runtime is
@@ -1832,6 +1830,7 @@ out: · 604 unmodified lines
1832 1830
1833static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused)1831static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused)
1834{1832{
1833 struct rq_flags rf;
1835 struct rq *rq;1834 struct rq *rq;
1836 1835
1837 if (READ_ONCE(p->__state) != TASK_WAKING)1836 if (READ_ONCE(p->__state) != TASK_WAKING)
@@ -1843,7 +1842,7 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused · 5 unmodified lines
1843 * from try_to_wake_up(). Hence, p->pi_lock is locked, but1842 * from try_to_wake_up(). Hence, p->pi_lock is locked, but
1844 * rq->lock is not... So, lock it1843 * rq->lock is not... So, lock it
1845 */1844 */
1846 raw_spin_rq_lock(rq);1845 rq_lock(rq, &rf);
1847 if (p->dl.dl_non_contending) {1846 if (p->dl.dl_non_contending) {
1848 update_rq_clock(rq);1847 update_rq_clock(rq);
1849 sub_running_bw(&p->dl, &rq->dl);1848 sub_running_bw(&p->dl, &rq->dl);
@@ -1859,7 +1858,7 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused · 9 unmodified lines
1859 put_task_struct(p);1858 put_task_struct(p);
1860 }1859 }
1861 sub_rq_bw(&p->dl, &rq->dl);1860 sub_rq_bw(&p->dl, &rq->dl);
1862 raw_spin_rq_unlock(rq);1861 rq_unlock(rq, &rf);
1863}1862}
1864 1863
1865static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)1864static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)
@@ -2319,13 +2318,7 @@ retry: · 453 unmodified lines
2319 2318
2320 deactivate_task(rq, next_task, 0);2319 deactivate_task(rq, next_task, 0);
2321 set_task_cpu(next_task, later_rq->cpu);2320 set_task_cpu(next_task, later_rq->cpu);
2322 2321 activate_task(later_rq, next_task, 0);
2323 /*
2324 * Update the later_rq clock here, because the clock is used
2325 * by the cpufreq_update_util() inside __add_running_bw().
2326 */
2327 update_rq_clock(later_rq);
2328 activate_task(later_rq, next_task, ENQUEUE_NOCLOCK);
2329 ret = 1;2322 ret = 1;
2330 2323
2331 resched_curr(later_rq);2324 resched_curr(later_rq);
kernel/sched/fair.c+67−225
@@ -36,6 +36,7 @@ · 35 unmodified lines
36#include <linux/sched/cond_resched.h>36#include <linux/sched/cond_resched.h>
37#include <linux/sched/cputime.h>37#include <linux/sched/cputime.h>
38#include <linux/sched/isolation.h>38#include <linux/sched/isolation.h>
39#include <linux/sched/nohz.h>
39 40
40#include <linux/cpuidle.h>41#include <linux/cpuidle.h>
41#include <linux/interrupt.h>42#include <linux/interrupt.h>
@@ -313,19 +314,6 @@ const struct sched_class fair_sched_class; · 271 unmodified lines
313#define for_each_sched_entity(se) \314#define for_each_sched_entity(se) \
314 for (; se; se = se->parent)315 for (; se; se = se->parent)
315 316
316static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len)
317{
318 if (!path)
319 return;
320
321 if (cfs_rq && task_group_is_autogroup(cfs_rq->tg))
322 autogroup_path(cfs_rq->tg, path, len);
323 else if (cfs_rq && cfs_rq->tg->css.cgroup)
324 cgroup_path(cfs_rq->tg->css.cgroup, path, len);
325 else
326 strlcpy(path, "(null)", len);
327}
328
329static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)317static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
330{318{
331 struct rq *rq = rq_of(cfs_rq);319 struct rq *rq = rq_of(cfs_rq);
@@ -493,12 +481,6 @@ static int se_is_idle(struct sched_entity *se) · 161 unmodified lines
493#define for_each_sched_entity(se) \481#define for_each_sched_entity(se) \
494 for (; se; se = NULL)482 for (; se; se = NULL)
495 483
496static inline void cfs_rq_tg_path(struct cfs_rq *cfs_rq, char *path, int len)
497{
498 if (path)
499 strlcpy(path, "(null)", len);
500}
501
502static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)484static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
503{485{
504 return true;486 return true;
@@ -4846,11 +4828,11 @@ static int tg_unthrottle_up(struct task_group *tg, void *data) · 4341 unmodified lines
4846 4828
4847 cfs_rq->throttle_count--;4829 cfs_rq->throttle_count--;
4848 if (!cfs_rq->throttle_count) {4830 if (!cfs_rq->throttle_count) {
4849 cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -4831 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) -
4850 cfs_rq->throttled_clock_task;4832 cfs_rq->throttled_clock_pelt;
4851 4833
4852 /* Add cfs_rq with load or one or more already running entities to the list */4834 /* Add cfs_rq with load or one or more already running entities to the list */
4853 if (!cfs_rq_is_decayed(cfs_rq) || cfs_rq->nr_running)4835 if (!cfs_rq_is_decayed(cfs_rq))
4854 list_add_leaf_cfs_rq(cfs_rq);4836 list_add_leaf_cfs_rq(cfs_rq);
4855 }4837 }
4856 4838
@@ -4864,7 +4846,7 @@ static int tg_throttle_down(struct task_group *tg, void *data) · 7 unmodified lines
4864 4846
4865 /* group is entering throttled state, stop time */4847 /* group is entering throttled state, stop time */
4866 if (!cfs_rq->throttle_count) {4848 if (!cfs_rq->throttle_count) {
4867 cfs_rq->throttled_clock_task = rq_clock_task(rq);4849 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);
4868 list_del_leaf_cfs_rq(cfs_rq);4850 list_del_leaf_cfs_rq(cfs_rq);
4869 }4851 }
4870 cfs_rq->throttle_count++;4852 cfs_rq->throttle_count++;
@@ -5308,7 +5290,7 @@ static void sync_throttle(struct task_group *tg, int cpu) · 437 unmodified lines
5308 pcfs_rq = tg->parent->cfs_rq[cpu];5290 pcfs_rq = tg->parent->cfs_rq[cpu];
5309 5291
5310 cfs_rq->throttle_count = pcfs_rq->throttle_count;5292 cfs_rq->throttle_count = pcfs_rq->throttle_count;
5311 cfs_rq->throttled_clock_task = rq_clock_task(cpu_rq(cpu));5293 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
5312}5294}
5313 5295
5314/* conditionally throttle active cfs_rq's from put_prev_entity() */5296/* conditionally throttle active cfs_rq's from put_prev_entity() */
@@ -6544,108 +6526,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) · 1229 unmodified lines
6544}6526}
6545 6527
6546/*6528/*
6547 * cpu_util_without: compute cpu utilization without any contributions from *p6529 * Predicts what cpu_util(@cpu) would return if @p was removed from @cpu
6548 * @cpu: the CPU which utilization is requested6530 * (@dst_cpu = -1) or migrated to @dst_cpu.
6549 * @p: the task which utilization should be discounted
6550 *
6551 * The utilization of a CPU is defined by the utilization of tasks currently
6552 * enqueued on that CPU as well as tasks which are currently sleeping after an
6553 * execution on that CPU.
6554 *
6555 * This method returns the utilization of the specified CPU by discounting the
6556 * utilization of the specified task, whenever the task is currently
6557 * contributing to the CPU utilization.
6558 */
6559static unsigned long cpu_util_without(int cpu, struct task_struct *p)
6560{
6561 struct cfs_rq *cfs_rq;
6562 unsigned int util;
6563
6564 /* Task has no contribution or is new */
6565 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6566 return cpu_util_cfs(cpu);
6567
6568 cfs_rq = &cpu_rq(cpu)->cfs;
6569 util = READ_ONCE(cfs_rq->avg.util_avg);
6570
6571 /* Discount task's util from CPU's util */
6572 lsub_positive(&util, task_util(p));
6573
6574 /*
6575 * Covered cases:
6576 *
6577 * a) if *p is the only task sleeping on this CPU, then:
6578 * cpu_util (== task_util) > util_est (== 0)
6579 * and thus we return:
6580 * cpu_util_without = (cpu_util - task_util) = 0
6581 *
6582 * b) if other tasks are SLEEPING on this CPU, which is now exiting
6583 * IDLE, then:
6584 * cpu_util >= task_util
6585 * cpu_util > util_est (== 0)
6586 * and thus we discount *p's blocked utilization to return:
6587 * cpu_util_without = (cpu_util - task_util) >= 0
6588 *
6589 * c) if other tasks are RUNNABLE on that CPU and
6590 * util_est > cpu_util
6591 * then we use util_est since it returns a more restrictive
6592 * estimation of the spare capacity on that CPU, by just
6593 * considering the expected utilization of tasks already
6594 * runnable on that CPU.
6595 *
6596 * Cases a) and b) are covered by the above code, while case c) is
6597 * covered by the following code when estimated utilization is
6598 * enabled.
6599 */
6600 if (sched_feat(UTIL_EST)) {
6601 unsigned int estimated =
6602 READ_ONCE(cfs_rq->avg.util_est.enqueued);
6603
6604 /*
6605 * Despite the following checks we still have a small window
6606 * for a possible race, when an execl's select_task_rq_fair()
6607 * races with LB's detach_task():
6608 *
6609 * detach_task()
6610 * p->on_rq = TASK_ON_RQ_MIGRATING;
6611 * ---------------------------------- A
6612 * deactivate_task() \
6613 * dequeue_task() + RaceTime
6614 * util_est_dequeue() /
6615 * ---------------------------------- B
6616 *
6617 * The additional check on "current == p" it's required to
6618 * properly fix the execl regression and it helps in further
6619 * reducing the chances for the above race.
6620 */
6621 if (unlikely(task_on_rq_queued(p) || current == p))
6622 lsub_positive(&estimated, _task_util_est(p));
6623
6624 util = max(util, estimated);
6625 }
6626
6627 /*
6628 * Utilization (estimated) can exceed the CPU capacity, thus let's
6629 * clamp to the maximum CPU capacity to ensure consistency with
6630 * cpu_util.
6631 */
6632 return min_t(unsigned long, util, capacity_orig_of(cpu));
6633}
6634
6635/*
6636 * Predicts what cpu_util(@cpu) would return if @p was migrated (and enqueued)
6637 * to @dst_cpu.
6638 */6531 */
6639static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu)6532static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu)
6640{6533{
6641 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;6534 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
6642 unsigned long util_est, util = READ_ONCE(cfs_rq->avg.util_avg);6535 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg);
6643 6536
6644 /*6537 /*
6645 * If @p migrates from @cpu to another, remove its contribution. Or,6538 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its
6646 * if @p migrates from another CPU to @cpu, add its contribution. In6539 * contribution. If @p migrates from another CPU to @cpu add its
6647 * the other cases, @cpu is not impacted by the migration, so the6540 * contribution. In all the other cases @cpu is not impacted by the
6648 * util_avg should already be correct.6541 * migration so its util_avg is already correct.
6649 */6542 */
6650 if (task_cpu(p) == cpu && dst_cpu != cpu)6543 if (task_cpu(p) == cpu && dst_cpu != cpu)
6651 lsub_positive(&util, task_util(p));6544 lsub_positive(&util, task_util(p));
@@ -6653,16 +6546,40 @@ static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) · 1 unmodified line
6653 util += task_util(p);6546 util += task_util(p);
6654 6547
6655 if (sched_feat(UTIL_EST)) {6548 if (sched_feat(UTIL_EST)) {
6549 unsigned long util_est;
6550
6656 util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);6551 util_est = READ_ONCE(cfs_rq->avg.util_est.enqueued);
6657 6552
6658 /*6553 /*
6659 * During wake-up, the task isn't enqueued yet and doesn't6554 * During wake-up @p isn't enqueued yet and doesn't contribute
6660 * appear in the cfs_rq->avg.util_est.enqueued of any rq,6555 * to any cpu_rq(cpu)->cfs.avg.util_est.enqueued.
6661 * so just add it (if needed) to "simulate" what will be6556 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p
6662 * cpu_util after the task has been enqueued.6557 * has been enqueued.
6558 *
6559 * During exec (@dst_cpu = -1) @p is enqueued and does
6560 * contribute to cpu_rq(cpu)->cfs.util_est.enqueued.
6561 * Remove it to "simulate" cpu_util without @p's contribution.
6562 *
6563 * Despite the task_on_rq_queued(@p) check there is still a
6564 * small window for a possible race when an exec
6565 * select_task_rq_fair() races with LB's detach_task().
6566 *
6567 * detach_task()
6568 * deactivate_task()
6569 * p->on_rq = TASK_ON_RQ_MIGRATING;
6570 * -------------------------------- A
6571 * dequeue_task() \
6572 * dequeue_task_fair() + Race Time
6573 * util_est_dequeue() /
6574 * -------------------------------- B
6575 *
6576 * The additional check "current == p" is required to further
6577 * reduce the race window.
6663 */6578 */
6664 if (dst_cpu == cpu)6579 if (dst_cpu == cpu)
6665 util_est += _task_util_est(p);6580 util_est += _task_util_est(p);
6581 else if (unlikely(task_on_rq_queued(p) || current == p))
6582 lsub_positive(&util_est, _task_util_est(p));
6666 6583
6667 util = max(util, util_est);6584 util = max(util, util_est);
6668 }6585 }
@@ -6670,6 +6587,28 @@ static unsigned long cpu_util_next(int cpu, struct task_struct *p, int dst_cpu) · 1 unmodified line
6670 return min(util, capacity_orig_of(cpu));6587 return min(util, capacity_orig_of(cpu));
6671}6588}
6672 6589
6590/*
6591 * cpu_util_without: compute cpu utilization without any contributions from *p
6592 * @cpu: the CPU which utilization is requested
6593 * @p: the task which utilization should be discounted
6594 *
6595 * The utilization of a CPU is defined by the utilization of tasks currently
6596 * enqueued on that CPU as well as tasks which are currently sleeping after an
6597 * execution on that CPU.
6598 *
6599 * This method returns the utilization of the specified CPU by discounting the
6600 * utilization of the specified task, whenever the task is currently
6601 * contributing to the CPU utilization.
6602 */
6603static unsigned long cpu_util_without(int cpu, struct task_struct *p)
6604{
6605 /* Task has no contribution or is new */
6606 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6607 return cpu_util_cfs(cpu);
6608
6609 return cpu_util_next(cpu, p, -1);
6610}
6611
6673/*6612/*
6674 * compute_energy(): Estimates the energy that @pd would consume if @p was6613 * compute_energy(): Estimates the energy that @pd would consume if @p was
6675 * migrated to @dst_cpu. compute_energy() predicts what will be the utilization6614 * migrated to @dst_cpu. compute_energy() predicts what will be the utilization
@@ -9460,8 +9399,6 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s · 2784 unmodified lines
9460 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /9399 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
9461 local->group_capacity;9400 local->group_capacity;
9462 9401
9463 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
9464 sds->total_capacity;
9465 /*9402 /*
9466 * If the local group is more loaded than the selected9403 * If the local group is more loaded than the selected
9467 * busiest group don't try to pull any tasks.9404 * busiest group don't try to pull any tasks.
@@ -9470,6 +9407,9 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s · 2 unmodified lines
9470 env->imbalance = 0;9407 env->imbalance = 0;
9471 return;9408 return;
9472 }9409 }
9410
9411 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
9412 sds->total_capacity;
9473 }9413 }
9474 9414
9475 /*9415 /*
@@ -9495,7 +9435,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s · 19 unmodified lines
9495 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded9435 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
9496 * has_spare nr_idle balanced N/A N/A balanced balanced9436 * has_spare nr_idle balanced N/A N/A balanced balanced
9497 * fully_busy nr_idle nr_idle N/A N/A balanced balanced9437 * fully_busy nr_idle nr_idle N/A N/A balanced balanced
9498 * misfit_task force N/A N/A N/A force force9438 * misfit_task force N/A N/A N/A N/A N/A
9499 * asym_packing force force N/A N/A force force9439 * asym_packing force force N/A N/A force force
9500 * imbalanced force force N/A N/A force force9440 * imbalanced force force N/A N/A force force
9501 * overloaded force force N/A N/A force avg_load9441 * overloaded force force N/A N/A force avg_load
@@ -11881,101 +11821,3 @@ __init void init_sched_fair_class(void) · 2379 unmodified lines
11881#endif /* SMP */11821#endif /* SMP */
11882 11822
11883}11823}
11884
11885/*
11886 * Helper functions to facilitate extracting info from tracepoints.
11887 */
11888
11889const struct sched_avg *sched_trace_cfs_rq_avg(struct cfs_rq *cfs_rq)
11890{
11891#ifdef CONFIG_SMP
11892 return cfs_rq ? &cfs_rq->avg : NULL;
11893#else
11894 return NULL;
11895#endif
11896}
11897EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_avg);
11898
11899char *sched_trace_cfs_rq_path(struct cfs_rq *cfs_rq, char *str, int len)
11900{
11901 if (!cfs_rq) {
11902 if (str)
11903 strlcpy(str, "(null)", len);
11904 else
11905 return NULL;
11906 }
11907
11908 cfs_rq_tg_path(cfs_rq, str, len);
11909 return str;
11910}
11911EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_path);
11912
11913int sched_trace_cfs_rq_cpu(struct cfs_rq *cfs_rq)
11914{
11915 return cfs_rq ? cpu_of(rq_of(cfs_rq)) : -1;
11916}
11917EXPORT_SYMBOL_GPL(sched_trace_cfs_rq_cpu);
11918
11919const struct sched_avg *sched_trace_rq_avg_rt(struct rq *rq)
11920{
11921#ifdef CONFIG_SMP
11922 return rq ? &rq->avg_rt : NULL;
11923#else
11924 return NULL;
11925#endif
11926}
11927EXPORT_SYMBOL_GPL(sched_trace_rq_avg_rt);
11928
11929const struct sched_avg *sched_trace_rq_avg_dl(struct rq *rq)
11930{
11931#ifdef CONFIG_SMP
11932 return rq ? &rq->avg_dl : NULL;
11933#else
11934 return NULL;
11935#endif
11936}
11937EXPORT_SYMBOL_GPL(sched_trace_rq_avg_dl);
11938
11939const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq)
11940{
11941#if defined(CONFIG_SMP) && defined(CONFIG_HAVE_SCHED_AVG_IRQ)
11942 return rq ? &rq->avg_irq : NULL;
11943#else
11944 return NULL;
11945#endif
11946}
11947EXPORT_SYMBOL_GPL(sched_trace_rq_avg_irq);
11948
11949int sched_trace_rq_cpu(struct rq *rq)
11950{
11951 return rq ? cpu_of(rq) : -1;
11952}
11953EXPORT_SYMBOL_GPL(sched_trace_rq_cpu);
11954
11955int sched_trace_rq_cpu_capacity(struct rq *rq)
11956{
11957 return rq ?
11958#ifdef CONFIG_SMP
11959 rq->cpu_capacity
11960#else
11961 SCHED_CAPACITY_SCALE
11962#endif
11963 : -1;
11964}
11965EXPORT_SYMBOL_GPL(sched_trace_rq_cpu_capacity);
11966
11967const struct cpumask *sched_trace_rd_span(struct root_domain *rd)
11968{
11969#ifdef CONFIG_SMP
11970 return rd ? rd->span : NULL;
11971#else
11972 return NULL;
11973#endif
11974}
11975EXPORT_SYMBOL_GPL(sched_trace_rd_span);
11976
11977int sched_trace_rq_nr_running(struct rq *rq)
11978{
11979 return rq ? rq->nr_running : -1;
11980}
11981EXPORT_SYMBOL_GPL(sched_trace_rq_nr_running);
kernel/sched/idle.c+1−1
@@ -327,7 +327,7 @@ static void do_idle(void) · 326 unmodified lines
327 * RCU relies on this call to be done outside of an RCU read-side327 * RCU relies on this call to be done outside of an RCU read-side
328 * critical section.328 * critical section.
329 */329 */
330 flush_smp_call_function_from_idle();330 flush_smp_call_function_queue();
331 schedule_idle();331 schedule_idle();
332 332
333 if (unlikely(klp_patch_pending(current)))333 if (unlikely(klp_patch_pending(current)))
kernel/sched/pelt.h+2−2
@@ -145,9 +145,9 @@ static inline u64 rq_clock_pelt(struct rq *rq) · 144 unmodified lines
145static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)145static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
146{146{
147 if (unlikely(cfs_rq->throttle_count))147 if (unlikely(cfs_rq->throttle_count))
148 return cfs_rq->throttled_clock_task - cfs_rq->throttled_clock_task_time;148 return cfs_rq->throttled_clock_pelt - cfs_rq->throttled_clock_pelt_time;
149 149
150 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;150 return rq_clock_pelt(rq_of(cfs_rq)) - cfs_rq->throttled_clock_pelt_time;
151}151}
152#else152#else
153static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)153static inline u64 cfs_rq_clock_pelt(struct cfs_rq *cfs_rq)
kernel/sched/psi.c+11−7
@@ -1060,14 +1060,17 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) · 1059 unmodified lines
1060 mutex_unlock(&group->avgs_lock);1060 mutex_unlock(&group->avgs_lock);
1061 1061
1062 for (full = 0; full < 2; full++) {1062 for (full = 0; full < 2; full++) {
1063 unsigned long avg[3];1063 unsigned long avg[3] = { 0, };
1064 u64 total;1064 u64 total = 0;
1065 int w;1065 int w;
1066 1066
1067 for (w = 0; w < 3; w++)1067 /* CPU FULL is undefined at the system level */
1068 avg[w] = group->avg[res * 2 + full][w];1068 if (!(group == &psi_system && res == PSI_CPU && full)) {
1069 total = div_u64(group->total[PSI_AVGS][res * 2 + full],1069 for (w = 0; w < 3; w++)
1070 NSEC_PER_USEC);1070 avg[w] = group->avg[res * 2 + full][w];
1071 total = div_u64(group->total[PSI_AVGS][res * 2 + full],
1072 NSEC_PER_USEC);
1073 }
1071 1074
1072 seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",1075 seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",
1073 full ? "full" : "some",1076 full ? "full" : "some",
@@ -1117,7 +1120,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, · 43 unmodified lines
1117 t->state = state;1120 t->state = state;
1118 t->threshold = threshold_us * NSEC_PER_USEC;1121 t->threshold = threshold_us * NSEC_PER_USEC;
1119 t->win.size = window_us * NSEC_PER_USEC;1122 t->win.size = window_us * NSEC_PER_USEC;
1120 window_reset(&t->win, 0, 0, 0);1123 window_reset(&t->win, sched_clock(),
1124 group->total[PSI_POLL][t->state], 0);
1121 1125
1122 t->event = 0;1126 t->event = 0;
1123 t->last_event_time = 0;1127 t->last_event_time = 0;
kernel/sched/rt.c+3−2
@@ -871,6 +871,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 870 unmodified lines
871 int enqueue = 0;871 int enqueue = 0;
872 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);872 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
873 struct rq *rq = rq_of_rt_rq(rt_rq);873 struct rq *rq = rq_of_rt_rq(rt_rq);
874 struct rq_flags rf;
874 int skip;875 int skip;
875 876
876 /*877 /*
@@ -885,7 +886,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 8 unmodified lines
885 if (skip)886 if (skip)
886 continue;887 continue;
887 888
888 raw_spin_rq_lock(rq);889 rq_lock(rq, &rf);
889 update_rq_clock(rq);890 update_rq_clock(rq);
890 891
891 if (rt_rq->rt_time) {892 if (rt_rq->rt_time) {
@@ -923,7 +924,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) · 31 unmodified lines
923 924
924 if (enqueue)925 if (enqueue)
925 sched_rt_rq_enqueue(rt_rq);926 sched_rt_rq_enqueue(rt_rq);
926 raw_spin_rq_unlock(rq);927 rq_unlock(rq, &rf);
927 }928 }
928 929
929 if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))930 if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
kernel/sched/sched.h+36−19
@@ -603,8 +603,8 @@ struct cfs_rq { · 602 unmodified lines
603 s64 runtime_remaining;603 s64 runtime_remaining;
604 604
605 u64 throttled_clock;605 u64 throttled_clock;
606 u64 throttled_clock_task;606 u64 throttled_clock_pelt;
607 u64 throttled_clock_task_time;607 u64 throttled_clock_pelt_time;
608 int throttled;608 int throttled;
609 int throttle_count;609 int throttle_count;
610 struct list_head throttled_list;610 struct list_head throttled_list;
@@ -1827,12 +1827,7 @@ static inline void dirty_sched_domain_sysctl(int cpu) · 1216 unmodified lines
1827#endif1827#endif
1828 1828
1829extern int sched_update_scaling(void);1829extern int sched_update_scaling(void);
1830 1830#endif /* CONFIG_SMP */
1831extern void flush_smp_call_function_from_idle(void);
1832
1833#else /* !CONFIG_SMP: */
1834static inline void flush_smp_call_function_from_idle(void) { }
1835#endif
1836 1831
1837#include "stats.h"1832#include "stats.h"
1838 1833
@@ -2182,6 +2177,8 @@ static inline void set_next_task(struct rq *rq, struct task_struct *next) · 343 unmodified lines
2182 *2177 *
2183 * include/asm-generic/vmlinux.lds.h2178 * include/asm-generic/vmlinux.lds.h
2184 *2179 *
2180 * *CAREFUL* they are laid out in *REVERSE* order!!!
2181 *
2185 * Also enforce alignment on the instance, not the type, to guarantee layout.2182 * Also enforce alignment on the instance, not the type, to guarantee layout.
2186 */2183 */
2187#define DEFINE_SCHED_CLASS(name) \2184#define DEFINE_SCHED_CLASS(name) \
@@ -2190,17 +2187,16 @@ const struct sched_class name##_sched_class \ · 2 unmodified lines
2190 __section("__" #name "_sched_class")2187 __section("__" #name "_sched_class")
2191 2188
2192/* Defined in include/asm-generic/vmlinux.lds.h */2189/* Defined in include/asm-generic/vmlinux.lds.h */
2193extern struct sched_class __begin_sched_classes[];2190extern struct sched_class __sched_class_highest[];
2194extern struct sched_class __end_sched_classes[];2191extern struct sched_class __sched_class_lowest[];
2195
2196#define sched_class_highest (__end_sched_classes - 1)
2197#define sched_class_lowest (__begin_sched_classes - 1)
2198 2192
2199#define for_class_range(class, _from, _to) \2193#define for_class_range(class, _from, _to) \
2200 for (class = (_from); class != (_to); class--)2194 for (class = (_from); class < (_to); class++)
2201 2195
2202#define for_each_class(class) \2196#define for_each_class(class) \
2203 for_class_range(class, sched_class_highest, sched_class_lowest)2197 for_class_range(class, __sched_class_highest, __sched_class_lowest)
2198
2199#define sched_class_above(_a, _b) ((_a) < (_b))
2204 2200
2205extern const struct sched_class stop_sched_class;2201extern const struct sched_class stop_sched_class;
2206extern const struct sched_class dl_sched_class;2202extern const struct sched_class dl_sched_class;
@@ -2309,6 +2305,7 @@ extern void resched_cpu(int cpu); · 102 unmodified lines
2309 2305
2310extern struct rt_bandwidth def_rt_bandwidth;2306extern struct rt_bandwidth def_rt_bandwidth;
2311extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);2307extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
2308extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
2312 2309
2313extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime);2310extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime);
2314extern void init_dl_task_timer(struct sched_dl_entity *dl_se);2311extern void init_dl_task_timer(struct sched_dl_entity *dl_se);
@@ -2478,6 +2475,24 @@ unsigned long arch_scale_freq_capacity(int cpu) · 163 unmodified lines
2478}2475}
2479#endif2476#endif
2480 2477
2478#ifdef CONFIG_SCHED_DEBUG
2479/*
2480 * In double_lock_balance()/double_rq_lock(), we use raw_spin_rq_lock() to
2481 * acquire rq lock instead of rq_lock(). So at the end of these two functions
2482 * we need to call double_rq_clock_clear_update() to clear RQCF_UPDATED of
2483 * rq->clock_update_flags to avoid the WARN_DOUBLE_CLOCK warning.
2484 */
2485static inline void double_rq_clock_clear_update(struct rq *rq1, struct rq *rq2)
2486{
2487 rq1->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);
2488 /* rq1 == rq2 for !CONFIG_SMP, so just clear RQCF_UPDATED once. */
2489#ifdef CONFIG_SMP
2490 rq2->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);
2491#endif
2492}
2493#else
2494static inline void double_rq_clock_clear_update(struct rq *rq1, struct rq *rq2) {}
2495#endif
2481 2496
2482#ifdef CONFIG_SMP2497#ifdef CONFIG_SMP
2483 2498
@@ -2543,14 +2558,15 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) · 59 unmodified lines
2543 __acquires(busiest->lock)2558 __acquires(busiest->lock)
2544 __acquires(this_rq->lock)2559 __acquires(this_rq->lock)
2545{2560{
2546 if (__rq_lockp(this_rq) == __rq_lockp(busiest))2561 if (__rq_lockp(this_rq) == __rq_lockp(busiest) ||
2547 return 0;2562 likely(raw_spin_rq_trylock(busiest))) {
2548 2563 double_rq_clock_clear_update(this_rq, busiest);
2549 if (likely(raw_spin_rq_trylock(busiest)))
2550 return 0;2564 return 0;
2565 }
2551 2566
2552 if (rq_order_less(this_rq, busiest)) {2567 if (rq_order_less(this_rq, busiest)) {
2553 raw_spin_rq_lock_nested(busiest, SINGLE_DEPTH_NESTING);2568 raw_spin_rq_lock_nested(busiest, SINGLE_DEPTH_NESTING);
2569 double_rq_clock_clear_update(this_rq, busiest);
2554 return 0;2570 return 0;
2555 }2571 }
2556 2572
@@ -2644,6 +2660,7 @@ static inline void double_rq_lock(struct rq *rq1, struct rq *rq2) · 87 unmodified lines
2644 BUG_ON(rq1 != rq2);2660 BUG_ON(rq1 != rq2);
2645 raw_spin_rq_lock(rq1);2661 raw_spin_rq_lock(rq1);
2646 __acquire(rq2->lock); /* Fake it out ;) */2662 __acquire(rq2->lock); /* Fake it out ;) */
2663 double_rq_clock_clear_update(rq1, rq2);
2647}2664}
2648 2665
2649/*2666/*
kernel/sched/smp.h+6−0
@@ -7,3 +7,9 @@ · 6 unmodified lines
7extern void sched_ttwu_pending(void *arg);7extern void sched_ttwu_pending(void *arg);
8 8
9extern void send_call_function_single_ipi(int cpu);9extern void send_call_function_single_ipi(int cpu);
10
11#ifdef CONFIG_SMP
12extern void flush_smp_call_function_queue(void);
13#else
14static inline void flush_smp_call_function_queue(void) { }
15#endif
kernel/smp.c+24−8
@@ -96,7 +96,7 @@ static DEFINE_PER_CPU_ALIGNED(struct call_function_data, cfd_data); · 95 unmodified lines
96 96
97static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);97static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
98 98
99static void flush_smp_call_function_queue(bool warn_cpu_offline);99static void __flush_smp_call_function_queue(bool warn_cpu_offline);
100 100
101int smpcfd_prepare_cpu(unsigned int cpu)101int smpcfd_prepare_cpu(unsigned int cpu)
102{102{
@@ -141,7 +141,7 @@ int smpcfd_dying_cpu(unsigned int cpu) · 38 unmodified lines
141 * ensure that the outgoing CPU doesn't go offline with work141 * ensure that the outgoing CPU doesn't go offline with work
142 * still pending.142 * still pending.
143 */143 */
144 flush_smp_call_function_queue(false);144 __flush_smp_call_function_queue(false);
145 irq_work_run();145 irq_work_run();
146 return 0;146 return 0;
147}147}
@@ -544,11 +544,11 @@ void generic_smp_call_function_single_interrupt(void) · 396 unmodified lines
544{544{
545 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->gotipi, CFD_SEQ_NOCPU,545 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->gotipi, CFD_SEQ_NOCPU,
546 smp_processor_id(), CFD_SEQ_GOTIPI);546 smp_processor_id(), CFD_SEQ_GOTIPI);
547 flush_smp_call_function_queue(true);547 __flush_smp_call_function_queue(true);
548}548}
549 549
550/**550/**
551 * flush_smp_call_function_queue - Flush pending smp-call-function callbacks551 * __flush_smp_call_function_queue - Flush pending smp-call-function callbacks
552 *552 *
553 * @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an553 * @warn_cpu_offline: If set to 'true', warn if callbacks were queued on an
554 * offline CPU. Skip this check if set to 'false'.554 * offline CPU. Skip this check if set to 'false'.
@@ -561,7 +561,7 @@ void generic_smp_call_function_single_interrupt(void) · 6 unmodified lines
561 * Loop through the call_single_queue and run all the queued callbacks.561 * Loop through the call_single_queue and run all the queued callbacks.
562 * Must be called with interrupts disabled.562 * Must be called with interrupts disabled.
563 */563 */
564static void flush_smp_call_function_queue(bool warn_cpu_offline)564static void __flush_smp_call_function_queue(bool warn_cpu_offline)
565{565{
566 call_single_data_t *csd, *csd_next;566 call_single_data_t *csd, *csd_next;
567 struct llist_node *entry, *prev;567 struct llist_node *entry, *prev;
@@ -684,8 +684,22 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) · 116 unmodified lines
684 smp_processor_id(), CFD_SEQ_HDLEND);684 smp_processor_id(), CFD_SEQ_HDLEND);
685}685}
686 686
687void flush_smp_call_function_from_idle(void)687
688/**
689 * flush_smp_call_function_queue - Flush pending smp-call-function callbacks
690 * from task context (idle, migration thread)
691 *
692 * When TIF_POLLING_NRFLAG is supported and a CPU is in idle and has it
693 * set, then remote CPUs can avoid sending IPIs and wake the idle CPU by
694 * setting TIF_NEED_RESCHED. The idle task on the woken up CPU has to
695 * handle queued SMP function calls before scheduling.
696 *
697 * The migration thread has to ensure that an eventually pending wakeup has
698 * been handled before it migrates a task.
699 */
700void flush_smp_call_function_queue(void)
688{701{
702 unsigned int was_pending;
689 unsigned long flags;703 unsigned long flags;
690 704
691 if (llist_empty(this_cpu_ptr(&call_single_queue)))705 if (llist_empty(this_cpu_ptr(&call_single_queue)))
@@ -694,9 +708,11 @@ void flush_smp_call_function_from_idle(void) · 2 unmodified lines
694 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->idle, CFD_SEQ_NOCPU,708 cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->idle, CFD_SEQ_NOCPU,
695 smp_processor_id(), CFD_SEQ_IDLE);709 smp_processor_id(), CFD_SEQ_IDLE);
696 local_irq_save(flags);710 local_irq_save(flags);
697 flush_smp_call_function_queue(true);711 /* Get the already pending soft interrupts for RT enabled kernels */
712 was_pending = local_softirq_pending();
713 __flush_smp_call_function_queue(true);
698 if (local_softirq_pending())714 if (local_softirq_pending())
699 do_softirq();715 do_softirq_post_smp_call_flush(was_pending);
700 716
701 local_irq_restore(flags);717 local_irq_restore(flags);
702}718}
kernel/softirq.c+13−0
@@ -294,6 +294,19 @@ static inline void invoke_softirq(void) · 293 unmodified lines
294 wakeup_softirqd();294 wakeup_softirqd();
295}295}
296 296
297/*
298 * flush_smp_call_function_queue() can raise a soft interrupt in a function
299 * call. On RT kernels this is undesired and the only known functionality
300 * in the block layer which does this is disabled on RT. If soft interrupts
301 * get raised which haven't been raised before the flush, warn so it can be
302 * investigated.
303 */
304void do_softirq_post_smp_call_flush(unsigned int was_pending)
305{
306 if (WARN_ON_ONCE(was_pending != local_softirq_pending()))
307 invoke_softirq();
308}
309
297#else /* CONFIG_PREEMPT_RT */310#else /* CONFIG_PREEMPT_RT */
298 311
299/*312/*
kernel/stop_machine.c+0−2
@@ -535,8 +535,6 @@ void stop_machine_park(int cpu) · 534 unmodified lines
535 kthread_park(stopper->thread);535 kthread_park(stopper->thread);
536}536}
537 537
538extern void sched_set_stop_task(int cpu, struct task_struct *stop);
539
540static void cpu_stop_create(unsigned int cpu)538static void cpu_stop_create(unsigned int cpu)
541{539{
542 sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));540 sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));
kernel/trace/trace.c+4−10
@@ -4289,17 +4289,11 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter) · 4288 unmodified lines
4289 entries,4289 entries,
4290 total,4290 total,
4291 buf->cpu,4291 buf->cpu,
4292#if defined(CONFIG_PREEMPT_NONE)4292 preempt_model_none() ? "server" :
4293 "server",4293 preempt_model_voluntary() ? "desktop" :
4294#elif defined(CONFIG_PREEMPT_VOLUNTARY)4294 preempt_model_full() ? "preempt" :
4295 "desktop",4295 preempt_model_rt() ? "preempt_rt" :
4296#elif defined(CONFIG_PREEMPT)
4297 "preempt",
4298#elif defined(CONFIG_PREEMPT_RT)
4299 "preempt_rt",
4300#else
4301 "unknown",4296 "unknown",
4302#endif
4303 /* These are reserved for later use */4297 /* These are reserved for later use */
4304 0, 0, 0, 0);4298 0, 0, 0, 0);
4305#ifdef CONFIG_SMP4299#ifdef CONFIG_SMP