History

Merge branch 'sched/fast-headers' into sched/core

Merge the scheduler build speedup of the fast-headers tree. Cumulative scheduler (kernel/sched/) build time speedup on a Linux distribution's config, which enables all scheduler features, compared to the vanilla kernel: _____________________________________________________________________________ | | Vanilla kernel (v5.13-rc7): |_____________________________________________________________________________ | | Performance counter stats for 'make -j96 kernel/sched/' (3 runs): | | 126,975,564,374 instructions # 1.45 insn per cycle ( +- 0.00% ) | 87,637,847,671 cycles # 3.959 GHz ( +- 0.30% ) | 22,136.96 msec cpu-clock # 7.499 CPUs utilized ( +- 0.29% ) | | 2.9520 +- 0.0169 seconds time elapsed ( +- 0.57% ) |_____________________________________________________________________________ | | Patched kernel: |_____________________________________________________________________________ | | Performance counter stats for 'make -j96 kernel/sched/' (3 runs): | | 50,420,496,914 instructions # 1.47 insn per cycle ( +- 0.00% ) | 34,234,322,038 cycles # 3.946 GHz ( +- 0.31% ) | 8,675.81 msec cpu-clock # 3.053 CPUs utilized ( +- 0.45% ) | | 2.8420 +- 0.0181 seconds time elapsed ( +- 0.64% ) |_____________________________________________________________________________ Summary: - CPU time used to build the scheduler dropped by -60.9%, a reduction from 22.1 clock-seconds to 8.7 clock-seconds. - Wall-clock time to build the scheduler dropped by -3.9%, a reduction from 2.95 seconds to 2.84 seconds.

Ingo Molnar committed 4 years ago

Commit ccacfe52 parents f2aa197 4ff8f2c69 files · +407 −163Browse files

Unified · Split

69 files
  1. arch/arm64/include/asm/paravirt_api_clock.h +1 −0
  2. arch/x86/include/asm/paravirt_api_clock.h +1 −0
  3. include/linux/cgroup_api.h +1 −0
  4. include/linux/cpumask_api.h +1 −0
  5. include/linux/fs_api.h +1 −0
  6. include/linux/gfp_api.h +1 −0
  7. include/linux/hashtable_api.h +1 −0
  8. include/linux/hrtimer_api.h +1 −0
  9. include/linux/kobject_api.h +1 −0
  10. include/linux/kref_api.h +1 −0
  11. include/linux/ktime_api.h +1 −0
  12. include/linux/llist_api.h +1 −0
  13. include/linux/lockdep_api.h +1 −0
  14. include/linux/mm_api.h +1 −0
  15. include/linux/mutex_api.h +1 −0
  16. include/linux/perf_event_api.h +1 −0
  17. include/linux/pgtable_api.h +1 −0
  18. include/linux/ptrace_api.h +1 −0
  19. include/linux/rcuwait_api.h +1 −0
  20. include/linux/refcount_api.h +1 −0
  21. include/linux/sched/affinity.h +1 −0
  22. include/linux/sched/cond_resched.h +1 −0
  23. include/linux/sched/deadline.h +2 −0
  24. include/linux/sched/posix-timers.h +1 −0
  25. include/linux/sched/rseq_api.h +1 −0
  26. include/linux/sched/task_flags.h +1 −0
  27. include/linux/sched/thread_info_api.h +1 −0
  28. include/linux/sched_clock.h +2 −0
  29. include/linux/seqlock_api.h +1 −0
  30. include/linux/softirq.h +1 −0
  31. include/linux/spinlock_api.h +1 −0
  32. include/linux/swait_api.h +1 −0
  33. include/linux/syscalls_api.h +1 −0
  34. include/linux/u64_stats_sync_api.h +1 −0
  35. include/linux/wait_api.h +1 −0
  36. include/linux/workqueue_api.h +1 −0
  37. kernel/sched/Makefile +10 −18
  38. kernel/sched/autogroup.c +1 −2
  39. kernel/sched/autogroup.h +5 −0
  40. kernel/sched/build_policy.c +52 −0
  41. kernel/sched/build_utility.c +109 −0
  42. kernel/sched/clock.c +21 −23
  43. kernel/sched/completion.c +1 −1
  44. kernel/sched/core.c +71 −10
  45. kernel/sched/core_sched.c +0 −3
  46. kernel/sched/cpuacct.c +1 −2
  47. kernel/sched/cpudeadline.c +1 −2
  48. kernel/sched/cpufreq.c +0 −3
  49. kernel/sched/cpufreq_schedutil.c +0 −7
  50. kernel/sched/cpupri.c +0 −1
  51. kernel/sched/cputime.c +0 −1
  52. kernel/sched/deadline.c +0 −2
  53. kernel/sched/debug.c +0 −1
  54. kernel/sched/fair.c +31 −0
  55. kernel/sched/idle.c +0 −3
  56. kernel/sched/isolation.c +0 −1
  57. kernel/sched/loadavg.c +0 −1
  58. kernel/sched/membarrier.c +0 −1
  59. kernel/sched/pelt.c +0 −4
  60. kernel/sched/psi.c +0 −15
  61. kernel/sched/rt.c +0 −3
  62. kernel/sched/sched.h +61 −53
  63. kernel/sched/stats.c +0 −1
  64. kernel/sched/stats.h +4 −0
  65. kernel/sched/stop_task.c +0 −1
  66. kernel/sched/swait.c +0 −1
  67. kernel/sched/topology.c +0 −1
  68. kernel/sched/wait.c +0 −1
  69. kernel/sched/wait_bit.c +1 −1
arch/arm64/include/asm/paravirt_api_clock.h+1−0
@@ -0,0 +1 @@
1#include <asm/paravirt.h>
arch/x86/include/asm/paravirt_api_clock.h+1−0
@@ -0,0 +1 @@
1#include <asm/paravirt.h>
include/linux/cgroup_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/cgroup.h>
include/linux/cpumask_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/cpumask.h>
include/linux/fs_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/fs.h>
include/linux/gfp_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/gfp.h>
include/linux/hashtable_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/hashtable.h>
include/linux/hrtimer_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/hrtimer.h>
include/linux/kobject_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/kobject.h>
include/linux/kref_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/kref.h>
include/linux/ktime_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/ktime.h>
include/linux/llist_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/llist.h>
include/linux/lockdep_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/lockdep.h>
include/linux/mm_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/mm.h>
include/linux/mutex_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/mutex.h>
include/linux/perf_event_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/perf_event.h>
include/linux/pgtable_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/pgtable.h>
include/linux/ptrace_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/ptrace.h>
include/linux/rcuwait_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/rcuwait.h>
include/linux/refcount_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/refcount.h>
include/linux/sched/affinity.h+1−0
@@ -0,0 +1 @@
1#include <linux/sched.h>
include/linux/sched/cond_resched.h+1−0
@@ -0,0 +1 @@
1#include <linux/sched.h>
include/linux/sched/deadline.h+2−0
@@ -6,6 +6,8 @@ · 5 unmodified lines
6 * NORMAL/BATCH tasks.6 * NORMAL/BATCH tasks.
7 */7 */
8 8
9#include <linux/sched.h>
10
9#define MAX_DL_PRIO 011#define MAX_DL_PRIO 0
10 12
11static inline int dl_prio(int prio)13static inline int dl_prio(int prio)
include/linux/sched/posix-timers.h+1−0
@@ -0,0 +1 @@
1#include <linux/posix-timers.h>
include/linux/sched/rseq_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/rseq.h>
include/linux/sched/task_flags.h+1−0
@@ -0,0 +1 @@
1#include <linux/sched.h>
include/linux/sched/thread_info_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/thread_info.h>
include/linux/sched_clock.h+2−0
@@ -5,6 +5,8 @@ · 4 unmodified lines
5#ifndef LINUX_SCHED_CLOCK5#ifndef LINUX_SCHED_CLOCK
6#define LINUX_SCHED_CLOCK6#define LINUX_SCHED_CLOCK
7 7
8#include <linux/types.h>
9
8#ifdef CONFIG_GENERIC_SCHED_CLOCK10#ifdef CONFIG_GENERIC_SCHED_CLOCK
9/**11/**
10 * struct clock_read_data - data required to read from sched_clock()12 * struct clock_read_data - data required to read from sched_clock()
include/linux/seqlock_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/seqlock.h>
include/linux/softirq.h+1−0
@@ -0,0 +1 @@
1#include <linux/interrupt.h>
include/linux/spinlock_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/spinlock.h>
include/linux/swait_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/swait.h>
include/linux/syscalls_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/syscalls.h>
include/linux/u64_stats_sync_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/u64_stats_sync.h>
include/linux/wait_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/wait.h>
include/linux/workqueue_api.h+1−0
@@ -0,0 +1 @@
1#include <linux/workqueue.h>
kernel/sched/Makefile+10−18
@@ -1,7 +1,4 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2ifdef CONFIG_FUNCTION_TRACER
3CFLAGS_REMOVE_clock.o = $(CC_FLAGS_FTRACE)
4endif
5 2
6# The compilers are complaining about unused variables inside an if(0) scope3# The compilers are complaining about unused variables inside an if(0) scope
7# block. This is daft, shut them up.4# block. This is daft, shut them up.
@@ -25,18 +22,13 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) · 17 unmodified lines
25CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer22CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer
26endif23endif
27 24
28obj-y += core.o loadavg.o clock.o cputime.o25#
29obj-y += idle.o fair.o rt.o deadline.o26# Build efficiency:
30obj-y += wait.o wait_bit.o swait.o completion.o27#
31 28# These compilation units have roughly the same size and complexity - so their
32obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o pelt.o29# build parallelizes well and finishes roughly at once:
33obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o30#
34obj-$(CONFIG_SCHEDSTATS) += stats.o31obj-y += core.o
35obj-$(CONFIG_SCHED_DEBUG) += debug.o32obj-y += fair.o
36obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o33obj-y += build_policy.o
37obj-$(CONFIG_CPU_FREQ) += cpufreq.o34obj-y += build_utility.o
38obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o
39obj-$(CONFIG_MEMBARRIER) += membarrier.o
40obj-$(CONFIG_CPU_ISOLATION) += isolation.o
41obj-$(CONFIG_PSI) += psi.o
42obj-$(CONFIG_SCHED_CORE) += core_sched.o
kernel/sched/autogroup.c+1−2
@@ -1,9 +1,8 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2
2/*3/*
3 * Auto-group scheduling implementation:4 * Auto-group scheduling implementation:
4 */5 */
5#include <linux/nospec.h>
6#include "sched.h"
7 6
8unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;7unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
9static struct autogroup autogroup_default;8static struct autogroup autogroup_default;
kernel/sched/autogroup.h+5−0
@@ -1,4 +1,7 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _KERNEL_SCHED_AUTOGROUP_H
3#define _KERNEL_SCHED_AUTOGROUP_H
4
2#ifdef CONFIG_SCHED_AUTOGROUP5#ifdef CONFIG_SCHED_AUTOGROUP
3 6
4struct autogroup {7struct autogroup {
@@ -59,3 +62,5 @@ static inline int autogroup_path(struct task_group *tg, char *buf, int buflen) · 54 unmodified lines
59}62}
60 63
61#endif /* CONFIG_SCHED_AUTOGROUP */64#endif /* CONFIG_SCHED_AUTOGROUP */
65
66#endif /* _KERNEL_SCHED_AUTOGROUP_H */
kernel/sched/build_policy.c+52−0
@@ -0,0 +1,52 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * These are the scheduling policy related scheduler files, built
4 * in a single compilation unit for build efficiency reasons.
5 *
6 * ( Incidentally, the size of the compilation unit is roughly
7 * comparable to core.c and fair.c, the other two big
8 * compilation units. This helps balance build time, while
9 * coalescing source files to amortize header inclusion
10 * cost. )
11 *
12 * core.c and fair.c are built separately.
13 */
14
15/* Headers: */
16#include <linux/sched/clock.h>
17#include <linux/sched/cputime.h>
18#include <linux/sched/posix-timers.h>
19#include <linux/sched/rt.h>
20
21#include <linux/cpuidle.h>
22#include <linux/jiffies.h>
23#include <linux/livepatch.h>
24#include <linux/psi.h>
25#include <linux/seqlock_api.h>
26#include <linux/slab.h>
27#include <linux/suspend.h>
28#include <linux/tsacct_kern.h>
29#include <linux/vtime.h>
30
31#include <uapi/linux/sched/types.h>
32
33#include "sched.h"
34
35#include "autogroup.h"
36#include "stats.h"
37#include "pelt.h"
38
39/* Source code modules: */
40
41#include "idle.c"
42
43#include "rt.c"
44
45#ifdef CONFIG_SMP
46# include "cpudeadline.c"
47# include "pelt.c"
48#endif
49
50#include "cputime.c"
51#include "deadline.c"
52
kernel/sched/build_utility.c+109−0
@@ -0,0 +1,109 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * These are various utility functions of the scheduler,
4 * built in a single compilation unit for build efficiency reasons.
5 *
6 * ( Incidentally, the size of the compilation unit is roughly
7 * comparable to core.c, fair.c, smp.c and policy.c, the other
8 * big compilation units. This helps balance build time, while
9 * coalescing source files to amortize header inclusion
10 * cost. )
11 */
12#include <linux/sched/clock.h>
13#include <linux/sched/cputime.h>
14#include <linux/sched/debug.h>
15#include <linux/sched/isolation.h>
16#include <linux/sched/loadavg.h>
17#include <linux/sched/mm.h>
18#include <linux/sched/rseq_api.h>
19#include <linux/sched/task_stack.h>
20
21#include <linux/cpufreq.h>
22#include <linux/cpumask_api.h>
23#include <linux/cpuset.h>
24#include <linux/ctype.h>
25#include <linux/debugfs.h>
26#include <linux/energy_model.h>
27#include <linux/hashtable_api.h>
28#include <linux/irq.h>
29#include <linux/kobject_api.h>
30#include <linux/membarrier.h>
31#include <linux/mempolicy.h>
32#include <linux/nmi.h>
33#include <linux/nospec.h>
34#include <linux/proc_fs.h>
35#include <linux/psi.h>
36#include <linux/psi.h>
37#include <linux/ptrace_api.h>
38#include <linux/sched_clock.h>
39#include <linux/security.h>
40#include <linux/spinlock_api.h>
41#include <linux/swait_api.h>
42#include <linux/timex.h>
43#include <linux/utsname.h>
44#include <linux/wait_api.h>
45#include <linux/workqueue_api.h>
46
47#include <uapi/linux/prctl.h>
48#include <uapi/linux/sched/types.h>
49
50#include <asm/switch_to.h>
51
52#include "sched.h"
53#include "sched-pelt.h"
54#include "stats.h"
55#include "autogroup.h"
56
57#include "clock.c"
58
59#ifdef CONFIG_CGROUP_CPUACCT
60# include "cpuacct.c"
61#endif
62
63#ifdef CONFIG_CPU_FREQ
64# include "cpufreq.c"
65#endif
66
67#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
68# include "cpufreq_schedutil.c"
69#endif
70
71#ifdef CONFIG_SCHED_DEBUG
72# include "debug.c"
73#endif
74
75#ifdef CONFIG_SCHEDSTATS
76# include "stats.c"
77#endif
78
79#include "loadavg.c"
80#include "completion.c"
81#include "swait.c"
82#include "wait_bit.c"
83#include "wait.c"
84
85#ifdef CONFIG_SMP
86# include "cpupri.c"
87# include "stop_task.c"
88# include "topology.c"
89#endif
90
91#ifdef CONFIG_SCHED_CORE
92# include "core_sched.c"
93#endif
94
95#ifdef CONFIG_PSI
96# include "psi.c"
97#endif
98
99#ifdef CONFIG_MEMBARRIER
100# include "membarrier.c"
101#endif
102
103#ifdef CONFIG_CPU_ISOLATION
104# include "isolation.c"
105#endif
106
107#ifdef CONFIG_SCHED_AUTOGROUP
108# include "autogroup.c"
109#endif
kernel/sched/clock.c+21−23
@@ -53,15 +53,13 @@ · 52 unmodified lines
53 * that is otherwise invisible (TSC gets stopped).53 * that is otherwise invisible (TSC gets stopped).
54 *54 *
55 */55 */
56#include "sched.h"
57#include <linux/sched_clock.h>
58 56
59/*57/*
60 * Scheduler clock - returns current time in nanosec units.58 * Scheduler clock - returns current time in nanosec units.
61 * This is default implementation.59 * This is default implementation.
62 * Architectures and sub-architectures can override this.60 * Architectures and sub-architectures can override this.
63 */61 */
64unsigned long long __weak sched_clock(void)62notrace unsigned long long __weak sched_clock(void)
65{63{
66 return (unsigned long long)(jiffies - INITIAL_JIFFIES)64 return (unsigned long long)(jiffies - INITIAL_JIFFIES)
67 * (NSEC_PER_SEC / HZ);65 * (NSEC_PER_SEC / HZ);
@@ -95,28 +93,28 @@ struct sched_clock_data { · 27 unmodified lines
95 93
96static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);94static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
97 95
98static inline struct sched_clock_data *this_scd(void)96notrace static inline struct sched_clock_data *this_scd(void)
99{97{
100 return this_cpu_ptr(&sched_clock_data);98 return this_cpu_ptr(&sched_clock_data);
101}99}
102 100
103static inline struct sched_clock_data *cpu_sdc(int cpu)101notrace static inline struct sched_clock_data *cpu_sdc(int cpu)
104{102{
105 return &per_cpu(sched_clock_data, cpu);103 return &per_cpu(sched_clock_data, cpu);
106}104}
107 105
108int sched_clock_stable(void)106notrace int sched_clock_stable(void)
109{107{
110 return static_branch_likely(&__sched_clock_stable);108 return static_branch_likely(&__sched_clock_stable);
111}109}
112 110
113static void __scd_stamp(struct sched_clock_data *scd)111notrace static void __scd_stamp(struct sched_clock_data *scd)
114{112{
115 scd->tick_gtod = ktime_get_ns();113 scd->tick_gtod = ktime_get_ns();
116 scd->tick_raw = sched_clock();114 scd->tick_raw = sched_clock();
117}115}
118 116
119static void __set_sched_clock_stable(void)117notrace static void __set_sched_clock_stable(void)
120{118{
121 struct sched_clock_data *scd;119 struct sched_clock_data *scd;
122 120
@@ -151,7 +149,7 @@ static void __set_sched_clock_stable(void) · 28 unmodified lines
151 * The only way to fully avoid random clock jumps is to boot with:149 * The only way to fully avoid random clock jumps is to boot with:
152 * "tsc=unstable".150 * "tsc=unstable".
153 */151 */
154static void __sched_clock_work(struct work_struct *work)152notrace static void __sched_clock_work(struct work_struct *work)
155{153{
156 struct sched_clock_data *scd;154 struct sched_clock_data *scd;
157 int cpu;155 int cpu;
@@ -177,7 +175,7 @@ static void __sched_clock_work(struct work_struct *work) · 19 unmodified lines
177 175
178static DECLARE_WORK(sched_clock_work, __sched_clock_work);176static DECLARE_WORK(sched_clock_work, __sched_clock_work);
179 177
180static void __clear_sched_clock_stable(void)178notrace static void __clear_sched_clock_stable(void)
181{179{
182 if (!sched_clock_stable())180 if (!sched_clock_stable())
183 return;181 return;
@@ -186,7 +184,7 @@ static void __clear_sched_clock_stable(void) · 2 unmodified lines
186 schedule_work(&sched_clock_work);184 schedule_work(&sched_clock_work);
187}185}
188 186
189void clear_sched_clock_stable(void)187notrace void clear_sched_clock_stable(void)
190{188{
191 __sched_clock_stable_early = 0;189 __sched_clock_stable_early = 0;
192 190
@@ -196,7 +194,7 @@ void clear_sched_clock_stable(void) · 3 unmodified lines
196 __clear_sched_clock_stable();194 __clear_sched_clock_stable();
197}195}
198 196
199static void __sched_clock_gtod_offset(void)197notrace static void __sched_clock_gtod_offset(void)
200{198{
201 struct sched_clock_data *scd = this_scd();199 struct sched_clock_data *scd = this_scd();
202 200
@@ -246,12 +244,12 @@ late_initcall(sched_clock_init_late); · 43 unmodified lines
246 * min, max except they take wrapping into account244 * min, max except they take wrapping into account
247 */245 */
248 246
249static inline u64 wrap_min(u64 x, u64 y)247notrace static inline u64 wrap_min(u64 x, u64 y)
250{248{
251 return (s64)(x - y) < 0 ? x : y;249 return (s64)(x - y) < 0 ? x : y;
252}250}
253 251
254static inline u64 wrap_max(u64 x, u64 y)252notrace static inline u64 wrap_max(u64 x, u64 y)
255{253{
256 return (s64)(x - y) > 0 ? x : y;254 return (s64)(x - y) > 0 ? x : y;
257}255}
@@ -262,7 +260,7 @@ static inline u64 wrap_max(u64 x, u64 y) · 4 unmodified lines
262 * - filter out backward motion260 * - filter out backward motion
263 * - use the GTOD tick value to create a window to filter crazy TSC values261 * - use the GTOD tick value to create a window to filter crazy TSC values
264 */262 */
265static u64 sched_clock_local(struct sched_clock_data *scd)263notrace static u64 sched_clock_local(struct sched_clock_data *scd)
266{264{
267 u64 now, clock, old_clock, min_clock, max_clock, gtod;265 u64 now, clock, old_clock, min_clock, max_clock, gtod;
268 s64 delta;266 s64 delta;
@@ -295,7 +293,7 @@ again: · 26 unmodified lines
295 return clock;293 return clock;
296}294}
297 295
298static u64 sched_clock_remote(struct sched_clock_data *scd)296notrace static u64 sched_clock_remote(struct sched_clock_data *scd)
299{297{
300 struct sched_clock_data *my_scd = this_scd();298 struct sched_clock_data *my_scd = this_scd();
301 u64 this_clock, remote_clock;299 u64 this_clock, remote_clock;
@@ -362,7 +360,7 @@ again: · 60 unmodified lines
362 *360 *
363 * See cpu_clock().361 * See cpu_clock().
364 */362 */
365u64 sched_clock_cpu(int cpu)363notrace u64 sched_clock_cpu(int cpu)
366{364{
367 struct sched_clock_data *scd;365 struct sched_clock_data *scd;
368 u64 clock;366 u64 clock;
@@ -386,7 +384,7 @@ u64 sched_clock_cpu(int cpu) · 17 unmodified lines
386}384}
387EXPORT_SYMBOL_GPL(sched_clock_cpu);385EXPORT_SYMBOL_GPL(sched_clock_cpu);
388 386
389void sched_clock_tick(void)387notrace void sched_clock_tick(void)
390{388{
391 struct sched_clock_data *scd;389 struct sched_clock_data *scd;
392 390
@@ -403,7 +401,7 @@ void sched_clock_tick(void) · 10 unmodified lines
403 sched_clock_local(scd);401 sched_clock_local(scd);
404}402}
405 403
406void sched_clock_tick_stable(void)404notrace void sched_clock_tick_stable(void)
407{405{
408 if (!sched_clock_stable())406 if (!sched_clock_stable())
409 return;407 return;
@@ -423,7 +421,7 @@ void sched_clock_tick_stable(void) · 13 unmodified lines
423/*421/*
424 * We are going deep-idle (irqs are disabled):422 * We are going deep-idle (irqs are disabled):
425 */423 */
426void sched_clock_idle_sleep_event(void)424notrace void sched_clock_idle_sleep_event(void)
427{425{
428 sched_clock_cpu(smp_processor_id());426 sched_clock_cpu(smp_processor_id());
429}427}
@@ -432,7 +430,7 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); · 2 unmodified lines
432/*430/*
433 * We just idled; resync with ktime.431 * We just idled; resync with ktime.
434 */432 */
435void sched_clock_idle_wakeup_event(void)433notrace void sched_clock_idle_wakeup_event(void)
436{434{
437 unsigned long flags;435 unsigned long flags;
438 436
@@ -458,7 +456,7 @@ void __init sched_clock_init(void) · 19 unmodified lines
458 local_irq_enable();456 local_irq_enable();
459}457}
460 458
461u64 sched_clock_cpu(int cpu)459notrace u64 sched_clock_cpu(int cpu)
462{460{
463 if (!static_branch_likely(&sched_clock_running))461 if (!static_branch_likely(&sched_clock_running))
464 return 0;462 return 0;
@@ -476,7 +474,7 @@ u64 sched_clock_cpu(int cpu) · 11 unmodified lines
476 * On bare metal this function should return the same as local_clock.474 * On bare metal this function should return the same as local_clock.
477 * Architectures and sub-architectures can override this.475 * Architectures and sub-architectures can override this.
478 */476 */
479u64 __weak running_clock(void)477notrace u64 __weak running_clock(void)
480{478{
481 return local_clock();479 return local_clock();
482}480}
kernel/sched/completion.c+1−1
@@ -1,4 +1,5 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2
2/*3/*
3 * Generic wait-for-completion handler;4 * Generic wait-for-completion handler;
4 *5 *
@@ -11,7 +12,6 @@ · 6 unmodified lines
11 * typically be used for exclusion which gives rise to priority inversion.12 * typically be used for exclusion which gives rise to priority inversion.
12 * Waiting for completion is a typically sync point, but not an exclusion point.13 * Waiting for completion is a typically sync point, but not an exclusion point.
13 */14 */
14#include "sched.h"
15 15
16/**16/**
17 * complete: - signals a single thread waiting on this completion17 * complete: - signals a single thread waiting on this completion
kernel/sched/core.c+71−10
@@ -6,27 +6,88 @@ · 5 unmodified lines
6 *6 *
7 * Copyright (C) 1991-2002 Linus Torvalds7 * Copyright (C) 1991-2002 Linus Torvalds
8 */8 */
9#define CREATE_TRACE_POINTS9#include <linux/highmem.h>
10#include <trace/events/sched.h>10#include <linux/hrtimer_api.h>
11#undef CREATE_TRACE_POINTS11#include <linux/ktime_api.h>
12#include <linux/sched/signal.h>
13#include <linux/syscalls_api.h>
14#include <linux/debug_locks.h>
15#include <linux/prefetch.h>
16#include <linux/capability.h>
17#include <linux/pgtable_api.h>
18#include <linux/wait_bit.h>
19#include <linux/jiffies.h>
20#include <linux/spinlock_api.h>
21#include <linux/cpumask_api.h>
22#include <linux/lockdep_api.h>
23#include <linux/hardirq.h>
24#include <linux/softirq.h>
25#include <linux/refcount_api.h>
26#include <linux/topology.h>
27#include <linux/sched/clock.h>
28#include <linux/sched/cond_resched.h>
29#include <linux/sched/debug.h>
30#include <linux/sched/isolation.h>
31#include <linux/sched/loadavg.h>
32#include <linux/sched/mm.h>
33#include <linux/sched/nohz.h>
34#include <linux/sched/rseq_api.h>
35#include <linux/sched/rt.h>
12 36
13#include "sched.h"
14
15#include <linux/nospec.h>
16#include <linux/blkdev.h>37#include <linux/blkdev.h>
17#include <linux/jump_label.h>38#include <linux/context_tracking.h>
39#include <linux/cpuset.h>
40#include <linux/delayacct.h>
41#include <linux/init_task.h>
42#include <linux/interrupt.h>
43#include <linux/ioprio.h>
44#include <linux/kallsyms.h>
18#include <linux/kcov.h>45#include <linux/kcov.h>
46#include <linux/kprobes.h>
47#include <linux/llist_api.h>
48#include <linux/mmu_context.h>
49#include <linux/mmzone.h>
50#include <linux/mutex_api.h>
51#include <linux/nmi.h>
52#include <linux/nospec.h>
53#include <linux/perf_event_api.h>
54#include <linux/profile.h>
55#include <linux/psi.h>
56#include <linux/rcuwait_api.h>
57#include <linux/sched/wake_q.h>
19#include <linux/scs.h>58#include <linux/scs.h>
59#include <linux/slab.h>
60#include <linux/syscalls.h>
61#include <linux/vtime.h>
62#include <linux/wait_api.h>
63#include <linux/workqueue_api.h>
64
65#ifdef CONFIG_PREEMPT_DYNAMIC
66# include <linux/entry-common.h>
67#endif
68
69#include <uapi/linux/sched/types.h>
20 70
21#include <asm/switch_to.h>71#include <asm/switch_to.h>
22#include <asm/tlb.h>72#include <asm/tlb.h>
23 73
24#include "../workqueue_internal.h"74#define CREATE_TRACE_POINTS
25#include "../../fs/io-wq.h"75#include <linux/sched/rseq_api.h>
26#include "../smpboot.h"76#include <trace/events/sched.h>
77#undef CREATE_TRACE_POINTS
27 78
79#include "sched.h"
80#include "stats.h"
81#include "autogroup.h"
82
83#include "autogroup.h"
28#include "pelt.h"84#include "pelt.h"
29#include "smp.h"85#include "smp.h"
86#include "stats.h"
87
88#include "../workqueue_internal.h"
89#include "../../fs/io-wq.h"
90#include "../smpboot.h"
30 91
31/*92/*
32 * Export tracepoints that act as a bare tracehook (ie: have no trace event93 * Export tracepoints that act as a bare tracehook (ie: have no trace event
kernel/sched/core_sched.c+0−3
@@ -1,8 +1,5 @@
1// SPDX-License-Identifier: GPL-2.0-only1// SPDX-License-Identifier: GPL-2.0-only
2 2
3#include <linux/prctl.h>
4#include "sched.h"
5
6/*3/*
7 * A simple wrapper around refcount. An allocated sched_core_cookie's4 * A simple wrapper around refcount. An allocated sched_core_cookie's
8 * address is used to compute the cookie of the task.5 * address is used to compute the cookie of the task.
kernel/sched/cpuacct.c+1−2
@@ -1,12 +1,11 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2
2/*3/*
3 * CPU accounting code for task groups.4 * CPU accounting code for task groups.
4 *5 *
5 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh6 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
6 * (balbir@in.ibm.com).7 * (balbir@in.ibm.com).
7 */8 */
8#include <asm/irq_regs.h>
9#include "sched.h"
10 9
11/* Time spent by the tasks of the CPU accounting group executing in ... */10/* Time spent by the tasks of the CPU accounting group executing in ... */
12enum cpuacct_stat_index {11enum cpuacct_stat_index {
kernel/sched/cpudeadline.c+1−2
@@ -1,12 +1,11 @@
1// SPDX-License-Identifier: GPL-2.0-only1// SPDX-License-Identifier: GPL-2.0-only
2/*2/*
3 * kernel/sched/cpudl.c3 * kernel/sched/cpudeadline.c
4 *4 *
5 * Global CPU deadline management5 * Global CPU deadline management
6 *6 *
7 * Author: Juri Lelli <j.lelli@sssup.it>7 * Author: Juri Lelli <j.lelli@sssup.it>
8 */8 */
9#include "sched.h"
10 9
11static inline int parent(int i)10static inline int parent(int i)
12{11{
kernel/sched/cpufreq.c+0−3
@@ -5,9 +5,6 @@ · 4 unmodified lines
5 * Copyright (C) 2016, Intel Corporation5 * Copyright (C) 2016, Intel Corporation
6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
7 */7 */
8#include <linux/cpufreq.h>
9
10#include "sched.h"
11 8
12DEFINE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data);9DEFINE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data);
13 10
kernel/sched/cpufreq_schedutil.c+0−7
@@ -6,13 +6,6 @@ · 5 unmodified lines
6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>6 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
7 */7 */
8 8
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include "sched.h"
12
13#include <linux/sched/cpufreq.h>
14#include <trace/events/power.h>
15
16#define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8)9#define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8)
17 10
18struct sugov_tunables {11struct sugov_tunables {
kernel/sched/cpupri.c+0−1
@@ -22,7 +22,6 @@ · 21 unmodified lines
22 * worst case complexity of O(min(101, nr_domcpus)), though the scenario that22 * worst case complexity of O(min(101, nr_domcpus)), though the scenario that
23 * yields the worst case search is fairly contrived.23 * yields the worst case search is fairly contrived.
24 */24 */
25#include "sched.h"
26 25
27/*26/*
28 * p->rt_priority p->prio newpri cpupri27 * p->rt_priority p->prio newpri cpupri
kernel/sched/cputime.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
2/*2/*
3 * Simple CPU accounting cgroup controller3 * Simple CPU accounting cgroup controller
4 */4 */
5#include "sched.h"
6 5
7#ifdef CONFIG_IRQ_TIME_ACCOUNTING6#ifdef CONFIG_IRQ_TIME_ACCOUNTING
8 7
kernel/sched/deadline.c+0−2
@@ -15,8 +15,6 @@ · 14 unmodified lines
15 * Michael Trimarchi <michael@amarulasolutions.com>,15 * Michael Trimarchi <michael@amarulasolutions.com>,
16 * Fabio Checconi <fchecconi@gmail.com>16 * Fabio Checconi <fchecconi@gmail.com>
17 */17 */
18#include "sched.h"
19#include "pelt.h"
20 18
21static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)19static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)
22{20{
kernel/sched/debug.c+0−1
@@ -6,7 +6,6 @@ · 5 unmodified lines
6 *6 *
7 * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar7 * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar
8 */8 */
9#include "sched.h"
10 9
11/*10/*
12 * This allows printing both to /proc/sched_debug and11 * This allows printing both to /proc/sched_debug and
kernel/sched/fair.c+31−0
@@ -20,7 +20,38 @@ · 19 unmodified lines
20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
22 */22 */
23#include <linux/energy_model.h>
24#include <linux/mmap_lock.h>
25#include <linux/hugetlb_inline.h>
26#include <linux/jiffies.h>
27#include <linux/mm_api.h>
28#include <linux/highmem.h>
29#include <linux/spinlock_api.h>
30#include <linux/cpumask_api.h>
31#include <linux/lockdep_api.h>
32#include <linux/softirq.h>
33#include <linux/refcount_api.h>
34#include <linux/topology.h>
35#include <linux/sched/clock.h>
36#include <linux/sched/cond_resched.h>
37#include <linux/sched/cputime.h>
38#include <linux/sched/isolation.h>
39
40#include <linux/cpuidle.h>
41#include <linux/interrupt.h>
42#include <linux/mempolicy.h>
43#include <linux/mutex_api.h>
44#include <linux/profile.h>
45#include <linux/psi.h>
46#include <linux/ratelimit.h>
47
48#include <asm/switch_to.h>
49
50#include <linux/sched/cond_resched.h>
51
23#include "sched.h"52#include "sched.h"
53#include "stats.h"
54#include "autogroup.h"
24 55
25/*56/*
26 * Targeted preemption latency for CPU-bound tasks:57 * Targeted preemption latency for CPU-bound tasks:
kernel/sched/idle.c+0−3
@@ -6,9 +6,6 @@ · 5 unmodified lines
6 * (NOTE: these are not related to SCHED_IDLE batch scheduled6 * (NOTE: these are not related to SCHED_IDLE batch scheduled
7 * tasks which are handled in sched/fair.c )7 * tasks which are handled in sched/fair.c )
8 */8 */
9#include "sched.h"
10
11#include <trace/events/power.h>
12 9
13/* Linker adds these: start and end of __cpuidle functions */10/* Linker adds these: start and end of __cpuidle functions */
14extern char __cpuidle_text_start[], __cpuidle_text_end[];11extern char __cpuidle_text_start[], __cpuidle_text_end[];
kernel/sched/isolation.c+0−1
@@ -7,7 +7,6 @@ · 6 unmodified lines
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *8 *
9 */9 */
10#include "sched.h"
11 10
12enum hk_flags {11enum hk_flags {
13 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
kernel/sched/loadavg.c+0−1
@@ -6,7 +6,6 @@ · 5 unmodified lines
6 * figure. Its a silly number but people think its important. We go through6 * figure. Its a silly number but people think its important. We go through
7 * great pains to make it work on big machines and tickless kernels.7 * great pains to make it work on big machines and tickless kernels.
8 */8 */
9#include "sched.h"
10 9
11/*10/*
12 * Global load-average calculations11 * Global load-average calculations
kernel/sched/membarrier.c+0−1
@@ -4,7 +4,6 @@ · 3 unmodified lines
4 *4 *
5 * membarrier system call5 * membarrier system call
6 */6 */
7#include "sched.h"
8 7
9/*8/*
10 * For documentation purposes, here are some membarrier ordering9 * For documentation purposes, here are some membarrier ordering
kernel/sched/pelt.c+0−4
@@ -24,10 +24,6 @@ · 23 unmodified lines
24 * Author: Vincent Guittot <vincent.guittot@linaro.org>24 * Author: Vincent Guittot <vincent.guittot@linaro.org>
25 */25 */
26 26
27#include <linux/sched.h>
28#include "sched.h"
29#include "pelt.h"
30
31/*27/*
32 * Approximate:28 * Approximate:
33 * val * y^n, where y^32 ~= 0.5 (~1 scheduling period)29 * val * y^n, where y^32 ~= 0.5 (~1 scheduling period)
kernel/sched/psi.c+0−15
@@ -137,21 +137,6 @@ · 136 unmodified lines
137 * sampling of the aggregate task states would be.137 * sampling of the aggregate task states would be.
138 */138 */
139 139
140#include "../workqueue_internal.h"
141#include <linux/sched/loadavg.h>
142#include <linux/seq_file.h>
143#include <linux/proc_fs.h>
144#include <linux/seqlock.h>
145#include <linux/uaccess.h>
146#include <linux/cgroup.h>
147#include <linux/module.h>
148#include <linux/sched.h>
149#include <linux/ctype.h>
150#include <linux/file.h>
151#include <linux/poll.h>
152#include <linux/psi.h>
153#include "sched.h"
154
155static int psi_bug __read_mostly;140static int psi_bug __read_mostly;
156 141
157DEFINE_STATIC_KEY_FALSE(psi_disabled);142DEFINE_STATIC_KEY_FALSE(psi_disabled);
kernel/sched/rt.c+0−3
@@ -3,9 +3,6 @@ · 2 unmodified lines
3 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR3 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
4 * policies)4 * policies)
5 */5 */
6#include "sched.h"
7
8#include "pelt.h"
9 6
10int sched_rr_timeslice = RR_TIMESLICE;7int sched_rr_timeslice = RR_TIMESLICE;
11int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;8int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;
kernel/sched/sched.h+61−53
@@ -2,86 +2,98 @@ · 1 unmodified line
2/*2/*
3 * Scheduler internal types and methods:3 * Scheduler internal types and methods:
4 */4 */
5#include <linux/sched.h>5#ifndef _KERNEL_SCHED_SCHED_H
6#define _KERNEL_SCHED_SCHED_H
6 7
8#include <linux/sched/affinity.h>
7#include <linux/sched/autogroup.h>9#include <linux/sched/autogroup.h>
8#include <linux/sched/clock.h>
9#include <linux/sched/coredump.h>
10#include <linux/sched/cpufreq.h>10#include <linux/sched/cpufreq.h>
11#include <linux/sched/cputime.h>
12#include <linux/sched/deadline.h>11#include <linux/sched/deadline.h>
13#include <linux/sched/debug.h>12#include <linux/sched.h>
14#include <linux/sched/hotplug.h>
15#include <linux/sched/idle.h>
16#include <linux/sched/init.h>
17#include <linux/sched/isolation.h>
18#include <linux/sched/jobctl.h>
19#include <linux/sched/loadavg.h>13#include <linux/sched/loadavg.h>
20#include <linux/sched/mm.h>14#include <linux/sched/mm.h>
21#include <linux/sched/nohz.h>15#include <linux/sched/rseq_api.h>
22#include <linux/sched/numa_balancing.h>
23#include <linux/sched/prio.h>
24#include <linux/sched/rt.h>
25#include <linux/sched/signal.h>16#include <linux/sched/signal.h>
26#include <linux/sched/smt.h>17#include <linux/sched/smt.h>
27#include <linux/sched/stat.h>18#include <linux/sched/stat.h>
28#include <linux/sched/sysctl.h>19#include <linux/sched/sysctl.h>
20#include <linux/sched/task_flags.h>
29#include <linux/sched/task.h>21#include <linux/sched/task.h>
30#include <linux/sched/task_stack.h>
31#include <linux/sched/topology.h>22#include <linux/sched/topology.h>
32#include <linux/sched/user.h>
33#include <linux/sched/wake_q.h>
34#include <linux/sched/xacct.h>
35
36#include <uapi/linux/sched/types.h>
37 23
38#include <linux/binfmts.h>24#include <linux/atomic.h>
39#include <linux/bitops.h>25#include <linux/bitmap.h>
40#include <linux/compat.h>26#include <linux/bug.h>
41#include <linux/context_tracking.h>27#include <linux/capability.h>
28#include <linux/cgroup_api.h>
29#include <linux/cgroup.h>
42#include <linux/cpufreq.h>30#include <linux/cpufreq.h>
43#include <linux/cpuidle.h>31#include <linux/cpumask_api.h>
44#include <linux/cpuset.h>
45#include <linux/ctype.h>32#include <linux/ctype.h>
46#include <linux/debugfs.h>33#include <linux/file.h>
47#include <linux/delayacct.h>34#include <linux/fs_api.h>
48#include <linux/energy_model.h>35#include <linux/hrtimer_api.h>
49#include <linux/init_task.h>36#include <linux/interrupt.h>
50#include <linux/kprobes.h>37#include <linux/irq_work.h>
38#include <linux/jiffies.h>
39#include <linux/kref_api.h>
51#include <linux/kthread.h>40#include <linux/kthread.h>
52#include <linux/membarrier.h>41#include <linux/ktime_api.h>
53#include <linux/migrate.h>42#include <linux/lockdep_api.h>
54#include <linux/mmu_context.h>43#include <linux/lockdep.h>
55#include <linux/nmi.h>44#include <linux/minmax.h>
45#include <linux/mm.h>
46#include <linux/module.h>
47#include <linux/mutex_api.h>
48#include <linux/plist.h>
49#include <linux/poll.h>
56#include <linux/proc_fs.h>50#include <linux/proc_fs.h>
57#include <linux/prefetch.h>
58#include <linux/profile.h>51#include <linux/profile.h>
59#include <linux/psi.h>52#include <linux/psi.h>
60#include <linux/ratelimit.h>53#include <linux/rcupdate.h>
61#include <linux/rcupdate_wait.h>54#include <linux/seq_file.h>
62#include <linux/security.h>55#include <linux/seqlock.h>
56#include <linux/softirq.h>
57#include <linux/spinlock_api.h>
58#include <linux/static_key.h>
63#include <linux/stop_machine.h>59#include <linux/stop_machine.h>
64#include <linux/suspend.h>60#include <linux/syscalls_api.h>
65#include <linux/swait.h>
66#include <linux/syscalls.h>61#include <linux/syscalls.h>
67#include <linux/task_work.h>62#include <linux/tick.h>
68#include <linux/tsacct_kern.h>63#include <linux/topology.h>
64#include <linux/types.h>
65#include <linux/u64_stats_sync_api.h>
66#include <linux/uaccess.h>
67#include <linux/wait_api.h>
68#include <linux/wait_bit.h>
69#include <linux/workqueue_api.h>
70
71#include <trace/events/power.h>
72#include <trace/events/sched.h>
73
74#include "../workqueue_internal.h"
75
76#ifdef CONFIG_CGROUP_SCHED
77#include <linux/cgroup.h>
78#include <linux/psi.h>
79#endif
69 80
70#include <asm/tlb.h>81#ifdef CONFIG_SCHED_DEBUG
82# include <linux/static_key.h>
83#endif
71 84
72#ifdef CONFIG_PARAVIRT85#ifdef CONFIG_PARAVIRT
73# include <asm/paravirt.h>86# include <asm/paravirt.h>
87# include <asm/paravirt_api_clock.h>
74#endif88#endif
75 89
76#include "cpupri.h"90#include "cpupri.h"
77#include "cpudeadline.h"91#include "cpudeadline.h"
78 92
79#include <trace/events/sched.h>
80
81#ifdef CONFIG_SCHED_DEBUG93#ifdef CONFIG_SCHED_DEBUG
82# define SCHED_WARN_ON(x) WARN_ONCE(x, #x)94# define SCHED_WARN_ON(x) WARN_ONCE(x, #x)
83#else95#else
84# define SCHED_WARN_ON(x) ({ (void)(x), 0; })96# define SCHED_WARN_ON(x) ({ (void)(x), 0; })
85#endif97#endif
86 98
87struct rq;99struct rq;
@@ -329,9 +341,6 @@ extern int dl_cpu_busy(int cpu, struct task_struct *p); · 241 unmodified lines
329 341
330#ifdef CONFIG_CGROUP_SCHED342#ifdef CONFIG_CGROUP_SCHED
331 343
332#include <linux/cgroup.h>
333#include <linux/psi.h>
334
335struct cfs_rq;344struct cfs_rq;
336struct rt_rq;345struct rt_rq;
337 346
@@ -1832,7 +1841,6 @@ static inline void flush_smp_call_function_from_idle(void) { } · 1494 unmodified lines
1832#endif1841#endif
1833 1842
1834#include "stats.h"1843#include "stats.h"
1835#include "autogroup.h"
1836 1844
1837#if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS)1845#if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS)
1838 1846
@@ -1928,7 +1936,6 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) · 89 unmodified lines
1928 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:1936 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
1929 */1937 */
1930#ifdef CONFIG_SCHED_DEBUG1938#ifdef CONFIG_SCHED_DEBUG
1931# include <linux/static_key.h>
1932# define const_debug __read_mostly1939# define const_debug __read_mostly
1933#else1940#else
1934# define const_debug const1941# define const_debug const
@@ -3086,3 +3093,4 @@ extern int sched_dynamic_mode(const char *str); · 1151 unmodified lines
3086extern void sched_dynamic_update(int mode);3093extern void sched_dynamic_update(int mode);
3087#endif3094#endif
3088 3095
3096#endif /* _KERNEL_SCHED_SCHED_H */
kernel/sched/stats.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
2/*2/*
3 * /proc/schedstat implementation3 * /proc/schedstat implementation
4 */4 */
5#include "sched.h"
6 5
7void __update_stats_wait_start(struct rq *rq, struct task_struct *p,6void __update_stats_wait_start(struct rq *rq, struct task_struct *p,
8 struct sched_statistics *stats)7 struct sched_statistics *stats)
kernel/sched/stats.h+4−0
@@ -1,4 +1,6 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _KERNEL_STATS_H
3#define _KERNEL_STATS_H
2 4
3#ifdef CONFIG_SCHEDSTATS5#ifdef CONFIG_SCHEDSTATS
4 6
@@ -298,3 +300,5 @@ sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *n · 293 unmodified lines
298# define sched_info_dequeue(rq, t) do { } while (0)300# define sched_info_dequeue(rq, t) do { } while (0)
299# define sched_info_switch(rq, t, next) do { } while (0)301# define sched_info_switch(rq, t, next) do { } while (0)
300#endif /* CONFIG_SCHED_INFO */302#endif /* CONFIG_SCHED_INFO */
303
304#endif /* _KERNEL_STATS_H */
kernel/sched/stop_task.c+0−1
@@ -7,7 +7,6 @@ · 6 unmodified lines
7 *7 *
8 * See kernel/stop_machine.c8 * See kernel/stop_machine.c
9 */9 */
10#include "sched.h"
11 10
12#ifdef CONFIG_SMP11#ifdef CONFIG_SMP
13static int12static int
kernel/sched/swait.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
2/*2/*
3 * <linux/swait.h> (simple wait queues ) implementation:3 * <linux/swait.h> (simple wait queues ) implementation:
4 */4 */
5#include "sched.h"
6 5
7void __init_swait_queue_head(struct swait_queue_head *q, const char *name,6void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
8 struct lock_class_key *key)7 struct lock_class_key *key)
kernel/sched/topology.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
2/*2/*
3 * Scheduler topology setup/handling methods3 * Scheduler topology setup/handling methods
4 */4 */
5#include "sched.h"
6 5
7DEFINE_MUTEX(sched_domains_mutex);6DEFINE_MUTEX(sched_domains_mutex);
8 7
kernel/sched/wait.c+0−1
@@ -4,7 +4,6 @@ · 3 unmodified lines
4 *4 *
5 * (C) 2004 Nadia Yvette Chambers, Oracle5 * (C) 2004 Nadia Yvette Chambers, Oracle
6 */6 */
7#include "sched.h"
8 7
9void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key)8void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key)
10{9{
kernel/sched/wait_bit.c+1−1
@@ -1,8 +1,8 @@
1// SPDX-License-Identifier: GPL-2.0-only1// SPDX-License-Identifier: GPL-2.0-only
2
2/*3/*
3 * The implementation of the wait_bit*() and related waiting APIs:4 * The implementation of the wait_bit*() and related waiting APIs:
4 */5 */
5#include "sched.h"
6 6
7#define WAIT_TABLE_BITS 87#define WAIT_TABLE_BITS 8
8#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)8#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)