History

Merge branch 'sched/fast-headers' into sched/core

Merge the scheduler build speedup of the fast-headers tree. Cumulative scheduler (kernel/sched/) build time speedup on a Linux distribution's config, which enables all scheduler features, compared to the vanilla kernel: _____________________________________________________________________________ | | Vanilla kernel (v5.13-rc7): |_____________________________________________________________________________ | | Performance counter stats for 'make -j96 kernel/sched/' (3 runs): | | 126,975,564,374 instructions # 1.45 insn per cycle ( +- 0.00% ) | 87,637,847,671 cycles # 3.959 GHz ( +- 0.30% ) | 22,136.96 msec cpu-clock # 7.499 CPUs utilized ( +- 0.29% ) | | 2.9520 +- 0.0169 seconds time elapsed ( +- 0.57% ) |_____________________________________________________________________________ | | Patched kernel: |_____________________________________________________________________________ | | Performance counter stats for 'make -j96 kernel/sched/' (3 runs): | | 50,420,496,914 instructions # 1.47 insn per cycle ( +- 0.00% ) | 34,234,322,038 cycles # 3.946 GHz ( +- 0.31% ) | 8,675.81 msec cpu-clock # 3.053 CPUs utilized ( +- 0.45% ) | | 2.8420 +- 0.0181 seconds time elapsed ( +- 0.64% ) |_____________________________________________________________________________ Summary: - CPU time used to build the scheduler dropped by -60.9%, a reduction from 22.1 clock-seconds to 8.7 clock-seconds. - Wall-clock time to build the scheduler dropped by -3.9%, a reduction from 2.95 seconds to 2.84 seconds.

Ingo Molnar committed 4 years ago

Commit ccacfe52 parents f2aa197 4ff8f2c69 files · +407 −163Browse files

Unified · Split

69 files
  1. arch/arm64/include/asm/paravirt_api_clock.h +1 −0
  2. arch/x86/include/asm/paravirt_api_clock.h +1 −0
  3. include/linux/cgroup_api.h +1 −0
  4. include/linux/cpumask_api.h +1 −0
  5. include/linux/fs_api.h +1 −0
  6. include/linux/gfp_api.h +1 −0
  7. include/linux/hashtable_api.h +1 −0
  8. include/linux/hrtimer_api.h +1 −0
  9. include/linux/kobject_api.h +1 −0
  10. include/linux/kref_api.h +1 −0
  11. include/linux/ktime_api.h +1 −0
  12. include/linux/llist_api.h +1 −0
  13. include/linux/lockdep_api.h +1 −0
  14. include/linux/mm_api.h +1 −0
  15. include/linux/mutex_api.h +1 −0
  16. include/linux/perf_event_api.h +1 −0
  17. include/linux/pgtable_api.h +1 −0
  18. include/linux/ptrace_api.h +1 −0
  19. include/linux/rcuwait_api.h +1 −0
  20. include/linux/refcount_api.h +1 −0
  21. include/linux/sched/affinity.h +1 −0
  22. include/linux/sched/cond_resched.h +1 −0
  23. include/linux/sched/deadline.h +2 −0
  24. include/linux/sched/posix-timers.h +1 −0
  25. include/linux/sched/rseq_api.h +1 −0
  26. include/linux/sched/task_flags.h +1 −0
  27. include/linux/sched/thread_info_api.h +1 −0
  28. include/linux/sched_clock.h +2 −0
  29. include/linux/seqlock_api.h +1 −0
  30. include/linux/softirq.h +1 −0
  31. include/linux/spinlock_api.h +1 −0
  32. include/linux/swait_api.h +1 −0
  33. include/linux/syscalls_api.h +1 −0
  34. include/linux/u64_stats_sync_api.h +1 −0
  35. include/linux/wait_api.h +1 −0
  36. include/linux/workqueue_api.h +1 −0
  37. kernel/sched/Makefile +10 −18
  38. kernel/sched/autogroup.c +1 −2
  39. kernel/sched/autogroup.h +5 −0
  40. kernel/sched/build_policy.c +52 −0
  41. kernel/sched/build_utility.c +109 −0
  42. kernel/sched/clock.c +21 −23
  43. kernel/sched/completion.c +1 −1
  44. kernel/sched/core.c +71 −10
  45. kernel/sched/core_sched.c +0 −3
  46. kernel/sched/cpuacct.c +1 −2
  47. kernel/sched/cpudeadline.c +1 −2
  48. kernel/sched/cpufreq.c +0 −3
  49. kernel/sched/cpufreq_schedutil.c +0 −7
  50. kernel/sched/cpupri.c +0 −1
  51. kernel/sched/cputime.c +0 −1
  52. kernel/sched/deadline.c +0 −2
  53. kernel/sched/debug.c +0 −1
  54. kernel/sched/fair.c +31 −0
  55. kernel/sched/idle.c +0 −3
  56. kernel/sched/isolation.c +0 −1
  57. kernel/sched/loadavg.c +0 −1
  58. kernel/sched/membarrier.c +0 −1
  59. kernel/sched/pelt.c +0 −4
  60. kernel/sched/psi.c +0 −15
  61. kernel/sched/rt.c +0 −3
  62. kernel/sched/sched.h +61 −53
  63. kernel/sched/stats.c +0 −1
  64. kernel/sched/stats.h +4 −0
  65. kernel/sched/stop_task.c +0 −1
  66. kernel/sched/swait.c +0 −1
  67. kernel/sched/topology.c +0 −1
  68. kernel/sched/wait.c +0 −1
  69. kernel/sched/wait_bit.c +1 −1
arch/arm64/include/asm/paravirt_api_clock.h+1−0
@@ -0,0 +1 @@
1+#include <asm/paravirt.h>
arch/x86/include/asm/paravirt_api_clock.h+1−0
@@ -0,0 +1 @@
1+#include <asm/paravirt.h>
include/linux/cgroup_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/cgroup.h>
include/linux/cpumask_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/cpumask.h>
include/linux/fs_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/fs.h>
include/linux/gfp_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/gfp.h>
include/linux/hashtable_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/hashtable.h>
include/linux/hrtimer_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/hrtimer.h>
include/linux/kobject_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/kobject.h>
include/linux/kref_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/kref.h>
include/linux/ktime_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/ktime.h>
include/linux/llist_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/llist.h>
include/linux/lockdep_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/lockdep.h>
include/linux/mm_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/mm.h>
include/linux/mutex_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/mutex.h>
include/linux/perf_event_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/perf_event.h>
include/linux/pgtable_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/pgtable.h>
include/linux/ptrace_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/ptrace.h>
include/linux/rcuwait_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/rcuwait.h>
include/linux/refcount_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/refcount.h>
include/linux/sched/affinity.h+1−0
@@ -0,0 +1 @@
1+#include <linux/sched.h>
include/linux/sched/cond_resched.h+1−0
@@ -0,0 +1 @@
1+#include <linux/sched.h>
include/linux/sched/deadline.h+2−0
@@ -6,6 +6,8 @@ · 5 unmodified lines
66 * NORMAL/BATCH tasks.
77 */
88
9+#include <linux/sched.h>
10+
911 #define MAX_DL_PRIO 0
1012
1113 static inline int dl_prio(int prio)
include/linux/sched/posix-timers.h+1−0
@@ -0,0 +1 @@
1+#include <linux/posix-timers.h>
include/linux/sched/rseq_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/rseq.h>
include/linux/sched/task_flags.h+1−0
@@ -0,0 +1 @@
1+#include <linux/sched.h>
include/linux/sched/thread_info_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/thread_info.h>
include/linux/sched_clock.h+2−0
@@ -5,6 +5,8 @@ · 4 unmodified lines
55 #ifndef LINUX_SCHED_CLOCK
66 #define LINUX_SCHED_CLOCK
77
8+#include <linux/types.h>
9+
810 #ifdef CONFIG_GENERIC_SCHED_CLOCK
911 /**
1012 * struct clock_read_data - data required to read from sched_clock()
include/linux/seqlock_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/seqlock.h>
include/linux/softirq.h+1−0
@@ -0,0 +1 @@
1+#include <linux/interrupt.h>
include/linux/spinlock_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/spinlock.h>
include/linux/swait_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/swait.h>
include/linux/syscalls_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/syscalls.h>
include/linux/u64_stats_sync_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/u64_stats_sync.h>
include/linux/wait_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/wait.h>
include/linux/workqueue_api.h+1−0
@@ -0,0 +1 @@
1+#include <linux/workqueue.h>
kernel/sched/Makefile+10−18
@@ -1,7 +1,4 @@
11 # SPDX-License-Identifier: GPL-2.0
2−ifdef CONFIG_FUNCTION_TRACER
3−CFLAGS_REMOVE_clock.o = $(CC_FLAGS_FTRACE)
4−endif
52
63 # The compilers are complaining about unused variables inside an if(0) scope
74 # block. This is daft, shut them up.
@@ -25,18 +22,13 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) · 17 unmodified lines
2522 CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer
2623 endif
2724
28−obj-y += core.o loadavg.o clock.o cputime.o
29−obj-y += idle.o fair.o rt.o deadline.o
30−obj-y += wait.o wait_bit.o swait.o completion.o
31−
32−obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o pelt.o
33−obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o
34−obj-$(CONFIG_SCHEDSTATS) += stats.o
35−obj-$(CONFIG_SCHED_DEBUG) += debug.o
36−obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o
37−obj-$(CONFIG_CPU_FREQ) += cpufreq.o
38−obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o
39−obj-$(CONFIG_MEMBARRIER) += membarrier.o
40−obj-$(CONFIG_CPU_ISOLATION) += isolation.o
41−obj-$(CONFIG_PSI) += psi.o
42−obj-$(CONFIG_SCHED_CORE) += core_sched.o
25+#
26+# Build efficiency:
27+#
28+# These compilation units have roughly the same size and complexity - so their
29+# build parallelizes well and finishes roughly at once:
30+#
31+obj-y += core.o
32+obj-y += fair.o
33+obj-y += build_policy.o
34+obj-y += build_utility.o
kernel/sched/autogroup.c+1−2
@@ -1,9 +1,8 @@
11 // SPDX-License-Identifier: GPL-2.0
2+
23 /*
34 * Auto-group scheduling implementation:
45 */
5−#include <linux/nospec.h>
6−#include "sched.h"
76
87 unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
98 static struct autogroup autogroup_default;
kernel/sched/autogroup.h+5−0
@@ -1,4 +1,7 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef _KERNEL_SCHED_AUTOGROUP_H
3+#define _KERNEL_SCHED_AUTOGROUP_H
4+
25 #ifdef CONFIG_SCHED_AUTOGROUP
36
47 struct autogroup {
@@ -59,3 +62,5 @@ static inline int autogroup_path(struct task_group *tg, char *buf, int buflen) · 54 unmodified lines
5962 }
6063
6164 #endif /* CONFIG_SCHED_AUTOGROUP */
65+
66+#endif /* _KERNEL_SCHED_AUTOGROUP_H */
kernel/sched/build_policy.c+52−0
@@ -0,0 +1,52 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * These are the scheduling policy related scheduler files, built
4+ * in a single compilation unit for build efficiency reasons.
5+ *
6+ * ( Incidentally, the size of the compilation unit is roughly
7+ * comparable to core.c and fair.c, the other two big
8+ * compilation units. This helps balance build time, while
9+ * coalescing source files to amortize header inclusion
10+ * cost. )
11+ *
12+ * core.c and fair.c are built separately.
13+ */
14+
15+/* Headers: */
16+#include <linux/sched/clock.h>
17+#include <linux/sched/cputime.h>
18+#include <linux/sched/posix-timers.h>
19+#include <linux/sched/rt.h>
20+
21+#include <linux/cpuidle.h>
22+#include <linux/jiffies.h>
23+#include <linux/livepatch.h>
24+#include <linux/psi.h>
25+#include <linux/seqlock_api.h>
26+#include <linux/slab.h>
27+#include <linux/suspend.h>
28+#include <linux/tsacct_kern.h>
29+#include <linux/vtime.h>
30+
31+#include <uapi/linux/sched/types.h>
32+
33+#include "sched.h"
34+
35+#include "autogroup.h"
36+#include "stats.h"
37+#include "pelt.h"
38+
39+/* Source code modules: */
40+
41+#include "idle.c"
42+
43+#include "rt.c"
44+
45+#ifdef CONFIG_SMP
46+# include "cpudeadline.c"
47+# include "pelt.c"
48+#endif
49+
50+#include "cputime.c"
51+#include "deadline.c"
52+
kernel/sched/build_utility.c+109−0
@@ -0,0 +1,109 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * These are various utility functions of the scheduler,
4+ * built in a single compilation unit for build efficiency reasons.
5+ *
6+ * ( Incidentally, the size of the compilation unit is roughly
7+ * comparable to core.c, fair.c, smp.c and policy.c, the other
8+ * big compilation units. This helps balance build time, while
9+ * coalescing source files to amortize header inclusion
10+ * cost. )
11+ */
12+#include <linux/sched/clock.h>
13+#include <linux/sched/cputime.h>
14+#include <linux/sched/debug.h>
15+#include <linux/sched/isolation.h>
16+#include <linux/sched/loadavg.h>
17+#include <linux/sched/mm.h>
18+#include <linux/sched/rseq_api.h>
19+#include <linux/sched/task_stack.h>
20+
21+#include <linux/cpufreq.h>
22+#include <linux/cpumask_api.h>
23+#include <linux/cpuset.h>
24+#include <linux/ctype.h>
25+#include <linux/debugfs.h>
26+#include <linux/energy_model.h>
27+#include <linux/hashtable_api.h>
28+#include <linux/irq.h>
29+#include <linux/kobject_api.h>
30+#include <linux/membarrier.h>
31+#include <linux/mempolicy.h>
32+#include <linux/nmi.h>
33+#include <linux/nospec.h>
34+#include <linux/proc_fs.h>
35+#include <linux/psi.h>
36+#include <linux/psi.h>
37+#include <linux/ptrace_api.h>
38+#include <linux/sched_clock.h>
39+#include <linux/security.h>
40+#include <linux/spinlock_api.h>
41+#include <linux/swait_api.h>
42+#include <linux/timex.h>
43+#include <linux/utsname.h>
44+#include <linux/wait_api.h>
45+#include <linux/workqueue_api.h>
46+
47+#include <uapi/linux/prctl.h>
48+#include <uapi/linux/sched/types.h>
49+
50+#include <asm/switch_to.h>
51+
52+#include "sched.h"
53+#include "sched-pelt.h"
54+#include "stats.h"
55+#include "autogroup.h"
56+
57+#include "clock.c"
58+
59+#ifdef CONFIG_CGROUP_CPUACCT
60+# include "cpuacct.c"
61+#endif
62+
63+#ifdef CONFIG_CPU_FREQ
64+# include "cpufreq.c"
65+#endif
66+
67+#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
68+# include "cpufreq_schedutil.c"
69+#endif
70+
71+#ifdef CONFIG_SCHED_DEBUG
72+# include "debug.c"
73+#endif
74+
75+#ifdef CONFIG_SCHEDSTATS
76+# include "stats.c"
77+#endif
78+
79+#include "loadavg.c"
80+#include "completion.c"
81+#include "swait.c"
82+#include "wait_bit.c"
83+#include "wait.c"
84+
85+#ifdef CONFIG_SMP
86+# include "cpupri.c"
87+# include "stop_task.c"
88+# include "topology.c"
89+#endif
90+
91+#ifdef CONFIG_SCHED_CORE
92+# include "core_sched.c"
93+#endif
94+
95+#ifdef CONFIG_PSI
96+# include "psi.c"
97+#endif
98+
99+#ifdef CONFIG_MEMBARRIER
100+# include "membarrier.c"
101+#endif
102+
103+#ifdef CONFIG_CPU_ISOLATION
104+# include "isolation.c"
105+#endif
106+
107+#ifdef CONFIG_SCHED_AUTOGROUP
108+# include "autogroup.c"
109+#endif
kernel/sched/clock.c+21−23
@@ -53,15 +53,13 @@ · 52 unmodified lines
5353 * that is otherwise invisible (TSC gets stopped).
5454 *
5555 */
56−#include "sched.h"
57−#include <linux/sched_clock.h>
5856
5957 /*
6058 * Scheduler clock - returns current time in nanosec units.
6159 * This is default implementation.
6260 * Architectures and sub-architectures can override this.
6361 */
64−unsigned long long __weak sched_clock(void)
62+notrace unsigned long long __weak sched_clock(void)
6563 {
6664 return (unsigned long long)(jiffies - INITIAL_JIFFIES)
6765 * (NSEC_PER_SEC / HZ);
@@ -95,28 +93,28 @@ struct sched_clock_data { · 27 unmodified lines
9593
9694 static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
9795
98−static inline struct sched_clock_data *this_scd(void)
96+notrace static inline struct sched_clock_data *this_scd(void)
9997 {
10098 return this_cpu_ptr(&sched_clock_data);
10199 }
102100
103−static inline struct sched_clock_data *cpu_sdc(int cpu)
101+notrace static inline struct sched_clock_data *cpu_sdc(int cpu)
104102 {
105103 return &per_cpu(sched_clock_data, cpu);
106104 }
107105
108−int sched_clock_stable(void)
106+notrace int sched_clock_stable(void)
109107 {
110108 return static_branch_likely(&__sched_clock_stable);
111109 }
112110
113−static void __scd_stamp(struct sched_clock_data *scd)
111+notrace static void __scd_stamp(struct sched_clock_data *scd)
114112 {
115113 scd->tick_gtod = ktime_get_ns();
116114 scd->tick_raw = sched_clock();
117115 }
118116
119−static void __set_sched_clock_stable(void)
117+notrace static void __set_sched_clock_stable(void)
120118 {
121119 struct sched_clock_data *scd;
122120
@@ -151,7 +149,7 @@ static void __set_sched_clock_stable(void) · 28 unmodified lines
151149 * The only way to fully avoid random clock jumps is to boot with:
152150 * "tsc=unstable".
153151 */
154−static void __sched_clock_work(struct work_struct *work)
152+notrace static void __sched_clock_work(struct work_struct *work)
155153 {
156154 struct sched_clock_data *scd;
157155 int cpu;
@@ -177,7 +175,7 @@ static void __sched_clock_work(struct work_struct *work) · 19 unmodified lines
177175
178176 static DECLARE_WORK(sched_clock_work, __sched_clock_work);
179177
180−static void __clear_sched_clock_stable(void)
178+notrace static void __clear_sched_clock_stable(void)
181179 {
182180 if (!sched_clock_stable())
183181 return;
@@ -186,7 +184,7 @@ static void __clear_sched_clock_stable(void) · 2 unmodified lines
186184 schedule_work(&sched_clock_work);
187185 }
188186
189−void clear_sched_clock_stable(void)
187+notrace void clear_sched_clock_stable(void)
190188 {
191189 __sched_clock_stable_early = 0;
192190
@@ -196,7 +194,7 @@ void clear_sched_clock_stable(void) · 3 unmodified lines
196194 __clear_sched_clock_stable();
197195 }
198196
199−static void __sched_clock_gtod_offset(void)
197+notrace static void __sched_clock_gtod_offset(void)
200198 {
201199 struct sched_clock_data *scd = this_scd();
202200
@@ -246,12 +244,12 @@ late_initcall(sched_clock_init_late); · 43 unmodified lines
246244 * min, max except they take wrapping into account
247245 */
248246
249−static inline u64 wrap_min(u64 x, u64 y)
247+notrace static inline u64 wrap_min(u64 x, u64 y)
250248 {
251249 return (s64)(x - y) < 0 ? x : y;
252250 }
253251
254−static inline u64 wrap_max(u64 x, u64 y)
252+notrace static inline u64 wrap_max(u64 x, u64 y)
255253 {
256254 return (s64)(x - y) > 0 ? x : y;
257255 }
@@ -262,7 +260,7 @@ static inline u64 wrap_max(u64 x, u64 y) · 4 unmodified lines
262260 * - filter out backward motion
263261 * - use the GTOD tick value to create a window to filter crazy TSC values
264262 */
265−static u64 sched_clock_local(struct sched_clock_data *scd)
263+notrace static u64 sched_clock_local(struct sched_clock_data *scd)
266264 {
267265 u64 now, clock, old_clock, min_clock, max_clock, gtod;
268266 s64 delta;
@@ -295,7 +293,7 @@ again: · 26 unmodified lines
295293 return clock;
296294 }
297295
298−static u64 sched_clock_remote(struct sched_clock_data *scd)
296+notrace static u64 sched_clock_remote(struct sched_clock_data *scd)
299297 {
300298 struct sched_clock_data *my_scd = this_scd();
301299 u64 this_clock, remote_clock;
@@ -362,7 +360,7 @@ again: · 60 unmodified lines
362360 *
363361 * See cpu_clock().
364362 */
365−u64 sched_clock_cpu(int cpu)
363+notrace u64 sched_clock_cpu(int cpu)
366364 {
367365 struct sched_clock_data *scd;
368366 u64 clock;
@@ -386,7 +384,7 @@ u64 sched_clock_cpu(int cpu) · 17 unmodified lines
386384 }
387385 EXPORT_SYMBOL_GPL(sched_clock_cpu);
388386
389−void sched_clock_tick(void)
387+notrace void sched_clock_tick(void)
390388 {
391389 struct sched_clock_data *scd;
392390
@@ -403,7 +401,7 @@ void sched_clock_tick(void) · 10 unmodified lines
403401 sched_clock_local(scd);
404402 }
405403
406−void sched_clock_tick_stable(void)
404+notrace void sched_clock_tick_stable(void)
407405 {
408406 if (!sched_clock_stable())
409407 return;
@@ -423,7 +421,7 @@ void sched_clock_tick_stable(void) · 13 unmodified lines
423421 /*
424422 * We are going deep-idle (irqs are disabled):
425423 */
426−void sched_clock_idle_sleep_event(void)
424+notrace void sched_clock_idle_sleep_event(void)
427425 {
428426 sched_clock_cpu(smp_processor_id());
429427 }
@@ -432,7 +430,7 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); · 2 unmodified lines
432430 /*
433431 * We just idled; resync with ktime.
434432 */
435−void sched_clock_idle_wakeup_event(void)
433+notrace void sched_clock_idle_wakeup_event(void)
436434 {
437435 unsigned long flags;
438436
@@ -458,7 +456,7 @@ void __init sched_clock_init(void) · 19 unmodified lines
458456 local_irq_enable();
459457 }
460458
461−u64 sched_clock_cpu(int cpu)
459+notrace u64 sched_clock_cpu(int cpu)
462460 {
463461 if (!static_branch_likely(&sched_clock_running))
464462 return 0;
@@ -476,7 +474,7 @@ u64 sched_clock_cpu(int cpu) · 11 unmodified lines
476474 * On bare metal this function should return the same as local_clock.
477475 * Architectures and sub-architectures can override this.
478476 */
479−u64 __weak running_clock(void)
477+notrace u64 __weak running_clock(void)
480478 {
481479 return local_clock();
482480 }
kernel/sched/completion.c+1−1
@@ -1,4 +1,5 @@
11 // SPDX-License-Identifier: GPL-2.0
2+
23 /*
34 * Generic wait-for-completion handler;
45 *
@@ -11,7 +12,6 @@ · 6 unmodified lines
1112 * typically be used for exclusion which gives rise to priority inversion.
1213 * Waiting for completion is a typically sync point, but not an exclusion point.
1314 */
14−#include "sched.h"
1515
1616 /**
1717 * complete: - signals a single thread waiting on this completion
kernel/sched/core.c+71−10
@@ -6,27 +6,88 @@ · 5 unmodified lines
66 *
77 * Copyright (C) 1991-2002 Linus Torvalds
88 */
9−#define CREATE_TRACE_POINTS
10−#include <trace/events/sched.h>
11−#undef CREATE_TRACE_POINTS
9+#include <linux/highmem.h>
10+#include <linux/hrtimer_api.h>
11+#include <linux/ktime_api.h>
12+#include <linux/sched/signal.h>
13+#include <linux/syscalls_api.h>
14+#include <linux/debug_locks.h>
15+#include <linux/prefetch.h>
16+#include <linux/capability.h>
17+#include <linux/pgtable_api.h>
18+#include <linux/wait_bit.h>
19+#include <linux/jiffies.h>
20+#include <linux/spinlock_api.h>
21+#include <linux/cpumask_api.h>
22+#include <linux/lockdep_api.h>
23+#include <linux/hardirq.h>
24+#include <linux/softirq.h>
25+#include <linux/refcount_api.h>
26+#include <linux/topology.h>
27+#include <linux/sched/clock.h>
28+#include <linux/sched/cond_resched.h>
29+#include <linux/sched/debug.h>
30+#include <linux/sched/isolation.h>
31+#include <linux/sched/loadavg.h>
32+#include <linux/sched/mm.h>
33+#include <linux/sched/nohz.h>
34+#include <linux/sched/rseq_api.h>
35+#include <linux/sched/rt.h>
1236
13−#include "sched.h"
14−
15−#include <linux/nospec.h>
1637 #include <linux/blkdev.h>
17−#include <linux/jump_label.h>
38+#include <linux/context_tracking.h>
39+#include <linux/cpuset.h>
40+#include <linux/delayacct.h>
41+#include <linux/init_task.h>
42+#include <linux/interrupt.h>
43+#include <linux/ioprio.h>
44+#include <linux/kallsyms.h>
1845 #include <linux/kcov.h>
46+#include <linux/kprobes.h>
47+#include <linux/llist_api.h>
48+#include <linux/mmu_context.h>
49+#include <linux/mmzone.h>
50+#include <linux/mutex_api.h>
51+#include <linux/nmi.h>
52+#include <linux/nospec.h>
53+#include <linux/perf_event_api.h>
54+#include <linux/profile.h>
55+#include <linux/psi.h>
56+#include <linux/rcuwait_api.h>
57+#include <linux/sched/wake_q.h>
1958 #include <linux/scs.h>
59+#include <linux/slab.h>
60+#include <linux/syscalls.h>
61+#include <linux/vtime.h>
62+#include <linux/wait_api.h>
63+#include <linux/workqueue_api.h>
64+
65+#ifdef CONFIG_PREEMPT_DYNAMIC
66+# include <linux/entry-common.h>
67+#endif
68+
69+#include <uapi/linux/sched/types.h>
2070
2171 #include <asm/switch_to.h>
2272 #include <asm/tlb.h>
2373
24−#include "../workqueue_internal.h"
25−#include "../../fs/io-wq.h"
26−#include "../smpboot.h"
74+#define CREATE_TRACE_POINTS
75+#include <linux/sched/rseq_api.h>
76+#include <trace/events/sched.h>
77+#undef CREATE_TRACE_POINTS
2778
79+#include "sched.h"
80+#include "stats.h"
81+#include "autogroup.h"
82+
83+#include "autogroup.h"
2884 #include "pelt.h"
2985 #include "smp.h"
86+#include "stats.h"
87+
88+#include "../workqueue_internal.h"
89+#include "../../fs/io-wq.h"
90+#include "../smpboot.h"
3091
3192 /*
3293 * Export tracepoints that act as a bare tracehook (ie: have no trace event
kernel/sched/core_sched.c+0−3
@@ -1,8 +1,5 @@
11 // SPDX-License-Identifier: GPL-2.0-only
22
3−#include <linux/prctl.h>
4−#include "sched.h"
5−
63 /*
74 * A simple wrapper around refcount. An allocated sched_core_cookie's
85 * address is used to compute the cookie of the task.
kernel/sched/cpuacct.c+1−2
@@ -1,12 +1,11 @@
11 // SPDX-License-Identifier: GPL-2.0
2+
23 /*
34 * CPU accounting code for task groups.
45 *
56 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
67 * (balbir@in.ibm.com).
78 */
8−#include <asm/irq_regs.h>
9−#include "sched.h"
109
1110 /* Time spent by the tasks of the CPU accounting group executing in ... */
1211 enum cpuacct_stat_index {
kernel/sched/cpudeadline.c+1−2
@@ -1,12 +1,11 @@
11 // SPDX-License-Identifier: GPL-2.0-only
22 /*
3− * kernel/sched/cpudl.c
3+ * kernel/sched/cpudeadline.c
44 *
55 * Global CPU deadline management
66 *
77 * Author: Juri Lelli <j.lelli@sssup.it>
88 */
9−#include "sched.h"
109
1110 static inline int parent(int i)
1211 {
kernel/sched/cpufreq.c+0−3
@@ -5,9 +5,6 @@ · 4 unmodified lines
55 * Copyright (C) 2016, Intel Corporation
66 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
77 */
8−#include <linux/cpufreq.h>
9−
10−#include "sched.h"
118
129 DEFINE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data);
1310
kernel/sched/cpufreq_schedutil.c+0−7
@@ -6,13 +6,6 @@ · 5 unmodified lines
66 * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
77 */
88
9−#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10−
11−#include "sched.h"
12−
13−#include <linux/sched/cpufreq.h>
14−#include <trace/events/power.h>
15−
169 #define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8)
1710
1811 struct sugov_tunables {
kernel/sched/cpupri.c+0−1
@@ -22,7 +22,6 @@ · 21 unmodified lines
2222 * worst case complexity of O(min(101, nr_domcpus)), though the scenario that
2323 * yields the worst case search is fairly contrived.
2424 */
25−#include "sched.h"
2625
2726 /*
2827 * p->rt_priority p->prio newpri cpupri
kernel/sched/cputime.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
22 /*
33 * Simple CPU accounting cgroup controller
44 */
5−#include "sched.h"
65
76 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
87
kernel/sched/deadline.c+0−2
@@ -15,8 +15,6 @@ · 14 unmodified lines
1515 * Michael Trimarchi <michael@amarulasolutions.com>,
1616 * Fabio Checconi <fchecconi@gmail.com>
1717 */
18−#include "sched.h"
19−#include "pelt.h"
2018
2119 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)
2220 {
kernel/sched/debug.c+0−1
@@ -6,7 +6,6 @@ · 5 unmodified lines
66 *
77 * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar
88 */
9−#include "sched.h"
109
1110 /*
1211 * This allows printing both to /proc/sched_debug and
kernel/sched/fair.c+31−0
@@ -20,7 +20,38 @@ · 19 unmodified lines
2020 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra
2121 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
2222 */
23+#include <linux/energy_model.h>
24+#include <linux/mmap_lock.h>
25+#include <linux/hugetlb_inline.h>
26+#include <linux/jiffies.h>
27+#include <linux/mm_api.h>
28+#include <linux/highmem.h>
29+#include <linux/spinlock_api.h>
30+#include <linux/cpumask_api.h>
31+#include <linux/lockdep_api.h>
32+#include <linux/softirq.h>
33+#include <linux/refcount_api.h>
34+#include <linux/topology.h>
35+#include <linux/sched/clock.h>
36+#include <linux/sched/cond_resched.h>
37+#include <linux/sched/cputime.h>
38+#include <linux/sched/isolation.h>
39+
40+#include <linux/cpuidle.h>
41+#include <linux/interrupt.h>
42+#include <linux/mempolicy.h>
43+#include <linux/mutex_api.h>
44+#include <linux/profile.h>
45+#include <linux/psi.h>
46+#include <linux/ratelimit.h>
47+
48+#include <asm/switch_to.h>
49+
50+#include <linux/sched/cond_resched.h>
51+
2352 #include "sched.h"
53+#include "stats.h"
54+#include "autogroup.h"
2455
2556 /*
2657 * Targeted preemption latency for CPU-bound tasks:
kernel/sched/idle.c+0−3
@@ -6,9 +6,6 @@ · 5 unmodified lines
66 * (NOTE: these are not related to SCHED_IDLE batch scheduled
77 * tasks which are handled in sched/fair.c )
88 */
9−#include "sched.h"
10−
11−#include <trace/events/power.h>
129
1310 /* Linker adds these: start and end of __cpuidle functions */
1411 extern char __cpuidle_text_start[], __cpuidle_text_end[];
kernel/sched/isolation.c+0−1
@@ -7,7 +7,6 @@ · 6 unmodified lines
77 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
88 *
99 */
10−#include "sched.h"
1110
1211 enum hk_flags {
1312 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
kernel/sched/loadavg.c+0−1
@@ -6,7 +6,6 @@ · 5 unmodified lines
66 * figure. Its a silly number but people think its important. We go through
77 * great pains to make it work on big machines and tickless kernels.
88 */
9−#include "sched.h"
109
1110 /*
1211 * Global load-average calculations
kernel/sched/membarrier.c+0−1
@@ -4,7 +4,6 @@ · 3 unmodified lines
44 *
55 * membarrier system call
66 */
7−#include "sched.h"
87
98 /*
109 * For documentation purposes, here are some membarrier ordering
kernel/sched/pelt.c+0−4
@@ -24,10 +24,6 @@ · 23 unmodified lines
2424 * Author: Vincent Guittot <vincent.guittot@linaro.org>
2525 */
2626
27−#include <linux/sched.h>
28−#include "sched.h"
29−#include "pelt.h"
30−
3127 /*
3228 * Approximate:
3329 * val * y^n, where y^32 ~= 0.5 (~1 scheduling period)
kernel/sched/psi.c+0−15
@@ -137,21 +137,6 @@ · 136 unmodified lines
137137 * sampling of the aggregate task states would be.
138138 */
139139
140−#include "../workqueue_internal.h"
141−#include <linux/sched/loadavg.h>
142−#include <linux/seq_file.h>
143−#include <linux/proc_fs.h>
144−#include <linux/seqlock.h>
145−#include <linux/uaccess.h>
146−#include <linux/cgroup.h>
147−#include <linux/module.h>
148−#include <linux/sched.h>
149−#include <linux/ctype.h>
150−#include <linux/file.h>
151−#include <linux/poll.h>
152−#include <linux/psi.h>
153−#include "sched.h"
154−
155140 static int psi_bug __read_mostly;
156141
157142 DEFINE_STATIC_KEY_FALSE(psi_disabled);
kernel/sched/rt.c+0−3
@@ -3,9 +3,6 @@ · 2 unmodified lines
33 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
44 * policies)
55 */
6−#include "sched.h"
7−
8−#include "pelt.h"
96
107 int sched_rr_timeslice = RR_TIMESLICE;
118 int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;
kernel/sched/sched.h+61−53
@@ -2,86 +2,98 @@ · 1 unmodified line
22 /*
33 * Scheduler internal types and methods:
44 */
5−#include <linux/sched.h>
5+#ifndef _KERNEL_SCHED_SCHED_H
6+#define _KERNEL_SCHED_SCHED_H
67
8+#include <linux/sched/affinity.h>
79 #include <linux/sched/autogroup.h>
8−#include <linux/sched/clock.h>
9−#include <linux/sched/coredump.h>
1010 #include <linux/sched/cpufreq.h>
11−#include <linux/sched/cputime.h>
1211 #include <linux/sched/deadline.h>
13−#include <linux/sched/debug.h>
14−#include <linux/sched/hotplug.h>
15−#include <linux/sched/idle.h>
16−#include <linux/sched/init.h>
17−#include <linux/sched/isolation.h>
18−#include <linux/sched/jobctl.h>
12+#include <linux/sched.h>
1913 #include <linux/sched/loadavg.h>
2014 #include <linux/sched/mm.h>
21−#include <linux/sched/nohz.h>
22−#include <linux/sched/numa_balancing.h>
23−#include <linux/sched/prio.h>
24−#include <linux/sched/rt.h>
15+#include <linux/sched/rseq_api.h>
2516 #include <linux/sched/signal.h>
2617 #include <linux/sched/smt.h>
2718 #include <linux/sched/stat.h>
2819 #include <linux/sched/sysctl.h>
20+#include <linux/sched/task_flags.h>
2921 #include <linux/sched/task.h>
30−#include <linux/sched/task_stack.h>
3122 #include <linux/sched/topology.h>
32−#include <linux/sched/user.h>
33−#include <linux/sched/wake_q.h>
34−#include <linux/sched/xacct.h>
35−
36−#include <uapi/linux/sched/types.h>
3723
38−#include <linux/binfmts.h>
39−#include <linux/bitops.h>
40−#include <linux/compat.h>
41−#include <linux/context_tracking.h>
24+#include <linux/atomic.h>
25+#include <linux/bitmap.h>
26+#include <linux/bug.h>
27+#include <linux/capability.h>
28+#include <linux/cgroup_api.h>
29+#include <linux/cgroup.h>
4230 #include <linux/cpufreq.h>
43−#include <linux/cpuidle.h>
44−#include <linux/cpuset.h>
31+#include <linux/cpumask_api.h>
4532 #include <linux/ctype.h>
46−#include <linux/debugfs.h>
47−#include <linux/delayacct.h>
48−#include <linux/energy_model.h>
49−#include <linux/init_task.h>
50−#include <linux/kprobes.h>
33+#include <linux/file.h>
34+#include <linux/fs_api.h>
35+#include <linux/hrtimer_api.h>
36+#include <linux/interrupt.h>
37+#include <linux/irq_work.h>
38+#include <linux/jiffies.h>
39+#include <linux/kref_api.h>
5140 #include <linux/kthread.h>
52−#include <linux/membarrier.h>
53−#include <linux/migrate.h>
54−#include <linux/mmu_context.h>
55−#include <linux/nmi.h>
41+#include <linux/ktime_api.h>
42+#include <linux/lockdep_api.h>
43+#include <linux/lockdep.h>
44+#include <linux/minmax.h>
45+#include <linux/mm.h>
46+#include <linux/module.h>
47+#include <linux/mutex_api.h>
48+#include <linux/plist.h>
49+#include <linux/poll.h>
5650 #include <linux/proc_fs.h>
57−#include <linux/prefetch.h>
5851 #include <linux/profile.h>
5952 #include <linux/psi.h>
60−#include <linux/ratelimit.h>
61−#include <linux/rcupdate_wait.h>
62−#include <linux/security.h>
53+#include <linux/rcupdate.h>
54+#include <linux/seq_file.h>
55+#include <linux/seqlock.h>
56+#include <linux/softirq.h>
57+#include <linux/spinlock_api.h>
58+#include <linux/static_key.h>
6359 #include <linux/stop_machine.h>
64−#include <linux/suspend.h>
65−#include <linux/swait.h>
60+#include <linux/syscalls_api.h>
6661 #include <linux/syscalls.h>
67−#include <linux/task_work.h>
68−#include <linux/tsacct_kern.h>
62+#include <linux/tick.h>
63+#include <linux/topology.h>
64+#include <linux/types.h>
65+#include <linux/u64_stats_sync_api.h>
66+#include <linux/uaccess.h>
67+#include <linux/wait_api.h>
68+#include <linux/wait_bit.h>
69+#include <linux/workqueue_api.h>
70+
71+#include <trace/events/power.h>
72+#include <trace/events/sched.h>
73+
74+#include "../workqueue_internal.h"
75+
76+#ifdef CONFIG_CGROUP_SCHED
77+#include <linux/cgroup.h>
78+#include <linux/psi.h>
79+#endif
6980
70−#include <asm/tlb.h>
81+#ifdef CONFIG_SCHED_DEBUG
82+# include <linux/static_key.h>
83+#endif
7184
7285 #ifdef CONFIG_PARAVIRT
7386 # include <asm/paravirt.h>
87+# include <asm/paravirt_api_clock.h>
7488 #endif
7589
7690 #include "cpupri.h"
7791 #include "cpudeadline.h"
7892
79−#include <trace/events/sched.h>
80−
8193 #ifdef CONFIG_SCHED_DEBUG
82−# define SCHED_WARN_ON(x) WARN_ONCE(x, #x)
94+# define SCHED_WARN_ON(x) WARN_ONCE(x, #x)
8395 #else
84−# define SCHED_WARN_ON(x) ({ (void)(x), 0; })
96+# define SCHED_WARN_ON(x) ({ (void)(x), 0; })
8597 #endif
8698
8799 struct rq;
@@ -329,9 +341,6 @@ extern int dl_cpu_busy(int cpu, struct task_struct *p); · 241 unmodified lines
329341
330342 #ifdef CONFIG_CGROUP_SCHED
331343
332−#include <linux/cgroup.h>
333−#include <linux/psi.h>
334−
335344 struct cfs_rq;
336345 struct rt_rq;
337346
@@ -1832,7 +1841,6 @@ static inline void flush_smp_call_function_from_idle(void) { } · 1494 unmodified lines
18321841 #endif
18331842
18341843 #include "stats.h"
1835−#include "autogroup.h"
18361844
18371845 #if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS)
18381846
@@ -1928,7 +1936,6 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) · 89 unmodified lines
19281936 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
19291937 */
19301938 #ifdef CONFIG_SCHED_DEBUG
1931−# include <linux/static_key.h>
19321939 # define const_debug __read_mostly
19331940 #else
19341941 # define const_debug const
@@ -3086,3 +3093,4 @@ extern int sched_dynamic_mode(const char *str); · 1151 unmodified lines
30863093 extern void sched_dynamic_update(int mode);
30873094 #endif
30883095
3096+#endif /* _KERNEL_SCHED_SCHED_H */
kernel/sched/stats.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
22 /*
33 * /proc/schedstat implementation
44 */
5−#include "sched.h"
65
76 void __update_stats_wait_start(struct rq *rq, struct task_struct *p,
87 struct sched_statistics *stats)
kernel/sched/stats.h+4−0
@@ -1,4 +1,6 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef _KERNEL_STATS_H
3+#define _KERNEL_STATS_H
24
35 #ifdef CONFIG_SCHEDSTATS
46
@@ -298,3 +300,5 @@ sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *n · 293 unmodified lines
298300 # define sched_info_dequeue(rq, t) do { } while (0)
299301 # define sched_info_switch(rq, t, next) do { } while (0)
300302 #endif /* CONFIG_SCHED_INFO */
303+
304+#endif /* _KERNEL_STATS_H */
kernel/sched/stop_task.c+0−1
@@ -7,7 +7,6 @@ · 6 unmodified lines
77 *
88 * See kernel/stop_machine.c
99 */
10−#include "sched.h"
1110
1211 #ifdef CONFIG_SMP
1312 static int
kernel/sched/swait.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
22 /*
33 * <linux/swait.h> (simple wait queues ) implementation:
44 */
5−#include "sched.h"
65
76 void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
87 struct lock_class_key *key)
kernel/sched/topology.c+0−1
@@ -2,7 +2,6 @@ · 1 unmodified line
22 /*
33 * Scheduler topology setup/handling methods
44 */
5−#include "sched.h"
65
76 DEFINE_MUTEX(sched_domains_mutex);
87
kernel/sched/wait.c+0−1
@@ -4,7 +4,6 @@ · 3 unmodified lines
44 *
55 * (C) 2004 Nadia Yvette Chambers, Oracle
66 */
7−#include "sched.h"
87
98 void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key)
109 {
kernel/sched/wait_bit.c+1−1
@@ -1,8 +1,8 @@
11 // SPDX-License-Identifier: GPL-2.0-only
2+
23 /*
34 * The implementation of the wait_bit*() and related waiting APIs:
45 */
5−#include "sched.h"
66
77 #define WAIT_TABLE_BITS 8
88 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)