Merge the scheduler build speedup of the fast-headers tree. Cumulative scheduler (kernel/sched/) build time speedup on a Linux distribution's config, which enables all scheduler features, compared to the vanilla kernel: _____________________________________________________________________________ | | Vanilla kernel (v5.13-rc7): |_____________________________________________________________________________ | | Performance counter stats for 'make -j96 kernel/sched/' (3 runs): | | 126,975,564,374 instructions # 1.45 insn per cycle ( +- 0.00% ) | 87,637,847,671 cycles # 3.959 GHz ( +- 0.30% ) | 22,136.96 msec cpu-clock # 7.499 CPUs utilized ( +- 0.29% ) | | 2.9520 +- 0.0169 seconds time elapsed ( +- 0.57% ) |_____________________________________________________________________________ | | Patched kernel: |_____________________________________________________________________________ | | Performance counter stats for 'make -j96 kernel/sched/' (3 runs): | | 50,420,496,914 instructions # 1.47 insn per cycle ( +- 0.00% ) | 34,234,322,038 cycles # 3.946 GHz ( +- 0.31% ) | 8,675.81 msec cpu-clock # 3.053 CPUs utilized ( +- 0.45% ) | | 2.8420 +- 0.0181 seconds time elapsed ( +- 0.64% ) |_____________________________________________________________________________ Summary: - CPU time used to build the scheduler dropped by -60.9%, a reduction from 22.1 clock-seconds to 8.7 clock-seconds. - Wall-clock time to build the scheduler dropped by -3.9%, a reduction from 2.95 seconds to 2.84 seconds.
Unified · Split
| @@ -0,0 +1 @@ | ||
| 1 | +#include <asm/paravirt.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <asm/paravirt.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/cgroup.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/cpumask.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/fs.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/gfp.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/hashtable.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/hrtimer.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/kobject.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/kref.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/ktime.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/llist.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/lockdep.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/mm.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/mutex.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/perf_event.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/pgtable.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/ptrace.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/rcuwait.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/refcount.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/sched.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/sched.h> | |
| @@ -6,6 +6,8 @@ · 5 unmodified lines | ||
| 6 | 6 | * NORMAL/BATCH tasks. |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | +#include <linux/sched.h> | |
| 10 | + | |
| 9 | 11 | #define MAX_DL_PRIO 0 |
| 10 | 12 | |
| 11 | 13 | static inline int dl_prio(int prio) |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/posix-timers.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/rseq.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/sched.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/thread_info.h> | |
| @@ -5,6 +5,8 @@ · 4 unmodified lines | ||
| 5 | 5 | #ifndef LINUX_SCHED_CLOCK |
| 6 | 6 | #define LINUX_SCHED_CLOCK |
| 7 | 7 | |
| 8 | +#include <linux/types.h> | |
| 9 | + | |
| 8 | 10 | #ifdef CONFIG_GENERIC_SCHED_CLOCK |
| 9 | 11 | /** |
| 10 | 12 | * struct clock_read_data - data required to read from sched_clock() |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/seqlock.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/interrupt.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/spinlock.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/swait.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/syscalls.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/u64_stats_sync.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/wait.h> | |
| @@ -0,0 +1 @@ | ||
| 1 | +#include <linux/workqueue.h> | |
| @@ -1,7 +1,4 @@ | ||
| 1 | 1 | # SPDX-License-Identifier: GPL-2.0 |
| 2 | −ifdef CONFIG_FUNCTION_TRACER | |
| 3 | −CFLAGS_REMOVE_clock.o = $(CC_FLAGS_FTRACE) | |
| 4 | −endif | |
| 5 | 2 | |
| 6 | 3 | # The compilers are complaining about unused variables inside an if(0) scope |
| 7 | 4 | # block. This is daft, shut them up. |
| @@ -25,18 +22,13 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) · 17 unmodified lines | ||
| 25 | 22 | CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer |
| 26 | 23 | endif |
| 27 | 24 | |
| 28 | −obj-y += core.o loadavg.o clock.o cputime.o | |
| 29 | −obj-y += idle.o fair.o rt.o deadline.o | |
| 30 | −obj-y += wait.o wait_bit.o swait.o completion.o | |
| 31 | − | |
| 32 | −obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o pelt.o | |
| 33 | −obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o | |
| 34 | −obj-$(CONFIG_SCHEDSTATS) += stats.o | |
| 35 | −obj-$(CONFIG_SCHED_DEBUG) += debug.o | |
| 36 | −obj-$(CONFIG_CGROUP_CPUACCT) += cpuacct.o | |
| 37 | −obj-$(CONFIG_CPU_FREQ) += cpufreq.o | |
| 38 | −obj-$(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) += cpufreq_schedutil.o | |
| 39 | −obj-$(CONFIG_MEMBARRIER) += membarrier.o | |
| 40 | −obj-$(CONFIG_CPU_ISOLATION) += isolation.o | |
| 41 | −obj-$(CONFIG_PSI) += psi.o | |
| 42 | −obj-$(CONFIG_SCHED_CORE) += core_sched.o | |
| 25 | +# | |
| 26 | +# Build efficiency: | |
| 27 | +# | |
| 28 | +# These compilation units have roughly the same size and complexity - so their | |
| 29 | +# build parallelizes well and finishes roughly at once: | |
| 30 | +# | |
| 31 | +obj-y += core.o | |
| 32 | +obj-y += fair.o | |
| 33 | +obj-y += build_policy.o | |
| 34 | +obj-y += build_utility.o | |
| @@ -1,9 +1,8 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | + | |
| 2 | 3 | /* |
| 3 | 4 | * Auto-group scheduling implementation: |
| 4 | 5 | */ |
| 5 | −#include <linux/nospec.h> | |
| 6 | −#include "sched.h" | |
| 7 | 6 | |
| 8 | 7 | unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1; |
| 9 | 8 | static struct autogroup autogroup_default; |
| @@ -1,4 +1,7 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | +#ifndef _KERNEL_SCHED_AUTOGROUP_H | |
| 3 | +#define _KERNEL_SCHED_AUTOGROUP_H | |
| 4 | + | |
| 2 | 5 | #ifdef CONFIG_SCHED_AUTOGROUP |
| 3 | 6 | |
| 4 | 7 | struct autogroup { |
| @@ -59,3 +62,5 @@ static inline int autogroup_path(struct task_group *tg, char *buf, int buflen) · 54 unmodified lines | ||
| 59 | 62 | } |
| 60 | 63 | |
| 61 | 64 | #endif /* CONFIG_SCHED_AUTOGROUP */ |
| 65 | + | |
| 66 | +#endif /* _KERNEL_SCHED_AUTOGROUP_H */ | |
| @@ -0,0 +1,52 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* | |
| 3 | + * These are the scheduling policy related scheduler files, built | |
| 4 | + * in a single compilation unit for build efficiency reasons. | |
| 5 | + * | |
| 6 | + * ( Incidentally, the size of the compilation unit is roughly | |
| 7 | + * comparable to core.c and fair.c, the other two big | |
| 8 | + * compilation units. This helps balance build time, while | |
| 9 | + * coalescing source files to amortize header inclusion | |
| 10 | + * cost. ) | |
| 11 | + * | |
| 12 | + * core.c and fair.c are built separately. | |
| 13 | + */ | |
| 14 | + | |
| 15 | +/* Headers: */ | |
| 16 | +#include <linux/sched/clock.h> | |
| 17 | +#include <linux/sched/cputime.h> | |
| 18 | +#include <linux/sched/posix-timers.h> | |
| 19 | +#include <linux/sched/rt.h> | |
| 20 | + | |
| 21 | +#include <linux/cpuidle.h> | |
| 22 | +#include <linux/jiffies.h> | |
| 23 | +#include <linux/livepatch.h> | |
| 24 | +#include <linux/psi.h> | |
| 25 | +#include <linux/seqlock_api.h> | |
| 26 | +#include <linux/slab.h> | |
| 27 | +#include <linux/suspend.h> | |
| 28 | +#include <linux/tsacct_kern.h> | |
| 29 | +#include <linux/vtime.h> | |
| 30 | + | |
| 31 | +#include <uapi/linux/sched/types.h> | |
| 32 | + | |
| 33 | +#include "sched.h" | |
| 34 | + | |
| 35 | +#include "autogroup.h" | |
| 36 | +#include "stats.h" | |
| 37 | +#include "pelt.h" | |
| 38 | + | |
| 39 | +/* Source code modules: */ | |
| 40 | + | |
| 41 | +#include "idle.c" | |
| 42 | + | |
| 43 | +#include "rt.c" | |
| 44 | + | |
| 45 | +#ifdef CONFIG_SMP | |
| 46 | +# include "cpudeadline.c" | |
| 47 | +# include "pelt.c" | |
| 48 | +#endif | |
| 49 | + | |
| 50 | +#include "cputime.c" | |
| 51 | +#include "deadline.c" | |
| 52 | + | |
| @@ -0,0 +1,109 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* | |
| 3 | + * These are various utility functions of the scheduler, | |
| 4 | + * built in a single compilation unit for build efficiency reasons. | |
| 5 | + * | |
| 6 | + * ( Incidentally, the size of the compilation unit is roughly | |
| 7 | + * comparable to core.c, fair.c, smp.c and policy.c, the other | |
| 8 | + * big compilation units. This helps balance build time, while | |
| 9 | + * coalescing source files to amortize header inclusion | |
| 10 | + * cost. ) | |
| 11 | + */ | |
| 12 | +#include <linux/sched/clock.h> | |
| 13 | +#include <linux/sched/cputime.h> | |
| 14 | +#include <linux/sched/debug.h> | |
| 15 | +#include <linux/sched/isolation.h> | |
| 16 | +#include <linux/sched/loadavg.h> | |
| 17 | +#include <linux/sched/mm.h> | |
| 18 | +#include <linux/sched/rseq_api.h> | |
| 19 | +#include <linux/sched/task_stack.h> | |
| 20 | + | |
| 21 | +#include <linux/cpufreq.h> | |
| 22 | +#include <linux/cpumask_api.h> | |
| 23 | +#include <linux/cpuset.h> | |
| 24 | +#include <linux/ctype.h> | |
| 25 | +#include <linux/debugfs.h> | |
| 26 | +#include <linux/energy_model.h> | |
| 27 | +#include <linux/hashtable_api.h> | |
| 28 | +#include <linux/irq.h> | |
| 29 | +#include <linux/kobject_api.h> | |
| 30 | +#include <linux/membarrier.h> | |
| 31 | +#include <linux/mempolicy.h> | |
| 32 | +#include <linux/nmi.h> | |
| 33 | +#include <linux/nospec.h> | |
| 34 | +#include <linux/proc_fs.h> | |
| 35 | +#include <linux/psi.h> | |
| 36 | +#include <linux/psi.h> | |
| 37 | +#include <linux/ptrace_api.h> | |
| 38 | +#include <linux/sched_clock.h> | |
| 39 | +#include <linux/security.h> | |
| 40 | +#include <linux/spinlock_api.h> | |
| 41 | +#include <linux/swait_api.h> | |
| 42 | +#include <linux/timex.h> | |
| 43 | +#include <linux/utsname.h> | |
| 44 | +#include <linux/wait_api.h> | |
| 45 | +#include <linux/workqueue_api.h> | |
| 46 | + | |
| 47 | +#include <uapi/linux/prctl.h> | |
| 48 | +#include <uapi/linux/sched/types.h> | |
| 49 | + | |
| 50 | +#include <asm/switch_to.h> | |
| 51 | + | |
| 52 | +#include "sched.h" | |
| 53 | +#include "sched-pelt.h" | |
| 54 | +#include "stats.h" | |
| 55 | +#include "autogroup.h" | |
| 56 | + | |
| 57 | +#include "clock.c" | |
| 58 | + | |
| 59 | +#ifdef CONFIG_CGROUP_CPUACCT | |
| 60 | +# include "cpuacct.c" | |
| 61 | +#endif | |
| 62 | + | |
| 63 | +#ifdef CONFIG_CPU_FREQ | |
| 64 | +# include "cpufreq.c" | |
| 65 | +#endif | |
| 66 | + | |
| 67 | +#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL | |
| 68 | +# include "cpufreq_schedutil.c" | |
| 69 | +#endif | |
| 70 | + | |
| 71 | +#ifdef CONFIG_SCHED_DEBUG | |
| 72 | +# include "debug.c" | |
| 73 | +#endif | |
| 74 | + | |
| 75 | +#ifdef CONFIG_SCHEDSTATS | |
| 76 | +# include "stats.c" | |
| 77 | +#endif | |
| 78 | + | |
| 79 | +#include "loadavg.c" | |
| 80 | +#include "completion.c" | |
| 81 | +#include "swait.c" | |
| 82 | +#include "wait_bit.c" | |
| 83 | +#include "wait.c" | |
| 84 | + | |
| 85 | +#ifdef CONFIG_SMP | |
| 86 | +# include "cpupri.c" | |
| 87 | +# include "stop_task.c" | |
| 88 | +# include "topology.c" | |
| 89 | +#endif | |
| 90 | + | |
| 91 | +#ifdef CONFIG_SCHED_CORE | |
| 92 | +# include "core_sched.c" | |
| 93 | +#endif | |
| 94 | + | |
| 95 | +#ifdef CONFIG_PSI | |
| 96 | +# include "psi.c" | |
| 97 | +#endif | |
| 98 | + | |
| 99 | +#ifdef CONFIG_MEMBARRIER | |
| 100 | +# include "membarrier.c" | |
| 101 | +#endif | |
| 102 | + | |
| 103 | +#ifdef CONFIG_CPU_ISOLATION | |
| 104 | +# include "isolation.c" | |
| 105 | +#endif | |
| 106 | + | |
| 107 | +#ifdef CONFIG_SCHED_AUTOGROUP | |
| 108 | +# include "autogroup.c" | |
| 109 | +#endif | |
| @@ -53,15 +53,13 @@ · 52 unmodified lines | ||
| 53 | 53 | * that is otherwise invisible (TSC gets stopped). |
| 54 | 54 | * |
| 55 | 55 | */ |
| 56 | −#include "sched.h" | |
| 57 | −#include <linux/sched_clock.h> | |
| 58 | 56 | |
| 59 | 57 | /* |
| 60 | 58 | * Scheduler clock - returns current time in nanosec units. |
| 61 | 59 | * This is default implementation. |
| 62 | 60 | * Architectures and sub-architectures can override this. |
| 63 | 61 | */ |
| 64 | −unsigned long long __weak sched_clock(void) | |
| 62 | +notrace unsigned long long __weak sched_clock(void) | |
| 65 | 63 | { |
| 66 | 64 | return (unsigned long long)(jiffies - INITIAL_JIFFIES) |
| 67 | 65 | * (NSEC_PER_SEC / HZ); |
| @@ -95,28 +93,28 @@ struct sched_clock_data { · 27 unmodified lines | ||
| 95 | 93 | |
| 96 | 94 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); |
| 97 | 95 | |
| 98 | −static inline struct sched_clock_data *this_scd(void) | |
| 96 | +notrace static inline struct sched_clock_data *this_scd(void) | |
| 99 | 97 | { |
| 100 | 98 | return this_cpu_ptr(&sched_clock_data); |
| 101 | 99 | } |
| 102 | 100 | |
| 103 | −static inline struct sched_clock_data *cpu_sdc(int cpu) | |
| 101 | +notrace static inline struct sched_clock_data *cpu_sdc(int cpu) | |
| 104 | 102 | { |
| 105 | 103 | return &per_cpu(sched_clock_data, cpu); |
| 106 | 104 | } |
| 107 | 105 | |
| 108 | −int sched_clock_stable(void) | |
| 106 | +notrace int sched_clock_stable(void) | |
| 109 | 107 | { |
| 110 | 108 | return static_branch_likely(&__sched_clock_stable); |
| 111 | 109 | } |
| 112 | 110 | |
| 113 | −static void __scd_stamp(struct sched_clock_data *scd) | |
| 111 | +notrace static void __scd_stamp(struct sched_clock_data *scd) | |
| 114 | 112 | { |
| 115 | 113 | scd->tick_gtod = ktime_get_ns(); |
| 116 | 114 | scd->tick_raw = sched_clock(); |
| 117 | 115 | } |
| 118 | 116 | |
| 119 | −static void __set_sched_clock_stable(void) | |
| 117 | +notrace static void __set_sched_clock_stable(void) | |
| 120 | 118 | { |
| 121 | 119 | struct sched_clock_data *scd; |
| 122 | 120 | |
| @@ -151,7 +149,7 @@ static void __set_sched_clock_stable(void) · 28 unmodified lines | ||
| 151 | 149 | * The only way to fully avoid random clock jumps is to boot with: |
| 152 | 150 | * "tsc=unstable". |
| 153 | 151 | */ |
| 154 | −static void __sched_clock_work(struct work_struct *work) | |
| 152 | +notrace static void __sched_clock_work(struct work_struct *work) | |
| 155 | 153 | { |
| 156 | 154 | struct sched_clock_data *scd; |
| 157 | 155 | int cpu; |
| @@ -177,7 +175,7 @@ static void __sched_clock_work(struct work_struct *work) · 19 unmodified lines | ||
| 177 | 175 | |
| 178 | 176 | static DECLARE_WORK(sched_clock_work, __sched_clock_work); |
| 179 | 177 | |
| 180 | −static void __clear_sched_clock_stable(void) | |
| 178 | +notrace static void __clear_sched_clock_stable(void) | |
| 181 | 179 | { |
| 182 | 180 | if (!sched_clock_stable()) |
| 183 | 181 | return; |
| @@ -186,7 +184,7 @@ static void __clear_sched_clock_stable(void) · 2 unmodified lines | ||
| 186 | 184 | schedule_work(&sched_clock_work); |
| 187 | 185 | } |
| 188 | 186 | |
| 189 | −void clear_sched_clock_stable(void) | |
| 187 | +notrace void clear_sched_clock_stable(void) | |
| 190 | 188 | { |
| 191 | 189 | __sched_clock_stable_early = 0; |
| 192 | 190 | |
| @@ -196,7 +194,7 @@ void clear_sched_clock_stable(void) · 3 unmodified lines | ||
| 196 | 194 | __clear_sched_clock_stable(); |
| 197 | 195 | } |
| 198 | 196 | |
| 199 | −static void __sched_clock_gtod_offset(void) | |
| 197 | +notrace static void __sched_clock_gtod_offset(void) | |
| 200 | 198 | { |
| 201 | 199 | struct sched_clock_data *scd = this_scd(); |
| 202 | 200 | |
| @@ -246,12 +244,12 @@ late_initcall(sched_clock_init_late); · 43 unmodified lines | ||
| 246 | 244 | * min, max except they take wrapping into account |
| 247 | 245 | */ |
| 248 | 246 | |
| 249 | −static inline u64 wrap_min(u64 x, u64 y) | |
| 247 | +notrace static inline u64 wrap_min(u64 x, u64 y) | |
| 250 | 248 | { |
| 251 | 249 | return (s64)(x - y) < 0 ? x : y; |
| 252 | 250 | } |
| 253 | 251 | |
| 254 | −static inline u64 wrap_max(u64 x, u64 y) | |
| 252 | +notrace static inline u64 wrap_max(u64 x, u64 y) | |
| 255 | 253 | { |
| 256 | 254 | return (s64)(x - y) > 0 ? x : y; |
| 257 | 255 | } |
| @@ -262,7 +260,7 @@ static inline u64 wrap_max(u64 x, u64 y) · 4 unmodified lines | ||
| 262 | 260 | * - filter out backward motion |
| 263 | 261 | * - use the GTOD tick value to create a window to filter crazy TSC values |
| 264 | 262 | */ |
| 265 | −static u64 sched_clock_local(struct sched_clock_data *scd) | |
| 263 | +notrace static u64 sched_clock_local(struct sched_clock_data *scd) | |
| 266 | 264 | { |
| 267 | 265 | u64 now, clock, old_clock, min_clock, max_clock, gtod; |
| 268 | 266 | s64 delta; |
| @@ -295,7 +293,7 @@ again: · 26 unmodified lines | ||
| 295 | 293 | return clock; |
| 296 | 294 | } |
| 297 | 295 | |
| 298 | −static u64 sched_clock_remote(struct sched_clock_data *scd) | |
| 296 | +notrace static u64 sched_clock_remote(struct sched_clock_data *scd) | |
| 299 | 297 | { |
| 300 | 298 | struct sched_clock_data *my_scd = this_scd(); |
| 301 | 299 | u64 this_clock, remote_clock; |
| @@ -362,7 +360,7 @@ again: · 60 unmodified lines | ||
| 362 | 360 | * |
| 363 | 361 | * See cpu_clock(). |
| 364 | 362 | */ |
| 365 | −u64 sched_clock_cpu(int cpu) | |
| 363 | +notrace u64 sched_clock_cpu(int cpu) | |
| 366 | 364 | { |
| 367 | 365 | struct sched_clock_data *scd; |
| 368 | 366 | u64 clock; |
| @@ -386,7 +384,7 @@ u64 sched_clock_cpu(int cpu) · 17 unmodified lines | ||
| 386 | 384 | } |
| 387 | 385 | EXPORT_SYMBOL_GPL(sched_clock_cpu); |
| 388 | 386 | |
| 389 | −void sched_clock_tick(void) | |
| 387 | +notrace void sched_clock_tick(void) | |
| 390 | 388 | { |
| 391 | 389 | struct sched_clock_data *scd; |
| 392 | 390 | |
| @@ -403,7 +401,7 @@ void sched_clock_tick(void) · 10 unmodified lines | ||
| 403 | 401 | sched_clock_local(scd); |
| 404 | 402 | } |
| 405 | 403 | |
| 406 | −void sched_clock_tick_stable(void) | |
| 404 | +notrace void sched_clock_tick_stable(void) | |
| 407 | 405 | { |
| 408 | 406 | if (!sched_clock_stable()) |
| 409 | 407 | return; |
| @@ -423,7 +421,7 @@ void sched_clock_tick_stable(void) · 13 unmodified lines | ||
| 423 | 421 | /* |
| 424 | 422 | * We are going deep-idle (irqs are disabled): |
| 425 | 423 | */ |
| 426 | −void sched_clock_idle_sleep_event(void) | |
| 424 | +notrace void sched_clock_idle_sleep_event(void) | |
| 427 | 425 | { |
| 428 | 426 | sched_clock_cpu(smp_processor_id()); |
| 429 | 427 | } |
| @@ -432,7 +430,7 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); · 2 unmodified lines | ||
| 432 | 430 | /* |
| 433 | 431 | * We just idled; resync with ktime. |
| 434 | 432 | */ |
| 435 | −void sched_clock_idle_wakeup_event(void) | |
| 433 | +notrace void sched_clock_idle_wakeup_event(void) | |
| 436 | 434 | { |
| 437 | 435 | unsigned long flags; |
| 438 | 436 | |
| @@ -458,7 +456,7 @@ void __init sched_clock_init(void) · 19 unmodified lines | ||
| 458 | 456 | local_irq_enable(); |
| 459 | 457 | } |
| 460 | 458 | |
| 461 | −u64 sched_clock_cpu(int cpu) | |
| 459 | +notrace u64 sched_clock_cpu(int cpu) | |
| 462 | 460 | { |
| 463 | 461 | if (!static_branch_likely(&sched_clock_running)) |
| 464 | 462 | return 0; |
| @@ -476,7 +474,7 @@ u64 sched_clock_cpu(int cpu) · 11 unmodified lines | ||
| 476 | 474 | * On bare metal this function should return the same as local_clock. |
| 477 | 475 | * Architectures and sub-architectures can override this. |
| 478 | 476 | */ |
| 479 | −u64 __weak running_clock(void) | |
| 477 | +notrace u64 __weak running_clock(void) | |
| 480 | 478 | { |
| 481 | 479 | return local_clock(); |
| 482 | 480 | } |
| @@ -1,4 +1,5 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | + | |
| 2 | 3 | /* |
| 3 | 4 | * Generic wait-for-completion handler; |
| 4 | 5 | * |
| @@ -11,7 +12,6 @@ · 6 unmodified lines | ||
| 11 | 12 | * typically be used for exclusion which gives rise to priority inversion. |
| 12 | 13 | * Waiting for completion is a typically sync point, but not an exclusion point. |
| 13 | 14 | */ |
| 14 | −#include "sched.h" | |
| 15 | 15 | |
| 16 | 16 | /** |
| 17 | 17 | * complete: - signals a single thread waiting on this completion |
| @@ -6,27 +6,88 @@ · 5 unmodified lines | ||
| 6 | 6 | * |
| 7 | 7 | * Copyright (C) 1991-2002 Linus Torvalds |
| 8 | 8 | */ |
| 9 | −#define CREATE_TRACE_POINTS | |
| 10 | −#include <trace/events/sched.h> | |
| 11 | −#undef CREATE_TRACE_POINTS | |
| 9 | +#include <linux/highmem.h> | |
| 10 | +#include <linux/hrtimer_api.h> | |
| 11 | +#include <linux/ktime_api.h> | |
| 12 | +#include <linux/sched/signal.h> | |
| 13 | +#include <linux/syscalls_api.h> | |
| 14 | +#include <linux/debug_locks.h> | |
| 15 | +#include <linux/prefetch.h> | |
| 16 | +#include <linux/capability.h> | |
| 17 | +#include <linux/pgtable_api.h> | |
| 18 | +#include <linux/wait_bit.h> | |
| 19 | +#include <linux/jiffies.h> | |
| 20 | +#include <linux/spinlock_api.h> | |
| 21 | +#include <linux/cpumask_api.h> | |
| 22 | +#include <linux/lockdep_api.h> | |
| 23 | +#include <linux/hardirq.h> | |
| 24 | +#include <linux/softirq.h> | |
| 25 | +#include <linux/refcount_api.h> | |
| 26 | +#include <linux/topology.h> | |
| 27 | +#include <linux/sched/clock.h> | |
| 28 | +#include <linux/sched/cond_resched.h> | |
| 29 | +#include <linux/sched/debug.h> | |
| 30 | +#include <linux/sched/isolation.h> | |
| 31 | +#include <linux/sched/loadavg.h> | |
| 32 | +#include <linux/sched/mm.h> | |
| 33 | +#include <linux/sched/nohz.h> | |
| 34 | +#include <linux/sched/rseq_api.h> | |
| 35 | +#include <linux/sched/rt.h> | |
| 12 | 36 | |
| 13 | −#include "sched.h" | |
| 14 | − | |
| 15 | −#include <linux/nospec.h> | |
| 16 | 37 | #include <linux/blkdev.h> |
| 17 | −#include <linux/jump_label.h> | |
| 38 | +#include <linux/context_tracking.h> | |
| 39 | +#include <linux/cpuset.h> | |
| 40 | +#include <linux/delayacct.h> | |
| 41 | +#include <linux/init_task.h> | |
| 42 | +#include <linux/interrupt.h> | |
| 43 | +#include <linux/ioprio.h> | |
| 44 | +#include <linux/kallsyms.h> | |
| 18 | 45 | #include <linux/kcov.h> |
| 46 | +#include <linux/kprobes.h> | |
| 47 | +#include <linux/llist_api.h> | |
| 48 | +#include <linux/mmu_context.h> | |
| 49 | +#include <linux/mmzone.h> | |
| 50 | +#include <linux/mutex_api.h> | |
| 51 | +#include <linux/nmi.h> | |
| 52 | +#include <linux/nospec.h> | |
| 53 | +#include <linux/perf_event_api.h> | |
| 54 | +#include <linux/profile.h> | |
| 55 | +#include <linux/psi.h> | |
| 56 | +#include <linux/rcuwait_api.h> | |
| 57 | +#include <linux/sched/wake_q.h> | |
| 19 | 58 | #include <linux/scs.h> |
| 59 | +#include <linux/slab.h> | |
| 60 | +#include <linux/syscalls.h> | |
| 61 | +#include <linux/vtime.h> | |
| 62 | +#include <linux/wait_api.h> | |
| 63 | +#include <linux/workqueue_api.h> | |
| 64 | + | |
| 65 | +#ifdef CONFIG_PREEMPT_DYNAMIC | |
| 66 | +# include <linux/entry-common.h> | |
| 67 | +#endif | |
| 68 | + | |
| 69 | +#include <uapi/linux/sched/types.h> | |
| 20 | 70 | |
| 21 | 71 | #include <asm/switch_to.h> |
| 22 | 72 | #include <asm/tlb.h> |
| 23 | 73 | |
| 24 | −#include "../workqueue_internal.h" | |
| 25 | −#include "../../fs/io-wq.h" | |
| 26 | −#include "../smpboot.h" | |
| 74 | +#define CREATE_TRACE_POINTS | |
| 75 | +#include <linux/sched/rseq_api.h> | |
| 76 | +#include <trace/events/sched.h> | |
| 77 | +#undef CREATE_TRACE_POINTS | |
| 27 | 78 | |
| 79 | +#include "sched.h" | |
| 80 | +#include "stats.h" | |
| 81 | +#include "autogroup.h" | |
| 82 | + | |
| 83 | +#include "autogroup.h" | |
| 28 | 84 | #include "pelt.h" |
| 29 | 85 | #include "smp.h" |
| 86 | +#include "stats.h" | |
| 87 | + | |
| 88 | +#include "../workqueue_internal.h" | |
| 89 | +#include "../../fs/io-wq.h" | |
| 90 | +#include "../smpboot.h" | |
| 30 | 91 | |
| 31 | 92 | /* |
| 32 | 93 | * Export tracepoints that act as a bare tracehook (ie: have no trace event |
| @@ -1,8 +1,5 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 2 | 2 | |
| 3 | −#include <linux/prctl.h> | |
| 4 | −#include "sched.h" | |
| 5 | − | |
| 6 | 3 | /* |
| 7 | 4 | * A simple wrapper around refcount. An allocated sched_core_cookie's |
| 8 | 5 | * address is used to compute the cookie of the task. |
| @@ -1,12 +1,11 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
| 2 | + | |
| 2 | 3 | /* |
| 3 | 4 | * CPU accounting code for task groups. |
| 4 | 5 | * |
| 5 | 6 | * Based on the work by Paul Menage (menage@google.com) and Balbir Singh |
| 6 | 7 | * (balbir@in.ibm.com). |
| 7 | 8 | */ |
| 8 | −#include <asm/irq_regs.h> | |
| 9 | −#include "sched.h" | |
| 10 | 9 | |
| 11 | 10 | /* Time spent by the tasks of the CPU accounting group executing in ... */ |
| 12 | 11 | enum cpuacct_stat_index { |
| @@ -1,12 +1,11 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 2 | 2 | /* |
| 3 | − * kernel/sched/cpudl.c | |
| 3 | + * kernel/sched/cpudeadline.c | |
| 4 | 4 | * |
| 5 | 5 | * Global CPU deadline management |
| 6 | 6 | * |
| 7 | 7 | * Author: Juri Lelli <j.lelli@sssup.it> |
| 8 | 8 | */ |
| 9 | −#include "sched.h" | |
| 10 | 9 | |
| 11 | 10 | static inline int parent(int i) |
| 12 | 11 | { |
| @@ -5,9 +5,6 @@ · 4 unmodified lines | ||
| 5 | 5 | * Copyright (C) 2016, Intel Corporation |
| 6 | 6 | * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> |
| 7 | 7 | */ |
| 8 | −#include <linux/cpufreq.h> | |
| 9 | − | |
| 10 | −#include "sched.h" | |
| 11 | 8 | |
| 12 | 9 | DEFINE_PER_CPU(struct update_util_data __rcu *, cpufreq_update_util_data); |
| 13 | 10 | |
| @@ -6,13 +6,6 @@ · 5 unmodified lines | ||
| 6 | 6 | * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com> |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | −#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
| 10 | − | |
| 11 | −#include "sched.h" | |
| 12 | − | |
| 13 | −#include <linux/sched/cpufreq.h> | |
| 14 | −#include <trace/events/power.h> | |
| 15 | − | |
| 16 | 9 | #define IOWAIT_BOOST_MIN (SCHED_CAPACITY_SCALE / 8) |
| 17 | 10 | |
| 18 | 11 | struct sugov_tunables { |
| @@ -22,7 +22,6 @@ · 21 unmodified lines | ||
| 22 | 22 | * worst case complexity of O(min(101, nr_domcpus)), though the scenario that |
| 23 | 23 | * yields the worst case search is fairly contrived. |
| 24 | 24 | */ |
| 25 | −#include "sched.h" | |
| 26 | 25 | |
| 27 | 26 | /* |
| 28 | 27 | * p->rt_priority p->prio newpri cpupri |
| @@ -2,7 +2,6 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * Simple CPU accounting cgroup controller |
| 4 | 4 | */ |
| 5 | −#include "sched.h" | |
| 6 | 5 | |
| 7 | 6 | #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
| 8 | 7 | |
| @@ -15,8 +15,6 @@ · 14 unmodified lines | ||
| 15 | 15 | * Michael Trimarchi <michael@amarulasolutions.com>, |
| 16 | 16 | * Fabio Checconi <fchecconi@gmail.com> |
| 17 | 17 | */ |
| 18 | −#include "sched.h" | |
| 19 | −#include "pelt.h" | |
| 20 | 18 | |
| 21 | 19 | static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) |
| 22 | 20 | { |
| @@ -6,7 +6,6 @@ · 5 unmodified lines | ||
| 6 | 6 | * |
| 7 | 7 | * Copyright(C) 2007, Red Hat, Inc., Ingo Molnar |
| 8 | 8 | */ |
| 9 | −#include "sched.h" | |
| 10 | 9 | |
| 11 | 10 | /* |
| 12 | 11 | * This allows printing both to /proc/sched_debug and |
| @@ -20,7 +20,38 @@ · 19 unmodified lines | ||
| 20 | 20 | * Adaptive scheduling granularity, math enhancements by Peter Zijlstra |
| 21 | 21 | * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra |
| 22 | 22 | */ |
| 23 | +#include <linux/energy_model.h> | |
| 24 | +#include <linux/mmap_lock.h> | |
| 25 | +#include <linux/hugetlb_inline.h> | |
| 26 | +#include <linux/jiffies.h> | |
| 27 | +#include <linux/mm_api.h> | |
| 28 | +#include <linux/highmem.h> | |
| 29 | +#include <linux/spinlock_api.h> | |
| 30 | +#include <linux/cpumask_api.h> | |
| 31 | +#include <linux/lockdep_api.h> | |
| 32 | +#include <linux/softirq.h> | |
| 33 | +#include <linux/refcount_api.h> | |
| 34 | +#include <linux/topology.h> | |
| 35 | +#include <linux/sched/clock.h> | |
| 36 | +#include <linux/sched/cond_resched.h> | |
| 37 | +#include <linux/sched/cputime.h> | |
| 38 | +#include <linux/sched/isolation.h> | |
| 39 | + | |
| 40 | +#include <linux/cpuidle.h> | |
| 41 | +#include <linux/interrupt.h> | |
| 42 | +#include <linux/mempolicy.h> | |
| 43 | +#include <linux/mutex_api.h> | |
| 44 | +#include <linux/profile.h> | |
| 45 | +#include <linux/psi.h> | |
| 46 | +#include <linux/ratelimit.h> | |
| 47 | + | |
| 48 | +#include <asm/switch_to.h> | |
| 49 | + | |
| 50 | +#include <linux/sched/cond_resched.h> | |
| 51 | + | |
| 23 | 52 | #include "sched.h" |
| 53 | +#include "stats.h" | |
| 54 | +#include "autogroup.h" | |
| 24 | 55 | |
| 25 | 56 | /* |
| 26 | 57 | * Targeted preemption latency for CPU-bound tasks: |
| @@ -6,9 +6,6 @@ · 5 unmodified lines | ||
| 6 | 6 | * (NOTE: these are not related to SCHED_IDLE batch scheduled |
| 7 | 7 | * tasks which are handled in sched/fair.c ) |
| 8 | 8 | */ |
| 9 | −#include "sched.h" | |
| 10 | − | |
| 11 | −#include <trace/events/power.h> | |
| 12 | 9 | |
| 13 | 10 | /* Linker adds these: start and end of __cpuidle functions */ |
| 14 | 11 | extern char __cpuidle_text_start[], __cpuidle_text_end[]; |
| @@ -7,7 +7,6 @@ · 6 unmodified lines | ||
| 7 | 7 | * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker |
| 8 | 8 | * |
| 9 | 9 | */ |
| 10 | −#include "sched.h" | |
| 11 | 10 | |
| 12 | 11 | enum hk_flags { |
| 13 | 12 | HK_FLAG_TIMER = BIT(HK_TYPE_TIMER), |
| @@ -6,7 +6,6 @@ · 5 unmodified lines | ||
| 6 | 6 | * figure. Its a silly number but people think its important. We go through |
| 7 | 7 | * great pains to make it work on big machines and tickless kernels. |
| 8 | 8 | */ |
| 9 | −#include "sched.h" | |
| 10 | 9 | |
| 11 | 10 | /* |
| 12 | 11 | * Global load-average calculations |
| @@ -4,7 +4,6 @@ · 3 unmodified lines | ||
| 4 | 4 | * |
| 5 | 5 | * membarrier system call |
| 6 | 6 | */ |
| 7 | −#include "sched.h" | |
| 8 | 7 | |
| 9 | 8 | /* |
| 10 | 9 | * For documentation purposes, here are some membarrier ordering |
| @@ -24,10 +24,6 @@ · 23 unmodified lines | ||
| 24 | 24 | * Author: Vincent Guittot <vincent.guittot@linaro.org> |
| 25 | 25 | */ |
| 26 | 26 | |
| 27 | −#include <linux/sched.h> | |
| 28 | −#include "sched.h" | |
| 29 | −#include "pelt.h" | |
| 30 | − | |
| 31 | 27 | /* |
| 32 | 28 | * Approximate: |
| 33 | 29 | * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) |
| @@ -137,21 +137,6 @@ · 136 unmodified lines | ||
| 137 | 137 | * sampling of the aggregate task states would be. |
| 138 | 138 | */ |
| 139 | 139 | |
| 140 | −#include "../workqueue_internal.h" | |
| 141 | −#include <linux/sched/loadavg.h> | |
| 142 | −#include <linux/seq_file.h> | |
| 143 | −#include <linux/proc_fs.h> | |
| 144 | −#include <linux/seqlock.h> | |
| 145 | −#include <linux/uaccess.h> | |
| 146 | −#include <linux/cgroup.h> | |
| 147 | −#include <linux/module.h> | |
| 148 | −#include <linux/sched.h> | |
| 149 | −#include <linux/ctype.h> | |
| 150 | −#include <linux/file.h> | |
| 151 | −#include <linux/poll.h> | |
| 152 | −#include <linux/psi.h> | |
| 153 | −#include "sched.h" | |
| 154 | − | |
| 155 | 140 | static int psi_bug __read_mostly; |
| 156 | 141 | |
| 157 | 142 | DEFINE_STATIC_KEY_FALSE(psi_disabled); |
| @@ -3,9 +3,6 @@ · 2 unmodified lines | ||
| 3 | 3 | * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR |
| 4 | 4 | * policies) |
| 5 | 5 | */ |
| 6 | −#include "sched.h" | |
| 7 | − | |
| 8 | −#include "pelt.h" | |
| 9 | 6 | |
| 10 | 7 | int sched_rr_timeslice = RR_TIMESLICE; |
| 11 | 8 | int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE; |
| @@ -2,86 +2,98 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * Scheduler internal types and methods: |
| 4 | 4 | */ |
| 5 | −#include <linux/sched.h> | |
| 5 | +#ifndef _KERNEL_SCHED_SCHED_H | |
| 6 | +#define _KERNEL_SCHED_SCHED_H | |
| 6 | 7 | |
| 8 | +#include <linux/sched/affinity.h> | |
| 7 | 9 | #include <linux/sched/autogroup.h> |
| 8 | −#include <linux/sched/clock.h> | |
| 9 | −#include <linux/sched/coredump.h> | |
| 10 | 10 | #include <linux/sched/cpufreq.h> |
| 11 | −#include <linux/sched/cputime.h> | |
| 12 | 11 | #include <linux/sched/deadline.h> |
| 13 | −#include <linux/sched/debug.h> | |
| 14 | −#include <linux/sched/hotplug.h> | |
| 15 | −#include <linux/sched/idle.h> | |
| 16 | −#include <linux/sched/init.h> | |
| 17 | −#include <linux/sched/isolation.h> | |
| 18 | −#include <linux/sched/jobctl.h> | |
| 12 | +#include <linux/sched.h> | |
| 19 | 13 | #include <linux/sched/loadavg.h> |
| 20 | 14 | #include <linux/sched/mm.h> |
| 21 | −#include <linux/sched/nohz.h> | |
| 22 | −#include <linux/sched/numa_balancing.h> | |
| 23 | −#include <linux/sched/prio.h> | |
| 24 | −#include <linux/sched/rt.h> | |
| 15 | +#include <linux/sched/rseq_api.h> | |
| 25 | 16 | #include <linux/sched/signal.h> |
| 26 | 17 | #include <linux/sched/smt.h> |
| 27 | 18 | #include <linux/sched/stat.h> |
| 28 | 19 | #include <linux/sched/sysctl.h> |
| 20 | +#include <linux/sched/task_flags.h> | |
| 29 | 21 | #include <linux/sched/task.h> |
| 30 | −#include <linux/sched/task_stack.h> | |
| 31 | 22 | #include <linux/sched/topology.h> |
| 32 | −#include <linux/sched/user.h> | |
| 33 | −#include <linux/sched/wake_q.h> | |
| 34 | −#include <linux/sched/xacct.h> | |
| 35 | − | |
| 36 | −#include <uapi/linux/sched/types.h> | |
| 37 | 23 | |
| 38 | −#include <linux/binfmts.h> | |
| 39 | −#include <linux/bitops.h> | |
| 40 | −#include <linux/compat.h> | |
| 41 | −#include <linux/context_tracking.h> | |
| 24 | +#include <linux/atomic.h> | |
| 25 | +#include <linux/bitmap.h> | |
| 26 | +#include <linux/bug.h> | |
| 27 | +#include <linux/capability.h> | |
| 28 | +#include <linux/cgroup_api.h> | |
| 29 | +#include <linux/cgroup.h> | |
| 42 | 30 | #include <linux/cpufreq.h> |
| 43 | −#include <linux/cpuidle.h> | |
| 44 | −#include <linux/cpuset.h> | |
| 31 | +#include <linux/cpumask_api.h> | |
| 45 | 32 | #include <linux/ctype.h> |
| 46 | −#include <linux/debugfs.h> | |
| 47 | −#include <linux/delayacct.h> | |
| 48 | −#include <linux/energy_model.h> | |
| 49 | −#include <linux/init_task.h> | |
| 50 | −#include <linux/kprobes.h> | |
| 33 | +#include <linux/file.h> | |
| 34 | +#include <linux/fs_api.h> | |
| 35 | +#include <linux/hrtimer_api.h> | |
| 36 | +#include <linux/interrupt.h> | |
| 37 | +#include <linux/irq_work.h> | |
| 38 | +#include <linux/jiffies.h> | |
| 39 | +#include <linux/kref_api.h> | |
| 51 | 40 | #include <linux/kthread.h> |
| 52 | −#include <linux/membarrier.h> | |
| 53 | −#include <linux/migrate.h> | |
| 54 | −#include <linux/mmu_context.h> | |
| 55 | −#include <linux/nmi.h> | |
| 41 | +#include <linux/ktime_api.h> | |
| 42 | +#include <linux/lockdep_api.h> | |
| 43 | +#include <linux/lockdep.h> | |
| 44 | +#include <linux/minmax.h> | |
| 45 | +#include <linux/mm.h> | |
| 46 | +#include <linux/module.h> | |
| 47 | +#include <linux/mutex_api.h> | |
| 48 | +#include <linux/plist.h> | |
| 49 | +#include <linux/poll.h> | |
| 56 | 50 | #include <linux/proc_fs.h> |
| 57 | −#include <linux/prefetch.h> | |
| 58 | 51 | #include <linux/profile.h> |
| 59 | 52 | #include <linux/psi.h> |
| 60 | −#include <linux/ratelimit.h> | |
| 61 | −#include <linux/rcupdate_wait.h> | |
| 62 | −#include <linux/security.h> | |
| 53 | +#include <linux/rcupdate.h> | |
| 54 | +#include <linux/seq_file.h> | |
| 55 | +#include <linux/seqlock.h> | |
| 56 | +#include <linux/softirq.h> | |
| 57 | +#include <linux/spinlock_api.h> | |
| 58 | +#include <linux/static_key.h> | |
| 63 | 59 | #include <linux/stop_machine.h> |
| 64 | −#include <linux/suspend.h> | |
| 65 | −#include <linux/swait.h> | |
| 60 | +#include <linux/syscalls_api.h> | |
| 66 | 61 | #include <linux/syscalls.h> |
| 67 | −#include <linux/task_work.h> | |
| 68 | −#include <linux/tsacct_kern.h> | |
| 62 | +#include <linux/tick.h> | |
| 63 | +#include <linux/topology.h> | |
| 64 | +#include <linux/types.h> | |
| 65 | +#include <linux/u64_stats_sync_api.h> | |
| 66 | +#include <linux/uaccess.h> | |
| 67 | +#include <linux/wait_api.h> | |
| 68 | +#include <linux/wait_bit.h> | |
| 69 | +#include <linux/workqueue_api.h> | |
| 70 | + | |
| 71 | +#include <trace/events/power.h> | |
| 72 | +#include <trace/events/sched.h> | |
| 73 | + | |
| 74 | +#include "../workqueue_internal.h" | |
| 75 | + | |
| 76 | +#ifdef CONFIG_CGROUP_SCHED | |
| 77 | +#include <linux/cgroup.h> | |
| 78 | +#include <linux/psi.h> | |
| 79 | +#endif | |
| 69 | 80 | |
| 70 | −#include <asm/tlb.h> | |
| 81 | +#ifdef CONFIG_SCHED_DEBUG | |
| 82 | +# include <linux/static_key.h> | |
| 83 | +#endif | |
| 71 | 84 | |
| 72 | 85 | #ifdef CONFIG_PARAVIRT |
| 73 | 86 | # include <asm/paravirt.h> |
| 87 | +# include <asm/paravirt_api_clock.h> | |
| 74 | 88 | #endif |
| 75 | 89 | |
| 76 | 90 | #include "cpupri.h" |
| 77 | 91 | #include "cpudeadline.h" |
| 78 | 92 | |
| 79 | −#include <trace/events/sched.h> | |
| 80 | − | |
| 81 | 93 | #ifdef CONFIG_SCHED_DEBUG |
| 82 | −# define SCHED_WARN_ON(x) WARN_ONCE(x, #x) | |
| 94 | +# define SCHED_WARN_ON(x) WARN_ONCE(x, #x) | |
| 83 | 95 | #else |
| 84 | −# define SCHED_WARN_ON(x) ({ (void)(x), 0; }) | |
| 96 | +# define SCHED_WARN_ON(x) ({ (void)(x), 0; }) | |
| 85 | 97 | #endif |
| 86 | 98 | |
| 87 | 99 | struct rq; |
| @@ -329,9 +341,6 @@ extern int dl_cpu_busy(int cpu, struct task_struct *p); · 241 unmodified lines | ||
| 329 | 341 | |
| 330 | 342 | #ifdef CONFIG_CGROUP_SCHED |
| 331 | 343 | |
| 332 | −#include <linux/cgroup.h> | |
| 333 | −#include <linux/psi.h> | |
| 334 | − | |
| 335 | 344 | struct cfs_rq; |
| 336 | 345 | struct rt_rq; |
| 337 | 346 | |
| @@ -1832,7 +1841,6 @@ static inline void flush_smp_call_function_from_idle(void) { } · 1494 unmodified lines | ||
| 1832 | 1841 | #endif |
| 1833 | 1842 | |
| 1834 | 1843 | #include "stats.h" |
| 1835 | −#include "autogroup.h" | |
| 1836 | 1844 | |
| 1837 | 1845 | #if defined(CONFIG_SCHED_CORE) && defined(CONFIG_SCHEDSTATS) |
| 1838 | 1846 | |
| @@ -1928,7 +1936,6 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) · 89 unmodified lines | ||
| 1928 | 1936 | * Tunables that become constants when CONFIG_SCHED_DEBUG is off: |
| 1929 | 1937 | */ |
| 1930 | 1938 | #ifdef CONFIG_SCHED_DEBUG |
| 1931 | −# include <linux/static_key.h> | |
| 1932 | 1939 | # define const_debug __read_mostly |
| 1933 | 1940 | #else |
| 1934 | 1941 | # define const_debug const |
| @@ -3086,3 +3093,4 @@ extern int sched_dynamic_mode(const char *str); · 1151 unmodified lines | ||
| 3086 | 3093 | extern void sched_dynamic_update(int mode); |
| 3087 | 3094 | #endif |
| 3088 | 3095 | |
| 3096 | +#endif /* _KERNEL_SCHED_SCHED_H */ | |
| @@ -2,7 +2,6 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * /proc/schedstat implementation |
| 4 | 4 | */ |
| 5 | −#include "sched.h" | |
| 6 | 5 | |
| 7 | 6 | void __update_stats_wait_start(struct rq *rq, struct task_struct *p, |
| 8 | 7 | struct sched_statistics *stats) |
| @@ -1,4 +1,6 @@ | ||
| 1 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | +#ifndef _KERNEL_STATS_H | |
| 3 | +#define _KERNEL_STATS_H | |
| 2 | 4 | |
| 3 | 5 | #ifdef CONFIG_SCHEDSTATS |
| 4 | 6 | |
| @@ -298,3 +300,5 @@ sched_info_switch(struct rq *rq, struct task_struct *prev, struct task_struct *n · 293 unmodified lines | ||
| 298 | 300 | # define sched_info_dequeue(rq, t) do { } while (0) |
| 299 | 301 | # define sched_info_switch(rq, t, next) do { } while (0) |
| 300 | 302 | #endif /* CONFIG_SCHED_INFO */ |
| 303 | + | |
| 304 | +#endif /* _KERNEL_STATS_H */ | |
| @@ -7,7 +7,6 @@ · 6 unmodified lines | ||
| 7 | 7 | * |
| 8 | 8 | * See kernel/stop_machine.c |
| 9 | 9 | */ |
| 10 | −#include "sched.h" | |
| 11 | 10 | |
| 12 | 11 | #ifdef CONFIG_SMP |
| 13 | 12 | static int |
| @@ -2,7 +2,6 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * <linux/swait.h> (simple wait queues ) implementation: |
| 4 | 4 | */ |
| 5 | −#include "sched.h" | |
| 6 | 5 | |
| 7 | 6 | void __init_swait_queue_head(struct swait_queue_head *q, const char *name, |
| 8 | 7 | struct lock_class_key *key) |
| @@ -2,7 +2,6 @@ · 1 unmodified line | ||
| 2 | 2 | /* |
| 3 | 3 | * Scheduler topology setup/handling methods |
| 4 | 4 | */ |
| 5 | −#include "sched.h" | |
| 6 | 5 | |
| 7 | 6 | DEFINE_MUTEX(sched_domains_mutex); |
| 8 | 7 | |
| @@ -4,7 +4,6 @@ · 3 unmodified lines | ||
| 4 | 4 | * |
| 5 | 5 | * (C) 2004 Nadia Yvette Chambers, Oracle |
| 6 | 6 | */ |
| 7 | −#include "sched.h" | |
| 8 | 7 | |
| 9 | 8 | void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key) |
| 10 | 9 | { |
| @@ -1,8 +1,8 @@ | ||
| 1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 2 | + | |
| 2 | 3 | /* |
| 3 | 4 | * The implementation of the wait_bit*() and related waiting APIs: |
| 4 | 5 | */ |
| 5 | −#include "sched.h" | |
| 6 | 6 | |
| 7 | 7 | #define WAIT_TABLE_BITS 8 |
| 8 | 8 | #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS) |