Merge branch 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler changes from Ingo Molnar: "Main changes: - scheduler side full-dynticks (user-space execution is undisturbed and receives no timer IRQs) preparation changes that convert the cputime accounting code to be full-dynticks ready, from Frederic Weisbecker. - Initial sched.h split-up changes, by Clark Williams - select_idle_sibling() performance improvement by Mike Galbraith: " 1 tbench pair (worst case) in a 10 core + SMT package: pre 15.22 MB/sec 1 procs post 252.01 MB/sec 1 procs " - sched_rr_get_interval() ABI fix/change. We think this detail is not used by apps (so it's not an ABI in practice), but lets keep it under observation. - misc RT scheduling cleanups, optimizations" * 'sched-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (24 commits) sched/rt: Add <linux/sched/rt.h> header to <linux/init_task.h> cputime: Remove irqsave from seqlock readers sched, powerpc: Fix sched.h split-up build failure cputime: Restore CPU_ACCOUNTING config defaults for PPC64 sched/rt: Move rt specific bits into new header file sched/rt: Add a tuning knob to allow changing SCHED_RR timeslice sched: Move sched.h sysctl bits into separate header sched: Fix signedness bug in yield_to() sched: Fix select_idle_sibling() bouncing cow syndrome sched/rt: Further simplify pick_rt_task() sched/rt: Do not account zero delta_exec in update_curr_rt() cputime: Safely read cputime of full dynticks CPUs kvm: Prepare to add generic guest entry/exit callbacks cputime: Use accessors to read task cputime stats cputime: Allow dynamic switch between tick/virtual based cputime accounting cputime: Generic on-demand virtual cputime accounting cputime: Move default nsecs_to_cputime() to jiffies based cputime file cputime: Librarize per nsecs resolution cputime definitions cputime: Avoid multiplication overflow on utime scaling context_tracking: Export context state for generic vtime ... Fix up conflict in kernel/context_tracking.c due to comment additions.

Linus Torvalds committed 13 years ago

Commit d652e1e2 parents 8f55cea 77852fe83 files · +1105 −507Browse files

Unified · Split

83 files

arch/alpha/kernel/osf_sys.c +4 −2
arch/ia64/include/asm/cputime.h +6 −86
arch/ia64/include/asm/thread_info.h +2 −2
arch/ia64/include/asm/xen/minstate.h +1 −1
arch/ia64/kernel/asm-offsets.c +1 −1
arch/ia64/kernel/entry.S +8 −8
arch/ia64/kernel/fsys.S +2 −2
arch/ia64/kernel/head.S +2 −2
arch/ia64/kernel/ivt.S +4 −4
arch/ia64/kernel/minstate.h +1 −1
arch/ia64/kernel/time.c +3 −2
arch/powerpc/configs/chroma_defconfig +1 −1
arch/powerpc/configs/corenet64_smp_defconfig +1 −1
arch/powerpc/configs/pasemi_defconfig +1 −1
arch/powerpc/include/asm/cputime.h +3 −3
arch/powerpc/include/asm/lppaca.h +1 −1
arch/powerpc/include/asm/ppc_asm.h +2 −2
arch/powerpc/kernel/entry_64.S +2 −2
arch/powerpc/kernel/time.c +3 −2
arch/powerpc/platforms/cell/spufs/sched.c +1 −0
arch/powerpc/platforms/pseries/dtl.c +3 −3
arch/powerpc/platforms/pseries/setup.c +3 −3
arch/s390/kernel/vtime.c +3 −3
arch/x86/kernel/apm_32.c +6 −5
block/blk-exec.c +1 −0
drivers/isdn/mISDN/stack.c +6 −1
drivers/spi/spi.c +1 −1
drivers/staging/csr/bh.c +1 −1
drivers/staging/csr/unifi_sme.c +1 −1
drivers/tty/sysrq.c +1 −0
fs/binfmt_elf.c +6 −2
fs/binfmt_elf_fdpic.c +5 −2
fs/proc/array.c +2 −2
fs/select.c +1 −0
include/asm-generic/cputime.h +6 −60
include/asm-generic/cputime_jiffies.h +72 −0
include/asm-generic/cputime_nsecs.h +104 −0
include/linux/context_tracking.h +28 −0
include/linux/hardirq.h +2 −2
include/linux/init_task.h +12 −0
include/linux/kernel_stat.h +1 −1
include/linux/kvm_host.h +46 −9
include/linux/sched.h +42 −143
include/linux/sched/rt.h +58 −0
include/linux/sched/sysctl.h +110 −0
include/linux/tsacct_kern.h +3 −0
include/linux/vtime.h +40 −19
init/Kconfig +23 −2
init/init_task.c +2 −0
kernel/acct.c +4 −2
kernel/context_tracking.c +25 −18
kernel/cpu.c +3 −1
kernel/delayacct.c +5 −2
kernel/exit.c +6 −4
kernel/fork.c +6 −0
kernel/futex.c +1 −0
kernel/hrtimer.c +2 −0
kernel/irq/manage.c +1 −0
kernel/mutex.c +1 −0
kernel/posix-cpu-timers.c +22 −6
kernel/rtmutex-debug.c +1 −0
kernel/rtmutex-tester.c +1 −0
kernel/rtmutex.c +1 −0
kernel/sched/core.c +21 −1
kernel/sched/cpupri.c +2 −0
kernel/sched/cputime.c +271 −43
kernel/sched/fair.c +9 −18
kernel/sched/rt.c +17 −9
kernel/sched/sched.h +2 −0
kernel/signal.c +8 −4
kernel/softirq.c +3 −3
kernel/sysctl.c +8 −0
kernel/time/tick-sched.c +4 −1
kernel/timer.c +1 −0
kernel/trace/trace.c +1 −0
kernel/trace/trace_sched_wakeup.c +1 −1
kernel/tsacct.c +34 −10
kernel/watchdog.c +1 −0
mm/mmap.c +1 −0
mm/mremap.c +1 −0
mm/nommu.c +1 −0
mm/page-writeback.c +1 −0
mm/page_alloc.c +1 −0

arch/alpha/kernel/osf_sys.c+4−2

@@ -1139,6 +1139,7 @@ struct rusage32 { · 1138 unmodified lines
1139	1139	SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru)
1140	1140	{
1141	1141	struct rusage32 r;
	1142	+ cputime_t utime, stime;
1142	1143
1143	1144	if (who != RUSAGE_SELF && who != RUSAGE_CHILDREN)
1144	1145	return -EINVAL;
@@ -1146,8 +1147,9 @@ SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru) · 1 unmodified line
1146	1147	memset(&r, 0, sizeof(r));
1147	1148	switch (who) {
1148	1149	case RUSAGE_SELF:
1149		− jiffies_to_timeval32(current->utime, &r.ru_utime);
1150		− jiffies_to_timeval32(current->stime, &r.ru_stime);
	1150	+ task_cputime(current, &utime, &stime);
	1151	+ jiffies_to_timeval32(utime, &r.ru_utime);
	1152	+ jiffies_to_timeval32(stime, &r.ru_stime);
1151	1153	r.ru_minflt = current->min_flt;
1152	1154	r.ru_majflt = current->maj_flt;
1153	1155	break;

arch/ia64/include/asm/cputime.h+6−86

@@ -11,99 +11,19 @@ · 10 unmodified lines
11	11	* as published by the Free Software Foundation; either version
12	12	* 2 of the License, or (at your option) any later version.
13	13	*
14		− * If we have CONFIG_VIRT_CPU_ACCOUNTING, we measure cpu time in nsec.
	14	+ * If we have CONFIG_VIRT_CPU_ACCOUNTING_NATIVE, we measure cpu time in nsec.
15	15	* Otherwise we measure cpu time in jiffies using the generic definitions.
16	16	*/
17	17
18	18	#ifndef __IA64_CPUTIME_H
19	19	#define __IA64_CPUTIME_H
20	20
21		−#ifndef CONFIG_VIRT_CPU_ACCOUNTING
22		−#include <asm-generic/cputime.h>
	21	+#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
	22	+# include <asm-generic/cputime.h>
23	23	#else
24		−
25		−#include <linux/time.h>
26		−#include <linux/jiffies.h>
27		−#include <asm/processor.h>
28		−
29		−typedef u64 __nocast cputime_t;
30		−typedef u64 __nocast cputime64_t;
31		−
32		−#define cputime_one_jiffy jiffies_to_cputime(1)
33		−
34		−/*
35		− * Convert cputime <-> jiffies (HZ)
36		− */
37		−#define cputime_to_jiffies(__ct) \
38		− ((__force u64)(__ct) / (NSEC_PER_SEC / HZ))
39		−#define jiffies_to_cputime(__jif) \
40		− (__force cputime_t)((__jif) * (NSEC_PER_SEC / HZ))
41		−#define cputime64_to_jiffies64(__ct) \
42		− ((__force u64)(__ct) / (NSEC_PER_SEC / HZ))
43		−#define jiffies64_to_cputime64(__jif) \
44		− (__force cputime64_t)((__jif) * (NSEC_PER_SEC / HZ))
45		−
46		−/*
47		− * Convert cputime <-> microseconds
48		− */
49		−#define cputime_to_usecs(__ct) \
50		− ((__force u64)(__ct) / NSEC_PER_USEC)
51		−#define usecs_to_cputime(__usecs) \
52		− (__force cputime_t)((__usecs) * NSEC_PER_USEC)
53		−#define usecs_to_cputime64(__usecs) \
54		− (__force cputime64_t)((__usecs) * NSEC_PER_USEC)
55		−
56		−/*
57		− * Convert cputime <-> seconds
58		− */
59		−#define cputime_to_secs(__ct) \
60		− ((__force u64)(__ct) / NSEC_PER_SEC)
61		−#define secs_to_cputime(__secs) \
62		− (__force cputime_t)((__secs) * NSEC_PER_SEC)
63		−
64		−/*
65		− * Convert cputime <-> timespec (nsec)
66		− */
67		−static inline cputime_t timespec_to_cputime(const struct timespec *val)
68		−{
69		− u64 ret = val->tv_sec * NSEC_PER_SEC + val->tv_nsec;
70		− return (__force cputime_t) ret;
71		−}
72		−static inline void cputime_to_timespec(const cputime_t ct, struct timespec *val)
73		−{
74		− val->tv_sec = (__force u64) ct / NSEC_PER_SEC;
75		− val->tv_nsec = (__force u64) ct % NSEC_PER_SEC;
76		−}
77		−
78		−/*
79		− * Convert cputime <-> timeval (msec)
80		− */
81		−static inline cputime_t timeval_to_cputime(struct timeval *val)
82		−{
83		− u64 ret = val->tv_sec * NSEC_PER_SEC + val->tv_usec * NSEC_PER_USEC;
84		− return (__force cputime_t) ret;
85		−}
86		−static inline void cputime_to_timeval(const cputime_t ct, struct timeval *val)
87		−{
88		− val->tv_sec = (__force u64) ct / NSEC_PER_SEC;
89		− val->tv_usec = ((__force u64) ct % NSEC_PER_SEC) / NSEC_PER_USEC;
90		−}
91		−
92		−/*
93		− * Convert cputime <-> clock (USER_HZ)
94		− */
95		−#define cputime_to_clock_t(__ct) \
96		− ((__force u64)(__ct) / (NSEC_PER_SEC / USER_HZ))
97		−#define clock_t_to_cputime(__x) \
98		− (__force cputime_t)((__x) * (NSEC_PER_SEC / USER_HZ))
99		−
100		−/*
101		− * Convert cputime64 to clock.
102		− */
103		−#define cputime64_to_clock_t(__ct) \
104		− cputime_to_clock_t((__force cputime_t)__ct)
105		−
	24	+# include <asm/processor.h>
	25	+# include <asm-generic/cputime_nsecs.h>
106	26	extern void arch_vtime_task_switch(struct task_struct *tsk);
	27	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
107	28
108		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
109	29	#endif /* __IA64_CPUTIME_H */

arch/ia64/include/asm/thread_info.h+2−2

@@ -31,7 +31,7 @@ struct thread_info { · 30 unmodified lines
31	31	mm_segment_t addr_limit; /* user-level address space limit */
32	32	int preempt_count; /* 0=premptable, <0=BUG; will also serve as bh-counter */
33	33	struct restart_block restart_block;
34		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	34	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
35	35	__u64 ac_stamp;
36	36	__u64 ac_leave;
37	37	__u64 ac_stime;
@@ -69,7 +69,7 @@ struct thread_info { · 31 unmodified lines
69	69	#define task_stack_page(tsk) ((void *)(tsk))
70	70
71	71	#define __HAVE_THREAD_FUNCTIONS
72		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	72	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
73	73	#define setup_thread_stack(p, org) \
74	74	task_thread_info(p) = task_thread_info(org); \
75	75	task_thread_info(p)->ac_stime = 0; \

arch/ia64/include/asm/xen/minstate.h+1−1

@@ -1,5 +1,5 @@
1	1
2		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	2	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
3	3	/* read ar.itc in advance, and use it before leaving bank 0 */
4	4	#define XEN_ACCOUNT_GET_STAMP \
5	5	MOV_FROM_ITC(pUStk, p6, r20, r2);

arch/ia64/kernel/asm-offsets.c+1−1

@@ -41,7 +41,7 @@ void foo(void) · 40 unmodified lines
41	41	DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
42	42	DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
43	43	DEFINE(TI_PRE_COUNT, offsetof(struct thread_info, preempt_count));
44		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	44	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
45	45	DEFINE(TI_AC_STAMP, offsetof(struct thread_info, ac_stamp));
46	46	DEFINE(TI_AC_LEAVE, offsetof(struct thread_info, ac_leave));
47	47	DEFINE(TI_AC_STIME, offsetof(struct thread_info, ac_stime));

arch/ia64/kernel/entry.S+8−8

@@ -724,7 +724,7 @@ GLOBAL_ENTRY(__paravirt_leave_syscall) · 723 unmodified lines
724	724	#endif
725	725	.global __paravirt_work_processed_syscall;
726	726	__paravirt_work_processed_syscall:
727		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	727	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
728	728	adds r2=PT(LOADRS)+16,r12
729	729	MOV_FROM_ITC(pUStk, p9, r22, r19) // fetch time at leave
730	730	adds r18=TI_FLAGS+IA64_TASK_SIZE,r13
@@ -762,7 +762,7 @@ __paravirt_work_processed_syscall: · 31 unmodified lines
762	762
763	763	ld8 r29=[r2],16 // M0\|1 load cr.ipsr
764	764	ld8 r28=[r3],16 // M0\|1 load cr.iip
765		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	765	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
766	766	(pUStk) add r14=TI_AC_LEAVE+IA64_TASK_SIZE,r13
767	767	;;
768	768	ld8 r30=[r2],16 // M0\|1 load cr.ifs
@@ -793,7 +793,7 @@ __paravirt_work_processed_syscall: · 24 unmodified lines
793	793	ld8.fill r1=[r3],16 // M0\|1 load r1
794	794	(pUStk) mov r17=1 // A
795	795	;;
796		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	796	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
797	797	(pUStk) st1 [r15]=r17 // M2\|3
798	798	#else
799	799	(pUStk) st1 [r14]=r17 // M2\|3
@@ -813,7 +813,7 @@ __paravirt_work_processed_syscall: · 13 unmodified lines
813	813	shr.u r18=r19,16 // I0\|1 get byte size of existing "dirty" partition
814	814	COVER // B add current frame into dirty partition & set cr.ifs
815	815	;;
816		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	816	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
817	817	mov r19=ar.bsp // M2 get new backing store pointer
818	818	st8 [r14]=r22 // M save time at leave
819	819	mov f10=f0 // F clear f10
@@ -948,7 +948,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) · 128 unmodified lines
948	948	adds r16=PT(CR_IPSR)+16,r12
949	949	adds r17=PT(CR_IIP)+16,r12
950	950
951		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	951	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
952	952	.pred.rel.mutex pUStk,pKStk
953	953	MOV_FROM_PSR(pKStk, r22, r29) // M2 read PSR now that interrupts are disabled
954	954	MOV_FROM_ITC(pUStk, p9, r22, r29) // M fetch time at leave
@@ -981,7 +981,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) · 26 unmodified lines
981	981	;;
982	982	ld8.fill r12=[r16],16
983	983	ld8.fill r13=[r17],16
984		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	984	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
985	985	(pUStk) adds r3=TI_AC_LEAVE+IA64_TASK_SIZE,r18
986	986	#else
987	987	(pUStk) adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18
@@ -989,7 +989,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) · 1 unmodified line
989	989	;;
990	990	ld8 r20=[r16],16 // ar.fpsr
991	991	ld8.fill r15=[r17],16
992		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	992	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
993	993	(pUStk) adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18 // deferred
994	994	#endif
995	995	;;
@@ -997,7 +997,7 @@ GLOBAL_ENTRY(__paravirt_leave_kernel) · 1 unmodified line
997	997	ld8.fill r2=[r17]
998	998	(pUStk) mov r17=1
999	999	;;
1000		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	1000	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1001	1001	// mmi_ : ld8 st1 shr;; mmi_ : st8 st1 shr;;
1002	1002	// mib : mov add br -> mib : ld8 add br
1003	1003	// bbb_ : br nop cover;; mbb_ : mov br cover;;

arch/ia64/kernel/fsys.S+2−2

@@ -529,7 +529,7 @@ GLOBAL_ENTRY(paravirt_fsys_bubble_down) · 528 unmodified lines
529	529	nop.i 0
530	530	;;
531	531	mov ar.rsc=0 // M2 set enforced lazy mode, pl 0, LE, loadrs=0
532		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	532	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
533	533	MOV_FROM_ITC(p0, p6, r30, r23) // M get cycle for accounting
534	534	#else
535	535	nop.m 0
@@ -555,7 +555,7 @@ GLOBAL_ENTRY(paravirt_fsys_bubble_down) · 19 unmodified lines
555	555	cmp.ne pKStk,pUStk=r0,r0 // A set pKStk <- 0, pUStk <- 1
556	556	br.call.sptk.many b7=ia64_syscall_setup // B
557	557	;;
558		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	558	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
559	559	// mov.m r30=ar.itc is called in advance
560	560	add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
561	561	add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2

arch/ia64/kernel/head.S+2−2

@@ -1073,7 +1073,7 @@ END(ia64_native_sched_clock) · 1072 unmodified lines
1073	1073	sched_clock = ia64_native_sched_clock
1074	1074	#endif
1075	1075
1076		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	1076	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1077	1077	GLOBAL_ENTRY(cycle_to_cputime)
1078	1078	alloc r16=ar.pfs,1,0,0,0
1079	1079	addl r8=THIS_CPU(ia64_cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0
@@ -1091,7 +1091,7 @@ GLOBAL_ENTRY(cycle_to_cputime) · 11 unmodified lines
1091	1091	shrp r8=r9,r8,IA64_NSEC_PER_CYC_SHIFT
1092	1092	br.ret.sptk.many rp
1093	1093	END(cycle_to_cputime)
1094		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
	1094	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
1095	1095
1096	1096	#ifdef CONFIG_IA64_BRL_EMU
1097	1097

arch/ia64/kernel/ivt.S+4−4

@@ -784,7 +784,7 @@ ENTRY(break_fault) · 783 unmodified lines
784	784
785	785	(p8) adds r28=16,r28 // A switch cr.iip to next bundle
786	786	(p9) adds r8=1,r8 // A increment ei to next slot
787		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	787	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
788	788	;;
789	789	mov b6=r30 // I0 setup syscall handler branch reg early
790	790	#else
@@ -801,7 +801,7 @@ ENTRY(break_fault) · 10 unmodified lines
801	801	//
802	802	///////////////////////////////////////////////////////////////////////
803	803	st1 [r16]=r0 // M2\|3 clear current->thread.on_ustack flag
804		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	804	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
805	805	MOV_FROM_ITC(p0, p14, r30, r18) // M get cycle for accounting
806	806	#else
807	807	mov b6=r30 // I0 setup syscall handler branch reg early
@@ -817,7 +817,7 @@ ENTRY(break_fault) · 9 unmodified lines
817	817	cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited?
818	818	br.call.sptk.many b7=ia64_syscall_setup // B
819	819	1:
820		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	820	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
821	821	// mov.m r30=ar.itc is called in advance, and r13 is current
822	822	add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 // A
823	823	add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 // A
@@ -1043,7 +1043,7 @@ END(ia64_syscall_setup) · 219 unmodified lines
1043	1043	DBG_FAULT(16)
1044	1044	FAULT(16)
1045	1045
1046		−#if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE)
	1046	+#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE)
1047	1047	/*
1048	1048	* There is no particular reason for this code to be here, other than
1049	1049	* that there happens to be space here that would go unused otherwise.

arch/ia64/kernel/minstate.h+1−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	4	#include "entry.h"
5	5	#include "paravirt_inst.h"
6	6
7		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	7	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
8	8	/* read ar.itc in advance, and use it before leaving bank 0 */
9	9	#define ACCOUNT_GET_STAMP \
10	10	(pUStk) mov.m r20=ar.itc;

arch/ia64/kernel/time.c+3−2

@@ -77,7 +77,7 @@ static struct clocksource clocksource_itc = { · 76 unmodified lines
77	77	};
78	78	static struct clocksource *itc_clocksource;
79	79
80		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	80	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
81	81
82	82	#include <linux/kernel_stat.h>
83	83
@@ -136,13 +136,14 @@ void vtime_account_system(struct task_struct *tsk) · 52 unmodified lines
136	136
137	137	account_system_time(tsk, 0, delta, delta);
138	138	}
	139	+EXPORT_SYMBOL_GPL(vtime_account_system);
139	140
140	141	void vtime_account_idle(struct task_struct *tsk)
141	142	{
142	143	account_idle_time(vtime_delta(tsk));
143	144	}
144	145
145		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
	146	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
146	147
147	148	static irqreturn_t
148	149	timer_interrupt (int irq, void *dev_id)

arch/powerpc/configs/chroma_defconfig+1−1

@@ -1,6 +1,6 @@
1	1	CONFIG_PPC64=y
2	2	CONFIG_PPC_BOOK3E_64=y
3		−# CONFIG_VIRT_CPU_ACCOUNTING is not set
	3	+# CONFIG_VIRT_CPU_ACCOUNTING_NATIVE is not set
4	4	CONFIG_SMP=y
5	5	CONFIG_NR_CPUS=256
6	6	CONFIG_EXPERIMENTAL=y

arch/powerpc/configs/corenet64_smp_defconfig+1−1

@@ -1,6 +1,6 @@
1	1	CONFIG_PPC64=y
2	2	CONFIG_PPC_BOOK3E_64=y
3		−# CONFIG_VIRT_CPU_ACCOUNTING is not set
	3	+# CONFIG_VIRT_CPU_ACCOUNTING_NATIVE is not set
4	4	CONFIG_SMP=y
5	5	CONFIG_NR_CPUS=2
6	6	CONFIG_EXPERIMENTAL=y

arch/powerpc/configs/pasemi_defconfig+1−1

@@ -1,6 +1,6 @@
1	1	CONFIG_PPC64=y
2	2	CONFIG_ALTIVEC=y
3		−# CONFIG_VIRT_CPU_ACCOUNTING is not set
	3	+# CONFIG_VIRT_CPU_ACCOUNTING_NATIVE is not set
4	4	CONFIG_SMP=y
5	5	CONFIG_NR_CPUS=2
6	6	CONFIG_EXPERIMENTAL=y

arch/powerpc/include/asm/cputime.h+3−3

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	8	* as published by the Free Software Foundation; either version
9	9	* 2 of the License, or (at your option) any later version.
10	10	*
11		− * If we have CONFIG_VIRT_CPU_ACCOUNTING, we measure cpu time in
	11	+ * If we have CONFIG_VIRT_CPU_ACCOUNTING_NATIVE, we measure cpu time in
12	12	* the same units as the timebase. Otherwise we measure cpu time
13	13	* in jiffies using the generic definitions.
14	14	*/
@@ -16,7 +16,7 @@ · 1 unmodified line
16	16	#ifndef __POWERPC_CPUTIME_H
17	17	#define __POWERPC_CPUTIME_H
18	18
19		−#ifndef CONFIG_VIRT_CPU_ACCOUNTING
	19	+#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
20	20	#include <asm-generic/cputime.h>
21	21	#ifdef __KERNEL__
22	22	static inline void setup_cputime_one_jiffy(void) { }
@@ -231,5 +231,5 @@ static inline cputime_t clock_t_to_cputime(const unsigned long clk) · 208 unmodified lines
231	231	static inline void arch_vtime_task_switch(struct task_struct *tsk) { }
232	232
233	233	#endif /* __KERNEL__ */
234		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
	234	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
235	235	#endif /* __POWERPC_CPUTIME_H */

arch/powerpc/include/asm/lppaca.h+1−1

@@ -145,7 +145,7 @@ struct dtl_entry { · 144 unmodified lines
145	145	extern struct kmem_cache *dtl_cache;
146	146
147	147	/*
148		− * When CONFIG_VIRT_CPU_ACCOUNTING = y, the cpu accounting code controls
	148	+ * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls
149	149	* reading from the dispatch trace log. If other code wants to consume
150	150	* DTL entries, it can set this pointer to a function that will get
151	151	* called once for each DTL entry that gets processed.

arch/powerpc/include/asm/ppc_asm.h+2−2

@@ -24,7 +24,7 @@ · 23 unmodified lines
24	24	* user_time and system_time fields in the paca.
25	25	*/
26	26
27		−#ifndef CONFIG_VIRT_CPU_ACCOUNTING
	27	+#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
28	28	#define ACCOUNT_CPU_USER_ENTRY(ra, rb)
29	29	#define ACCOUNT_CPU_USER_EXIT(ra, rb)
30	30	#define ACCOUNT_STOLEN_TIME
@@ -70,7 +70,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) · 39 unmodified lines
70	70
71	71	#endif /* CONFIG_PPC_SPLPAR */
72	72
73		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
	73	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
74	74
75	75	/*
76	76	* Macros for storing registers into and loading registers from

arch/powerpc/kernel/entry_64.S+2−2

@@ -94,7 +94,7 @@ system_call_common: · 93 unmodified lines
94	94	addi r9,r1,STACK_FRAME_OVERHEAD
95	95	ld r11,exception_marker@toc(r2)
96	96	std r11,-16(r9) /* "regshere" marker */
97		−#if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR)
	97	+#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
98	98	BEGIN_FW_FTR_SECTION
99	99	beq 33f
100	100	/* if from user, see if there are any DTL entries to process */
@@ -110,7 +110,7 @@ BEGIN_FW_FTR_SECTION · 9 unmodified lines
110	110	addi r9,r1,STACK_FRAME_OVERHEAD
111	111	33:
112	112	END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
113		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING && CONFIG_PPC_SPLPAR */
	113	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
114	114
115	115	/*
116	116	* A syscall should always be called with interrupts enabled

arch/powerpc/kernel/time.c+3−2

@@ -143,7 +143,7 @@ EXPORT_SYMBOL_GPL(ppc_proc_freq); · 142 unmodified lines
143	143	unsigned long ppc_tb_freq;
144	144	EXPORT_SYMBOL_GPL(ppc_tb_freq);
145	145
146		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	146	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
147	147	/*
148	148	* Factors for converting from cputime_t (timebase ticks) to
149	149	* jiffies, microseconds, seconds, and clock_t (1/USER_HZ seconds).
@@ -347,6 +347,7 @@ void vtime_account_system(struct task_struct *tsk) · 197 unmodified lines
347	347	if (stolen)
348	348	account_steal_time(stolen);
349	349	}
	350	+EXPORT_SYMBOL_GPL(vtime_account_system);
350	351
351	352	void vtime_account_idle(struct task_struct *tsk)
352	353	{
@@ -377,7 +378,7 @@ void vtime_account_user(struct task_struct *tsk) · 24 unmodified lines
377	378	account_user_time(tsk, utime, utimescaled);
378	379	}
379	380
380		−#else /* ! CONFIG_VIRT_CPU_ACCOUNTING */
	381	+#else /* ! CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
381	382	#define calc_cputime_factors()
382	383	#endif
383	384

arch/powerpc/platforms/cell/spufs/sched.c+1−0

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24
25	25	#include <linux/errno.h>
26	26	#include <linux/sched.h>
	27	+#include <linux/sched/rt.h>
27	28	#include <linux/kernel.h>
28	29	#include <linux/mm.h>
29	30	#include <linux/slab.h>

arch/powerpc/platforms/pseries/dtl.c+3−3

@@ -57,7 +57,7 @@ static u8 dtl_event_mask = 0x7; · 56 unmodified lines
57	57	*/
58	58	static int dtl_buf_entries = N_DISPATCH_LOG;
59	59
60		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	60	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
61	61	struct dtl_ring {
62	62	u64 write_index;
63	63	struct dtl_entry *write_ptr;
@@ -142,7 +142,7 @@ static u64 dtl_current_index(struct dtl *dtl) · 78 unmodified lines
142	142	return per_cpu(dtl_rings, dtl->cpu).write_index;
143	143	}
144	144
145		−#else /* CONFIG_VIRT_CPU_ACCOUNTING */
	145	+#else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
146	146
147	147	static int dtl_start(struct dtl *dtl)
148	148	{
@@ -188,7 +188,7 @@ static u64 dtl_current_index(struct dtl *dtl) · 39 unmodified lines
188	188	{
189	189	return lppaca_of(dtl->cpu).dtl_idx;
190	190	}
191		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
	191	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
192	192
193	193	static int dtl_enable(struct dtl *dtl)
194	194	{

arch/powerpc/platforms/pseries/setup.c+3−3

@@ -281,7 +281,7 @@ static struct notifier_block pci_dn_reconfig_nb = { · 280 unmodified lines
281	281
282	282	struct kmem_cache *dtl_cache;
283	283
284		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	284	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
285	285	/*
286	286	* Allocate space for the dispatch trace log for all possible cpus
287	287	* and register the buffers with the hypervisor. This is used for
@@ -332,12 +332,12 @@ static int alloc_dispatch_logs(void) · 44 unmodified lines
332	332
333	333	return 0;
334	334	}
335		−#else /* !CONFIG_VIRT_CPU_ACCOUNTING */
	335	+#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
336	336	static inline int alloc_dispatch_logs(void)
337	337	{
338	338	return 0;
339	339	}
340		−#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
	340	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
341	341
342	342	static int alloc_dispatch_log_kmem_cache(void)
343	343	{

arch/s390/kernel/vtime.c+3−3

@@ -127,7 +127,7 @@ void vtime_account_user(struct task_struct *tsk) · 126 unmodified lines
127	127	* Update process times based on virtual cpu times stored by entry.S
128	128	* to the lowcore fields user_timer, system_timer & steal_clock.
129	129	*/
130		−void vtime_account(struct task_struct *tsk)
	130	+void vtime_account_irq_enter(struct task_struct *tsk)
131	131	{
132	132	struct thread_info *ti = task_thread_info(tsk);
133	133	u64 timer, system;
@@ -145,10 +145,10 @@ void vtime_account(struct task_struct *tsk) · 11 unmodified lines
145	145
146	146	virt_timer_forward(system);
147	147	}
148		−EXPORT_SYMBOL_GPL(vtime_account);
	148	+EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
149	149
150	150	void vtime_account_system(struct task_struct *tsk)
151		−__attribute__((alias("vtime_account")));
	151	+__attribute__((alias("vtime_account_irq_enter")));
152	152	EXPORT_SYMBOL_GPL(vtime_account_system);
153	153
154	154	void __kprobes vtime_stop_cpu(void)

arch/x86/kernel/apm_32.c+6−5

@@ -899,6 +899,7 @@ static void apm_cpu_idle(void) · 898 unmodified lines
899	899	static int use_apm_idle; /* = 0 */
900	900	static unsigned int last_jiffies; /* = 0 */
901	901	static unsigned int last_stime; /* = 0 */
	902	+ cputime_t stime;
902	903
903	904	int apm_idle_done = 0;
904	905	unsigned int jiffies_since_last_check = jiffies - last_jiffies;
@@ -906,23 +907,23 @@ static void apm_cpu_idle(void) · 1 unmodified line
906	907
907	908	WARN_ONCE(1, "deprecated apm_cpu_idle will be deleted in 2012");
908	909	recalc:
	910	+ task_cputime(current, NULL, &stime);
909	911	if (jiffies_since_last_check > IDLE_CALC_LIMIT) {
910	912	use_apm_idle = 0;
911		− last_jiffies = jiffies;
912		− last_stime = current->stime;
913	913	} else if (jiffies_since_last_check > idle_period) {
914	914	unsigned int idle_percentage;
915	915
916		− idle_percentage = current->stime - last_stime;
	916	+ idle_percentage = stime - last_stime;
917	917	idle_percentage *= 100;
918	918	idle_percentage /= jiffies_since_last_check;
919	919	use_apm_idle = (idle_percentage > idle_threshold);
920	920	if (apm_info.forbid_idle)
921	921	use_apm_idle = 0;
922		− last_jiffies = jiffies;
923		− last_stime = current->stime;
924	922	}
925	923
	924	+ last_jiffies = jiffies;
	925	+ last_stime = stime;
	926	+
926	927	bucket = IDLE_LEAKY_MAX;
927	928
928	929	while (!need_resched()) {

block/blk-exec.c+1−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	5	#include <linux/module.h>
6	6	#include <linux/bio.h>
7	7	#include <linux/blkdev.h>
	8	+#include <linux/sched/sysctl.h>
8	9
9	10	#include "blk.h"
10	11

drivers/isdn/mISDN/stack.c+6−1

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	#include <linux/slab.h>
19	19	#include <linux/mISDNif.h>
20	20	#include <linux/kthread.h>
	21	+#include <linux/sched.h>
21	22	#include "core.h"
22	23
23	24	static u_int *debug;
@@ -202,6 +203,9 @@ static int · 178 unmodified lines
202	203	mISDNStackd(void *data)
203	204	{
204	205	struct mISDNstack *st = data;
	206	+#ifdef MISDN_MSG_STATS
	207	+ cputime_t utime, stime;
	208	+#endif
205	209	int err = 0;
206	210
207	211	sigfillset(&current->blocked);
@@ -303,9 +307,10 @@ mISDNStackd(void *data) · 95 unmodified lines
303	307	"msg %d sleep %d stopped\n",
304	308	dev_name(&st->dev->dev), st->msg_cnt, st->sleep_cnt,
305	309	st->stopped_cnt);
	310	+ task_cputime(st->thread, &utime, &stime);
306	311	printk(KERN_DEBUG
307	312	"mISDNStackd daemon for %s utime(%ld) stime(%ld)\n",
308		− dev_name(&st->dev->dev), st->thread->utime, st->thread->stime);
	313	+ dev_name(&st->dev->dev), utime, stime);
309	314	printk(KERN_DEBUG
310	315	"mISDNStackd daemon for %s nvcsw(%ld) nivcsw(%ld)\n",
311	316	dev_name(&st->dev->dev), st->thread->nvcsw, st->thread->nivcsw);

drivers/spi/spi.c+1−1

@@ -33,7 +33,7 @@ · 32 unmodified lines
33	33	#include <linux/of_gpio.h>
34	34	#include <linux/pm_runtime.h>
35	35	#include <linux/export.h>
36		−#include <linux/sched.h>
	36	+#include <linux/sched/rt.h>
37	37	#include <linux/delay.h>
38	38	#include <linux/kthread.h>
39	39	#include <linux/ioport.h>

drivers/staging/csr/bh.c+1−1

@@ -15,7 +15,7 @@ · 14 unmodified lines
15	15	*/
16	16	#include "csr_wifi_hip_unifi.h"
17	17	#include "unifi_priv.h"
18		−
	18	+#include <linux/sched/rt.h>
19	19
20	20	/*
21	21	* ---------------------------------------------------------------------------

drivers/staging/csr/unifi_sme.c+1−1

@@ -15,7 +15,7 @@ · 14 unmodified lines
15	15	#include "unifi_priv.h"
16	16	#include "csr_wifi_hip_unifi.h"
17	17	#include "csr_wifi_hip_conversions.h"
18		−
	18	+#include <linux/sched/rt.h>
19	19
20	20
21	21

drivers/tty/sysrq.c+1−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16	16
17	17	#include <linux/sched.h>
	18	+#include <linux/sched/rt.h>
18	19	#include <linux/interrupt.h>
19	20	#include <linux/mm.h>
20	21	#include <linux/fs.h>

fs/binfmt_elf.c+6−2

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33	#include <linux/elf.h>
34	34	#include <linux/utsname.h>
35	35	#include <linux/coredump.h>
	36	+#include <linux/sched.h>
36	37	#include <asm/uaccess.h>
37	38	#include <asm/param.h>
38	39	#include <asm/page.h>
@@ -1320,8 +1321,11 @@ static void fill_prstatus(struct elf_prstatus *prstatus, · 1281 unmodified lines
1320	1321	cputime_to_timeval(cputime.utime, &prstatus->pr_utime);
1321	1322	cputime_to_timeval(cputime.stime, &prstatus->pr_stime);
1322	1323	} else {
1323		− cputime_to_timeval(p->utime, &prstatus->pr_utime);
1324		− cputime_to_timeval(p->stime, &prstatus->pr_stime);
	1324	+ cputime_t utime, stime;
	1325	+
	1326	+ task_cputime(p, &utime, &stime);
	1327	+ cputime_to_timeval(utime, &prstatus->pr_utime);
	1328	+ cputime_to_timeval(stime, &prstatus->pr_stime);
1325	1329	}
1326	1330	cputime_to_timeval(p->signal->cutime, &prstatus->pr_cutime);
1327	1331	cputime_to_timeval(p->signal->cstime, &prstatus->pr_cstime);

fs/binfmt_elf_fdpic.c+5−2

@@ -1375,8 +1375,11 @@ static void fill_prstatus(struct elf_prstatus *prstatus, · 1374 unmodified lines
1375	1375	cputime_to_timeval(cputime.utime, &prstatus->pr_utime);
1376	1376	cputime_to_timeval(cputime.stime, &prstatus->pr_stime);
1377	1377	} else {
1378		− cputime_to_timeval(p->utime, &prstatus->pr_utime);
1379		− cputime_to_timeval(p->stime, &prstatus->pr_stime);
	1378	+ cputime_t utime, stime;
	1379	+
	1380	+ task_cputime(p, &utime, &stime);
	1381	+ cputime_to_timeval(utime, &prstatus->pr_utime);
	1382	+ cputime_to_timeval(stime, &prstatus->pr_stime);
1380	1383	}
1381	1384	cputime_to_timeval(p->signal->cutime, &prstatus->pr_cutime);
1382	1385	cputime_to_timeval(p->signal->cstime, &prstatus->pr_cstime);

fs/proc/array.c+2−2

@@ -449,7 +449,7 @@ static int do_task_stat(struct seq_file m, struct pid_namespace ns, · 448 unmodified lines
449	449	do {
450	450	min_flt += t->min_flt;
451	451	maj_flt += t->maj_flt;
452		− gtime += t->gtime;
	452	+ gtime += task_gtime(t);
453	453	t = next_thread(t);
454	454	} while (t != task);
455	455
@@ -472,7 +472,7 @@ static int do_task_stat(struct seq_file m, struct pid_namespace ns, · 16 unmodified lines
472	472	min_flt = task->min_flt;
473	473	maj_flt = task->maj_flt;
474	474	task_cputime_adjusted(task, &utime, &stime);
475		− gtime = task->gtime;
	475	+ gtime = task_gtime(task);
476	476	}
477	477
478	478	/* scale priority and nice values from timeslices to -20..20 */

fs/select.c+1−0

@@ -26,6 +26,7 @@ · 25 unmodified lines
26	26	#include <linux/fs.h>
27	27	#include <linux/rcupdate.h>
28	28	#include <linux/hrtimer.h>
	29	+#include <linux/sched/rt.h>
29	30
30	31	#include <asm/uaccess.h>
31	32

include/asm-generic/cputime.h+6−60

@@ -4,66 +4,12 @@ · 3 unmodified lines
4	4	#include <linux/time.h>
5	5	#include <linux/jiffies.h>
6	6
7		−typedef unsigned long __nocast cputime_t;
8		−
9		−#define cputime_one_jiffy jiffies_to_cputime(1)
10		−#define cputime_to_jiffies(__ct) (__force unsigned long)(__ct)
11		−#define cputime_to_scaled(__ct) (__ct)
12		−#define jiffies_to_cputime(__hz) (__force cputime_t)(__hz)
13		−
14		−typedef u64 __nocast cputime64_t;
15		−
16		−#define cputime64_to_jiffies64(__ct) (__force u64)(__ct)
17		−#define jiffies64_to_cputime64(__jif) (__force cputime64_t)(__jif)
18		−
19		−#define nsecs_to_cputime64(__ct) \
20		− jiffies64_to_cputime64(nsecs_to_jiffies64(__ct))
21		−
22		−
23		−/*
24		− * Convert cputime to microseconds and back.
25		− */
26		−#define cputime_to_usecs(__ct) \
27		− jiffies_to_usecs(cputime_to_jiffies(__ct))
28		−#define usecs_to_cputime(__usec) \
29		− jiffies_to_cputime(usecs_to_jiffies(__usec))
30		−#define usecs_to_cputime64(__usec) \
31		− jiffies64_to_cputime64(nsecs_to_jiffies64((__usec) * 1000))
32		−
33		−/*
34		− * Convert cputime to seconds and back.
35		− */
36		−#define cputime_to_secs(jif) (cputime_to_jiffies(jif) / HZ)
37		−#define secs_to_cputime(sec) jiffies_to_cputime((sec) * HZ)
38		−
39		−/*
40		− * Convert cputime to timespec and back.
41		− */
42		−#define timespec_to_cputime(__val) \
43		− jiffies_to_cputime(timespec_to_jiffies(__val))
44		−#define cputime_to_timespec(__ct,__val) \
45		− jiffies_to_timespec(cputime_to_jiffies(__ct),__val)
46		−
47		−/*
48		− * Convert cputime to timeval and back.
49		− */
50		−#define timeval_to_cputime(__val) \
51		− jiffies_to_cputime(timeval_to_jiffies(__val))
52		−#define cputime_to_timeval(__ct,__val) \
53		− jiffies_to_timeval(cputime_to_jiffies(__ct),__val)
54		−
55		−/*
56		− * Convert cputime to clock and back.
57		− */
58		−#define cputime_to_clock_t(__ct) \
59		− jiffies_to_clock_t(cputime_to_jiffies(__ct))
60		−#define clock_t_to_cputime(__x) \
61		− jiffies_to_cputime(clock_t_to_jiffies(__x))
	7	+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
	8	+# include <asm-generic/cputime_jiffies.h>
	9	+#endif
62	10
63		−/*
64		− * Convert cputime64 to clock.
65		− */
66		−#define cputime64_to_clock_t(__ct) \
67		− jiffies_64_to_clock_t(cputime64_to_jiffies64(__ct))
	11	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
	12	+# include <asm-generic/cputime_nsecs.h>
	13	+#endif
68	14
69	15	#endif

include/asm-generic/cputime_jiffies.h+72−0

@@ -0,0 +1,72 @@
	1	+#ifndef _ASM_GENERIC_CPUTIME_JIFFIES_H
	2	+#define _ASM_GENERIC_CPUTIME_JIFFIES_H
	3	+
	4	+typedef unsigned long __nocast cputime_t;
	5	+
	6	+#define cputime_one_jiffy jiffies_to_cputime(1)
	7	+#define cputime_to_jiffies(__ct) (__force unsigned long)(__ct)
	8	+#define cputime_to_scaled(__ct) (__ct)
	9	+#define jiffies_to_cputime(__hz) (__force cputime_t)(__hz)
	10	+
	11	+typedef u64 __nocast cputime64_t;
	12	+
	13	+#define cputime64_to_jiffies64(__ct) (__force u64)(__ct)
	14	+#define jiffies64_to_cputime64(__jif) (__force cputime64_t)(__jif)
	15	+
	16	+
	17	+/*
	18	+ * Convert nanoseconds to cputime
	19	+ */
	20	+#define nsecs_to_cputime64(__nsec) \
	21	+ jiffies64_to_cputime64(nsecs_to_jiffies64(__nsec))
	22	+#define nsecs_to_cputime(__nsec) \
	23	+ jiffies_to_cputime(nsecs_to_jiffies(__nsec))
	24	+
	25	+
	26	+/*
	27	+ * Convert cputime to microseconds and back.
	28	+ */
	29	+#define cputime_to_usecs(__ct) \
	30	+ jiffies_to_usecs(cputime_to_jiffies(__ct))
	31	+#define usecs_to_cputime(__usec) \
	32	+ jiffies_to_cputime(usecs_to_jiffies(__usec))
	33	+#define usecs_to_cputime64(__usec) \
	34	+ jiffies64_to_cputime64(nsecs_to_jiffies64((__usec) * 1000))
	35	+
	36	+/*
	37	+ * Convert cputime to seconds and back.
	38	+ */
	39	+#define cputime_to_secs(jif) (cputime_to_jiffies(jif) / HZ)
	40	+#define secs_to_cputime(sec) jiffies_to_cputime((sec) * HZ)
	41	+
	42	+/*
	43	+ * Convert cputime to timespec and back.
	44	+ */
	45	+#define timespec_to_cputime(__val) \
	46	+ jiffies_to_cputime(timespec_to_jiffies(__val))
	47	+#define cputime_to_timespec(__ct,__val) \
	48	+ jiffies_to_timespec(cputime_to_jiffies(__ct),__val)
	49	+
	50	+/*
	51	+ * Convert cputime to timeval and back.
	52	+ */
	53	+#define timeval_to_cputime(__val) \
	54	+ jiffies_to_cputime(timeval_to_jiffies(__val))
	55	+#define cputime_to_timeval(__ct,__val) \
	56	+ jiffies_to_timeval(cputime_to_jiffies(__ct),__val)
	57	+
	58	+/*
	59	+ * Convert cputime to clock and back.
	60	+ */
	61	+#define cputime_to_clock_t(__ct) \
	62	+ jiffies_to_clock_t(cputime_to_jiffies(__ct))
	63	+#define clock_t_to_cputime(__x) \
	64	+ jiffies_to_cputime(clock_t_to_jiffies(__x))
	65	+
	66	+/*
	67	+ * Convert cputime64 to clock.
	68	+ */
	69	+#define cputime64_to_clock_t(__ct) \
	70	+ jiffies_64_to_clock_t(cputime64_to_jiffies64(__ct))
	71	+
	72	+#endif

include/asm-generic/cputime_nsecs.h+104−0

@@ -0,0 +1,104 @@
	1	+/*
	2	+ * Definitions for measuring cputime in nsecs resolution.
	3	+ *
	4	+ * Based on <arch/ia64/include/asm/cputime.h>
	5	+ *
	6	+ * Copyright (C) 2007 FUJITSU LIMITED
	7	+ * Copyright (C) 2007 Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or
	10	+ * modify it under the terms of the GNU General Public License
	11	+ * as published by the Free Software Foundation; either version
	12	+ * 2 of the License, or (at your option) any later version.
	13	+ *
	14	+ */
	15	+
	16	+#ifndef _ASM_GENERIC_CPUTIME_NSECS_H
	17	+#define _ASM_GENERIC_CPUTIME_NSECS_H
	18	+
	19	+typedef u64 __nocast cputime_t;
	20	+typedef u64 __nocast cputime64_t;
	21	+
	22	+#define cputime_one_jiffy jiffies_to_cputime(1)
	23	+
	24	+/*
	25	+ * Convert cputime <-> jiffies (HZ)
	26	+ */
	27	+#define cputime_to_jiffies(__ct) \
	28	+ ((__force u64)(__ct) / (NSEC_PER_SEC / HZ))
	29	+#define cputime_to_scaled(__ct) (__ct)
	30	+#define jiffies_to_cputime(__jif) \
	31	+ (__force cputime_t)((__jif) * (NSEC_PER_SEC / HZ))
	32	+#define cputime64_to_jiffies64(__ct) \
	33	+ ((__force u64)(__ct) / (NSEC_PER_SEC / HZ))
	34	+#define jiffies64_to_cputime64(__jif) \
	35	+ (__force cputime64_t)((__jif) * (NSEC_PER_SEC / HZ))
	36	+
	37	+
	38	+/*
	39	+ * Convert cputime <-> nanoseconds
	40	+ */
	41	+#define nsecs_to_cputime(__nsecs) ((__force u64)(__nsecs))
	42	+
	43	+
	44	+/*
	45	+ * Convert cputime <-> microseconds
	46	+ */
	47	+#define cputime_to_usecs(__ct) \
	48	+ ((__force u64)(__ct) / NSEC_PER_USEC)
	49	+#define usecs_to_cputime(__usecs) \
	50	+ (__force cputime_t)((__usecs) * NSEC_PER_USEC)
	51	+#define usecs_to_cputime64(__usecs) \
	52	+ (__force cputime64_t)((__usecs) * NSEC_PER_USEC)
	53	+
	54	+/*
	55	+ * Convert cputime <-> seconds
	56	+ */
	57	+#define cputime_to_secs(__ct) \
	58	+ ((__force u64)(__ct) / NSEC_PER_SEC)
	59	+#define secs_to_cputime(__secs) \
	60	+ (__force cputime_t)((__secs) * NSEC_PER_SEC)
	61	+
	62	+/*
	63	+ * Convert cputime <-> timespec (nsec)
	64	+ */
	65	+static inline cputime_t timespec_to_cputime(const struct timespec *val)
	66	+{
	67	+ u64 ret = val->tv_sec * NSEC_PER_SEC + val->tv_nsec;
	68	+ return (__force cputime_t) ret;
	69	+}
	70	+static inline void cputime_to_timespec(const cputime_t ct, struct timespec *val)
	71	+{
	72	+ val->tv_sec = (__force u64) ct / NSEC_PER_SEC;
	73	+ val->tv_nsec = (__force u64) ct % NSEC_PER_SEC;
	74	+}
	75	+
	76	+/*
	77	+ * Convert cputime <-> timeval (msec)
	78	+ */
	79	+static inline cputime_t timeval_to_cputime(struct timeval *val)
	80	+{
	81	+ u64 ret = val->tv_sec * NSEC_PER_SEC + val->tv_usec * NSEC_PER_USEC;
	82	+ return (__force cputime_t) ret;
	83	+}
	84	+static inline void cputime_to_timeval(const cputime_t ct, struct timeval *val)
	85	+{
	86	+ val->tv_sec = (__force u64) ct / NSEC_PER_SEC;
	87	+ val->tv_usec = ((__force u64) ct % NSEC_PER_SEC) / NSEC_PER_USEC;
	88	+}
	89	+
	90	+/*
	91	+ * Convert cputime <-> clock (USER_HZ)
	92	+ */
	93	+#define cputime_to_clock_t(__ct) \
	94	+ ((__force u64)(__ct) / (NSEC_PER_SEC / USER_HZ))
	95	+#define clock_t_to_cputime(__x) \
	96	+ (__force cputime_t)((__x) * (NSEC_PER_SEC / USER_HZ))
	97	+
	98	+/*
	99	+ * Convert cputime64 to clock.
	100	+ */
	101	+#define cputime64_to_clock_t(__ct) \
	102	+ cputime_to_clock_t((__force cputime_t)__ct)
	103	+
	104	+#endif

include/linux/context_tracking.h+28−0

@@ -3,12 +3,40 @@ · 2 unmodified lines
3	3
4	4	#ifdef CONFIG_CONTEXT_TRACKING
5	5	#include <linux/sched.h>
	6	+#include <linux/percpu.h>
	7	+
	8	+struct context_tracking {
	9	+ /*
	10	+ * When active is false, probes are unset in order
	11	+ * to minimize overhead: TIF flags are cleared
	12	+ * and calls to user_enter/exit are ignored. This
	13	+ * may be further optimized using static keys.
	14	+ */
	15	+ bool active;
	16	+ enum {
	17	+ IN_KERNEL = 0,
	18	+ IN_USER,
	19	+ } state;
	20	+};
	21	+
	22	+DECLARE_PER_CPU(struct context_tracking, context_tracking);
	23	+
	24	+static inline bool context_tracking_in_user(void)
	25	+{
	26	+ return __this_cpu_read(context_tracking.state) == IN_USER;
	27	+}
	28	+
	29	+static inline bool context_tracking_active(void)
	30	+{
	31	+ return __this_cpu_read(context_tracking.active);
	32	+}
6	33
7	34	extern void user_enter(void);
8	35	extern void user_exit(void);
9	36	extern void context_tracking_task_switch(struct task_struct *prev,
10	37	struct task_struct *next);
11	38	#else
	39	+static inline bool context_tracking_in_user(void) { return false; }
12	40	static inline void user_enter(void) { }
13	41	static inline void user_exit(void) { }
14	42	static inline void context_tracking_task_switch(struct task_struct *prev,

include/linux/hardirq.h+2−2

@@ -153,7 +153,7 @@ extern void rcu_nmi_exit(void); · 152 unmodified lines
153	153	*/
154	154	#define __irq_enter() \
155	155	do { \
156		− vtime_account_irq_enter(current); \
	156	+ account_irq_enter_time(current); \
157	157	add_preempt_count(HARDIRQ_OFFSET); \
158	158	trace_hardirq_enter(); \
159	159	} while (0)
@@ -169,7 +169,7 @@ extern void irq_enter(void); · 9 unmodified lines
169	169	#define __irq_exit() \
170	170	do { \
171	171	trace_hardirq_exit(); \
172		− vtime_account_irq_exit(current); \
	172	+ account_irq_exit_time(current); \
173	173	sub_preempt_count(HARDIRQ_OFFSET); \
174	174	} while (0)
175	175

include/linux/init_task.h+12−0

@@ -10,7 +10,9 @@ · 9 unmodified lines
10	10	#include <linux/pid_namespace.h>
11	11	#include <linux/user_namespace.h>
12	12	#include <linux/securebits.h>
	13	+#include <linux/seqlock.h>
13	14	#include <net/net_namespace.h>
	15	+#include <linux/sched/rt.h>
14	16
15	17	#ifdef CONFIG_SMP
16	18	# define INIT_PUSHABLE_TASKS(tsk) \
@@ -141,6 +143,15 @@ extern struct task_group root_task_group; · 124 unmodified lines
141	143	# define INIT_PERF_EVENTS(tsk)
142	144	#endif
143	145
	146	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
	147	+# define INIT_VTIME(tsk) \
	148	+ .vtime_seqlock = __SEQLOCK_UNLOCKED(tsk.vtime_seqlock), \
	149	+ .vtime_snap = 0, \
	150	+ .vtime_snap_whence = VTIME_SYS,
	151	+#else
	152	+# define INIT_VTIME(tsk)
	153	+#endif
	154	+
144	155	#define INIT_TASK_COMM "swapper"
145	156
146	157	/*
@@ -210,6 +221,7 @@ extern struct task_group root_task_group; · 63 unmodified lines
210	221	INIT_TRACE_RECURSION \
211	222	INIT_TASK_RCU_PREEMPT(tsk) \
212	223	INIT_CPUSET_SEQ \
	224	+ INIT_VTIME(tsk) \
213	225	}
214	226
215	227

include/linux/kernel_stat.h+1−1

@@ -127,7 +127,7 @@ extern void account_system_time(struct task_struct *, int, cputime_t, cputime_t) · 126 unmodified lines
127	127	extern void account_steal_time(cputime_t);
128	128	extern void account_idle_time(cputime_t);
129	129
130		−#ifdef CONFIG_VIRT_CPU_ACCOUNTING
	130	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
131	131	static inline void account_process_tick(struct task_struct *tsk, int user)
132	132	{
133	133	vtime_account_user(tsk);

include/linux/kvm_host.h+46−9

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	#include <linux/rcupdate.h>
23	23	#include <linux/ratelimit.h>
24	24	#include <linux/err.h>
	25	+#include <linux/irqflags.h>
25	26	#include <asm/signal.h>
26	27
27	28	#include <linux/kvm.h>
@@ -740,15 +741,52 @@ static inline int kvm_deassign_device(struct kvm *kvm, · 712 unmodified lines
740	741	}
741	742	#endif /* CONFIG_IOMMU_API */
742	743
743		−static inline void kvm_guest_enter(void)
	744	+static inline void __guest_enter(void)
744	745	{
745		− BUG_ON(preemptible());
746	746	/*
747	747	* This is running in ioctl context so we can avoid
748	748	* the call to vtime_account() with its unnecessary idle check.
749	749	*/
750		− vtime_account_system_irqsafe(current);
	750	+ vtime_account_system(current);
751	751	current->flags \|= PF_VCPU;
	752	+}
	753	+
	754	+static inline void __guest_exit(void)
	755	+{
	756	+ /*
	757	+ * This is running in ioctl context so we can avoid
	758	+ * the call to vtime_account() with its unnecessary idle check.
	759	+ */
	760	+ vtime_account_system(current);
	761	+ current->flags &= ~PF_VCPU;
	762	+}
	763	+
	764	+#ifdef CONFIG_CONTEXT_TRACKING
	765	+extern void guest_enter(void);
	766	+extern void guest_exit(void);
	767	+
	768	+#else /* !CONFIG_CONTEXT_TRACKING */
	769	+static inline void guest_enter(void)
	770	+{
	771	+ __guest_enter();
	772	+}
	773	+
	774	+static inline void guest_exit(void)
	775	+{
	776	+ __guest_exit();
	777	+}
	778	+#endif /* !CONFIG_CONTEXT_TRACKING */
	779	+
	780	+static inline void kvm_guest_enter(void)
	781	+{
	782	+ unsigned long flags;
	783	+
	784	+ BUG_ON(preemptible());
	785	+
	786	+ local_irq_save(flags);
	787	+ guest_enter();
	788	+ local_irq_restore(flags);
	789	+
752	790	/* KVM does not hold any references to rcu protected data when it
753	791	* switches CPU into a guest mode. In fact switching to a guest mode
754	792	* is very similar to exiting to userspase from rcu point of view. In
@@ -761,12 +799,11 @@ static inline void kvm_guest_enter(void) · 6 unmodified lines
761	799
762	800	static inline void kvm_guest_exit(void)
763	801	{
764		− /*
765		− * This is running in ioctl context so we can avoid
766		− * the call to vtime_account() with its unnecessary idle check.
767		− */
768		− vtime_account_system_irqsafe(current);
769		− current->flags &= ~PF_VCPU;
	802	+ unsigned long flags;
	803	+
	804	+ local_irq_save(flags);
	805	+ guest_exit();
	806	+ local_irq_restore(flags);
770	807	}
771	808
772	809	/*

include/linux/sched.h+42−143

@@ -304,19 +304,6 @@ static inline void lockup_detector_init(void) · 303 unmodified lines
304	304	}
305	305	#endif
306	306
307		−#ifdef CONFIG_DETECT_HUNG_TASK
308		−extern unsigned int sysctl_hung_task_panic;
309		−extern unsigned long sysctl_hung_task_check_count;
310		−extern unsigned long sysctl_hung_task_timeout_secs;
311		−extern unsigned long sysctl_hung_task_warnings;
312		−extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
313		− void __user *buffer,
314		− size_t lenp, loff_t ppos);
315		−#else
316		−/* Avoid need for ifdefs elsewhere in the code */
317		−enum { sysctl_hung_task_timeout_secs = 0 };
318		−#endif
319		−
320	307	/* Attach to any functions which should be ignored in wchan output. */
321	308	#define __sched __attribute__((__section__(".sched.text")))
322	309
@@ -338,23 +325,6 @@ extern int mutex_spin_on_owner(struct mutex lock, struct task_struct owner); · 15 unmodified lines
338	325	struct nsproxy;
339	326	struct user_namespace;
340	327
341		−/*
342		− * Default maximum number of active map areas, this limits the number of vmas
343		− * per mm struct. Users can overwrite this number by sysctl but there is a
344		− * problem.
345		− *
346		− * When a program's coredump is generated as ELF format, a section is created
347		− * per a vma. In ELF, the number of sections is represented in unsigned short.
348		− * This means the number of sections should be smaller than 65535 at coredump.
349		− * Because the kernel adds some informative sections to a image of program at
350		− * generating coredump, we need some margin. The number of extra sections is
351		− * 1-3 now and depends on arch. We use "5" as safe margin, here.
352		− */
353		−#define MAPCOUNT_ELF_CORE_MARGIN (5)
354		−#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
355		−
356		−extern int sysctl_max_map_count;
357		−
358	328	#include <linux/aio.h>
359	329
360	330	#ifdef CONFIG_MMU
@@ -1194,6 +1164,7 @@ struct sched_entity { · 833 unmodified lines
1194	1164	/* rq "owned" by this entity/group: */
1195	1165	struct cfs_rq *my_q;
1196	1166	#endif
	1167	+
1197	1168	/*
1198	1169	* Load-tracking only depends on SMP, FAIR_GROUP_SCHED dependency below may be
1199	1170	* removed when useful for applications beyond shares distribution (e.g.
@@ -1208,6 +1179,7 @@ struct sched_entity { · 8 unmodified lines
1208	1179	struct sched_rt_entity {
1209	1180	struct list_head run_list;
1210	1181	unsigned long timeout;
	1182	+ unsigned long watchdog_stamp;
1211	1183	unsigned int time_slice;
1212	1184
1213	1185	struct sched_rt_entity *back;
@@ -1220,11 +1192,6 @@ struct sched_rt_entity { · 6 unmodified lines
1220	1192	#endif
1221	1193	};
1222	1194
1223		−/*
1224		− * default timeslice is 100 msecs (used only for SCHED_RR tasks).
1225		− * Timeslices get refilled after they expire.
1226		− */
1227		−#define RR_TIMESLICE (100 * HZ / 1000)
1228	1195
1229	1196	struct rcu_node;
1230	1197
@@ -1367,6 +1334,15 @@ struct task_struct { · 136 unmodified lines
1367	1334	cputime_t gtime;
1368	1335	#ifndef CONFIG_VIRT_CPU_ACCOUNTING
1369	1336	struct cputime prev_cputime;
	1337	+#endif
	1338	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
	1339	+ seqlock_t vtime_seqlock;
	1340	+ unsigned long long vtime_snap;
	1341	+ enum {
	1342	+ VTIME_SLEEPING = 0,
	1343	+ VTIME_USER,
	1344	+ VTIME_SYS,
	1345	+ } vtime_snap_whence;
1370	1346	#endif
1371	1347	unsigned long nvcsw, nivcsw; /* context switch counts */
1372	1348	struct timespec start_time; /* monotonic time */
@@ -1622,37 +1598,6 @@ static inline void set_numabalancing_state(bool enabled) · 249 unmodified lines
1622	1598	}
1623	1599	#endif
1624	1600
1625		−/*
1626		− * Priority of a process goes from 0..MAX_PRIO-1, valid RT
1627		− * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH
1628		− * tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority
1629		− * values are inverted: lower p->prio value means higher priority.
1630		− *
1631		− * The MAX_USER_RT_PRIO value allows the actual maximum
1632		− * RT priority to be separate from the value exported to
1633		− * user-space. This allows kernel threads to set their
1634		− * priority to a value higher than any user task. Note:
1635		− * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO.
1636		− */
1637		−
1638		−#define MAX_USER_RT_PRIO 100
1639		−#define MAX_RT_PRIO MAX_USER_RT_PRIO
1640		−
1641		−#define MAX_PRIO (MAX_RT_PRIO + 40)
1642		−#define DEFAULT_PRIO (MAX_RT_PRIO + 20)
1643		−
1644		−static inline int rt_prio(int prio)
1645		−{
1646		− if (unlikely(prio < MAX_RT_PRIO))
1647		− return 1;
1648		− return 0;
1649		−}
1650		−
1651		−static inline int rt_task(struct task_struct *p)
1652		−{
1653		− return rt_prio(p->prio);
1654		−}
1655		−
1656	1601	static inline struct pid task_pid(struct task_struct task)
1657	1602	{
1658	1603	return task->pids[PIDTYPE_PID].pid;
@@ -1792,6 +1737,37 @@ static inline void put_task_struct(struct task_struct *t) · 133 unmodified lines
1792	1737	__put_task_struct(t);
1793	1738	}
1794	1739
	1740	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
	1741	+extern void task_cputime(struct task_struct *t,
	1742	+ cputime_t utime, cputime_t stime);
	1743	+extern void task_cputime_scaled(struct task_struct *t,
	1744	+ cputime_t utimescaled, cputime_t stimescaled);
	1745	+extern cputime_t task_gtime(struct task_struct *t);
	1746	+#else
	1747	+static inline void task_cputime(struct task_struct *t,
	1748	+ cputime_t utime, cputime_t stime)
	1749	+{
	1750	+ if (utime)
	1751	+ *utime = t->utime;
	1752	+ if (stime)
	1753	+ *stime = t->stime;
	1754	+}
	1755	+
	1756	+static inline void task_cputime_scaled(struct task_struct *t,
	1757	+ cputime_t *utimescaled,
	1758	+ cputime_t *stimescaled)
	1759	+{
	1760	+ if (utimescaled)
	1761	+ *utimescaled = t->utimescaled;
	1762	+ if (stimescaled)
	1763	+ *stimescaled = t->stimescaled;
	1764	+}
	1765	+
	1766	+static inline cputime_t task_gtime(struct task_struct *t)
	1767	+{
	1768	+ return t->gtime;
	1769	+}
	1770	+#endif
1795	1771	extern void task_cputime_adjusted(struct task_struct p, cputime_t ut, cputime_t *st);
1796	1772	extern void thread_group_cputime_adjusted(struct task_struct p, cputime_t ut, cputime_t *st);
1797	1773
@@ -2033,58 +2009,7 @@ extern void wake_up_idle_cpu(int cpu); · 235 unmodified lines
2033	2009	static inline void wake_up_idle_cpu(int cpu) { }
2034	2010	#endif
2035	2011
2036		−extern unsigned int sysctl_sched_latency;
2037		−extern unsigned int sysctl_sched_min_granularity;
2038		−extern unsigned int sysctl_sched_wakeup_granularity;
2039		−extern unsigned int sysctl_sched_child_runs_first;
2040		−
2041		−enum sched_tunable_scaling {
2042		− SCHED_TUNABLESCALING_NONE,
2043		− SCHED_TUNABLESCALING_LOG,
2044		− SCHED_TUNABLESCALING_LINEAR,
2045		− SCHED_TUNABLESCALING_END,
2046		−};
2047		−extern enum sched_tunable_scaling sysctl_sched_tunable_scaling;
2048		−
2049		−extern unsigned int sysctl_numa_balancing_scan_delay;
2050		−extern unsigned int sysctl_numa_balancing_scan_period_min;
2051		−extern unsigned int sysctl_numa_balancing_scan_period_max;
2052		−extern unsigned int sysctl_numa_balancing_scan_period_reset;
2053		−extern unsigned int sysctl_numa_balancing_scan_size;
2054		−extern unsigned int sysctl_numa_balancing_settle_count;
2055		−
2056		−#ifdef CONFIG_SCHED_DEBUG
2057		−extern unsigned int sysctl_sched_migration_cost;
2058		−extern unsigned int sysctl_sched_nr_migrate;
2059		−extern unsigned int sysctl_sched_time_avg;
2060		−extern unsigned int sysctl_timer_migration;
2061		−extern unsigned int sysctl_sched_shares_window;
2062		−
2063		−int sched_proc_update_handler(struct ctl_table *table, int write,
2064		− void __user buffer, size_t length,
2065		− loff_t *ppos);
2066		−#endif
2067		−#ifdef CONFIG_SCHED_DEBUG
2068		−static inline unsigned int get_sysctl_timer_migration(void)
2069		−{
2070		− return sysctl_timer_migration;
2071		−}
2072		−#else
2073		−static inline unsigned int get_sysctl_timer_migration(void)
2074		−{
2075		− return 1;
2076		−}
2077		−#endif
2078		−extern unsigned int sysctl_sched_rt_period;
2079		−extern int sysctl_sched_rt_runtime;
2080		−
2081		−int sched_rt_handler(struct ctl_table *table, int write,
2082		− void __user buffer, size_t lenp,
2083		− loff_t *ppos);
2084		−
2085	2012	#ifdef CONFIG_SCHED_AUTOGROUP
2086		−extern unsigned int sysctl_sched_autogroup_enabled;
2087		−
2088	2013	extern void sched_autogroup_create_attach(struct task_struct *p);
2089	2014	extern void sched_autogroup_detach(struct task_struct *p);
2090	2015	extern void sched_autogroup_fork(struct signal_struct *sig);
@@ -2100,30 +2025,6 @@ static inline void sched_autogroup_fork(struct signal_struct *sig) { } · 9 unmodified lines
2100	2025	static inline void sched_autogroup_exit(struct signal_struct *sig) { }
2101	2026	#endif
2102	2027
2103		−#ifdef CONFIG_CFS_BANDWIDTH
2104		−extern unsigned int sysctl_sched_cfs_bandwidth_slice;
2105		−#endif
2106		−
2107		−#ifdef CONFIG_RT_MUTEXES
2108		−extern int rt_mutex_getprio(struct task_struct *p);
2109		−extern void rt_mutex_setprio(struct task_struct *p, int prio);
2110		−extern void rt_mutex_adjust_pi(struct task_struct *p);
2111		−static inline bool tsk_is_pi_blocked(struct task_struct *tsk)
2112		−{
2113		− return tsk->pi_blocked_on != NULL;
2114		−}
2115		−#else
2116		−static inline int rt_mutex_getprio(struct task_struct *p)
2117		−{
2118		− return p->normal_prio;
2119		−}
2120		−# define rt_mutex_adjust_pi(p) do { } while (0)
2121		−static inline bool tsk_is_pi_blocked(struct task_struct *tsk)
2122		−{
2123		− return false;
2124		−}
2125		−#endif
2126		−
2127	2028	extern bool yield_to(struct task_struct *p, bool preempt);
2128	2029	extern void set_user_nice(struct task_struct *p, long nice);
2129	2030	extern int task_prio(const struct task_struct *p);
@@ -2753,8 +2654,6 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) · 623 unmodified lines
2753	2654	extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
2754	2655	extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
2755	2656
2756		−extern void normalize_rt_tasks(void);
2757		−
2758	2657	#ifdef CONFIG_CGROUP_SCHED
2759	2658
2760	2659	extern struct task_group root_task_group;

include/linux/sched/rt.h+58−0

@@ -0,0 +1,58 @@
	1	+#ifndef _SCHED_RT_H
	2	+#define _SCHED_RT_H
	3	+
	4	+/*
	5	+ * Priority of a process goes from 0..MAX_PRIO-1, valid RT
	6	+ * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH
	7	+ * tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority
	8	+ * values are inverted: lower p->prio value means higher priority.
	9	+ *
	10	+ * The MAX_USER_RT_PRIO value allows the actual maximum
	11	+ * RT priority to be separate from the value exported to
	12	+ * user-space. This allows kernel threads to set their
	13	+ * priority to a value higher than any user task. Note:
	14	+ * MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO.
	15	+ */
	16	+
	17	+#define MAX_USER_RT_PRIO 100
	18	+#define MAX_RT_PRIO MAX_USER_RT_PRIO
	19	+
	20	+#define MAX_PRIO (MAX_RT_PRIO + 40)
	21	+#define DEFAULT_PRIO (MAX_RT_PRIO + 20)
	22	+
	23	+static inline int rt_prio(int prio)
	24	+{
	25	+ if (unlikely(prio < MAX_RT_PRIO))
	26	+ return 1;
	27	+ return 0;
	28	+}
	29	+
	30	+static inline int rt_task(struct task_struct *p)
	31	+{
	32	+ return rt_prio(p->prio);
	33	+}
	34	+
	35	+#ifdef CONFIG_RT_MUTEXES
	36	+extern int rt_mutex_getprio(struct task_struct *p);
	37	+extern void rt_mutex_setprio(struct task_struct *p, int prio);
	38	+extern void rt_mutex_adjust_pi(struct task_struct *p);
	39	+static inline bool tsk_is_pi_blocked(struct task_struct *tsk)
	40	+{
	41	+ return tsk->pi_blocked_on != NULL;
	42	+}
	43	+#else
	44	+static inline int rt_mutex_getprio(struct task_struct *p)
	45	+{
	46	+ return p->normal_prio;
	47	+}
	48	+# define rt_mutex_adjust_pi(p) do { } while (0)
	49	+static inline bool tsk_is_pi_blocked(struct task_struct *tsk)
	50	+{
	51	+ return false;
	52	+}
	53	+#endif
	54	+
	55	+extern void normalize_rt_tasks(void);
	56	+
	57	+
	58	+#endif /* _SCHED_RT_H */

include/linux/sched/sysctl.h+110−0

@@ -0,0 +1,110 @@
	1	+#ifndef _SCHED_SYSCTL_H
	2	+#define _SCHED_SYSCTL_H
	3	+
	4	+#ifdef CONFIG_DETECT_HUNG_TASK
	5	+extern unsigned int sysctl_hung_task_panic;
	6	+extern unsigned long sysctl_hung_task_check_count;
	7	+extern unsigned long sysctl_hung_task_timeout_secs;
	8	+extern unsigned long sysctl_hung_task_warnings;
	9	+extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
	10	+ void __user *buffer,
	11	+ size_t lenp, loff_t ppos);
	12	+#else
	13	+/* Avoid need for ifdefs elsewhere in the code */
	14	+enum { sysctl_hung_task_timeout_secs = 0 };
	15	+#endif
	16	+
	17	+/*
	18	+ * Default maximum number of active map areas, this limits the number of vmas
	19	+ * per mm struct. Users can overwrite this number by sysctl but there is a
	20	+ * problem.
	21	+ *
	22	+ * When a program's coredump is generated as ELF format, a section is created
	23	+ * per a vma. In ELF, the number of sections is represented in unsigned short.
	24	+ * This means the number of sections should be smaller than 65535 at coredump.
	25	+ * Because the kernel adds some informative sections to a image of program at
	26	+ * generating coredump, we need some margin. The number of extra sections is
	27	+ * 1-3 now and depends on arch. We use "5" as safe margin, here.
	28	+ */
	29	+#define MAPCOUNT_ELF_CORE_MARGIN (5)
	30	+#define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
	31	+
	32	+extern int sysctl_max_map_count;
	33	+
	34	+extern unsigned int sysctl_sched_latency;
	35	+extern unsigned int sysctl_sched_min_granularity;
	36	+extern unsigned int sysctl_sched_wakeup_granularity;
	37	+extern unsigned int sysctl_sched_child_runs_first;
	38	+
	39	+enum sched_tunable_scaling {
	40	+ SCHED_TUNABLESCALING_NONE,
	41	+ SCHED_TUNABLESCALING_LOG,
	42	+ SCHED_TUNABLESCALING_LINEAR,
	43	+ SCHED_TUNABLESCALING_END,
	44	+};
	45	+extern enum sched_tunable_scaling sysctl_sched_tunable_scaling;
	46	+
	47	+extern unsigned int sysctl_numa_balancing_scan_delay;
	48	+extern unsigned int sysctl_numa_balancing_scan_period_min;
	49	+extern unsigned int sysctl_numa_balancing_scan_period_max;
	50	+extern unsigned int sysctl_numa_balancing_scan_period_reset;
	51	+extern unsigned int sysctl_numa_balancing_scan_size;
	52	+extern unsigned int sysctl_numa_balancing_settle_count;
	53	+
	54	+#ifdef CONFIG_SCHED_DEBUG
	55	+extern unsigned int sysctl_sched_migration_cost;
	56	+extern unsigned int sysctl_sched_nr_migrate;
	57	+extern unsigned int sysctl_sched_time_avg;
	58	+extern unsigned int sysctl_timer_migration;
	59	+extern unsigned int sysctl_sched_shares_window;
	60	+
	61	+int sched_proc_update_handler(struct ctl_table *table, int write,
	62	+ void __user buffer, size_t length,
	63	+ loff_t *ppos);
	64	+#endif
	65	+#ifdef CONFIG_SCHED_DEBUG
	66	+static inline unsigned int get_sysctl_timer_migration(void)
	67	+{
	68	+ return sysctl_timer_migration;
	69	+}
	70	+#else
	71	+static inline unsigned int get_sysctl_timer_migration(void)
	72	+{
	73	+ return 1;
	74	+}
	75	+#endif
	76	+
	77	+/*
	78	+ * control realtime throttling:
	79	+ *
	80	+ * /proc/sys/kernel/sched_rt_period_us
	81	+ * /proc/sys/kernel/sched_rt_runtime_us
	82	+ */
	83	+extern unsigned int sysctl_sched_rt_period;
	84	+extern int sysctl_sched_rt_runtime;
	85	+
	86	+#ifdef CONFIG_CFS_BANDWIDTH
	87	+extern unsigned int sysctl_sched_cfs_bandwidth_slice;
	88	+#endif
	89	+
	90	+#ifdef CONFIG_SCHED_AUTOGROUP
	91	+extern unsigned int sysctl_sched_autogroup_enabled;
	92	+#endif
	93	+
	94	+/*
	95	+ * default timeslice is 100 msecs (used only for SCHED_RR tasks).
	96	+ * Timeslices get refilled after they expire.
	97	+ */
	98	+#define RR_TIMESLICE (100 * HZ / 1000)
	99	+
	100	+extern int sched_rr_timeslice;
	101	+
	102	+extern int sched_rr_handler(struct ctl_table *table, int write,
	103	+ void __user buffer, size_t lenp,
	104	+ loff_t *ppos);
	105	+
	106	+extern int sched_rt_handler(struct ctl_table *table, int write,
	107	+ void __user buffer, size_t lenp,
	108	+ loff_t *ppos);
	109	+
	110	+#endif /* _SCHED_SYSCTL_H */

include/linux/tsacct_kern.h+3−0

@@ -23,12 +23,15 @@ static inline void bacct_add_tsk(struct user_namespace *user_ns, · 22 unmodified lines
23	23	#ifdef CONFIG_TASK_XACCT
24	24	extern void xacct_add_tsk(struct taskstats stats, struct task_struct p);
25	25	extern void acct_update_integrals(struct task_struct *tsk);
	26	+extern void acct_account_cputime(struct task_struct *tsk);
26	27	extern void acct_clear_integrals(struct task_struct *tsk);
27	28	#else
28	29	static inline void xacct_add_tsk(struct taskstats stats, struct task_struct p)
29	30	{}
30	31	static inline void acct_update_integrals(struct task_struct *tsk)
31	32	{}
	33	+static inline void acct_account_cputime(struct task_struct *tsk)
	34	+{}
32	35	static inline void acct_clear_integrals(struct task_struct *tsk)
33	36	{}
34	37	#endif /* CONFIG_TASK_XACCT */

include/linux/vtime.h+40−19

@@ -6,15 +6,46 @@ struct task_struct; · 5 unmodified lines
6	6	#ifdef CONFIG_VIRT_CPU_ACCOUNTING
7	7	extern void vtime_task_switch(struct task_struct *prev);
8	8	extern void vtime_account_system(struct task_struct *tsk);
9		−extern void vtime_account_system_irqsafe(struct task_struct *tsk);
10	9	extern void vtime_account_idle(struct task_struct *tsk);
11	10	extern void vtime_account_user(struct task_struct *tsk);
12		−extern void vtime_account(struct task_struct *tsk);
13		−#else
	11	+extern void vtime_account_irq_enter(struct task_struct *tsk);
	12	+
	13	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
	14	+static inline bool vtime_accounting_enabled(void) { return true; }
	15	+#endif
	16	+
	17	+#else /* !CONFIG_VIRT_CPU_ACCOUNTING */
	18	+
14	19	static inline void vtime_task_switch(struct task_struct *prev) { }
15	20	static inline void vtime_account_system(struct task_struct *tsk) { }
16		−static inline void vtime_account_system_irqsafe(struct task_struct *tsk) { }
17		−static inline void vtime_account(struct task_struct *tsk) { }
	21	+static inline void vtime_account_user(struct task_struct *tsk) { }
	22	+static inline void vtime_account_irq_enter(struct task_struct *tsk) { }
	23	+static inline bool vtime_accounting_enabled(void) { return false; }
	24	+#endif
	25	+
	26	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
	27	+extern void arch_vtime_task_switch(struct task_struct *tsk);
	28	+extern void vtime_account_irq_exit(struct task_struct *tsk);
	29	+extern bool vtime_accounting_enabled(void);
	30	+extern void vtime_user_enter(struct task_struct *tsk);
	31	+static inline void vtime_user_exit(struct task_struct *tsk)
	32	+{
	33	+ vtime_account_user(tsk);
	34	+}
	35	+extern void vtime_guest_enter(struct task_struct *tsk);
	36	+extern void vtime_guest_exit(struct task_struct *tsk);
	37	+extern void vtime_init_idle(struct task_struct *tsk);
	38	+#else
	39	+static inline void vtime_account_irq_exit(struct task_struct *tsk)
	40	+{
	41	+ /* On hard\|softirq exit we always account to hard\|softirq cputime */
	42	+ vtime_account_system(tsk);
	43	+}
	44	+static inline void vtime_user_enter(struct task_struct *tsk) { }
	45	+static inline void vtime_user_exit(struct task_struct *tsk) { }
	46	+static inline void vtime_guest_enter(struct task_struct *tsk) { }
	47	+static inline void vtime_guest_exit(struct task_struct *tsk) { }
	48	+static inline void vtime_init_idle(struct task_struct *tsk) { }
18	49	#endif
19	50
20	51	#ifdef CONFIG_IRQ_TIME_ACCOUNTING
@@ -23,25 +54,15 @@ extern void irqtime_account_irq(struct task_struct *tsk); · 2 unmodified lines
23	54	static inline void irqtime_account_irq(struct task_struct *tsk) { }
24	55	#endif
25	56
26		−static inline void vtime_account_irq_enter(struct task_struct *tsk)
	57	+static inline void account_irq_enter_time(struct task_struct *tsk)
27	58	{
28		− /*
29		− * Hardirq can interrupt idle task anytime. So we need vtime_account()
30		− * that performs the idle check in CONFIG_VIRT_CPU_ACCOUNTING.
31		− * Softirq can also interrupt idle task directly if it calls
32		− * local_bh_enable(). Such case probably don't exist but we never know.
33		− * Ksoftirqd is not concerned because idle time is flushed on context
34		− * switch. Softirqs in the end of hardirqs are also not a problem because
35		− * the idle time is flushed on hardirq time already.
36		− */
37		− vtime_account(tsk);
	59	+ vtime_account_irq_enter(tsk);
38	60	irqtime_account_irq(tsk);
39	61	}
40	62
41		−static inline void vtime_account_irq_exit(struct task_struct *tsk)
	63	+static inline void account_irq_exit_time(struct task_struct *tsk)
42	64	{
43		− /* On hard\|softirq exit we always account to hard\|softirq cputime */
44		− vtime_account_system(tsk);
	65	+ vtime_account_irq_exit(tsk);
45	66	irqtime_account_irq(tsk);
46	67	}
47	68

init/Kconfig+23−2

@@ -322,10 +322,13 @@ source "kernel/time/Kconfig" · 321 unmodified lines
322	322
323	323	menu "CPU/Task time and stats accounting"
324	324
	325	+config VIRT_CPU_ACCOUNTING
	326	+ bool
	327	+
325	328	choice
326	329	prompt "Cputime accounting"
327	330	default TICK_CPU_ACCOUNTING if !PPC64
328		− default VIRT_CPU_ACCOUNTING if PPC64
	331	+ default VIRT_CPU_ACCOUNTING_NATIVE if PPC64
329	332
330	333	# Kind of a stub config for the pure tick based cputime accounting
331	334	config TICK_CPU_ACCOUNTING
@@ -338,9 +341,10 @@ config TICK_CPU_ACCOUNTING · 6 unmodified lines
338	341
339	342	If unsure, say Y.
340	343
341		−config VIRT_CPU_ACCOUNTING
	344	+config VIRT_CPU_ACCOUNTING_NATIVE
342	345	bool "Deterministic task and CPU time accounting"
343	346	depends on HAVE_VIRT_CPU_ACCOUNTING
	347	+ select VIRT_CPU_ACCOUNTING
344	348	help
345	349	Select this option to enable more accurate task and CPU time
346	350	accounting. This is done by reading a CPU counter on each
@@ -350,6 +354,23 @@ config VIRT_CPU_ACCOUNTING · 3 unmodified lines
350	354	this also enables accounting of stolen time on logically-partitioned
351	355	systems.
352	356
	357	+config VIRT_CPU_ACCOUNTING_GEN
	358	+ bool "Full dynticks CPU time accounting"
	359	+ depends on HAVE_CONTEXT_TRACKING && 64BIT
	360	+ select VIRT_CPU_ACCOUNTING
	361	+ select CONTEXT_TRACKING
	362	+ help
	363	+ Select this option to enable task and CPU time accounting on full
	364	+ dynticks systems. This accounting is implemented by watching every
	365	+ kernel-user boundaries using the context tracking subsystem.
	366	+ The accounting is thus performed at the expense of some significant
	367	+ overhead.
	368	+
	369	+ For now this is only useful if you are working on the full
	370	+ dynticks subsystem development.
	371	+
	372	+ If unsure, say N.
	373	+
353	374	config IRQ_TIME_ACCOUNTING
354	375	bool "Fine granularity task level IRQ time accounting"
355	376	depends on HAVE_IRQ_TIME_ACCOUNTING

init/init_task.c+2−0

@@ -2,6 +2,8 @@ · 1 unmodified line
2	2	#include <linux/export.h>
3	3	#include <linux/mqueue.h>
4	4	#include <linux/sched.h>
	5	+#include <linux/sched/sysctl.h>
	6	+#include <linux/sched/rt.h>
5	7	#include <linux/init.h>
6	8	#include <linux/fs.h>
7	9	#include <linux/mm.h>

kernel/acct.c+4−2

@@ -566,6 +566,7 @@ out: · 565 unmodified lines
566	566	void acct_collect(long exitcode, int group_dead)
567	567	{
568	568	struct pacct_struct *pacct = &current->signal->pacct;
	569	+ cputime_t utime, stime;
569	570	unsigned long vsize = 0;
570	571
571	572	if (group_dead && current->mm) {
@@ -593,8 +594,9 @@ void acct_collect(long exitcode, int group_dead) · 21 unmodified lines
593	594	pacct->ac_flag \|= ACORE;
594	595	if (current->flags & PF_SIGNALED)
595	596	pacct->ac_flag \|= AXSIG;
596		− pacct->ac_utime += current->utime;
597		− pacct->ac_stime += current->stime;
	597	+ task_cputime(current, &utime, &stime);
	598	+ pacct->ac_utime += utime;
	599	+ pacct->ac_stime += stime;
598	600	pacct->ac_minflt += current->min_flt;
599	601	pacct->ac_majflt += current->maj_flt;
600	602	spin_unlock_irq(&current->sighand->siglock);

kernel/context_tracking.c+25−18

@@ -15,26 +15,13 @@ · 14 unmodified lines
15	15	*/
16	16
17	17	#include <linux/context_tracking.h>
	18	+#include <linux/kvm_host.h>
18	19	#include <linux/rcupdate.h>
19	20	#include <linux/sched.h>
20		−#include <linux/percpu.h>
21	21	#include <linux/hardirq.h>
	22	+#include <linux/export.h>
22	23
23		−struct context_tracking {
24		− /*
25		− * When active is false, probes are unset in order
26		− * to minimize overhead: TIF flags are cleared
27		− * and calls to user_enter/exit are ignored. This
28		− * may be further optimized using static keys.
29		− */
30		− bool active;
31		− enum {
32		− IN_KERNEL = 0,
33		− IN_USER,
34		− } state;
35		−};
36		−
37		−static DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
	24	+DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
38	25	#ifdef CONFIG_CONTEXT_TRACKING_FORCE
39	26	.active = true,
40	27	#endif
@@ -70,7 +57,6 @@ void user_enter(void) · 29 unmodified lines
70	57	local_irq_save(flags);
71	58	if (__this_cpu_read(context_tracking.active) &&
72	59	__this_cpu_read(context_tracking.state) != IN_USER) {
73		− __this_cpu_write(context_tracking.state, IN_USER);
74	60	/*
75	61	* At this stage, only low level arch entry code remains and
76	62	* then we'll run in userspace. We can assume there won't be
@@ -78,7 +64,9 @@ void user_enter(void) · 1 unmodified line
78	64	* user_exit() or rcu_irq_enter(). Let's remove RCU's dependency
79	65	* on the tick.
80	66	*/
	67	+ vtime_user_enter(current);
81	68	rcu_user_enter();
	69	+ __this_cpu_write(context_tracking.state, IN_USER);
82	70	}
83	71	local_irq_restore(flags);
84	72	}
@@ -104,16 +92,35 @@ void user_exit(void) · 19 unmodified lines
104	92
105	93	local_irq_save(flags);
106	94	if (__this_cpu_read(context_tracking.state) == IN_USER) {
107		− __this_cpu_write(context_tracking.state, IN_KERNEL);
108	95	/*
109	96	* We are going to run code that may use RCU. Inform
110	97	* RCU core about that (ie: we may need the tick again).
111	98	*/
112	99	rcu_user_exit();
	100	+ vtime_user_exit(current);
	101	+ __this_cpu_write(context_tracking.state, IN_KERNEL);
113	102	}
114	103	local_irq_restore(flags);
115	104	}
116	105
	106	+void guest_enter(void)
	107	+{
	108	+ if (vtime_accounting_enabled())
	109	+ vtime_guest_enter(current);
	110	+ else
	111	+ __guest_enter();
	112	+}
	113	+EXPORT_SYMBOL_GPL(guest_enter);
	114	+
	115	+void guest_exit(void)
	116	+{
	117	+ if (vtime_accounting_enabled())
	118	+ vtime_guest_exit(current);
	119	+ else
	120	+ __guest_exit();
	121	+}
	122	+EXPORT_SYMBOL_GPL(guest_exit);
	123	+
117	124
118	125	/**
119	126	* context_tracking_task_switch - context switch the syscall callbacks

kernel/cpu.c+3−1

@@ -224,11 +224,13 @@ void clear_tasks_mm_cpumask(int cpu) · 223 unmodified lines
224	224	static inline void check_for_tasks(int cpu)
225	225	{
226	226	struct task_struct *p;
	227	+ cputime_t utime, stime;
227	228
228	229	write_lock_irq(&tasklist_lock);
229	230	for_each_process(p) {
	231	+ task_cputime(p, &utime, &stime);
230	232	if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
231		− (p->utime \|\| p->stime))
	233	+ (utime \|\| stime))
232	234	printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
233	235	"(state = %ld, flags = %x)\n",
234	236	p->comm, task_pid_nr(p), cpu,

kernel/delayacct.c+5−2

@@ -106,6 +106,7 @@ int __delayacct_add_tsk(struct taskstats d, struct task_struct tsk) · 105 unmodified lines
106	106	unsigned long long t2, t3;
107	107	unsigned long flags;
108	108	struct timespec ts;
	109	+ cputime_t utime, stime, stimescaled, utimescaled;
109	110
110	111	/* Though tsk->delays accessed later, early exit avoids
111	112	* unnecessary returning of other data
@@ -114,12 +115,14 @@ int __delayacct_add_tsk(struct taskstats d, struct task_struct tsk) · 2 unmodified lines
114	115	goto done;
115	116
116	117	tmp = (s64)d->cpu_run_real_total;
117		− cputime_to_timespec(tsk->utime + tsk->stime, &ts);
	118	+ task_cputime(tsk, &utime, &stime);
	119	+ cputime_to_timespec(utime + stime, &ts);
118	120	tmp += timespec_to_ns(&ts);
119	121	d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp;
120	122
121	123	tmp = (s64)d->cpu_scaled_run_real_total;
122		− cputime_to_timespec(tsk->utimescaled + tsk->stimescaled, &ts);
	124	+ task_cputime_scaled(tsk, &utimescaled, &stimescaled);
	125	+ cputime_to_timespec(utimescaled + stimescaled, &ts);
123	126	tmp += timespec_to_ns(&ts);
124	127	d->cpu_scaled_run_real_total =
125	128	(tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp;

kernel/exit.c+6−4

@@ -85,6 +85,7 @@ static void __exit_signal(struct task_struct *tsk) · 84 unmodified lines
85	85	bool group_dead = thread_group_leader(tsk);
86	86	struct sighand_struct *sighand;
87	87	struct tty_struct *uninitialized_var(tty);
	88	+ cputime_t utime, stime;
88	89
89	90	sighand = rcu_dereference_check(tsk->sighand,
90	91	lockdep_tasklist_lock_is_held());
@@ -123,9 +124,10 @@ static void __exit_signal(struct task_struct *tsk) · 32 unmodified lines
123	124	* We won't ever get here for the group leader, since it
124	125	* will have been the last reference on the signal_struct.
125	126	*/
126		− sig->utime += tsk->utime;
127		− sig->stime += tsk->stime;
128		− sig->gtime += tsk->gtime;
	127	+ task_cputime(tsk, &utime, &stime);
	128	+ sig->utime += utime;
	129	+ sig->stime += stime;
	130	+ sig->gtime += task_gtime(tsk);
129	131	sig->min_flt += tsk->min_flt;
130	132	sig->maj_flt += tsk->maj_flt;
131	133	sig->nvcsw += tsk->nvcsw;
@@ -1092,7 +1094,7 @@ static int wait_task_zombie(struct wait_opts wo, struct task_struct p) · 960 unmodified lines
1092	1094	sig = p->signal;
1093	1095	psig->cutime += tgutime + sig->cutime;
1094	1096	psig->cstime += tgstime + sig->cstime;
1095		− psig->cgtime += p->gtime + sig->gtime + sig->cgtime;
	1097	+ psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime;
1096	1098	psig->cmin_flt +=
1097	1099	p->min_flt + sig->min_flt + sig->cmin_flt;
1098	1100	psig->cmaj_flt +=

kernel/fork.c+6−0

@@ -1233,6 +1233,12 @@ static struct task_struct *copy_process(unsigned long clone_flags, · 1232 unmodified lines
1233	1233	#ifndef CONFIG_VIRT_CPU_ACCOUNTING
1234	1234	p->prev_cputime.utime = p->prev_cputime.stime = 0;
1235	1235	#endif
	1236	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
	1237	+ seqlock_init(&p->vtime_seqlock);
	1238	+ p->vtime_snap = 0;
	1239	+ p->vtime_snap_whence = VTIME_SLEEPING;
	1240	+#endif
	1241	+
1236	1242	#if defined(SPLIT_RSS_COUNTING)
1237	1243	memset(&p->rss_stat, 0, sizeof(p->rss_stat));
1238	1244	#endif

kernel/futex.c+1−0

@@ -60,6 +60,7 @@ · 59 unmodified lines
60	60	#include <linux/pid.h>
61	61	#include <linux/nsproxy.h>
62	62	#include <linux/ptrace.h>
	63	+#include <linux/sched/rt.h>
63	64
64	65	#include <asm/futex.h>
65	66

kernel/hrtimer.c+2−0

@@ -44,6 +44,8 @@ · 43 unmodified lines
44	44	#include <linux/err.h>
45	45	#include <linux/debugobjects.h>
46	46	#include <linux/sched.h>
	47	+#include <linux/sched/sysctl.h>
	48	+#include <linux/sched/rt.h>
47	49	#include <linux/timer.h>
48	50
49	51	#include <asm/uaccess.h>

kernel/irq/manage.c+1−0

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#include <linux/interrupt.h>
17	17	#include <linux/slab.h>
18	18	#include <linux/sched.h>
	19	+#include <linux/sched/rt.h>
19	20	#include <linux/task_work.h>
20	21
21	22	#include "internals.h"

kernel/mutex.c+1−0

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	*/
20	20	#include <linux/mutex.h>
21	21	#include <linux/sched.h>
	22	+#include <linux/sched/rt.h>
22	23	#include <linux/export.h>
23	24	#include <linux/spinlock.h>
24	25	#include <linux/interrupt.h>

kernel/posix-cpu-timers.c+22−6

@@ -155,11 +155,19 @@ static void bump_cpu_timer(struct k_itimer *timer, · 154 unmodified lines
155	155
156	156	static inline cputime_t prof_ticks(struct task_struct *p)
157	157	{
158		− return p->utime + p->stime;
	158	+ cputime_t utime, stime;
	159	+
	160	+ task_cputime(p, &utime, &stime);
	161	+
	162	+ return utime + stime;
159	163	}
160	164	static inline cputime_t virt_ticks(struct task_struct *p)
161	165	{
162		− return p->utime;
	166	+ cputime_t utime;
	167	+
	168	+ task_cputime(p, &utime, NULL);
	169	+
	170	+ return utime;
163	171	}
164	172
165	173	static int
@@ -471,18 +479,23 @@ static void cleanup_timers(struct list_head *head, · 305 unmodified lines
471	479	*/
472	480	void posix_cpu_timers_exit(struct task_struct *tsk)
473	481	{
	482	+ cputime_t utime, stime;
	483	+
474	484	add_device_randomness((const void*) &tsk->se.sum_exec_runtime,
475	485	sizeof(unsigned long long));
	486	+ task_cputime(tsk, &utime, &stime);
476	487	cleanup_timers(tsk->cpu_timers,
477		− tsk->utime, tsk->stime, tsk->se.sum_exec_runtime);
	488	+ utime, stime, tsk->se.sum_exec_runtime);
478	489
479	490	}
480	491	void posix_cpu_timers_exit_group(struct task_struct *tsk)
481	492	{
482	493	struct signal_struct *const sig = tsk->signal;
	494	+ cputime_t utime, stime;
483	495
	496	+ task_cputime(tsk, &utime, &stime);
484	497	cleanup_timers(tsk->signal->cpu_timers,
485		− tsk->utime + sig->utime, tsk->stime + sig->stime,
	498	+ utime + sig->utime, stime + sig->stime,
486	499	tsk->se.sum_exec_runtime + sig->sum_sched_runtime);
487	500	}
488	501
@@ -1226,11 +1239,14 @@ static inline int task_cputime_expired(const struct task_cputime *sample, · 737 unmodified lines
1226	1239	static inline int fastpath_timer_check(struct task_struct *tsk)
1227	1240	{
1228	1241	struct signal_struct *sig;
	1242	+ cputime_t utime, stime;
	1243	+
	1244	+ task_cputime(tsk, &utime, &stime);
1229	1245
1230	1246	if (!task_cputime_zero(&tsk->cputime_expires)) {
1231	1247	struct task_cputime task_sample = {
1232		− .utime = tsk->utime,
1233		− .stime = tsk->stime,
	1248	+ .utime = utime,
	1249	+ .stime = stime,
1234	1250	.sum_exec_runtime = tsk->se.sum_exec_runtime
1235	1251	};
1236	1252

kernel/rtmutex-debug.c+1−0

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	* See rt.c in preempt-rt for proper credits and further information
18	18	*/
19	19	#include <linux/sched.h>
	20	+#include <linux/sched/rt.h>
20	21	#include <linux/delay.h>
21	22	#include <linux/export.h>
22	23	#include <linux/spinlock.h>

kernel/rtmutex-tester.c+1−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/kthread.h>
11	11	#include <linux/export.h>
12	12	#include <linux/sched.h>
	13	+#include <linux/sched/rt.h>
13	14	#include <linux/spinlock.h>
14	15	#include <linux/timer.h>
15	16	#include <linux/freezer.h>

kernel/rtmutex.c+1−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <linux/spinlock.h>
14	14	#include <linux/export.h>
15	15	#include <linux/sched.h>
	16	+#include <linux/sched/rt.h>
16	17	#include <linux/timer.h>
17	18
18	19	#include "rtmutex_common.h"

kernel/sched/core.c+21−1

@@ -4371,7 +4371,7 @@ bool __sched yield_to(struct task_struct *p, bool preempt) · 4370 unmodified lines
4371	4371	struct task_struct *curr = current;
4372	4372	struct rq rq, p_rq;
4373	4373	unsigned long flags;
4374		− bool yielded = 0;
	4374	+ int yielded = 0;
4375	4375
4376	4376	local_irq_save(flags);
4377	4377	rq = this_rq();
@@ -4667,6 +4667,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu) · 289 unmodified lines
4667	4667	*/
4668	4668	idle->sched_class = &idle_sched_class;
4669	4669	ftrace_graph_init_idle_task(idle, cpu);
	4670	+ vtime_init_idle(idle);
4670	4671	#if defined(CONFIG_SMP)
4671	4672	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
4672	4673	#endif
@@ -7508,6 +7509,25 @@ static int sched_rt_global_constraints(void) · 2835 unmodified lines
7508	7509	}
7509	7510	#endif /* CONFIG_RT_GROUP_SCHED */
7510	7511
	7512	+int sched_rr_handler(struct ctl_table *table, int write,
	7513	+ void __user buffer, size_t lenp,
	7514	+ loff_t *ppos)
	7515	+{
	7516	+ int ret;
	7517	+ static DEFINE_MUTEX(mutex);
	7518	+
	7519	+ mutex_lock(&mutex);
	7520	+ ret = proc_dointvec(table, write, buffer, lenp, ppos);
	7521	+ /* make sure that internally we keep jiffies */
	7522	+ /* also, writing zero resets timeslice to default */
	7523	+ if (!ret && write) {
	7524	+ sched_rr_timeslice = sched_rr_timeslice <= 0 ?
	7525	+ RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
	7526	+ }
	7527	+ mutex_unlock(&mutex);
	7528	+ return ret;
	7529	+}
	7530	+
7511	7531	int sched_rt_handler(struct ctl_table *table, int write,
7512	7532	void __user buffer, size_t lenp,
7513	7533	loff_t *ppos)

kernel/sched/cpupri.c+2−0

@@ -28,6 +28,8 @@ · 27 unmodified lines
28	28	*/
29	29
30	30	#include <linux/gfp.h>
	31	+#include <linux/sched.h>
	32	+#include <linux/sched/rt.h>
31	33	#include "cpupri.h"
32	34
33	35	/* Convert between a 140 based task->prio, and our 102 based cpupri */

kernel/sched/cputime.c+271−43

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#include <linux/tsacct_kern.h>
4	4	#include <linux/kernel_stat.h>
5	5	#include <linux/static_key.h>
	6	+#include <linux/context_tracking.h>
6	7	#include "sched.h"
7	8
8	9
@@ -163,7 +164,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime, · 154 unmodified lines
163	164	task_group_account_field(p, index, (__force u64) cputime);
164	165
165	166	/* Account for user time used */
166		− acct_update_integrals(p);
	167	+ acct_account_cputime(p);
167	168	}
168	169
169	170	/*
@@ -213,7 +214,7 @@ void __account_system_time(struct task_struct *p, cputime_t cputime, · 43 unmodified lines
213	214	task_group_account_field(p, index, (__force u64) cputime);
214	215
215	216	/* Account for system time used */
216		− acct_update_integrals(p);
	217	+ acct_account_cputime(p);
217	218	}
218	219
219	220	/*
@@ -295,6 +296,7 @@ static __always_inline bool steal_account_process_tick(void) · 75 unmodified lines
295	296	void thread_group_cputime(struct task_struct tsk, struct task_cputime times)
296	297	{
297	298	struct signal_struct *sig = tsk->signal;
	299	+ cputime_t utime, stime;
298	300	struct task_struct *t;
299	301
300	302	times->utime = sig->utime;
@@ -308,16 +310,15 @@ void thread_group_cputime(struct task_struct tsk, struct task_cputime times) · 7 unmodified lines
308	310
309	311	t = tsk;
310	312	do {
311		− times->utime += t->utime;
312		− times->stime += t->stime;
	313	+ task_cputime(tsk, &utime, &stime);
	314	+ times->utime += utime;
	315	+ times->stime += stime;
313	316	times->sum_exec_runtime += task_sched_runtime(t);
314	317	} while_each_thread(tsk, t);
315	318	out:
316	319	rcu_read_unlock();
317	320	}
318	321
319		−#ifndef CONFIG_VIRT_CPU_ACCOUNTING
320		−
321	322	#ifdef CONFIG_IRQ_TIME_ACCOUNTING
322	323	/*
323	324	* Account a tick to a process and cpustat
@@ -382,11 +383,12 @@ static void irqtime_account_idle_ticks(int ticks) · 58 unmodified lines
382	383	irqtime_account_process_tick(current, 0, rq);
383	384	}
384	385	#else /* CONFIG_IRQ_TIME_ACCOUNTING */
385		−static void irqtime_account_idle_ticks(int ticks) {}
386		−static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
	386	+static inline void irqtime_account_idle_ticks(int ticks) {}
	387	+static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
387	388	struct rq *rq) {}
388	389	#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
389	390
	391	+#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
390	392	/*
391	393	* Account a single tick of cpu time.
392	394	* @p: the process that the cpu time gets accounted to
@@ -397,6 +399,9 @@ void account_process_tick(struct task_struct *p, int user_tick) · 4 unmodified lines
397	399	cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
398	400	struct rq *rq = this_rq();
399	401
	402	+ if (vtime_accounting_enabled())
	403	+ return;
	404	+
400	405	if (sched_clock_irqtime) {
401	406	irqtime_account_process_tick(p, user_tick, rq);
402	407	return;
@@ -438,8 +443,7 @@ void account_idle_ticks(unsigned long ticks) · 35 unmodified lines
438	443
439	444	account_idle_time(jiffies_to_cputime(ticks));
440	445	}
441		−
442		−#endif
	446	+#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
443	447
444	448	/*
445	449	* Use precise platform statistics if available:
@@ -461,25 +465,20 @@ void thread_group_cputime_adjusted(struct task_struct p, cputime_t ut, cputime · 15 unmodified lines
461	465	*st = cputime.stime;
462	466	}
463	467
464		−void vtime_account_system_irqsafe(struct task_struct *tsk)
465		−{
466		− unsigned long flags;
467		−
468		− local_irq_save(flags);
469		− vtime_account_system(tsk);
470		− local_irq_restore(flags);
471		−}
472		−EXPORT_SYMBOL_GPL(vtime_account_system_irqsafe);
473		−
474	468	#ifndef __ARCH_HAS_VTIME_TASK_SWITCH
475	469	void vtime_task_switch(struct task_struct *prev)
476	470	{
	471	+ if (!vtime_accounting_enabled())
	472	+ return;
	473	+
477	474	if (is_idle_task(prev))
478	475	vtime_account_idle(prev);
479	476	else
480	477	vtime_account_system(prev);
481	478
	479	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
482	480	vtime_account_user(prev);
	481	+#endif
483	482	arch_vtime_task_switch(prev);
484	483	}
485	484	#endif
@@ -493,27 +492,40 @@ void vtime_task_switch(struct task_struct *prev) · 7 unmodified lines
493	492	* vtime_account().
494	493	*/
495	494	#ifndef __ARCH_HAS_VTIME_ACCOUNT
496		−void vtime_account(struct task_struct *tsk)
	495	+void vtime_account_irq_enter(struct task_struct *tsk)
497	496	{
498		− if (in_interrupt() \|\| !is_idle_task(tsk))
499		− vtime_account_system(tsk);
500		− else
501		− vtime_account_idle(tsk);
	497	+ if (!vtime_accounting_enabled())
	498	+ return;
	499	+
	500	+ if (!in_interrupt()) {
	501	+ /*
	502	+ * If we interrupted user, context_tracking_in_user()
	503	+ * is 1 because the context tracking don't hook
	504	+ * on irq entry/exit. This way we know if
	505	+ * we need to flush user time on kernel entry.
	506	+ */
	507	+ if (context_tracking_in_user()) {
	508	+ vtime_account_user(tsk);
	509	+ return;
	510	+ }
	511	+
	512	+ if (is_idle_task(tsk)) {
	513	+ vtime_account_idle(tsk);
	514	+ return;
	515	+ }
	516	+ }
	517	+ vtime_account_system(tsk);
502	518	}
503		−EXPORT_SYMBOL_GPL(vtime_account);
	519	+EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
504	520	#endif /* __ARCH_HAS_VTIME_ACCOUNT */
505	521
506		−#else
507		−
508		−#ifndef nsecs_to_cputime
509		−# define nsecs_to_cputime(__nsecs) nsecs_to_jiffies(__nsecs)
510		−#endif
	522	+#else /* !CONFIG_VIRT_CPU_ACCOUNTING */
511	523
512		−static cputime_t scale_utime(cputime_t utime, cputime_t rtime, cputime_t total)
	524	+static cputime_t scale_stime(cputime_t stime, cputime_t rtime, cputime_t total)
513	525	{
514	526	u64 temp = (__force u64) rtime;
515	527
516		− temp *= (__force u64) utime;
	528	+ temp *= (__force u64) stime;
517	529
518	530	if (sizeof(cputime_t) == 4)
519	531	temp = div_u64(temp, (__force u32) total);
@@ -531,10 +543,10 @@ static void cputime_adjust(struct task_cputime *curr, · 11 unmodified lines
531	543	struct cputime *prev,
532	544	cputime_t ut, cputime_t st)
533	545	{
534		− cputime_t rtime, utime, total;
	546	+ cputime_t rtime, stime, total;
535	547
536		− utime = curr->utime;
537		− total = utime + curr->stime;
	548	+ stime = curr->stime;
	549	+ total = stime + curr->utime;
538	550
539	551	/*
540	552	* Tick based cputime accounting depend on random scheduling
@@ -549,17 +561,17 @@ static void cputime_adjust(struct task_cputime *curr, · 8 unmodified lines
549	561	rtime = nsecs_to_cputime(curr->sum_exec_runtime);
550	562
551	563	if (total)
552		− utime = scale_utime(utime, rtime, total);
	564	+ stime = scale_stime(stime, rtime, total);
553	565	else
554		− utime = rtime;
	566	+ stime = rtime;
555	567
556	568	/*
557	569	* If the tick based count grows faster than the scheduler one,
558	570	* the result of the scaling may go backward.
559	571	* Let's enforce monotonicity.
560	572	*/
561		− prev->utime = max(prev->utime, utime);
562		− prev->stime = max(prev->stime, rtime - prev->utime);
	573	+ prev->stime = max(prev->stime, stime);
	574	+ prev->utime = max(prev->utime, rtime - prev->stime);
563	575
564	576	*ut = prev->utime;
565	577	*st = prev->stime;
@@ -568,11 +580,10 @@ static void cputime_adjust(struct task_cputime *curr, · 2 unmodified lines
568	580	void task_cputime_adjusted(struct task_struct p, cputime_t ut, cputime_t *st)
569	581	{
570	582	struct task_cputime cputime = {
571		− .utime = p->utime,
572		− .stime = p->stime,
573	583	.sum_exec_runtime = p->se.sum_exec_runtime,
574	584	};
575	585
	586	+ task_cputime(p, &cputime.utime, &cputime.stime);
576	587	cputime_adjust(&cputime, &p->prev_cputime, ut, st);
577	588	}
578	589
@@ -586,4 +597,221 @@ void thread_group_cputime_adjusted(struct task_struct p, cputime_t ut, cputime · 7 unmodified lines
586	597	thread_group_cputime(p, &cputime);
587	598	cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
588	599	}
589		−#endif
	600	+#endif /* !CONFIG_VIRT_CPU_ACCOUNTING */
	601	+
	602	+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
	603	+static unsigned long long vtime_delta(struct task_struct *tsk)
	604	+{
	605	+ unsigned long long clock;
	606	+
	607	+ clock = sched_clock();
	608	+ if (clock < tsk->vtime_snap)
	609	+ return 0;
	610	+
	611	+ return clock - tsk->vtime_snap;
	612	+}
	613	+
	614	+static cputime_t get_vtime_delta(struct task_struct *tsk)
	615	+{
	616	+ unsigned long long delta = vtime_delta(tsk);
	617	+
	618	+ WARN_ON_ONCE(tsk->vtime_snap_whence == VTIME_SLEEPING);
	619	+ tsk->vtime_snap += delta;
	620	+
	621	+ /* CHECKME: always safe to convert nsecs to cputime? */
	622	+ return nsecs_to_cputime(delta);
	623	+}
	624	+
	625	+static void __vtime_account_system(struct task_struct *tsk)
	626	+{
	627	+ cputime_t delta_cpu = get_vtime_delta(tsk);
	628	+
	629	+ account_system_time(tsk, irq_count(), delta_cpu, cputime_to_scaled(delta_cpu));
	630	+}
	631	+
	632	+void vtime_account_system(struct task_struct *tsk)
	633	+{
	634	+ if (!vtime_accounting_enabled())
	635	+ return;
	636	+
	637	+ write_seqlock(&tsk->vtime_seqlock);
	638	+ __vtime_account_system(tsk);
	639	+ write_sequnlock(&tsk->vtime_seqlock);
	640	+}
	641	+
	642	+void vtime_account_irq_exit(struct task_struct *tsk)
	643	+{
	644	+ if (!vtime_accounting_enabled())
	645	+ return;
	646	+
	647	+ write_seqlock(&tsk->vtime_seqlock);
	648	+ if (context_tracking_in_user())
	649	+ tsk->vtime_snap_whence = VTIME_USER;
	650	+ __vtime_account_system(tsk);
	651	+ write_sequnlock(&tsk->vtime_seqlock);
	652	+}
	653	+
	654	+void vtime_account_user(struct task_struct *tsk)
	655	+{
	656	+ cputime_t delta_cpu;
	657	+
	658	+ if (!vtime_accounting_enabled())
	659	+ return;
	660	+
	661	+ delta_cpu = get_vtime_delta(tsk);
	662	+
	663	+ write_seqlock(&tsk->vtime_seqlock);
	664	+ tsk->vtime_snap_whence = VTIME_SYS;
	665	+ account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu));
	666	+ write_sequnlock(&tsk->vtime_seqlock);
	667	+}
	668	+
	669	+void vtime_user_enter(struct task_struct *tsk)
	670	+{
	671	+ if (!vtime_accounting_enabled())
	672	+ return;
	673	+
	674	+ write_seqlock(&tsk->vtime_seqlock);
	675	+ tsk->vtime_snap_whence = VTIME_USER;
	676	+ __vtime_account_system(tsk);
	677	+ write_sequnlock(&tsk->vtime_seqlock);
	678	+}
	679	+
	680	+void vtime_guest_enter(struct task_struct *tsk)
	681	+{
	682	+ write_seqlock(&tsk->vtime_seqlock);
	683	+ __vtime_account_system(tsk);
	684	+ current->flags \|= PF_VCPU;
	685	+ write_sequnlock(&tsk->vtime_seqlock);
	686	+}
	687	+
	688	+void vtime_guest_exit(struct task_struct *tsk)
	689	+{
	690	+ write_seqlock(&tsk->vtime_seqlock);
	691	+ __vtime_account_system(tsk);
	692	+ current->flags &= ~PF_VCPU;
	693	+ write_sequnlock(&tsk->vtime_seqlock);
	694	+}
	695	+
	696	+void vtime_account_idle(struct task_struct *tsk)
	697	+{
	698	+ cputime_t delta_cpu = get_vtime_delta(tsk);
	699	+
	700	+ account_idle_time(delta_cpu);
	701	+}
	702	+
	703	+bool vtime_accounting_enabled(void)
	704	+{
	705	+ return context_tracking_active();
	706	+}
	707	+
	708	+void arch_vtime_task_switch(struct task_struct *prev)
	709	+{
	710	+ write_seqlock(&prev->vtime_seqlock);
	711	+ prev->vtime_snap_whence = VTIME_SLEEPING;
	712	+ write_sequnlock(&prev->vtime_seqlock);
	713	+
	714	+ write_seqlock(&current->vtime_seqlock);
	715	+ current->vtime_snap_whence = VTIME_SYS;
	716	+ current->vtime_snap = sched_clock();
	717	+ write_sequnlock(&current->vtime_seqlock);
	718	+}
	719	+
	720	+void vtime_init_idle(struct task_struct *t)
	721	+{
	722	+ unsigned long flags;
	723	+
	724	+ write_seqlock_irqsave(&t->vtime_seqlock, flags);
	725	+ t->vtime_snap_whence = VTIME_SYS;
	726	+ t->vtime_snap = sched_clock();
	727	+ write_sequnlock_irqrestore(&t->vtime_seqlock, flags);
	728	+}
	729	+
	730	+cputime_t task_gtime(struct task_struct *t)
	731	+{
	732	+ unsigned int seq;
	733	+ cputime_t gtime;
	734	+
	735	+ do {
	736	+ seq = read_seqbegin(&t->vtime_seqlock);
	737	+
	738	+ gtime = t->gtime;
	739	+ if (t->flags & PF_VCPU)
	740	+ gtime += vtime_delta(t);
	741	+
	742	+ } while (read_seqretry(&t->vtime_seqlock, seq));
	743	+
	744	+ return gtime;
	745	+}
	746	+
	747	+/*
	748	+ * Fetch cputime raw values from fields of task_struct and
	749	+ * add up the pending nohz execution time since the last
	750	+ * cputime snapshot.
	751	+ */
	752	+static void
	753	+fetch_task_cputime(struct task_struct *t,
	754	+ cputime_t u_dst, cputime_t s_dst,
	755	+ cputime_t u_src, cputime_t s_src,
	756	+ cputime_t udelta, cputime_t sdelta)
	757	+{
	758	+ unsigned int seq;
	759	+ unsigned long long delta;
	760	+
	761	+ do {
	762	+ *udelta = 0;
	763	+ *sdelta = 0;
	764	+
	765	+ seq = read_seqbegin(&t->vtime_seqlock);
	766	+
	767	+ if (u_dst)
	768	+ u_dst = u_src;
	769	+ if (s_dst)
	770	+ s_dst = s_src;
	771	+
	772	+ /* Task is sleeping, nothing to add */
	773	+ if (t->vtime_snap_whence == VTIME_SLEEPING \|\|
	774	+ is_idle_task(t))
	775	+ continue;
	776	+
	777	+ delta = vtime_delta(t);
	778	+
	779	+ /*
	780	+ * Task runs either in user or kernel space, add pending nohz time to
	781	+ * the right place.
	782	+ */
	783	+ if (t->vtime_snap_whence == VTIME_USER \|\| t->flags & PF_VCPU) {
	784	+ *udelta = delta;
	785	+ } else {
	786	+ if (t->vtime_snap_whence == VTIME_SYS)
	787	+ *sdelta = delta;
	788	+ }
	789	+ } while (read_seqretry(&t->vtime_seqlock, seq));
	790	+}
	791	+
	792	+
	793	+void task_cputime(struct task_struct t, cputime_t utime, cputime_t *stime)
	794	+{
	795	+ cputime_t udelta, sdelta;
	796	+
	797	+ fetch_task_cputime(t, utime, stime, &t->utime,
	798	+ &t->stime, &udelta, &sdelta);
	799	+ if (utime)
	800	+ *utime += udelta;
	801	+ if (stime)
	802	+ *stime += sdelta;
	803	+}
	804	+
	805	+void task_cputime_scaled(struct task_struct *t,
	806	+ cputime_t utimescaled, cputime_t stimescaled)
	807	+{
	808	+ cputime_t udelta, sdelta;
	809	+
	810	+ fetch_task_cputime(t, utimescaled, stimescaled,
	811	+ &t->utimescaled, &t->stimescaled, &udelta, &sdelta);
	812	+ if (utimescaled)
	813	+ *utimescaled += cputime_to_scaled(udelta);
	814	+ if (stimescaled)
	815	+ *stimescaled += cputime_to_scaled(sdelta);
	816	+}
	817	+#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */

kernel/sched/fair.c+9−18

@@ -1680,9 +1680,7 @@ place_entity(struct cfs_rq cfs_rq, struct sched_entity se, int initial) · 1679 unmodified lines
1680	1680	}
1681	1681
1682	1682	/* ensure we never gain time by being placed backwards. */
1683		− vruntime = max_vruntime(se->vruntime, vruntime);
1684		−
1685		− se->vruntime = vruntime;
	1683	+ se->vruntime = max_vruntime(se->vruntime, vruntime);
1686	1684	}
1687	1685
1688	1686	static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
@@ -3254,25 +3252,18 @@ find_idlest_cpu(struct sched_group group, struct task_struct p, int this_cpu) · 1565 unmodified lines
3254	3252	*/
3255	3253	static int select_idle_sibling(struct task_struct *p, int target)
3256	3254	{
3257		− int cpu = smp_processor_id();
3258		− int prev_cpu = task_cpu(p);
3259	3255	struct sched_domain *sd;
3260	3256	struct sched_group *sg;
3261		− int i;
	3257	+ int i = task_cpu(p);
3262	3258
3263		− /*
3264		− * If the task is going to be woken-up on this cpu and if it is
3265		− * already idle, then it is the right target.
3266		− */
3267		− if (target == cpu && idle_cpu(cpu))
3268		− return cpu;
	3259	+ if (idle_cpu(target))
	3260	+ return target;
3269	3261
3270	3262	/*
3271		− * If the task is going to be woken-up on the cpu where it previously
3272		− * ran and if it is currently idle, then it the right target.
	3263	+ * If the prevous cpu is cache affine and idle, don't be stupid.
3273	3264	*/
3274		− if (target == prev_cpu && idle_cpu(prev_cpu))
3275		− return prev_cpu;
	3265	+ if (i != target && cpus_share_cache(i, target) && idle_cpu(i))
	3266	+ return i;
3276	3267
3277	3268	/*
3278	3269	* Otherwise, iterate the domains and find an elegible idle cpu.
@@ -3286,7 +3277,7 @@ static int select_idle_sibling(struct task_struct *p, int target) · 7 unmodified lines
3286	3277	goto next;
3287	3278
3288	3279	for_each_cpu(i, sched_group_cpus(sg)) {
3289		− if (!idle_cpu(i))
	3280	+ if (i == target \|\| !idle_cpu(i))
3290	3281	goto next;
3291	3282	}
3292	3283
@@ -6101,7 +6092,7 @@ static unsigned int get_rr_interval_fair(struct rq rq, struct task_struct task · 2808 unmodified lines
6101	6092	* idle runqueue:
6102	6093	*/
6103	6094	if (rq->cfs.load.weight)
6104		− rr_interval = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
	6095	+ rr_interval = NS_TO_JIFFIES(sched_slice(cfs_rq_of(se), se));
6105	6096
6106	6097	return rr_interval;
6107	6098	}

kernel/sched/rt.c+17−9

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	7
8	8	#include <linux/slab.h>
9	9
	10	+int sched_rr_timeslice = RR_TIMESLICE;
	11	+
10	12	static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
11	13
12	14	struct rt_bandwidth def_rt_bandwidth;
@@ -925,8 +927,8 @@ static void update_curr_rt(struct rq *rq) · 912 unmodified lines
925	927	return;
926	928
927	929	delta_exec = rq->clock_task - curr->se.exec_start;
928		− if (unlikely((s64)delta_exec < 0))
929		− delta_exec = 0;
	930	+ if (unlikely((s64)delta_exec <= 0))
	931	+ return;
930	932
931	933	schedstat_set(curr->se.statistics.exec_max,
932	934	max(curr->se.statistics.exec_max, delta_exec));
@@ -1427,8 +1429,7 @@ static void put_prev_task_rt(struct rq rq, struct task_struct p) · 494 unmodified lines
1427	1429	static int pick_rt_task(struct rq rq, struct task_struct p, int cpu)
1428	1430	{
1429	1431	if (!task_running(rq, p) &&
1430		− (cpu < 0 \|\| cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) &&
1431		− (p->nr_cpus_allowed > 1))
	1432	+ cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
1432	1433	return 1;
1433	1434	return 0;
1434	1435	}
@@ -1889,8 +1890,11 @@ static void switched_from_rt(struct rq rq, struct task_struct p) · 454 unmodified lines
1889	1890	* we may need to handle the pulling of RT tasks
1890	1891	* now.
1891	1892	*/
1892		− if (p->on_rq && !rq->rt.rt_nr_running)
1893		− pull_rt_task(rq);
	1893	+ if (!p->on_rq \|\| rq->rt.rt_nr_running)
	1894	+ return;
	1895	+
	1896	+ if (pull_rt_task(rq))
	1897	+ resched_task(rq->curr);
1894	1898	}
1895	1899
1896	1900	void init_sched_rt_class(void)
@@ -1985,7 +1989,11 @@ static void watchdog(struct rq rq, struct task_struct p) · 88 unmodified lines
1985	1989	if (soft != RLIM_INFINITY) {
1986	1990	unsigned long next;
1987	1991
1988		− p->rt.timeout++;
	1992	+ if (p->rt.watchdog_stamp != jiffies) {
	1993	+ p->rt.timeout++;
	1994	+ p->rt.watchdog_stamp = jiffies;
	1995	+ }
	1996	+
1989	1997	next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
1990	1998	if (p->rt.timeout > next)
1991	1999	p->cputime_expires.sched_exp = p->se.sum_exec_runtime;
@@ -2010,7 +2018,7 @@ static void task_tick_rt(struct rq rq, struct task_struct p, int queued) · 18 unmodified lines
2010	2018	if (--p->rt.time_slice)
2011	2019	return;
2012	2020
2013		− p->rt.time_slice = RR_TIMESLICE;
	2021	+ p->rt.time_slice = sched_rr_timeslice;
2014	2022
2015	2023	/*
2016	2024	* Requeue to the end of queue if we (and all of our ancestors) are the
@@ -2041,7 +2049,7 @@ static unsigned int get_rr_interval_rt(struct rq rq, struct task_struct task) · 24 unmodified lines
2041	2049	* Time slice is 0 for SCHED_FIFO tasks
2042	2050	*/
2043	2051	if (task->policy == SCHED_RR)
2044		− return RR_TIMESLICE;
	2052	+ return sched_rr_timeslice;
2045	2053	else
2046	2054	return 0;
2047	2055	}

kernel/sched/sched.h+2−0

@@ -1,5 +1,7 @@
1	1
2	2	#include <linux/sched.h>
	3	+#include <linux/sched/sysctl.h>
	4	+#include <linux/sched/rt.h>
3	5	#include <linux/mutex.h>
4	6	#include <linux/spinlock.h>
5	7	#include <linux/stop_machine.h>

kernel/signal.c+8−4

@@ -1632,6 +1632,7 @@ bool do_notify_parent(struct task_struct *tsk, int sig) · 1631 unmodified lines
1632	1632	unsigned long flags;
1633	1633	struct sighand_struct *psig;
1634	1634	bool autoreap = false;
	1635	+ cputime_t utime, stime;
1635	1636
1636	1637	BUG_ON(sig == -1);
1637	1638
@@ -1669,8 +1670,9 @@ bool do_notify_parent(struct task_struct *tsk, int sig) · 31 unmodified lines
1669	1670	task_uid(tsk));
1670	1671	rcu_read_unlock();
1671	1672
1672		− info.si_utime = cputime_to_clock_t(tsk->utime + tsk->signal->utime);
1673		− info.si_stime = cputime_to_clock_t(tsk->stime + tsk->signal->stime);
	1673	+ task_cputime(tsk, &utime, &stime);
	1674	+ info.si_utime = cputime_to_clock_t(utime + tsk->signal->utime);
	1675	+ info.si_stime = cputime_to_clock_t(stime + tsk->signal->stime);
1674	1676
1675	1677	info.si_status = tsk->exit_code & 0x7f;
1676	1678	if (tsk->exit_code & 0x80)
@@ -1734,6 +1736,7 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, · 57 unmodified lines
1734	1736	unsigned long flags;
1735	1737	struct task_struct *parent;
1736	1738	struct sighand_struct *sighand;
	1739	+ cputime_t utime, stime;
1737	1740
1738	1741	if (for_ptracer) {
1739	1742	parent = tsk->parent;
@@ -1752,8 +1755,9 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, · 12 unmodified lines
1752	1755	info.si_uid = from_kuid_munged(task_cred_xxx(parent, user_ns), task_uid(tsk));
1753	1756	rcu_read_unlock();
1754	1757
1755		− info.si_utime = cputime_to_clock_t(tsk->utime);
1756		− info.si_stime = cputime_to_clock_t(tsk->stime);
	1758	+ task_cputime(tsk, &utime, &stime);
	1759	+ info.si_utime = cputime_to_clock_t(utime);
	1760	+ info.si_stime = cputime_to_clock_t(stime);
1757	1761
1758	1762	info.si_code = why;
1759	1763	switch (why) {

kernel/softirq.c+3−3

@@ -221,7 +221,7 @@ asmlinkage void __do_softirq(void) · 220 unmodified lines
221	221	current->flags &= ~PF_MEMALLOC;
222	222
223	223	pending = local_softirq_pending();
224		− vtime_account_irq_enter(current);
	224	+ account_irq_enter_time(current);
225	225
226	226	__local_bh_disable((unsigned long)__builtin_return_address(0),
227	227	SOFTIRQ_OFFSET);
@@ -272,7 +272,7 @@ restart: · 44 unmodified lines
272	272
273	273	lockdep_softirq_exit();
274	274
275		− vtime_account_irq_exit(current);
	275	+ account_irq_exit_time(current);
276	276	__local_bh_enable(SOFTIRQ_OFFSET);
277	277	tsk_restore_flags(current, old_flags, PF_MEMALLOC);
278	278	}
@@ -341,7 +341,7 @@ static inline void invoke_softirq(void) · 62 unmodified lines
341	341	*/
342	342	void irq_exit(void)
343	343	{
344		− vtime_account_irq_exit(current);
	344	+ account_irq_exit_time(current);
345	345	trace_hardirq_exit();
346	346	sub_preempt_count(IRQ_EXIT_OFFSET);
347	347	if (!in_interrupt() && local_softirq_pending())

kernel/sysctl.c+8−0

@@ -61,6 +61,7 @@ · 60 unmodified lines
61	61	#include <linux/kmod.h>
62	62	#include <linux/capability.h>
63	63	#include <linux/binfmts.h>
	64	+#include <linux/sched/sysctl.h>
64	65
65	66	#include <asm/uaccess.h>
66	67	#include <asm/processor.h>
@@ -403,6 +404,13 @@ static struct ctl_table kern_table[] = { · 336 unmodified lines
403	404	.mode = 0644,
404	405	.proc_handler = sched_rt_handler,
405	406	},
	407	+ {
	408	+ .procname = "sched_rr_timeslice_ms",
	409	+ .data = &sched_rr_timeslice,
	410	+ .maxlen = sizeof(int),
	411	+ .mode = 0644,
	412	+ .proc_handler = sched_rr_handler,
	413	+ },
406	414	#ifdef CONFIG_SCHED_AUTOGROUP
407	415	{
408	416	.procname = "sched_autogroup_enabled",

kernel/time/tick-sched.c+4−1

@@ -632,8 +632,11 @@ static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) · 631 unmodified lines
632	632
633	633	static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
634	634	{
635		−#ifndef CONFIG_VIRT_CPU_ACCOUNTING
	635	+#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
636	636	unsigned long ticks;
	637	+
	638	+ if (vtime_accounting_enabled())
	639	+ return;
637	640	/*
638	641	* We stopped the tick in idle. Update process times would miss the
639	642	* time we slept as update_process_times does only a 1 tick

kernel/timer.c+1−0

@@ -39,6 +39,7 @@ · 38 unmodified lines
39	39	#include <linux/kallsyms.h>
40	40	#include <linux/irq_work.h>
41	41	#include <linux/sched.h>
	42	+#include <linux/sched/sysctl.h>
42	43	#include <linux/slab.h>
43	44
44	45	#include <asm/uaccess.h>

kernel/trace/trace.c+1−0

@@ -39,6 +39,7 @@ · 38 unmodified lines
39	39	#include <linux/poll.h>
40	40	#include <linux/nmi.h>
41	41	#include <linux/fs.h>
	42	+#include <linux/sched/rt.h>
42	43
43	44	#include "trace.h"
44	45	#include "trace_output.h"

kernel/trace/trace_sched_wakeup.c+1−1

@@ -15,8 +15,8 @@ · 14 unmodified lines
15	15	#include <linux/kallsyms.h>
16	16	#include <linux/uaccess.h>
17	17	#include <linux/ftrace.h>
	18	+#include <linux/sched/rt.h>
18	19	#include <trace/events/sched.h>
19		−
20	20	#include "trace.h"
21	21
22	22	static struct trace_array *wakeup_trace;

kernel/tsacct.c+34−10

@@ -32,6 +32,7 @@ void bacct_add_tsk(struct user_namespace *user_ns, · 31 unmodified lines
32	32	{
33	33	const struct cred *tcred;
34	34	struct timespec uptime, ts;
	35	+ cputime_t utime, stime, utimescaled, stimescaled;
35	36	u64 ac_etime;
36	37
37	38	BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN);
@@ -65,10 +66,15 @@ void bacct_add_tsk(struct user_namespace *user_ns, · 27 unmodified lines
65	66	stats->ac_ppid = pid_alive(tsk) ?
66	67	task_tgid_nr_ns(rcu_dereference(tsk->real_parent), pid_ns) : 0;
67	68	rcu_read_unlock();
68		− stats->ac_utime = cputime_to_usecs(tsk->utime);
69		− stats->ac_stime = cputime_to_usecs(tsk->stime);
70		− stats->ac_utimescaled = cputime_to_usecs(tsk->utimescaled);
71		− stats->ac_stimescaled = cputime_to_usecs(tsk->stimescaled);
	69	+
	70	+ task_cputime(tsk, &utime, &stime);
	71	+ stats->ac_utime = cputime_to_usecs(utime);
	72	+ stats->ac_stime = cputime_to_usecs(stime);
	73	+
	74	+ task_cputime_scaled(tsk, &utimescaled, &stimescaled);
	75	+ stats->ac_utimescaled = cputime_to_usecs(utimescaled);
	76	+ stats->ac_stimescaled = cputime_to_usecs(stimescaled);
	77	+
72	78	stats->ac_minflt = tsk->min_flt;
73	79	stats->ac_majflt = tsk->maj_flt;
74	80
@@ -115,11 +121,8 @@ void xacct_add_tsk(struct taskstats stats, struct task_struct p) · 40 unmodified lines
115	121	#undef KB
116	122	#undef MB
117	123
118		−/**
119		− * acct_update_integrals - update mm integral fields in task_struct
120		− * @tsk: task_struct for accounting
121		− */
122		−void acct_update_integrals(struct task_struct *tsk)
	124	+static void __acct_update_integrals(struct task_struct *tsk,
	125	+ cputime_t utime, cputime_t stime)
123	126	{
124	127	if (likely(tsk->mm)) {
125	128	cputime_t time, dtime;
@@ -128,7 +131,7 @@ void acct_update_integrals(struct task_struct *tsk) · 2 unmodified lines
128	131	u64 delta;
129	132
130	133	local_irq_save(flags);
131		− time = tsk->stime + tsk->utime;
	134	+ time = stime + utime;
132	135	dtime = time - tsk->acct_timexpd;
133	136	jiffies_to_timeval(cputime_to_jiffies(dtime), &value);
134	137	delta = value.tv_sec;
@@ -144,6 +147,27 @@ void acct_update_integrals(struct task_struct *tsk) · 9 unmodified lines
144	147	}
145	148	}
146	149
	150	+/**
	151	+ * acct_update_integrals - update mm integral fields in task_struct
	152	+ * @tsk: task_struct for accounting
	153	+ */
	154	+void acct_update_integrals(struct task_struct *tsk)
	155	+{
	156	+ cputime_t utime, stime;
	157	+
	158	+ task_cputime(tsk, &utime, &stime);
	159	+ __acct_update_integrals(tsk, utime, stime);
	160	+}
	161	+
	162	+/**
	163	+ * acct_account_cputime - update mm integral after cputime update
	164	+ * @tsk: task_struct for accounting
	165	+ */
	166	+void acct_account_cputime(struct task_struct *tsk)
	167	+{
	168	+ __acct_update_integrals(tsk, tsk->utime, tsk->stime);
	169	+}
	170	+
147	171	/**
148	172	* acct_clear_integrals - clear the mm integral fields in task_struct
149	173	* @tsk: task_struct whose accounting fields are cleared

kernel/watchdog.c+1−0

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	23	#include <linux/module.h>
24	24	#include <linux/sysctl.h>
25	25	#include <linux/smpboot.h>
	26	+#include <linux/sched/rt.h>
26	27
27	28	#include <asm/irq_regs.h>
28	29	#include <linux/kvm_para.h>

mm/mmap.c+1−0

@@ -32,6 +32,7 @@ · 31 unmodified lines
32	32	#include <linux/khugepaged.h>
33	33	#include <linux/uprobes.h>
34	34	#include <linux/rbtree_augmented.h>
	35	+#include <linux/sched/sysctl.h>
35	36
36	37	#include <asm/uaccess.h>
37	38	#include <asm/cacheflush.h>

mm/mremap.c+1−0

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#include <linux/security.h>
20	20	#include <linux/syscalls.h>
21	21	#include <linux/mmu_notifier.h>
	22	+#include <linux/sched/sysctl.h>
22	23
23	24	#include <asm/uaccess.h>
24	25	#include <asm/cacheflush.h>

mm/nommu.c+1−0

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	29	#include <linux/security.h>
30	30	#include <linux/syscalls.h>
31	31	#include <linux/audit.h>
	32	+#include <linux/sched/sysctl.h>
32	33
33	34	#include <asm/uaccess.h>
34	35	#include <asm/tlb.h>

mm/page-writeback.c+1−0

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	35	#include <linux/buffer_head.h> /* __set_page_dirty_buffers */
36	36	#include <linux/pagevec.h>
37	37	#include <linux/timer.h>
	38	+#include <linux/sched/rt.h>
38	39	#include <trace/events/writeback.h>
39	40
40	41	/*

mm/page_alloc.c+1−0

@@ -58,6 +58,7 @@ · 57 unmodified lines
58	58	#include <linux/prefetch.h>
59	59	#include <linux/migrate.h>
60	60	#include <linux/page-debug-flags.h>
	61	+#include <linux/sched/rt.h>
61	62
62	63	#include <asm/tlbflush.h>
63	64	#include <asm/div64.h>