History

Merge branch 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull core timer updates from Thomas Gleixner: "Timers and timekeeping updates: - A large overhaul of the posix CPU timer code which is a preparation for moving the CPU timer expiry out into task work so it can be properly accounted on the task/process. An update to the bogus permission checks will come later during the merge window as feedback was not complete before heading of for travel. - Switch the timerqueue code to use cached rbtrees and get rid of the homebrewn caching of the leftmost node. - Consolidate hrtimer_init() + hrtimer_init_sleeper() calls into a single function - Implement the separation of hrtimers to be forced to expire in hard interrupt context even when PREEMPT_RT is enabled and mark the affected timers accordingly. - Implement a mechanism for hrtimers and the timer wheel to protect RT against priority inversion and live lock issues when a (hr)timer which should be canceled is currently executing the callback. Instead of infinitely spinning, the task which tries to cancel the timer blocks on a per cpu base expiry lock which is held and released by the (hr)timer expiry code. - Enable the Hyper-V TSC page based sched_clock for Hyper-V guests resulting in faster access to timekeeping functions. - Updates to various clocksource/clockevent drivers and their device tree bindings. - The usual small improvements all over the place" * 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (101 commits) posix-cpu-timers: Fix permission check regression posix-cpu-timers: Always clear head pointer on dequeue hrtimer: Add a missing bracket and hide `migration_base' on !SMP posix-cpu-timers: Make expiry_active check actually work correctly posix-timers: Unbreak CONFIG_POSIX_TIMERS=n build tick: Mark sched_timer to expire in hard interrupt context hrtimer: Add kernel doc annotation for HRTIMER_MODE_HARD x86/hyperv: Hide pv_ops access for CONFIG_PARAVIRT=n posix-cpu-timers: Utilize timerqueue for storage posix-cpu-timers: Move state tracking to struct posix_cputimers posix-cpu-timers: Deduplicate rlimit handling posix-cpu-timers: Remove pointless comparisons posix-cpu-timers: Get rid of 64bit divisions posix-cpu-timers: Consolidate timer expiry further posix-cpu-timers: Get rid of zero checks rlimit: Rewrite non-sensical RLIMIT_CPU comment posix-cpu-timers: Respect INFINITY for hard RTTIME limit posix-cpu-timers: Switch thread group sampling to array posix-cpu-timers: Restructure expiry array posix-cpu-timers: Remove cputime_expires ...

Linus Torvalds committed 7 years ago

Commit 7f2444d2 parents c5f12fd 77b4b5461 files · +1476 −895Browse files

Unified · Split

61 files
  1. Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml +102 −0
  2. Documentation/devicetree/bindings/timer/allwinner,sun4i-timer.txt +0 −19
  3. Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.txt +0 −26
  4. Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml +79 −0
  5. Documentation/devicetree/bindings/timer/renesas,cmt.txt +23 −17
  6. arch/arm64/boot/dts/freescale/imx8mm.dtsi +8 −0
  7. arch/arm64/boot/dts/freescale/imx8mq.dtsi +8 −0
  8. arch/x86/entry/vdso/vma.c +1 −1
  9. arch/x86/hyperv/hv_init.c +0 −2
  10. arch/x86/include/asm/vdso/gettimeofday.h +3 −3
  11. arch/x86/kernel/cpu/mshyperv.c +10 −0
  12. arch/x86/kvm/lapic.c +3 −3
  13. block/blk-mq.c +2 −3
  14. drivers/clocksource/Kconfig +1 −1
  15. drivers/clocksource/em_sti.c +1 −3
  16. drivers/clocksource/hyperv_timer.c +16 −29
  17. drivers/clocksource/renesas-ostm.c +1 −1
  18. drivers/clocksource/sh_cmt.c +14 −5
  19. drivers/clocksource/sh_tmu.c +1 −4
  20. drivers/clocksource/timer-atmel-tcb.c +18 −0
  21. drivers/clocksource/timer-imx-sysctr.c +5 −0
  22. drivers/clocksource/timer-npcm7xx.c +3 −6
  23. drivers/clocksource/timer-of.c +4 −2
  24. drivers/clocksource/timer-probe.c +3 −1
  25. drivers/clocksource/timer-sun4i.c +4 −0
  26. drivers/hv/Kconfig +0 −3
  27. drivers/staging/android/vsoc.c +3 −5
  28. fs/timerfd.c +5 −1
  29. include/asm-generic/mshyperv.h +1 −0
  30. include/clocksource/hyperv_timer.h +3 −5
  31. include/linux/alarmtimer.h +2 −1
  32. include/linux/hrtimer.h +44 −4
  33. include/linux/init_task.h +0 −11
  34. include/linux/posix-timers.h +117 −14
  35. include/linux/sched.h +4 −25
  36. include/linux/sched/cputime.h +7 −5
  37. include/linux/sched/signal.h +3 −11
  38. include/linux/sched/types.h +23 −0
  39. include/linux/timer.h +1 −1
  40. include/linux/timerqueue.h +16 −7
  41. include/linux/wait.h +2 −2
  42. init/init_task.c +0 −2
  43. kernel/events/core.c +4 −4
  44. kernel/fork.c +3 −31
  45. kernel/futex.c +5 −7
  46. kernel/sched/core.c +3 −3
  47. kernel/sched/deadline.c +4 −4
  48. kernel/sched/rt.c +8 −5
  49. kernel/sys.c +3 −13
  50. kernel/time/alarmtimer.c +15 −1
  51. kernel/time/hrtimer.c +212 −23
  52. kernel/time/itimer.c +4 −8
  53. kernel/time/posix-cpu-timers.c +492 −518
  54. kernel/time/posix-timers.c +50 −11
  55. kernel/time/posix-timers.h +1 −0
  56. kernel/time/tick-broadcast-hrtimer.c +9 −4
  57. kernel/time/tick-sched.c +10 −7
  58. kernel/time/timer.c +96 −9
  59. kernel/watchdog.c +2 −2
  60. lib/timerqueue.c +12 −18
  61. net/core/pktgen.c +2 −4
Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml+102−0
@@ -0,0 +1,102 @@
1+# SPDX-License-Identifier: GPL-2.0
2+%YAML 1.2
3+---
4+$id: http://devicetree.org/schemas/timer/allwinner,sun4i-a10-timer.yaml#
5+$schema: http://devicetree.org/meta-schemas/core.yaml#
6+
7+title: Allwinner A10 Timer Device Tree Bindings
8+
9+maintainers:
10+ - Chen-Yu Tsai <wens@csie.org>
11+ - Maxime Ripard <maxime.ripard@bootlin.com>
12+
13+properties:
14+ compatible:
15+ enum:
16+ - allwinner,sun4i-a10-timer
17+ - allwinner,sun8i-a23-timer
18+ - allwinner,sun8i-v3s-timer
19+ - allwinner,suniv-f1c100s-timer
20+
21+ reg:
22+ maxItems: 1
23+
24+ interrupts:
25+ description:
26+ List of timers interrupts
27+
28+ clocks:
29+ maxItems: 1
30+
31+allOf:
32+ - if:
33+ properties:
34+ compatible:
35+ items:
36+ const: allwinner,sun4i-a10-timer
37+
38+ then:
39+ properties:
40+ interrupts:
41+ minItems: 6
42+ maxItems: 6
43+
44+ - if:
45+ properties:
46+ compatible:
47+ items:
48+ const: allwinner,sun8i-a23-timer
49+
50+ then:
51+ properties:
52+ interrupts:
53+ minItems: 2
54+ maxItems: 2
55+
56+ - if:
57+ properties:
58+ compatible:
59+ items:
60+ const: allwinner,sun8i-v3s-timer
61+
62+ then:
63+ properties:
64+ interrupts:
65+ minItems: 3
66+ maxItems: 3
67+
68+ - if:
69+ properties:
70+ compatible:
71+ items:
72+ const: allwinner,suniv-f1c100s-timer
73+
74+ then:
75+ properties:
76+ interrupts:
77+ minItems: 3
78+ maxItems: 3
79+
80+required:
81+ - compatible
82+ - reg
83+ - interrupts
84+ - clocks
85+
86+additionalProperties: false
87+
88+examples:
89+ - |
90+ timer {
91+ compatible = "allwinner,sun4i-a10-timer";
92+ reg = <0x01c20c00 0x400>;
93+ interrupts = <22>,
94+ <23>,
95+ <24>,
96+ <25>,
97+ <67>,
98+ <68>;
99+ clocks = <&osc>;
100+ };
101+
102+...
Documentation/devicetree/bindings/timer/allwinner,sun4i-timer.txt+0−19
@@ -1,19 +0,0 @@
1−Allwinner A1X SoCs Timer Controller
2−
3−Required properties:
4−
5−- compatible : should be one of the following:
6− "allwinner,sun4i-a10-timer"
7− "allwinner,suniv-f1c100s-timer"
8−- reg : Specifies base physical address and size of the registers.
9−- interrupts : The interrupt of the first timer
10−- clocks: phandle to the source clock (usually a 24 MHz fixed clock)
11−
12−Example:
13−
14−timer {
15− compatible = "allwinner,sun4i-a10-timer";
16− reg = <0x01c20c00 0x400>;
17− interrupts = <22>;
18− clocks = <&osc>;
19−};
Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.txt+0−26
@@ -1,26 +0,0 @@
1−Allwinner SoCs High Speed Timer Controller
2−
3−Required properties:
4−
5−- compatible : should be "allwinner,sun5i-a13-hstimer" or
6− "allwinner,sun7i-a20-hstimer"
7−- reg : Specifies base physical address and size of the registers.
8−- interrupts : The interrupts of these timers (2 for the sun5i IP, 4 for the sun7i
9− one)
10−- clocks: phandle to the source clock (usually the AHB clock)
11−
12−Optional properties:
13−- resets: phandle to a reset controller asserting the timer
14−
15−Example:
16−
17−timer@1c60000 {
18− compatible = "allwinner,sun7i-a20-hstimer";
19− reg = <0x01c60000 0x1000>;
20− interrupts = <0 51 1>,
21− <0 52 1>,
22− <0 53 1>,
23− <0 54 1>;
24− clocks = <&ahb1_gates 19>;
25− resets = <&ahb1rst 19>;
26−};
Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml+79−0
@@ -0,0 +1,79 @@
1+# SPDX-License-Identifier: GPL-2.0
2+%YAML 1.2
3+---
4+$id: http://devicetree.org/schemas/timer/allwinner,sun5i-a13-hstimer.yaml#
5+$schema: http://devicetree.org/meta-schemas/core.yaml#
6+
7+title: Allwinner A13 High-Speed Timer Device Tree Bindings
8+
9+maintainers:
10+ - Chen-Yu Tsai <wens@csie.org>
11+ - Maxime Ripard <maxime.ripard@bootlin.com>
12+
13+properties:
14+ compatible:
15+ oneOf:
16+ - const: allwinner,sun5i-a13-hstimer
17+ - const: allwinner,sun7i-a20-hstimer
18+ - items:
19+ - const: allwinner,sun6i-a31-hstimer
20+ - const: allwinner,sun7i-a20-hstimer
21+
22+ reg:
23+ maxItems: 1
24+
25+ interrupts:
26+ minItems: 2
27+ maxItems: 4
28+ items:
29+ - description: Timer 0 Interrupt
30+ - description: Timer 1 Interrupt
31+ - description: Timer 2 Interrupt
32+ - description: Timer 3 Interrupt
33+
34+ clocks:
35+ maxItems: 1
36+
37+ resets:
38+ maxItems: 1
39+
40+required:
41+ - compatible
42+ - reg
43+ - interrupts
44+ - clocks
45+
46+if:
47+ properties:
48+ compatible:
49+ items:
50+ const: allwinner,sun5i-a13-hstimer
51+
52+then:
53+ properties:
54+ interrupts:
55+ minItems: 2
56+ maxItems: 2
57+
58+else:
59+ properties:
60+ interrupts:
61+ minItems: 4
62+ maxItems: 4
63+
64+additionalProperties: false
65+
66+examples:
67+ - |
68+ timer@1c60000 {
69+ compatible = "allwinner,sun7i-a20-hstimer";
70+ reg = <0x01c60000 0x1000>;
71+ interrupts = <0 51 1>,
72+ <0 52 1>,
73+ <0 53 1>,
74+ <0 54 1>;
75+ clocks = <&ahb1_gates 19>;
76+ resets = <&ahb1rst 19>;
77+ };
78+
79+...
Documentation/devicetree/bindings/timer/renesas,cmt.txt+23−17
@@ -12,16 +12,13 @@ datasheets. · 11 unmodified lines
1212 Required Properties:
1313
1414 - compatible: must contain one or more of the following:
15− - "renesas,cmt-48-sh73a0" for the sh73A0 48-bit CMT
16− (CMT1)
17− - "renesas,cmt-48-r8a7740" for the r8a7740 48-bit CMT
18− (CMT1)
19− - "renesas,cmt-48" for all non-second generation 48-bit CMT
20− (CMT1 on sh73a0 and r8a7740)
21− This is a fallback for the above renesas,cmt-48-* entries.
22−
2315 - "renesas,r8a73a4-cmt0" for the 32-bit CMT0 device included in r8a73a4.
2416 - "renesas,r8a73a4-cmt1" for the 48-bit CMT1 device included in r8a73a4.
17+ - "renesas,r8a7740-cmt0" for the 32-bit CMT0 device included in r8a7740.
18+ - "renesas,r8a7740-cmt1" for the 48-bit CMT1 device included in r8a7740.
19+ - "renesas,r8a7740-cmt2" for the 32-bit CMT2 device included in r8a7740.
20+ - "renesas,r8a7740-cmt3" for the 32-bit CMT3 device included in r8a7740.
21+ - "renesas,r8a7740-cmt4" for the 32-bit CMT4 device included in r8a7740.
2522 - "renesas,r8a7743-cmt0" for the 32-bit CMT0 device included in r8a7743.
2623 - "renesas,r8a7743-cmt1" for the 48-bit CMT1 device included in r8a7743.
2724 - "renesas,r8a7744-cmt0" for the 32-bit CMT0 device included in r8a7744.
@@ -31,29 +28,38 @@ Required Properties: · 3 unmodified lines
3128 - "renesas,r8a77470-cmt0" for the 32-bit CMT0 device included in r8a77470.
3229 - "renesas,r8a77470-cmt1" for the 48-bit CMT1 device included in r8a77470.
3330 - "renesas,r8a774a1-cmt0" for the 32-bit CMT0 device included in r8a774a1.
34− - "renesas,r8a774a1-cmt1" for the 48-bit CMT1 device included in r8a774a1.
31+ - "renesas,r8a774a1-cmt1" for the 48-bit CMT devices included in r8a774a1.
3532 - "renesas,r8a774c0-cmt0" for the 32-bit CMT0 device included in r8a774c0.
36− - "renesas,r8a774c0-cmt1" for the 48-bit CMT1 device included in r8a774c0.
33+ - "renesas,r8a774c0-cmt1" for the 48-bit CMT devices included in r8a774c0.
3734 - "renesas,r8a7790-cmt0" for the 32-bit CMT0 device included in r8a7790.
3835 - "renesas,r8a7790-cmt1" for the 48-bit CMT1 device included in r8a7790.
3936 - "renesas,r8a7791-cmt0" for the 32-bit CMT0 device included in r8a7791.
4037 - "renesas,r8a7791-cmt1" for the 48-bit CMT1 device included in r8a7791.
38+ - "renesas,r8a7792-cmt0" for the 32-bit CMT0 device included in r8a7792.
39+ - "renesas,r8a7792-cmt1" for the 48-bit CMT1 device included in r8a7792.
4140 - "renesas,r8a7793-cmt0" for the 32-bit CMT0 device included in r8a7793.
4241 - "renesas,r8a7793-cmt1" for the 48-bit CMT1 device included in r8a7793.
4342 - "renesas,r8a7794-cmt0" for the 32-bit CMT0 device included in r8a7794.
4443 - "renesas,r8a7794-cmt1" for the 48-bit CMT1 device included in r8a7794.
4544 - "renesas,r8a7795-cmt0" for the 32-bit CMT0 device included in r8a7795.
46− - "renesas,r8a7795-cmt1" for the 48-bit CMT1 device included in r8a7795.
45+ - "renesas,r8a7795-cmt1" for the 48-bit CMT devices included in r8a7795.
4746 - "renesas,r8a7796-cmt0" for the 32-bit CMT0 device included in r8a7796.
48− - "renesas,r8a7796-cmt1" for the 48-bit CMT1 device included in r8a7796.
47+ - "renesas,r8a7796-cmt1" for the 48-bit CMT devices included in r8a7796.
4948 - "renesas,r8a77965-cmt0" for the 32-bit CMT0 device included in r8a77965.
50− - "renesas,r8a77965-cmt1" for the 48-bit CMT1 device included in r8a77965.
49+ - "renesas,r8a77965-cmt1" for the 48-bit CMT devices included in r8a77965.
5150 - "renesas,r8a77970-cmt0" for the 32-bit CMT0 device included in r8a77970.
52− - "renesas,r8a77970-cmt1" for the 48-bit CMT1 device included in r8a77970.
51+ - "renesas,r8a77970-cmt1" for the 48-bit CMT devices included in r8a77970.
5352 - "renesas,r8a77980-cmt0" for the 32-bit CMT0 device included in r8a77980.
54− - "renesas,r8a77980-cmt1" for the 48-bit CMT1 device included in r8a77980.
53+ - "renesas,r8a77980-cmt1" for the 48-bit CMT devices included in r8a77980.
5554 - "renesas,r8a77990-cmt0" for the 32-bit CMT0 device included in r8a77990.
56− - "renesas,r8a77990-cmt1" for the 48-bit CMT1 device included in r8a77990.
55+ - "renesas,r8a77990-cmt1" for the 48-bit CMT devices included in r8a77990.
56+ - "renesas,r8a77995-cmt0" for the 32-bit CMT0 device included in r8a77995.
57+ - "renesas,r8a77995-cmt1" for the 48-bit CMT devices included in r8a77995.
58+ - "renesas,sh73a0-cmt0" for the 32-bit CMT0 device included in sh73a0.
59+ - "renesas,sh73a0-cmt1" for the 48-bit CMT1 device included in sh73a0.
60+ - "renesas,sh73a0-cmt2" for the 32-bit CMT2 device included in sh73a0.
61+ - "renesas,sh73a0-cmt3" for the 32-bit CMT3 device included in sh73a0.
62+ - "renesas,sh73a0-cmt4" for the 32-bit CMT4 device included in sh73a0.
5763
5864 - "renesas,rcar-gen2-cmt0" for 32-bit CMT0 devices included in R-Car Gen2
5965 and RZ/G1.
@@ -63,7 +69,7 @@ Required Properties: · 3 unmodified lines
6369 listed above.
6470 - "renesas,rcar-gen3-cmt0" for 32-bit CMT0 devices included in R-Car Gen3
6571 and RZ/G2.
66− - "renesas,rcar-gen3-cmt1" for 48-bit CMT1 devices included in R-Car Gen3
72+ - "renesas,rcar-gen3-cmt1" for 48-bit CMT devices included in R-Car Gen3
6773 and RZ/G2.
6874 These are fallbacks for R-Car Gen3 and RZ/G2 entries listed
6975 above.
arch/arm64/boot/dts/freescale/imx8mm.dtsi+8−0
@@ -546,6 +546,14 @@ · 545 unmodified lines
546546 #pwm-cells = <2>;
547547 status = "disabled";
548548 };
549+
550+ system_counter: timer@306a0000 {
551+ compatible = "nxp,sysctr-timer";
552+ reg = <0x306a0000 0x20000>;
553+ interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
554+ clocks = <&osc_24m>;
555+ clock-names = "per";
556+ };
549557 };
550558
551559 aips3: bus@30800000 {
arch/arm64/boot/dts/freescale/imx8mq.dtsi+8−0
@@ -651,6 +651,14 @@ · 650 unmodified lines
651651 #pwm-cells = <2>;
652652 status = "disabled";
653653 };
654+
655+ system_counter: timer@306a0000 {
656+ compatible = "nxp,sysctr-timer";
657+ reg = <0x306a0000 0x20000>;
658+ interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
659+ clocks = <&osc_25m>;
660+ clock-names = "per";
661+ };
654662 };
655663
656664 bus@30800000 { /* AIPS3 */
arch/x86/entry/vdso/vma.c+1−1
@@ -122,7 +122,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, · 121 unmodified lines
122122
123123 if (tsc_pg && vclock_was_used(VCLOCK_HVCLOCK))
124124 return vmf_insert_pfn(vma, vmf->address,
125− vmalloc_to_pfn(tsc_pg));
125+ virt_to_phys(tsc_pg) >> PAGE_SHIFT);
126126 }
127127
128128 return VM_FAULT_SIGBUS;
arch/x86/hyperv/hv_init.c+0−2
@@ -315,8 +315,6 @@ void __init hyperv_init(void) · 314 unmodified lines
315315
316316 x86_init.pci.arch_init = hv_pci_init;
317317
318− /* Register Hyper-V specific clocksource */
319− hv_init_clocksource();
320318 return;
321319
322320 remove_cpuhp_state:
arch/x86/include/asm/vdso/gettimeofday.h+3−3
@@ -51,7 +51,7 @@ extern struct pvclock_vsyscall_time_info pvclock_page · 50 unmodified lines
5151 __attribute__((visibility("hidden")));
5252 #endif
5353
54−#ifdef CONFIG_HYPERV_TSCPAGE
54+#ifdef CONFIG_HYPERV_TIMER
5555 extern struct ms_hyperv_tsc_page hvclock_page
5656 __attribute__((visibility("hidden")));
5757 #endif
@@ -228,7 +228,7 @@ static u64 vread_pvclock(void) · 170 unmodified lines
228228 }
229229 #endif
230230
231−#ifdef CONFIG_HYPERV_TSCPAGE
231+#ifdef CONFIG_HYPERV_TIMER
232232 static u64 vread_hvclock(void)
233233 {
234234 return hv_read_tsc_page(&hvclock_page);
@@ -251,7 +251,7 @@ static inline u64 __arch_get_hw_counter(s32 clock_mode) · 16 unmodified lines
251251 return vread_pvclock();
252252 }
253253 #endif
254−#ifdef CONFIG_HYPERV_TSCPAGE
254+#ifdef CONFIG_HYPERV_TIMER
255255 if (clock_mode == VCLOCK_HVCLOCK) {
256256 barrier();
257257 return vread_hvclock();
arch/x86/kernel/cpu/mshyperv.c+10−0
@@ -29,6 +29,7 @@ · 28 unmodified lines
2929 #include <asm/timer.h>
3030 #include <asm/reboot.h>
3131 #include <asm/nmi.h>
32+#include <clocksource/hyperv_timer.h>
3233
3334 struct ms_hyperv_info ms_hyperv;
3435 EXPORT_SYMBOL_GPL(ms_hyperv);
@@ -338,6 +339,15 @@ static void __init ms_hyperv_init_platform(void) · 303 unmodified lines
338339 x2apic_phys = 1;
339340 # endif
340341
342+ /* Register Hyper-V specific clocksource */
343+ hv_init_clocksource();
344+#endif
345+}
346+
347+void hv_setup_sched_clock(void *sched_clock)
348+{
349+#ifdef CONFIG_PARAVIRT
350+ pv_ops.time.sched_clock = sched_clock;
341351 #endif
342352 }
343353
arch/x86/kvm/lapic.c+3−3
@@ -1598,7 +1598,7 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic) · 1597 unmodified lines
15981598 likely(ns > apic->lapic_timer.timer_advance_ns)) {
15991599 expire = ktime_add_ns(now, ns);
16001600 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1601− hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS);
1601+ hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
16021602 } else
16031603 apic_timer_expired(apic);
16041604
@@ -2299,7 +2299,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns) · 694 unmodified lines
22992299 apic->vcpu = vcpu;
23002300
23012301 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2302− HRTIMER_MODE_ABS);
2302+ HRTIMER_MODE_ABS_HARD);
23032303 apic->lapic_timer.timer.function = apic_timer_fn;
23042304 if (timer_advance_ns == -1) {
23052305 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
@@ -2484,7 +2484,7 @@ void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) · 178 unmodified lines
24842484
24852485 timer = &vcpu->arch.apic->lapic_timer.timer;
24862486 if (hrtimer_cancel(timer))
2487− hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
2487+ hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
24882488 }
24892489
24902490 /*
block/blk-mq.c+2−3
@@ -3411,15 +3411,14 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, · 3410 unmodified lines
34113411 kt = nsecs;
34123412
34133413 mode = HRTIMER_MODE_REL;
3414− hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
3414+ hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
34153415 hrtimer_set_expires(&hs.timer, kt);
34163416
3417− hrtimer_init_sleeper(&hs, current);
34183417 do {
34193418 if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
34203419 break;
34213420 set_current_state(TASK_UNINTERRUPTIBLE);
3422− hrtimer_start_expires(&hs.timer, mode);
3421+ hrtimer_sleeper_start_expires(&hs, mode);
34233422 if (hs.task)
34243423 io_schedule();
34253424 hrtimer_cancel(&hs.timer);
drivers/clocksource/Kconfig+1−1
@@ -429,7 +429,7 @@ config ATMEL_ST · 428 unmodified lines
429429
430430 config ATMEL_TCB_CLKSRC
431431 bool "Atmel TC Block timer driver" if COMPILE_TEST
432− depends on HAS_IOMEM
432+ depends on ARM && HAS_IOMEM
433433 select TIMER_OF if OF
434434 help
435435 Support for Timer Counter Blocks on Atmel SoCs.
drivers/clocksource/em_sti.c+1−3
@@ -291,10 +291,8 @@ static int em_sti_probe(struct platform_device *pdev) · 290 unmodified lines
291291 platform_set_drvdata(pdev, p);
292292
293293 irq = platform_get_irq(pdev, 0);
294− if (irq < 0) {
295− dev_err(&pdev->dev, "failed to get irq\n");
294+ if (irq < 0)
296295 return irq;
297− }
298296
299297 /* map memory, let base point to the STI instance */
300298 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
drivers/clocksource/hyperv_timer.c+16−29
@@ -22,6 +22,7 @@ · 21 unmodified lines
2222 #include <asm/mshyperv.h>
2323
2424 static struct clock_event_device __percpu *hv_clock_event;
25+static u64 hv_sched_clock_offset __ro_after_init;
2526
2627 /*
2728 * If false, we're using the old mechanism for stimer0 interrupts
@@ -212,19 +213,17 @@ EXPORT_SYMBOL_GPL(hv_stimer_global_cleanup); · 184 unmodified lines
212213 struct clocksource *hyperv_cs;
213214 EXPORT_SYMBOL_GPL(hyperv_cs);
214215
215−#ifdef CONFIG_HYPERV_TSCPAGE
216−
217−static struct ms_hyperv_tsc_page *tsc_pg;
216+static struct ms_hyperv_tsc_page tsc_pg __aligned(PAGE_SIZE);
218217
219218 struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
220219 {
221− return tsc_pg;
220+ return &tsc_pg;
222221 }
223222 EXPORT_SYMBOL_GPL(hv_get_tsc_page);
224223
225−static u64 notrace read_hv_sched_clock_tsc(void)
224+static u64 notrace read_hv_clock_tsc(struct clocksource *arg)
226225 {
227− u64 current_tick = hv_read_tsc_page(tsc_pg);
226+ u64 current_tick = hv_read_tsc_page(&tsc_pg);
228227
229228 if (current_tick == U64_MAX)
230229 hv_get_time_ref_count(current_tick);
@@ -232,9 +231,9 @@ static u64 notrace read_hv_sched_clock_tsc(void) · 1 unmodified line
232231 return current_tick;
233232 }
234233
235−static u64 read_hv_clock_tsc(struct clocksource *arg)
234+static u64 read_hv_sched_clock_tsc(void)
236235 {
237− return read_hv_sched_clock_tsc();
236+ return read_hv_clock_tsc(NULL) - hv_sched_clock_offset;
238237 }
239238
240239 static struct clocksource hyperv_cs_tsc = {
@@ -244,9 +243,8 @@ static struct clocksource hyperv_cs_tsc = { · 3 unmodified lines
244243 .mask = CLOCKSOURCE_MASK(64),
245244 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
246245 };
247−#endif
248246
249−static u64 notrace read_hv_sched_clock_msr(void)
247+static u64 notrace read_hv_clock_msr(struct clocksource *arg)
250248 {
251249 u64 current_tick;
252250 /*
@@ -258,9 +256,9 @@ static u64 notrace read_hv_sched_clock_msr(void) · 5 unmodified lines
258256 return current_tick;
259257 }
260258
261−static u64 read_hv_clock_msr(struct clocksource *arg)
259+static u64 read_hv_sched_clock_msr(void)
262260 {
263− return read_hv_sched_clock_msr();
261+ return read_hv_clock_msr(NULL) - hv_sched_clock_offset;
264262 }
265263
266264 static struct clocksource hyperv_cs_msr = {
@@ -271,7 +269,6 @@ static struct clocksource hyperv_cs_msr = { · 4 unmodified lines
271269 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
272270 };
273271
274−#ifdef CONFIG_HYPERV_TSCPAGE
275272 static bool __init hv_init_tsc_clocksource(void)
276273 {
277274 u64 tsc_msr;
@@ -280,12 +277,8 @@ static bool __init hv_init_tsc_clocksource(void) · 2 unmodified lines
280277 if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE))
281278 return false;
282279
283− tsc_pg = vmalloc(PAGE_SIZE);
284− if (!tsc_pg)
285− return false;
286−
287280 hyperv_cs = &hyperv_cs_tsc;
288− phys_addr = page_to_phys(vmalloc_to_page(tsc_pg));
281+ phys_addr = virt_to_phys(&tsc_pg);
289282
290283 /*
291284 * The Hyper-V TLFS specifies to preserve the value of reserved
@@ -302,17 +295,11 @@ static bool __init hv_init_tsc_clocksource(void) · 10 unmodified lines
302295 hv_set_clocksource_vdso(hyperv_cs_tsc);
303296 clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100);
304297
305− /* sched_clock_register is needed on ARM64 but is a no-op on x86 */
306− sched_clock_register(read_hv_sched_clock_tsc, 64, HV_CLOCK_HZ);
298+ hv_sched_clock_offset = hyperv_cs->read(hyperv_cs);
299+ hv_setup_sched_clock(read_hv_sched_clock_tsc);
300+
307301 return true;
308302 }
309−#else
310−static bool __init hv_init_tsc_clocksource(void)
311−{
312− return false;
313−}
314−#endif
315−
316303
317304 void __init hv_init_clocksource(void)
318305 {
@@ -333,7 +320,7 @@ void __init hv_init_clocksource(void) · 14 unmodified lines
333320 hyperv_cs = &hyperv_cs_msr;
334321 clocksource_register_hz(&hyperv_cs_msr, NSEC_PER_SEC/100);
335322
336− /* sched_clock_register is needed on ARM64 but is a no-op on x86 */
337− sched_clock_register(read_hv_sched_clock_msr, 64, HV_CLOCK_HZ);
323+ hv_sched_clock_offset = hyperv_cs->read(hyperv_cs);
324+ hv_setup_sched_clock(read_hv_sched_clock_msr);
338325 }
339326 EXPORT_SYMBOL_GPL(hv_init_clocksource);
drivers/clocksource/renesas-ostm.c+1−1
@@ -221,7 +221,7 @@ static int __init ostm_init(struct device_node *np) · 220 unmodified lines
221221 }
222222
223223 rate = clk_get_rate(ostm_clk);
224− ostm->ticks_per_jiffy = (rate + HZ / 2) / HZ;
224+ ostm->ticks_per_jiffy = DIV_ROUND_CLOSEST(rate, HZ);
225225
226226 /*
227227 * First probed device will be used as system clocksource. Any
drivers/clocksource/sh_cmt.c+14−5
@@ -776,11 +776,8 @@ static int sh_cmt_register_clockevent(struct sh_cmt_channel *ch, · 775 unmodified lines
776776 int ret;
777777
778778 irq = platform_get_irq(ch->cmt->pdev, ch->index);
779− if (irq < 0) {
780− dev_err(&ch->cmt->pdev->dev, "ch%u: failed to get irq\n",
781− ch->index);
779+ if (irq < 0)
782780 return irq;
783− }
784781
785782 ret = request_irq(irq, sh_cmt_interrupt,
786783 IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING,
@@ -921,12 +918,24 @@ static const struct platform_device_id sh_cmt_id_table[] = { · 134 unmodified lines
921918 MODULE_DEVICE_TABLE(platform, sh_cmt_id_table);
922919
923920 static const struct of_device_id sh_cmt_of_table[] __maybe_unused = {
924− { .compatible = "renesas,cmt-48", .data = &sh_cmt_info[SH_CMT_48BIT] },
921+ {
922+ /* deprecated, preserved for backward compatibility */
923+ .compatible = "renesas,cmt-48",
924+ .data = &sh_cmt_info[SH_CMT_48BIT]
925+ },
925926 {
926927 /* deprecated, preserved for backward compatibility */
927928 .compatible = "renesas,cmt-48-gen2",
928929 .data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
929930 },
931+ {
932+ .compatible = "renesas,r8a7740-cmt1",
933+ .data = &sh_cmt_info[SH_CMT_48BIT]
934+ },
935+ {
936+ .compatible = "renesas,sh73a0-cmt1",
937+ .data = &sh_cmt_info[SH_CMT_48BIT]
938+ },
930939 {
931940 .compatible = "renesas,rcar-gen2-cmt0",
932941 .data = &sh_cmt_info[SH_CMT0_RCAR_GEN2]
drivers/clocksource/sh_tmu.c+1−4
@@ -462,11 +462,8 @@ static int sh_tmu_channel_setup(struct sh_tmu_channel *ch, unsigned int index, · 461 unmodified lines
462462 ch->base = tmu->mapbase + 8 + ch->index * 12;
463463
464464 ch->irq = platform_get_irq(tmu->pdev, index);
465− if (ch->irq < 0) {
466− dev_err(&tmu->pdev->dev, "ch%u: failed to get irq\n",
467− ch->index);
465+ if (ch->irq < 0)
468466 return ch->irq;
469− }
470467
471468 ch->cs_enabled = false;
472469 ch->enable_count = 0;
drivers/clocksource/timer-atmel-tcb.c+18−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #include <linux/irq.h>
77
88 #include <linux/clk.h>
9+#include <linux/delay.h>
910 #include <linux/err.h>
1011 #include <linux/ioport.h>
1112 #include <linux/io.h>
@@ -125,6 +126,18 @@ static u64 notrace tc_sched_clock_read32(void) · 113 unmodified lines
125126 return tc_get_cycles32(&clksrc);
126127 }
127128
129+static struct delay_timer tc_delay_timer;
130+
131+static unsigned long tc_delay_timer_read(void)
132+{
133+ return tc_get_cycles(&clksrc);
134+}
135+
136+static unsigned long notrace tc_delay_timer_read32(void)
137+{
138+ return tc_get_cycles32(&clksrc);
139+}
140+
128141 #ifdef CONFIG_GENERIC_CLOCKEVENTS
129142
130143 struct tc_clkevt_device {
@@ -432,6 +445,7 @@ static int __init tcb_clksrc_init(struct device_node *node) · 301 unmodified lines
432445 /* setup ony channel 0 */
433446 tcb_setup_single_chan(&tc, best_divisor_idx);
434447 tc_sched_clock = tc_sched_clock_read32;
448+ tc_delay_timer.read_current_timer = tc_delay_timer_read32;
435449 } else {
436450 /* we have three clocks no matter what the
437451 * underlying platform supports.
@@ -444,6 +458,7 @@ static int __init tcb_clksrc_init(struct device_node *node) · 6 unmodified lines
444458 /* setup both channel 0 & 1 */
445459 tcb_setup_dual_chan(&tc, best_divisor_idx);
446460 tc_sched_clock = tc_sched_clock_read;
461+ tc_delay_timer.read_current_timer = tc_delay_timer_read;
447462 }
448463
449464 /* and away we go! */
@@ -458,6 +473,9 @@ static int __init tcb_clksrc_init(struct device_node *node) · 8 unmodified lines
458473
459474 sched_clock_register(tc_sched_clock, 32, divided_rate);
460475
476+ tc_delay_timer.freq = divided_rate;
477+ register_current_timer_delay(&tc_delay_timer);
478+
461479 return 0;
462480
463481 err_unregister_clksrc:
drivers/clocksource/timer-imx-sysctr.c+5−0
@@ -20,6 +20,8 @@ · 19 unmodified lines
2020 #define SYS_CTR_EN 0x1
2121 #define SYS_CTR_IRQ_MASK 0x2
2222
23+#define SYS_CTR_CLK_DIV 0x3
24+
2325 static void __iomem *sys_ctr_base;
2426 static u32 cmpcr;
2527
@@ -134,6 +136,9 @@ static int __init sysctr_timer_init(struct device_node *np) · 108 unmodified lines
134136 if (ret)
135137 return ret;
136138
139+ /* system counter clock is divided by 3 internally */
140+ to_sysctr.of_clk.rate /= SYS_CTR_CLK_DIV;
141+
137142 sys_ctr_base = timer_of_base(&to_sysctr);
138143 cmpcr = readl(sys_ctr_base + CMPCR);
139144 cmpcr &= ~SYS_CTR_EN;
drivers/clocksource/timer-npcm7xx.c+3−6
@@ -32,7 +32,7 @@ · 31 unmodified lines
3232 #define NPCM7XX_Tx_INTEN BIT(29)
3333 #define NPCM7XX_Tx_COUNTEN BIT(30)
3434 #define NPCM7XX_Tx_ONESHOT 0x0
35−#define NPCM7XX_Tx_OPER GENMASK(27, 3)
35+#define NPCM7XX_Tx_OPER GENMASK(28, 27)
3636 #define NPCM7XX_Tx_MIN_PRESCALE 0x1
3737 #define NPCM7XX_Tx_TDR_MASK_BITS 24
3838 #define NPCM7XX_Tx_MAX_CNT 0xFFFFFF
@@ -84,8 +84,6 @@ static int npcm7xx_timer_oneshot(struct clock_event_device *evt) · 45 unmodified lines
8484
8585 val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
8686 val &= ~NPCM7XX_Tx_OPER;
87−
88− val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
8987 val |= NPCM7XX_START_ONESHOT_Tx;
9088 writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
9189
@@ -97,12 +95,11 @@ static int npcm7xx_timer_periodic(struct clock_event_device *evt) · 5 unmodified lines
9795 struct timer_of *to = to_timer_of(evt);
9896 u32 val;
9997
98+ writel(timer_of_period(to), timer_of_base(to) + NPCM7XX_REG_TICR0);
99+
100100 val = readl(timer_of_base(to) + NPCM7XX_REG_TCSR0);
101101 val &= ~NPCM7XX_Tx_OPER;
102−
103− writel(timer_of_period(to), timer_of_base(to) + NPCM7XX_REG_TICR0);
104102 val |= NPCM7XX_START_PERIODIC_Tx;
105−
106103 writel(val, timer_of_base(to) + NPCM7XX_REG_TCSR0);
107104
108105 return 0;
drivers/clocksource/timer-of.c+4−2
@@ -113,8 +113,10 @@ static __init int timer_of_clk_init(struct device_node *np, · 112 unmodified lines
113113 of_clk->clk = of_clk->name ? of_clk_get_by_name(np, of_clk->name) :
114114 of_clk_get(np, of_clk->index);
115115 if (IS_ERR(of_clk->clk)) {
116− pr_err("Failed to get clock for %pOF\n", np);
117− return PTR_ERR(of_clk->clk);
116+ ret = PTR_ERR(of_clk->clk);
117+ if (ret != -EPROBE_DEFER)
118+ pr_err("Failed to get clock for %pOF\n", np);
119+ goto out;
118120 }
119121
120122 ret = clk_prepare_enable(of_clk->clk);
drivers/clocksource/timer-probe.c+3−1
@@ -29,7 +29,9 @@ void __init timer_probe(void) · 28 unmodified lines
2929
3030 ret = init_func_ret(np);
3131 if (ret) {
32− pr_err("Failed to initialize '%pOF': %d\n", np, ret);
32+ if (ret != -EPROBE_DEFER)
33+ pr_err("Failed to initialize '%pOF': %d\n", np,
34+ ret);
3335 continue;
3436 }
3537
drivers/clocksource/timer-sun4i.c+4−0
@@ -219,5 +219,9 @@ static int __init sun4i_timer_init(struct device_node *node) · 218 unmodified lines
219219 }
220220 TIMER_OF_DECLARE(sun4i, "allwinner,sun4i-a10-timer",
221221 sun4i_timer_init);
222+TIMER_OF_DECLARE(sun8i_a23, "allwinner,sun8i-a23-timer",
223+ sun4i_timer_init);
224+TIMER_OF_DECLARE(sun8i_v3s, "allwinner,sun8i-v3s-timer",
225+ sun4i_timer_init);
222226 TIMER_OF_DECLARE(suniv, "allwinner,suniv-f1c100s-timer",
223227 sun4i_timer_init);
drivers/hv/Kconfig+0−3
@@ -14,9 +14,6 @@ config HYPERV · 13 unmodified lines
1414 config HYPERV_TIMER
1515 def_bool HYPERV
1616
17−config HYPERV_TSCPAGE
18− def_bool HYPERV && X86_64
19−
2017 config HYPERV_UTILS
2118 tristate "Microsoft Hyper-V Utilities driver"
2219 depends on HYPERV && CONNECTOR && NLS
drivers/staging/android/vsoc.c+3−5
@@ -437,12 +437,10 @@ static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg) · 436 unmodified lines
437437 return -EINVAL;
438438 wake_time = ktime_set(arg->wake_time_sec, arg->wake_time_nsec);
439439
440− hrtimer_init_on_stack(&to->timer, CLOCK_MONOTONIC,
441− HRTIMER_MODE_ABS);
440+ hrtimer_init_sleeper_on_stack(to, CLOCK_MONOTONIC,
441+ HRTIMER_MODE_ABS);
442442 hrtimer_set_expires_range_ns(&to->timer, wake_time,
443443 current->timer_slack_ns);
444−
445− hrtimer_init_sleeper(to, current);
446444 }
447445
448446 while (1) {
@@ -460,7 +458,7 @@ static int handle_vsoc_cond_wait(struct file *filp, struct vsoc_cond_wait *arg) · 11 unmodified lines
460458 break;
461459 }
462460 if (to) {
463− hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
461+ hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
464462 if (likely(to->task))
465463 freezable_schedule();
466464 hrtimer_cancel(&to->timer);
fs/timerfd.c+5−1
@@ -471,7 +471,11 @@ static int do_timerfd_settime(int ufd, int flags, · 470 unmodified lines
471471 break;
472472 }
473473 spin_unlock_irq(&ctx->wqh.lock);
474− cpu_relax();
474+
475+ if (isalarm(ctx))
476+ hrtimer_cancel_wait_running(&ctx->t.alarm.timer);
477+ else
478+ hrtimer_cancel_wait_running(&ctx->t.tmr);
475479 }
476480
477481 /*
include/asm-generic/mshyperv.h+1−0
@@ -167,6 +167,7 @@ void hyperv_report_panic(struct pt_regs *regs, long err); · 166 unmodified lines
167167 void hyperv_report_panic_msg(phys_addr_t pa, size_t size);
168168 bool hv_is_hyperv_initialized(void);
169169 void hyperv_cleanup(void);
170+void hv_setup_sched_clock(void *sched_clock);
170171 #else /* CONFIG_HYPERV */
171172 static inline bool hv_is_hyperv_initialized(void) { return false; }
172173 static inline void hyperv_cleanup(void) {}
include/clocksource/hyperv_timer.h+3−5
@@ -28,12 +28,10 @@ extern void hv_stimer_cleanup(unsigned int cpu); · 27 unmodified lines
2828 extern void hv_stimer_global_cleanup(void);
2929 extern void hv_stimer0_isr(void);
3030
31−#if IS_ENABLED(CONFIG_HYPERV)
31+#ifdef CONFIG_HYPERV_TIMER
3232 extern struct clocksource *hyperv_cs;
3333 extern void hv_init_clocksource(void);
34−#endif /* CONFIG_HYPERV */
3534
36−#ifdef CONFIG_HYPERV_TSCPAGE
3735 extern struct ms_hyperv_tsc_page *hv_get_tsc_page(void);
3836
3937 static inline notrace u64
@@ -91,7 +89,7 @@ hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg) · 51 unmodified lines
9189 return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc);
9290 }
9391
94−#else /* CONFIG_HYPERV_TSC_PAGE */
92+#else /* CONFIG_HYPERV_TIMER */
9593 static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
9694 {
9795 return NULL;
@@ -102,6 +100,6 @@ static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg, · 4 unmodified lines
102100 {
103101 return U64_MAX;
104102 }
105−#endif /* CONFIG_HYPERV_TSCPAGE */
103+#endif /* CONFIG_HYPERV_TIMER */
106104
107105 #endif
include/linux/alarmtimer.h+2−1
@@ -5,7 +5,8 @@ · 4 unmodified lines
55 #include <linux/time.h>
66 #include <linux/hrtimer.h>
77 #include <linux/timerqueue.h>
8−#include <linux/rtc.h>
8+
9+struct rtc_device;
910
1011 enum alarmtimer_type {
1112 ALARM_REALTIME,
include/linux/hrtimer.h+44−4
@@ -32,12 +32,15 @@ struct hrtimer_cpu_base; · 31 unmodified lines
3232 * when starting the timer)
3333 * HRTIMER_MODE_SOFT - Timer callback function will be executed in
3434 * soft irq context
35+ * HRTIMER_MODE_HARD - Timer callback function will be executed in
36+ * hard irq context even on PREEMPT_RT.
3537 */
3638 enum hrtimer_mode {
3739 HRTIMER_MODE_ABS = 0x00,
3840 HRTIMER_MODE_REL = 0x01,
3941 HRTIMER_MODE_PINNED = 0x02,
4042 HRTIMER_MODE_SOFT = 0x04,
43+ HRTIMER_MODE_HARD = 0x08,
4144
4245 HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
4346 HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
@@ -48,6 +51,11 @@ enum hrtimer_mode { · 4 unmodified lines
4851 HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
4952 HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
5053
54+ HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD,
55+ HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD,
56+
57+ HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD,
58+ HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
5159 };
5260
5361 /*
@@ -101,6 +109,8 @@ enum hrtimer_restart { · 47 unmodified lines
101109 * @state: state information (See bit values above)
102110 * @is_rel: Set if the timer was armed relative
103111 * @is_soft: Set if hrtimer will be expired in soft interrupt context.
112+ * @is_hard: Set if hrtimer will be expired in hard interrupt context
113+ * even on RT.
104114 *
105115 * The hrtimer structure must be initialized by hrtimer_init()
106116 */
@@ -112,6 +122,7 @@ struct hrtimer { · 5 unmodified lines
112122 u8 state;
113123 u8 is_rel;
114124 u8 is_soft;
125+ u8 is_hard;
115126 };
116127
117128 /**
@@ -183,6 +194,10 @@ enum hrtimer_base_type { · 65 unmodified lines
183194 * @nr_retries: Total number of hrtimer interrupt retries
184195 * @nr_hangs: Total number of hrtimer interrupt hangs
185196 * @max_hang_time: Maximum time spent in hrtimer_interrupt
197+ * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
198+ * expired
199+ * @timer_waiters: A hrtimer_cancel() invocation waits for the timer
200+ * callback to finish.
186201 * @expires_next: absolute time of the next event, is required for remote
187202 * hrtimer enqueue; it is the total first expiry time (hard
188203 * and soft hrtimer are taken into account)
@@ -209,6 +224,10 @@ struct hrtimer_cpu_base { · 20 unmodified lines
209224 unsigned short nr_retries;
210225 unsigned short nr_hangs;
211226 unsigned int max_hang_time;
227+#endif
228+#ifdef CONFIG_PREEMPT_RT
229+ spinlock_t softirq_expiry_lock;
230+ atomic_t timer_waiters;
212231 #endif
213232 ktime_t expires_next;
214233 struct hrtimer *next_timer;
@@ -341,16 +360,29 @@ extern void hrtimers_resume(void); · 126 unmodified lines
341360
342361 DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
343362
363+#ifdef CONFIG_PREEMPT_RT
364+void hrtimer_cancel_wait_running(const struct hrtimer *timer);
365+#else
366+static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
367+{
368+ cpu_relax();
369+}
370+#endif
344371
345372 /* Exported timer functions: */
346373
347374 /* Initialize timers: */
348375 extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
349376 enum hrtimer_mode mode);
377+extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
378+ enum hrtimer_mode mode);
350379
351380 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
352381 extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
353382 enum hrtimer_mode mode);
383+extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
384+ clockid_t clock_id,
385+ enum hrtimer_mode mode);
354386
355387 extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
356388 #else
@@ -360,6 +392,14 @@ static inline void hrtimer_init_on_stack(struct hrtimer *timer, · 3 unmodified lines
360392 {
361393 hrtimer_init(timer, which_clock, mode);
362394 }
395+
396+static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
397+ clockid_t clock_id,
398+ enum hrtimer_mode mode)
399+{
400+ hrtimer_init_sleeper(sl, clock_id, mode);
401+}
402+
363403 static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
364404 #endif
365405
@@ -395,6 +435,9 @@ static inline void hrtimer_start_expires(struct hrtimer *timer, · 29 unmodified lines
395435 hrtimer_start_range_ns(timer, soft, delta, mode);
396436 }
397437
438+void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
439+ enum hrtimer_mode mode);
440+
398441 static inline void hrtimer_restart(struct hrtimer *timer)
399442 {
400443 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
@@ -463,11 +506,8 @@ extern long hrtimer_nanosleep(const struct timespec64 *rqtp, · 62 unmodified lines
463506 const enum hrtimer_mode mode,
464507 const clockid_t clockid);
465508
466−extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
467− struct task_struct *tsk);
468−
469509 extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
470− const enum hrtimer_mode mode);
510+ const enum hrtimer_mode mode);
471511 extern int schedule_hrtimeout_range_clock(ktime_t *expires,
472512 u64 delta,
473513 const enum hrtimer_mode mode,
include/linux/init_task.h+0−11
@@ -36,17 +36,6 @@ extern struct cred init_cred; · 35 unmodified lines
3636 #define INIT_PREV_CPUTIME(x)
3737 #endif
3838
39−#ifdef CONFIG_POSIX_TIMERS
40−#define INIT_CPU_TIMERS(s) \
41− .cpu_timers = { \
42− LIST_HEAD_INIT(s.cpu_timers[0]), \
43− LIST_HEAD_INIT(s.cpu_timers[1]), \
44− LIST_HEAD_INIT(s.cpu_timers[2]), \
45− },
46−#else
47−#define INIT_CPU_TIMERS(s)
48−#endif
49−
5039 #define INIT_TASK_COMM "swapper"
5140
5241 /* Attach to the init_task data structure for proper alignment */
include/linux/posix-timers.h+117−14
@@ -4,18 +4,11 @@ · 3 unmodified lines
44
55 #include <linux/spinlock.h>
66 #include <linux/list.h>
7−#include <linux/sched.h>
8−#include <linux/timex.h>
97 #include <linux/alarmtimer.h>
8+#include <linux/timerqueue.h>
109
11−struct siginfo;
12−
13−struct cpu_timer_list {
14− struct list_head entry;
15− u64 expires;
16− struct task_struct *task;
17− int firing;
18−};
10+struct kernel_siginfo;
11+struct task_struct;
1912
2013 /*
2114 * Bit fields within a clockid:
@@ -63,6 +56,115 @@ static inline int clockid_to_fd(const clockid_t clk) · 41 unmodified lines
6356 return ~(clk >> 3);
6457 }
6558
59+#ifdef CONFIG_POSIX_TIMERS
60+
61+/**
62+ * cpu_timer - Posix CPU timer representation for k_itimer
63+ * @node: timerqueue node to queue in the task/sig
64+ * @head: timerqueue head on which this timer is queued
65+ * @task: Pointer to target task
66+ * @elist: List head for the expiry list
67+ * @firing: Timer is currently firing
68+ */
69+struct cpu_timer {
70+ struct timerqueue_node node;
71+ struct timerqueue_head *head;
72+ struct task_struct *task;
73+ struct list_head elist;
74+ int firing;
75+};
76+
77+static inline bool cpu_timer_enqueue(struct timerqueue_head *head,
78+ struct cpu_timer *ctmr)
79+{
80+ ctmr->head = head;
81+ return timerqueue_add(head, &ctmr->node);
82+}
83+
84+static inline void cpu_timer_dequeue(struct cpu_timer *ctmr)
85+{
86+ if (ctmr->head) {
87+ timerqueue_del(ctmr->head, &ctmr->node);
88+ ctmr->head = NULL;
89+ }
90+}
91+
92+static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr)
93+{
94+ return ctmr->node.expires;
95+}
96+
97+static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp)
98+{
99+ ctmr->node.expires = exp;
100+}
101+
102+/**
103+ * posix_cputimer_base - Container per posix CPU clock
104+ * @nextevt: Earliest-expiration cache
105+ * @tqhead: timerqueue head for cpu_timers
106+ */
107+struct posix_cputimer_base {
108+ u64 nextevt;
109+ struct timerqueue_head tqhead;
110+};
111+
112+/**
113+ * posix_cputimers - Container for posix CPU timer related data
114+ * @bases: Base container for posix CPU clocks
115+ * @timers_active: Timers are queued.
116+ * @expiry_active: Timer expiry is active. Used for
117+ * process wide timers to avoid multiple
118+ * task trying to handle expiry concurrently
119+ *
120+ * Used in task_struct and signal_struct
121+ */
122+struct posix_cputimers {
123+ struct posix_cputimer_base bases[CPUCLOCK_MAX];
124+ unsigned int timers_active;
125+ unsigned int expiry_active;
126+};
127+
128+static inline void posix_cputimers_init(struct posix_cputimers *pct)
129+{
130+ memset(pct, 0, sizeof(*pct));
131+ pct->bases[0].nextevt = U64_MAX;
132+ pct->bases[1].nextevt = U64_MAX;
133+ pct->bases[2].nextevt = U64_MAX;
134+}
135+
136+void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit);
137+
138+static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct,
139+ u64 runtime)
140+{
141+ pct->bases[CPUCLOCK_SCHED].nextevt = runtime;
142+}
143+
144+/* Init task static initializer */
145+#define INIT_CPU_TIMERBASE(b) { \
146+ .nextevt = U64_MAX, \
147+}
148+
149+#define INIT_CPU_TIMERBASES(b) { \
150+ INIT_CPU_TIMERBASE(b[0]), \
151+ INIT_CPU_TIMERBASE(b[1]), \
152+ INIT_CPU_TIMERBASE(b[2]), \
153+}
154+
155+#define INIT_CPU_TIMERS(s) \
156+ .posix_cputimers = { \
157+ .bases = INIT_CPU_TIMERBASES(s.posix_cputimers.bases), \
158+ },
159+#else
160+struct posix_cputimers { };
161+struct cpu_timer { };
162+#define INIT_CPU_TIMERS(s)
163+static inline void posix_cputimers_init(struct posix_cputimers *pct) { }
164+static inline void posix_cputimers_group_init(struct posix_cputimers *pct,
165+ u64 cpu_limit) { }
166+#endif
167+
66168 #define REQUEUE_PENDING 1
67169
68170 /**
@@ -85,7 +187,8 @@ static inline int clockid_to_fd(const clockid_t clk) · 16 unmodified lines
85187 * @it_process: The task to wakeup on clock_nanosleep (CPU timers)
86188 * @sigq: Pointer to preallocated sigqueue
87189 * @it: Union representing the various posix timer type
88− * internals. Also used for rcu freeing the timer.
190+ * internals.
191+ * @rcu: RCU head for freeing the timer.
89192 */
90193 struct k_itimer {
91194 struct list_head list;
@@ -110,15 +213,15 @@ struct k_itimer { · 18 unmodified lines
110213 struct {
111214 struct hrtimer timer;
112215 } real;
113− struct cpu_timer_list cpu;
216+ struct cpu_timer cpu;
114217 struct {
115218 struct alarm alarmtimer;
116219 } alarm;
117− struct rcu_head rcu;
118220 } it;
221+ struct rcu_head rcu;
119222 };
120223
121−void run_posix_cpu_timers(struct task_struct *task);
224+void run_posix_cpu_timers(void);
122225 void posix_cpu_timers_exit(struct task_struct *task);
123226 void posix_cpu_timers_exit_group(struct task_struct *task);
124227 void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx,
include/linux/sched.h+4−25
@@ -25,9 +25,11 @@ · 24 unmodified lines
2525 #include <linux/resource.h>
2626 #include <linux/latencytop.h>
2727 #include <linux/sched/prio.h>
28+#include <linux/sched/types.h>
2829 #include <linux/signal_types.h>
2930 #include <linux/mm_types_task.h>
3031 #include <linux/task_io_accounting.h>
32+#include <linux/posix-timers.h>
3133 #include <linux/rseq.h>
3234
3335 /* task_struct member predeclarations (sorted alphabetically): */
@@ -244,27 +246,6 @@ struct prev_cputime { · 210 unmodified lines
244246 #endif
245247 };
246248
247−/**
248− * struct task_cputime - collected CPU time counts
249− * @utime: time spent in user mode, in nanoseconds
250− * @stime: time spent in kernel mode, in nanoseconds
251− * @sum_exec_runtime: total time spent on the CPU, in nanoseconds
252− *
253− * This structure groups together three kinds of CPU time that are tracked for
254− * threads and thread groups. Most things considering CPU time want to group
255− * these counts together and treat all three of them in parallel.
256− */
257−struct task_cputime {
258− u64 utime;
259− u64 stime;
260− unsigned long long sum_exec_runtime;
261−};
262−
263−/* Alternate field names when used on cache expirations: */
264−#define virt_exp utime
265−#define prof_exp stime
266−#define sched_exp sum_exec_runtime
267−
268249 enum vtime_state {
269250 /* Task is sleeping or running in a CPU with VTIME inactive: */
270251 VTIME_INACTIVE = 0,
@@ -881,10 +862,8 @@ struct task_struct { · 610 unmodified lines
881862 unsigned long min_flt;
882863 unsigned long maj_flt;
883864
884−#ifdef CONFIG_POSIX_TIMERS
885− struct task_cputime cputime_expires;
886− struct list_head cpu_timers[3];
887−#endif
865+ /* Empty if CONFIG_POSIX_CPUTIMERS=n */
866+ struct posix_cputimers posix_cputimers;
888867
889868 /* Process credentials: */
890869
include/linux/sched/cputime.h+7−5
@@ -61,8 +61,7 @@ extern void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, · 60 unmodified lines
6161 * Thread group CPU time accounting.
6262 */
6363 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
64−void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
65−
64+void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples);
6665
6766 /*
6867 * The following are functions that support scheduler-internal time accounting.
@@ -71,7 +70,7 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); · 2 unmodified lines
7170 */
7271
7372 /**
74− * get_running_cputimer - return &tsk->signal->cputimer if cputimer is running
73+ * get_running_cputimer - return &tsk->signal->cputimer if cputimers are active
7574 *
7675 * @tsk: Pointer to target task.
7776 */
@@ -81,8 +80,11 @@ struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk) · 3 unmodified lines
8180 {
8281 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
8382
84− /* Check if cputimer isn't running. This is accessed without locking. */
85− if (!READ_ONCE(cputimer->running))
83+ /*
84+ * Check whether posix CPU timers are active. If not the thread
85+ * group accounting is not active either. Lockless check.
86+ */
87+ if (!READ_ONCE(tsk->signal->posix_cputimers.timers_active))
8688 return NULL;
8789
8890 /*
include/linux/sched/signal.h+3−11
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include <linux/sched/task.h>
1010 #include <linux/cred.h>
1111 #include <linux/refcount.h>
12+#include <linux/posix-timers.h>
1213
1314 /*
1415 * Types defining task->signal and task->sighand and APIs using them:
@@ -56,18 +57,12 @@ struct task_cputime_atomic { · 41 unmodified lines
5657 /**
5758 * struct thread_group_cputimer - thread group interval timer counts
5859 * @cputime_atomic: atomic thread group interval timers.
59− * @running: true when there are timers running and
60− * @cputime_atomic receives updates.
61− * @checking_timer: true when a thread in the group is in the
62− * process of checking for thread group timers.
6360 *
6461 * This structure contains the version of task_cputime, above, that is
6562 * used for thread group CPU timer calculations.
6663 */
6764 struct thread_group_cputimer {
6865 struct task_cputime_atomic cputime_atomic;
69− bool running;
70− bool checking_timer;
7166 };
7267
7368 struct multiprocess_signals {
@@ -148,12 +143,9 @@ struct signal_struct { · 74 unmodified lines
148143 */
149144 struct thread_group_cputimer cputimer;
150145
151− /* Earliest-expiration cache. */
152− struct task_cputime cputime_expires;
153−
154− struct list_head cpu_timers[3];
155−
156146 #endif
147+ /* Empty if CONFIG_POSIX_TIMERS=n */
148+ struct posix_cputimers posix_cputimers;
157149
158150 /* PID/PID hash table linkage. */
159151 struct pid *pids[PIDTYPE_MAX];
include/linux/sched/types.h+23−0
@@ -0,0 +1,23 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef _LINUX_SCHED_TYPES_H
3+#define _LINUX_SCHED_TYPES_H
4+
5+#include <linux/types.h>
6+
7+/**
8+ * struct task_cputime - collected CPU time counts
9+ * @stime: time spent in kernel mode, in nanoseconds
10+ * @utime: time spent in user mode, in nanoseconds
11+ * @sum_exec_runtime: total time spent on the CPU, in nanoseconds
12+ *
13+ * This structure groups together three kinds of CPU time that are tracked for
14+ * threads and thread groups. Most things considering CPU time want to group
15+ * these counts together and treat all three of them in parallel.
16+ */
17+struct task_cputime {
18+ u64 stime;
19+ u64 utime;
20+ unsigned long long sum_exec_runtime;
21+};
22+
23+#endif
include/linux/timer.h+1−1
@@ -183,7 +183,7 @@ extern void add_timer(struct timer_list *timer); · 182 unmodified lines
183183
184184 extern int try_to_del_timer_sync(struct timer_list *timer);
185185
186−#ifdef CONFIG_SMP
186+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
187187 extern int del_timer_sync(struct timer_list *timer);
188188 #else
189189 # define del_timer_sync(t) del_timer(t)
include/linux/timerqueue.h+16−7
@@ -12,8 +12,7 @@ struct timerqueue_node { · 11 unmodified lines
1212 };
1313
1414 struct timerqueue_head {
15− struct rb_root head;
16− struct timerqueue_node *next;
15+ struct rb_root_cached rb_root;
1716 };
1817
1918
@@ -29,13 +28,14 @@ extern struct timerqueue_node *timerqueue_iterate_next( · 9 unmodified lines
2928 *
3029 * @head: head of timerqueue
3130 *
32− * Returns a pointer to the timer node that has the
33− * earliest expiration time.
31+ * Returns a pointer to the timer node that has the earliest expiration time.
3432 */
3533 static inline
3634 struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
3735 {
38− return head->next;
36+ struct rb_node *leftmost = rb_first_cached(&head->rb_root);
37+
38+ return rb_entry(leftmost, struct timerqueue_node, node);
3939 }
4040
4141 static inline void timerqueue_init(struct timerqueue_node *node)
@@ -43,9 +43,18 @@ static inline void timerqueue_init(struct timerqueue_node *node) · 1 unmodified line
4343 RB_CLEAR_NODE(&node->node);
4444 }
4545
46+static inline bool timerqueue_node_queued(struct timerqueue_node *node)
47+{
48+ return !RB_EMPTY_NODE(&node->node);
49+}
50+
51+static inline bool timerqueue_node_expires(struct timerqueue_node *node)
52+{
53+ return node->expires;
54+}
55+
4656 static inline void timerqueue_init_head(struct timerqueue_head *head)
4757 {
48− head->head = RB_ROOT;
49− head->next = NULL;
58+ head->rb_root = RB_ROOT_CACHED;
5059 }
5160 #endif /* _LINUX_TIMERQUEUE_H */
include/linux/wait.h+2−2
@@ -501,8 +501,8 @@ do { \ · 500 unmodified lines
501501 int __ret = 0; \
502502 struct hrtimer_sleeper __t; \
503503 \
504− hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); \
505− hrtimer_init_sleeper(&__t, current); \
504+ hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
505+ HRTIMER_MODE_REL); \
506506 if ((timeout) != KTIME_MAX) \
507507 hrtimer_start_range_ns(&__t.timer, timeout, \
508508 current->timer_slack_ns, \
init/init_task.c+0−2
@@ -30,8 +30,6 @@ static struct signal_struct init_signals = { · 29 unmodified lines
3030 .posix_timers = LIST_HEAD_INIT(init_signals.posix_timers),
3131 .cputimer = {
3232 .cputime_atomic = INIT_CPUTIME_ATOMIC,
33− .running = false,
34− .checking_timer = false,
3533 },
3634 #endif
3735 INIT_CPU_TIMERS(init_signals)
kernel/events/core.c+4−4
@@ -1103,7 +1103,7 @@ static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) · 1102 unmodified lines
11031103 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
11041104
11051105 raw_spin_lock_init(&cpuctx->hrtimer_lock);
1106− hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
1106+ hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
11071107 timer->function = perf_mux_hrtimer_handler;
11081108 }
11091109
@@ -1121,7 +1121,7 @@ static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx) · 11 unmodified lines
11211121 if (!cpuctx->hrtimer_active) {
11221122 cpuctx->hrtimer_active = 1;
11231123 hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
1124− hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
1124+ hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
11251125 }
11261126 raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
11271127
@@ -9574,7 +9574,7 @@ static void perf_swevent_start_hrtimer(struct perf_event *event) · 8446 unmodified lines
95749574 period = max_t(u64, 10000, hwc->sample_period);
95759575 }
95769576 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period),
9577− HRTIMER_MODE_REL_PINNED);
9577+ HRTIMER_MODE_REL_PINNED_HARD);
95789578 }
95799579
95809580 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
@@ -9596,7 +9596,7 @@ static void perf_swevent_init_hrtimer(struct perf_event *event) · 15 unmodified lines
95969596 if (!is_sampling_event(event))
95979597 return;
95989598
9599− hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
9599+ hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
96009600 hwc->hrtimer.function = perf_swevent_hrtimer;
96019601
96029602 /*
kernel/fork.c+3−31
@@ -1519,28 +1519,17 @@ void __cleanup_sighand(struct sighand_struct *sighand) · 1518 unmodified lines
15191519 }
15201520 }
15211521
1522−#ifdef CONFIG_POSIX_TIMERS
15231522 /*
15241523 * Initialize POSIX timer handling for a thread group.
15251524 */
15261525 static void posix_cpu_timers_init_group(struct signal_struct *sig)
15271526 {
1527+ struct posix_cputimers *pct = &sig->posix_cputimers;
15281528 unsigned long cpu_limit;
15291529
15301530 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
1531− if (cpu_limit != RLIM_INFINITY) {
1532− sig->cputime_expires.prof_exp = cpu_limit * NSEC_PER_SEC;
1533− sig->cputimer.running = true;
1534− }
1535−
1536− /* The timer lists. */
1537− INIT_LIST_HEAD(&sig->cpu_timers[0]);
1538− INIT_LIST_HEAD(&sig->cpu_timers[1]);
1539− INIT_LIST_HEAD(&sig->cpu_timers[2]);
1531+ posix_cputimers_group_init(pct, cpu_limit);
15401532 }
1541−#else
1542−static inline void posix_cpu_timers_init_group(struct signal_struct *sig) { }
1543−#endif
15441533
15451534 static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
15461535 {
@@ -1642,23 +1631,6 @@ static void rt_mutex_init_task(struct task_struct *p) · 95 unmodified lines
16421631 #endif
16431632 }
16441633
1645−#ifdef CONFIG_POSIX_TIMERS
1646−/*
1647− * Initialize POSIX timer handling for a single task.
1648− */
1649−static void posix_cpu_timers_init(struct task_struct *tsk)
1650−{
1651− tsk->cputime_expires.prof_exp = 0;
1652− tsk->cputime_expires.virt_exp = 0;
1653− tsk->cputime_expires.sched_exp = 0;
1654− INIT_LIST_HEAD(&tsk->cpu_timers[0]);
1655− INIT_LIST_HEAD(&tsk->cpu_timers[1]);
1656− INIT_LIST_HEAD(&tsk->cpu_timers[2]);
1657−}
1658−#else
1659−static inline void posix_cpu_timers_init(struct task_struct *tsk) { }
1660−#endif
1661−
16621634 static inline void init_task_pid_links(struct task_struct *task)
16631635 {
16641636 enum pid_type type;
@@ -1945,7 +1917,7 @@ static __latent_entropy struct task_struct *copy_process( · 280 unmodified lines
19451917 task_io_accounting_init(&p->ioac);
19461918 acct_clear_integrals(p);
19471919
1948− posix_cpu_timers_init(p);
1920+ posix_cputimers_init(&p->posix_cputimers);
19491921
19501922 p->io_context = NULL;
19511923 audit_set_context(p, NULL);
kernel/futex.c+5−7
@@ -487,11 +487,9 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, · 486 unmodified lines
487487 if (!time)
488488 return NULL;
489489
490− hrtimer_init_on_stack(&timeout->timer, (flags & FLAGS_CLOCKRT) ?
491− CLOCK_REALTIME : CLOCK_MONOTONIC,
492− HRTIMER_MODE_ABS);
493− hrtimer_init_sleeper(timeout, current);
494−
490+ hrtimer_init_sleeper_on_stack(timeout, (flags & FLAGS_CLOCKRT) ?
491+ CLOCK_REALTIME : CLOCK_MONOTONIC,
492+ HRTIMER_MODE_ABS);
495493 /*
496494 * If range_ns is 0, calling hrtimer_set_expires_range_ns() is
497495 * effectively the same as calling hrtimer_set_expires().
@@ -2613,7 +2611,7 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q, · 2115 unmodified lines
26132611
26142612 /* Arm the timer */
26152613 if (timeout)
2616− hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
2614+ hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
26172615
26182616 /*
26192617 * If we have been removed from the hash list, then another task
@@ -2899,7 +2897,7 @@ retry_private: · 279 unmodified lines
28992897 }
29002898
29012899 if (unlikely(to))
2902− hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS);
2900+ hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
29032901
29042902 ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
29052903
kernel/sched/core.c+3−3
@@ -255,7 +255,7 @@ static void __hrtick_restart(struct rq *rq) · 254 unmodified lines
255255 {
256256 struct hrtimer *timer = &rq->hrtick_timer;
257257
258− hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
258+ hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD);
259259 }
260260
261261 /*
@@ -314,7 +314,7 @@ void hrtick_start(struct rq *rq, u64 delay) · 52 unmodified lines
314314 */
315315 delay = max_t(u64, delay, 10000LL);
316316 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
317− HRTIMER_MODE_REL_PINNED);
317+ HRTIMER_MODE_REL_PINNED_HARD);
318318 }
319319 #endif /* CONFIG_SMP */
320320
@@ -328,7 +328,7 @@ static void hrtick_rq_init(struct rq *rq) · 7 unmodified lines
328328 rq->hrtick_csd.info = rq;
329329 #endif
330330
331− hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
331+ hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
332332 rq->hrtick_timer.function = hrtick;
333333 }
334334 #else /* CONFIG_SCHED_HRTICK */
kernel/sched/deadline.c+4−4
@@ -287,7 +287,7 @@ static void task_non_contending(struct task_struct *p) · 286 unmodified lines
287287
288288 dl_se->dl_non_contending = 1;
289289 get_task_struct(p);
290− hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL);
290+ hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL_HARD);
291291 }
292292
293293 static void task_contending(struct sched_dl_entity *dl_se, int flags)
@@ -956,7 +956,7 @@ static int start_dl_timer(struct task_struct *p) · 662 unmodified lines
956956 */
957957 if (!hrtimer_is_queued(timer)) {
958958 get_task_struct(p);
959− hrtimer_start(timer, act, HRTIMER_MODE_ABS);
959+ hrtimer_start(timer, act, HRTIMER_MODE_ABS_HARD);
960960 }
961961
962962 return 1;
@@ -1086,7 +1086,7 @@ void init_dl_task_timer(struct sched_dl_entity *dl_se) · 123 unmodified lines
10861086 {
10871087 struct hrtimer *timer = &dl_se->dl_timer;
10881088
1089− hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1089+ hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
10901090 timer->function = dl_task_timer;
10911091 }
10921092
@@ -1325,7 +1325,7 @@ void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se) · 232 unmodified lines
13251325 {
13261326 struct hrtimer *timer = &dl_se->inactive_timer;
13271327
1328− hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1328+ hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
13291329 timer->function = inactive_task_timer;
13301330 }
13311331
kernel/sched/rt.c+8−5
@@ -45,8 +45,8 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) · 44 unmodified lines
4545
4646 raw_spin_lock_init(&rt_b->rt_runtime_lock);
4747
48− hrtimer_init(&rt_b->rt_period_timer,
49− CLOCK_MONOTONIC, HRTIMER_MODE_REL);
48+ hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC,
49+ HRTIMER_MODE_REL_HARD);
5050 rt_b->rt_period_timer.function = sched_rt_period_timer;
5151 }
5252
@@ -67,7 +67,8 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) · 14 unmodified lines
6767 * to update the period.
6868 */
6969 hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
70− hrtimer_start_expires(&rt_b->rt_period_timer, HRTIMER_MODE_ABS_PINNED);
70+ hrtimer_start_expires(&rt_b->rt_period_timer,
71+ HRTIMER_MODE_ABS_PINNED_HARD);
7172 }
7273 raw_spin_unlock(&rt_b->rt_runtime_lock);
7374 }
@@ -2289,8 +2290,10 @@ static void watchdog(struct rq *rq, struct task_struct *p) · 2215 unmodified lines
22892290 }
22902291
22912292 next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
2292− if (p->rt.timeout > next)
2293− p->cputime_expires.sched_exp = p->se.sum_exec_runtime;
2293+ if (p->rt.timeout > next) {
2294+ posix_cputimers_rt_watchdog(&p->posix_cputimers,
2295+ p->se.sum_exec_runtime);
2296+ }
22942297 }
22952298 }
22962299 #else
kernel/sys.c+3−13
@@ -1557,15 +1557,6 @@ int do_prlimit(struct task_struct *tsk, unsigned int resource, · 1556 unmodified lines
15571557 retval = -EPERM;
15581558 if (!retval)
15591559 retval = security_task_setrlimit(tsk, resource, new_rlim);
1560− if (resource == RLIMIT_CPU && new_rlim->rlim_cur == 0) {
1561− /*
1562− * The caller is asking for an immediate RLIMIT_CPU
1563− * expiry. But we use the zero value to mean "it was
1564− * never set". So let's cheat and make it one second
1565− * instead
1566− */
1567− new_rlim->rlim_cur = 1;
1568− }
15691560 }
15701561 if (!retval) {
15711562 if (old_rlim)
@@ -1576,10 +1567,9 @@ int do_prlimit(struct task_struct *tsk, unsigned int resource, · 4 unmodified lines
15761567 task_unlock(tsk->group_leader);
15771568
15781569 /*
1579− * RLIMIT_CPU handling. Note that the kernel fails to return an error
1580− * code if it rejected the user's attempt to set RLIMIT_CPU. This is a
1581− * very long-standing error, and fixing it now risks breakage of
1582− * applications, so we live with it
1570+ * RLIMIT_CPU handling. Arm the posix CPU timer if the limit is not
1571+ * infite. In case of RLIM_INFINITY the posix CPU timer code
1572+ * ignores the rlimit.
15831573 */
15841574 if (!retval && new_rlim && resource == RLIMIT_CPU &&
15851575 new_rlim->rlim_cur != RLIM_INFINITY &&
kernel/time/alarmtimer.c+15−1
@@ -432,7 +432,7 @@ int alarm_cancel(struct alarm *alarm) · 431 unmodified lines
432432 int ret = alarm_try_to_cancel(alarm);
433433 if (ret >= 0)
434434 return ret;
435− cpu_relax();
435+ hrtimer_cancel_wait_running(&alarm->timer);
436436 }
437437 }
438438 EXPORT_SYMBOL_GPL(alarm_cancel);
@@ -605,6 +605,19 @@ static int alarm_timer_try_to_cancel(struct k_itimer *timr) · 166 unmodified lines
605605 return alarm_try_to_cancel(&timr->it.alarm.alarmtimer);
606606 }
607607
608+/**
609+ * alarm_timer_wait_running - Posix timer callback to wait for a timer
610+ * @timr: Pointer to the posixtimer data struct
611+ *
612+ * Called from the core code when timer cancel detected that the callback
613+ * is running. @timr is unlocked and rcu read lock is held to prevent it
614+ * from being freed.
615+ */
616+static void alarm_timer_wait_running(struct k_itimer *timr)
617+{
618+ hrtimer_cancel_wait_running(&timr->it.alarm.alarmtimer.timer);
619+}
620+
608621 /**
609622 * alarm_timer_arm - Posix timer callback to arm a timer
610623 * @timr: Pointer to the posixtimer data struct
@@ -834,6 +847,7 @@ const struct k_clock alarm_clock = { · 223 unmodified lines
834847 .timer_forward = alarm_timer_forward,
835848 .timer_remaining = alarm_timer_remaining,
836849 .timer_try_to_cancel = alarm_timer_try_to_cancel,
850+ .timer_wait_running = alarm_timer_wait_running,
837851 .nsleep = alarm_timer_nsleep,
838852 };
839853 #endif /* CONFIG_POSIX_TIMERS */
kernel/time/hrtimer.c+212−23
@@ -140,6 +140,11 @@ static struct hrtimer_cpu_base migration_cpu_base = { · 139 unmodified lines
140140
141141 #define migration_base migration_cpu_base.clock_base[0]
142142
143+static inline bool is_migration_base(struct hrtimer_clock_base *base)
144+{
145+ return base == &migration_base;
146+}
147+
143148 /*
144149 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
145150 * means that all timers which are tied to this base via timer->base are
@@ -264,6 +269,11 @@ again: · 118 unmodified lines
264269
265270 #else /* CONFIG_SMP */
266271
272+static inline bool is_migration_base(struct hrtimer_clock_base *base)
273+{
274+ return false;
275+}
276+
267277 static inline struct hrtimer_clock_base *
268278 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
269279 {
@@ -427,6 +437,17 @@ void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, · 157 unmodified lines
427437 }
428438 EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
429439
440+static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
441+ clockid_t clock_id, enum hrtimer_mode mode);
442+
443+void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
444+ clockid_t clock_id, enum hrtimer_mode mode)
445+{
446+ debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
447+ __hrtimer_init_sleeper(sl, clock_id, mode);
448+}
449+EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
450+
430451 void destroy_hrtimer_on_stack(struct hrtimer *timer)
431452 {
432453 debug_object_free(timer, &hrtimer_debug_descr);
@@ -1096,9 +1117,13 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, · 663 unmodified lines
10961117
10971118 /*
10981119 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
1099− * match.
1120+ * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
1121+ * expiry mode because unmarked timers are moved to softirq expiry.
11001122 */
1101− WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1123+ if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1124+ WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1125+ else
1126+ WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
11021127
11031128 base = lock_hrtimer_base(timer, &flags);
11041129
@@ -1147,6 +1172,93 @@ int hrtimer_try_to_cancel(struct hrtimer *timer) · 42 unmodified lines
11471172 }
11481173 EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
11491174
1175+#ifdef CONFIG_PREEMPT_RT
1176+static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
1177+{
1178+ spin_lock_init(&base->softirq_expiry_lock);
1179+}
1180+
1181+static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
1182+{
1183+ spin_lock(&base->softirq_expiry_lock);
1184+}
1185+
1186+static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
1187+{
1188+ spin_unlock(&base->softirq_expiry_lock);
1189+}
1190+
1191+/*
1192+ * The counterpart to hrtimer_cancel_wait_running().
1193+ *
1194+ * If there is a waiter for cpu_base->expiry_lock, then it was waiting for
1195+ * the timer callback to finish. Drop expiry_lock and reaquire it. That
1196+ * allows the waiter to acquire the lock and make progress.
1197+ */
1198+static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,
1199+ unsigned long flags)
1200+{
1201+ if (atomic_read(&cpu_base->timer_waiters)) {
1202+ raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1203+ spin_unlock(&cpu_base->softirq_expiry_lock);
1204+ spin_lock(&cpu_base->softirq_expiry_lock);
1205+ raw_spin_lock_irq(&cpu_base->lock);
1206+ }
1207+}
1208+
1209+/*
1210+ * This function is called on PREEMPT_RT kernels when the fast path
1211+ * deletion of a timer failed because the timer callback function was
1212+ * running.
1213+ *
1214+ * This prevents priority inversion: if the soft irq thread is preempted
1215+ * in the middle of a timer callback, then calling del_timer_sync() can
1216+ * lead to two issues:
1217+ *
1218+ * - If the caller is on a remote CPU then it has to spin wait for the timer
1219+ * handler to complete. This can result in unbound priority inversion.
1220+ *
1221+ * - If the caller originates from the task which preempted the timer
1222+ * handler on the same CPU, then spin waiting for the timer handler to
1223+ * complete is never going to end.
1224+ */
1225+void hrtimer_cancel_wait_running(const struct hrtimer *timer)
1226+{
1227+ /* Lockless read. Prevent the compiler from reloading it below */
1228+ struct hrtimer_clock_base *base = READ_ONCE(timer->base);
1229+
1230+ /*
1231+ * Just relax if the timer expires in hard interrupt context or if
1232+ * it is currently on the migration base.
1233+ */
1234+ if (!timer->is_soft || is_migration_base(base)) {
1235+ cpu_relax();
1236+ return;
1237+ }
1238+
1239+ /*
1240+ * Mark the base as contended and grab the expiry lock, which is
1241+ * held by the softirq across the timer callback. Drop the lock
1242+ * immediately so the softirq can expire the next timer. In theory
1243+ * the timer could already be running again, but that's more than
1244+ * unlikely and just causes another wait loop.
1245+ */
1246+ atomic_inc(&base->cpu_base->timer_waiters);
1247+ spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
1248+ atomic_dec(&base->cpu_base->timer_waiters);
1249+ spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
1250+}
1251+#else
1252+static inline void
1253+hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
1254+static inline void
1255+hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
1256+static inline void
1257+hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
1258+static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base,
1259+ unsigned long flags) { }
1260+#endif
1261+
11501262 /**
11511263 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
11521264 * @timer: the timer to be cancelled
@@ -1157,13 +1269,15 @@ EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); · 4 unmodified lines
11571269 */
11581270 int hrtimer_cancel(struct hrtimer *timer)
11591271 {
1160− for (;;) {
1161− int ret = hrtimer_try_to_cancel(timer);
1272+ int ret;
11621273
1163− if (ret >= 0)
1164− return ret;
1165− cpu_relax();
1166− }
1274+ do {
1275+ ret = hrtimer_try_to_cancel(timer);
1276+
1277+ if (ret < 0)
1278+ hrtimer_cancel_wait_running(timer);
1279+ } while (ret < 0);
1280+ return ret;
11671281 }
11681282 EXPORT_SYMBOL_GPL(hrtimer_cancel);
11691283
@@ -1260,8 +1374,17 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, · 90 unmodified lines
12601374 enum hrtimer_mode mode)
12611375 {
12621376 bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1263− int base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
12641377 struct hrtimer_cpu_base *cpu_base;
1378+ int base;
1379+
1380+ /*
1381+ * On PREEMPT_RT enabled kernels hrtimers which are not explicitely
1382+ * marked for hard interrupt expiry mode are moved into soft
1383+ * interrupt context for latency reasons and because the callbacks
1384+ * can invoke functions which might sleep on RT, e.g. spin_lock().
1385+ */
1386+ if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
1387+ softtimer = true;
12651388
12661389 memset(timer, 0, sizeof(struct hrtimer));
12671390
@@ -1275,8 +1398,10 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, · 7 unmodified lines
12751398 if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
12761399 clock_id = CLOCK_MONOTONIC;
12771400
1401+ base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
12781402 base += hrtimer_clockid_to_base(clock_id);
12791403 timer->is_soft = softtimer;
1404+ timer->is_hard = !softtimer;
12801405 timer->base = &cpu_base->clock_base[base];
12811406 timerqueue_init(&timer->node);
12821407 }
@@ -1449,6 +1574,8 @@ static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, · 166 unmodified lines
14491574 break;
14501575
14511576 __run_hrtimer(cpu_base, base, timer, &basenow, flags);
1577+ if (active_mask == HRTIMER_ACTIVE_SOFT)
1578+ hrtimer_sync_wait_running(cpu_base, flags);
14521579 }
14531580 }
14541581 }
@@ -1459,6 +1586,7 @@ static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h) · 4 unmodified lines
14591586 unsigned long flags;
14601587 ktime_t now;
14611588
1589+ hrtimer_cpu_base_lock_expiry(cpu_base);
14621590 raw_spin_lock_irqsave(&cpu_base->lock, flags);
14631591
14641592 now = hrtimer_update_base(cpu_base);
@@ -1468,6 +1596,7 @@ static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h) · 3 unmodified lines
14681596 hrtimer_update_softirq_timer(cpu_base, true);
14691597
14701598 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1599+ hrtimer_cpu_base_unlock_expiry(cpu_base);
14711600 }
14721601
14731602 #ifdef CONFIG_HIGH_RES_TIMERS
@@ -1639,10 +1768,75 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) · 165 unmodified lines
16391768 return HRTIMER_NORESTART;
16401769 }
16411770
1642−void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
1771+/**
1772+ * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
1773+ * @sl: sleeper to be started
1774+ * @mode: timer mode abs/rel
1775+ *
1776+ * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
1777+ * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
1778+ */
1779+void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
1780+ enum hrtimer_mode mode)
1781+{
1782+ /*
1783+ * Make the enqueue delivery mode check work on RT. If the sleeper
1784+ * was initialized for hard interrupt delivery, force the mode bit.
1785+ * This is a special case for hrtimer_sleepers because
1786+ * hrtimer_init_sleeper() determines the delivery mode on RT so the
1787+ * fiddling with this decision is avoided at the call sites.
1788+ */
1789+ if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
1790+ mode |= HRTIMER_MODE_HARD;
1791+
1792+ hrtimer_start_expires(&sl->timer, mode);
1793+}
1794+EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
1795+
1796+static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
1797+ clockid_t clock_id, enum hrtimer_mode mode)
16431798 {
1799+ /*
1800+ * On PREEMPT_RT enabled kernels hrtimers which are not explicitely
1801+ * marked for hard interrupt expiry mode are moved into soft
1802+ * interrupt context either for latency reasons or because the
1803+ * hrtimer callback takes regular spinlocks or invokes other
1804+ * functions which are not suitable for hard interrupt context on
1805+ * PREEMPT_RT.
1806+ *
1807+ * The hrtimer_sleeper callback is RT compatible in hard interrupt
1808+ * context, but there is a latency concern: Untrusted userspace can
1809+ * spawn many threads which arm timers for the same expiry time on
1810+ * the same CPU. That causes a latency spike due to the wakeup of
1811+ * a gazillion threads.
1812+ *
1813+ * OTOH, priviledged real-time user space applications rely on the
1814+ * low latency of hard interrupt wakeups. If the current task is in
1815+ * a real-time scheduling class, mark the mode for hard interrupt
1816+ * expiry.
1817+ */
1818+ if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
1819+ if (task_is_realtime(current) && !(mode & HRTIMER_MODE_SOFT))
1820+ mode |= HRTIMER_MODE_HARD;
1821+ }
1822+
1823+ __hrtimer_init(&sl->timer, clock_id, mode);
16441824 sl->timer.function = hrtimer_wakeup;
1645− sl->task = task;
1825+ sl->task = current;
1826+}
1827+
1828+/**
1829+ * hrtimer_init_sleeper - initialize sleeper to the given clock
1830+ * @sl: sleeper to be initialized
1831+ * @clock_id: the clock to be used
1832+ * @mode: timer mode abs/rel
1833+ */
1834+void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
1835+ enum hrtimer_mode mode)
1836+{
1837+ debug_init(&sl->timer, clock_id, mode);
1838+ __hrtimer_init_sleeper(sl, clock_id, mode);
1839+
16461840 }
16471841 EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
16481842
@@ -1669,11 +1863,9 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod · 20 unmodified lines
16691863 {
16701864 struct restart_block *restart;
16711865
1672− hrtimer_init_sleeper(t, current);
1673−
16741866 do {
16751867 set_current_state(TASK_INTERRUPTIBLE);
1676− hrtimer_start_expires(&t->timer, mode);
1868+ hrtimer_sleeper_start_expires(t, mode);
16771869
16781870 if (likely(t->task))
16791871 freezable_schedule();
@@ -1707,10 +1899,9 @@ static long __sched hrtimer_nanosleep_restart(struct restart_block *restart) · 27 unmodified lines
17071899 struct hrtimer_sleeper t;
17081900 int ret;
17091901
1710− hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,
1711− HRTIMER_MODE_ABS);
1902+ hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
1903+ HRTIMER_MODE_ABS);
17121904 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
1713−
17141905 ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
17151906 destroy_hrtimer_on_stack(&t.timer);
17161907 return ret;
@@ -1728,7 +1919,7 @@ long hrtimer_nanosleep(const struct timespec64 *rqtp, · 11 unmodified lines
17281919 if (dl_task(current) || rt_task(current))
17291920 slack = 0;
17301921
1731− hrtimer_init_on_stack(&t.timer, clockid, mode);
1922+ hrtimer_init_sleeper_on_stack(&t, clockid, mode);
17321923 hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
17331924 ret = do_nanosleep(&t, mode);
17341925 if (ret != -ERESTART_RESTARTBLOCK)
@@ -1809,6 +2000,7 @@ int hrtimers_prepare_cpu(unsigned int cpu) · 74 unmodified lines
18092000 cpu_base->softirq_next_timer = NULL;
18102001 cpu_base->expires_next = KTIME_MAX;
18112002 cpu_base->softirq_expires_next = KTIME_MAX;
2003+ hrtimer_cpu_base_init_expiry_lock(cpu_base);
18122004 return 0;
18132005 }
18142006
@@ -1927,12 +2119,9 @@ schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, · 112 unmodified lines
19272119 return -EINTR;
19282120 }
19292121
1930− hrtimer_init_on_stack(&t.timer, clock_id, mode);
2122+ hrtimer_init_sleeper_on_stack(&t, clock_id, mode);
19312123 hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
1932−
1933− hrtimer_init_sleeper(&t, current);
1934−
1935− hrtimer_start_expires(&t.timer, mode);
2124+ hrtimer_sleeper_start_expires(&t, mode);
19362125
19372126 if (likely(t.task))
19382127 schedule();
kernel/time/itimer.c+4−8
@@ -55,15 +55,10 @@ static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, · 54 unmodified lines
5555 val = it->expires;
5656 interval = it->incr;
5757 if (val) {
58− struct task_cputime cputime;
59− u64 t;
58+ u64 t, samples[CPUCLOCK_MAX];
6059
61− thread_group_cputimer(tsk, &cputime);
62− if (clock_id == CPUCLOCK_PROF)
63− t = cputime.utime + cputime.stime;
64− else
65− /* CPUCLOCK_VIRT */
66− t = cputime.utime;
60+ thread_group_sample_cputime(tsk, samples);
61+ t = samples[clock_id];
6762
6863 if (val < t)
6964 /* about to fire */
@@ -213,6 +208,7 @@ again: · 143 unmodified lines
213208 /* We are sharing ->siglock with it_real_fn() */
214209 if (hrtimer_try_to_cancel(timer) < 0) {
215210 spin_unlock_irq(&tsk->sighand->siglock);
211+ hrtimer_cancel_wait_running(timer);
216212 goto again;
217213 }
218214 expires = timeval_to_ktime(value->it_value);
kernel/time/posix-cpu-timers.c+492−518
@@ -20,11 +20,20 @@ · 19 unmodified lines
2020
2121 static void posix_cpu_timer_rearm(struct k_itimer *timer);
2222
23+void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit)
24+{
25+ posix_cputimers_init(pct);
26+ if (cpu_limit != RLIM_INFINITY) {
27+ pct->bases[CPUCLOCK_PROF].nextevt = cpu_limit * NSEC_PER_SEC;
28+ pct->timers_active = true;
29+ }
30+}
31+
2332 /*
2433 * Called after updating RLIMIT_CPU to run cpu timer and update
25− * tsk->signal->cputime_expires expiration cache if necessary. Needs
26− * siglock protection since other code may update expiration cache as
27− * well.
34+ * tsk->signal->posix_cputimers.bases[clock].nextevt expiration cache if
35+ * necessary. Needs siglock protection since other code may update the
36+ * expiration cache as well.
2837 */
2938 void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new)
3039 {
@@ -35,46 +44,97 @@ void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) · 4 unmodified lines
3544 spin_unlock_irq(&task->sighand->siglock);
3645 }
3746
38−static int check_clock(const clockid_t which_clock)
47+/*
48+ * Functions for validating access to tasks.
49+ */
50+static struct task_struct *lookup_task(const pid_t pid, bool thread,
51+ bool gettime)
3952 {
40− int error = 0;
4153 struct task_struct *p;
42− const pid_t pid = CPUCLOCK_PID(which_clock);
43−
44− if (CPUCLOCK_WHICH(which_clock) >= CPUCLOCK_MAX)
45− return -EINVAL;
4654
47− if (pid == 0)
48− return 0;
55+ /*
56+ * If the encoded PID is 0, then the timer is targeted at current
57+ * or the process to which current belongs.
58+ */
59+ if (!pid)
60+ return thread ? current : current->group_leader;
4961
50− rcu_read_lock();
5162 p = find_task_by_vpid(pid);
52− if (!p || !(CPUCLOCK_PERTHREAD(which_clock) ?
53− same_thread_group(p, current) : has_group_leader_pid(p))) {
54− error = -EINVAL;
63+ if (!p)
64+ return p;
65+
66+ if (thread)
67+ return same_thread_group(p, current) ? p : NULL;
68+
69+ if (gettime) {
70+ /*
71+ * For clock_gettime(PROCESS) the task does not need to be
72+ * the actual group leader. tsk->sighand gives
73+ * access to the group's clock.
74+ *
75+ * Timers need the group leader because they take a
76+ * reference on it and store the task pointer until the
77+ * timer is destroyed.
78+ */
79+ return (p == current || thread_group_leader(p)) ? p : NULL;
5580 }
81+
82+ /*
83+ * For processes require that p is group leader.
84+ */
85+ return has_group_leader_pid(p) ? p : NULL;
86+}
87+
88+static struct task_struct *__get_task_for_clock(const clockid_t clock,
89+ bool getref, bool gettime)
90+{
91+ const bool thread = !!CPUCLOCK_PERTHREAD(clock);
92+ const pid_t pid = CPUCLOCK_PID(clock);
93+ struct task_struct *p;
94+
95+ if (CPUCLOCK_WHICH(clock) >= CPUCLOCK_MAX)
96+ return NULL;
97+
98+ rcu_read_lock();
99+ p = lookup_task(pid, thread, gettime);
100+ if (p && getref)
101+ get_task_struct(p);
56102 rcu_read_unlock();
103+ return p;
104+}
57105
58− return error;
106+static inline struct task_struct *get_task_for_clock(const clockid_t clock)
107+{
108+ return __get_task_for_clock(clock, true, false);
109+}
110+
111+static inline struct task_struct *get_task_for_clock_get(const clockid_t clock)
112+{
113+ return __get_task_for_clock(clock, true, true);
114+}
115+
116+static inline int validate_clock_permissions(const clockid_t clock)
117+{
118+ return __get_task_for_clock(clock, false, false) ? 0 : -EINVAL;
59119 }
60120
61121 /*
62122 * Update expiry time from increment, and increase overrun count,
63123 * given the current clock sample.
64124 */
65−static void bump_cpu_timer(struct k_itimer *timer, u64 now)
125+static u64 bump_cpu_timer(struct k_itimer *timer, u64 now)
66126 {
127+ u64 delta, incr, expires = timer->it.cpu.node.expires;
67128 int i;
68− u64 delta, incr;
69129
70130 if (!timer->it_interval)
71− return;
131+ return expires;
72132
73− if (now < timer->it.cpu.expires)
74− return;
133+ if (now < expires)
134+ return expires;
75135
76136 incr = timer->it_interval;
77− delta = now + incr - timer->it.cpu.expires;
137+ delta = now + incr - expires;
78138
79139 /* Don't use (incr*2 < delta), incr*2 might overflow. */
80140 for (i = 0; incr < delta - incr; i++)
@@ -84,48 +144,26 @@ static void bump_cpu_timer(struct k_itimer *timer, u64 now) · 3 unmodified lines
84144 if (delta < incr)
85145 continue;
86146
87− timer->it.cpu.expires += incr;
147+ timer->it.cpu.node.expires += incr;
88148 timer->it_overrun += 1LL << i;
89149 delta -= incr;
90150 }
151+ return timer->it.cpu.node.expires;
91152 }
92153
93−/**
94− * task_cputime_zero - Check a task_cputime struct for all zero fields.
95− *
96− * @cputime: The struct to compare.
97− *
98− * Checks @cputime to see if all fields are zero. Returns true if all fields
99− * are zero, false if any field is nonzero.
100− */
101−static inline int task_cputime_zero(const struct task_cputime *cputime)
154+/* Check whether all cache entries contain U64_MAX, i.e. eternal expiry time */
155+static inline bool expiry_cache_is_inactive(const struct posix_cputimers *pct)
102156 {
103− if (!cputime->utime && !cputime->stime && !cputime->sum_exec_runtime)
104− return 1;
105− return 0;
106−}
107−
108−static inline u64 prof_ticks(struct task_struct *p)
109−{
110− u64 utime, stime;
111−
112− task_cputime(p, &utime, &stime);
113−
114− return utime + stime;
115−}
116−static inline u64 virt_ticks(struct task_struct *p)
117−{
118− u64 utime, stime;
119−
120− task_cputime(p, &utime, &stime);
121−
122− return utime;
157+ return !(~pct->bases[CPUCLOCK_PROF].nextevt |
158+ ~pct->bases[CPUCLOCK_VIRT].nextevt |
159+ ~pct->bases[CPUCLOCK_SCHED].nextevt);
123160 }
124161
125162 static int
126163 posix_cpu_clock_getres(const clockid_t which_clock, struct timespec64 *tp)
127164 {
128− int error = check_clock(which_clock);
165+ int error = validate_clock_permissions(which_clock);
166+
129167 if (!error) {
130168 tp->tv_sec = 0;
131169 tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ);
@@ -142,42 +180,66 @@ posix_cpu_clock_getres(const clockid_t which_clock, struct timespec64 *tp) · 10 unmodified lines
142180 }
143181
144182 static int
145−posix_cpu_clock_set(const clockid_t which_clock, const struct timespec64 *tp)
183+posix_cpu_clock_set(const clockid_t clock, const struct timespec64 *tp)
146184 {
185+ int error = validate_clock_permissions(clock);
186+
147187 /*
148188 * You can never reset a CPU clock, but we check for other errors
149189 * in the call before failing with EPERM.
150190 */
151− int error = check_clock(which_clock);
152− if (error == 0) {
153− error = -EPERM;
154− }
155− return error;
191+ return error ? : -EPERM;
156192 }
157193
158−
159194 /*
160− * Sample a per-thread clock for the given task.
195+ * Sample a per-thread clock for the given task. clkid is validated.
161196 */
162−static int cpu_clock_sample(const clockid_t which_clock,
163− struct task_struct *p, u64 *sample)
197+static u64 cpu_clock_sample(const clockid_t clkid, struct task_struct *p)
164198 {
165− switch (CPUCLOCK_WHICH(which_clock)) {
166− default:
167− return -EINVAL;
199+ u64 utime, stime;
200+
201+ if (clkid == CPUCLOCK_SCHED)
202+ return task_sched_runtime(p);
203+
204+ task_cputime(p, &utime, &stime);
205+
206+ switch (clkid) {
168207 case CPUCLOCK_PROF:
169− *sample = prof_ticks(p);
170− break;
208+ return utime + stime;
171209 case CPUCLOCK_VIRT:
172− *sample = virt_ticks(p);
173− break;
174− case CPUCLOCK_SCHED:
175− *sample = task_sched_runtime(p);
176− break;
210+ return utime;
211+ default:
212+ WARN_ON_ONCE(1);
177213 }
178214 return 0;
179215 }
180216
217+static inline void store_samples(u64 *samples, u64 stime, u64 utime, u64 rtime)
218+{
219+ samples[CPUCLOCK_PROF] = stime + utime;
220+ samples[CPUCLOCK_VIRT] = utime;
221+ samples[CPUCLOCK_SCHED] = rtime;
222+}
223+
224+static void task_sample_cputime(struct task_struct *p, u64 *samples)
225+{
226+ u64 stime, utime;
227+
228+ task_cputime(p, &utime, &stime);
229+ store_samples(samples, stime, utime, p->se.sum_exec_runtime);
230+}
231+
232+static void proc_sample_cputime_atomic(struct task_cputime_atomic *at,
233+ u64 *samples)
234+{
235+ u64 stime, utime, rtime;
236+
237+ utime = atomic64_read(&at->utime);
238+ stime = atomic64_read(&at->stime);
239+ rtime = atomic64_read(&at->sum_exec_runtime);
240+ store_samples(samples, stime, utime, rtime);
241+}
242+
181243 /*
182244 * Set cputime to sum_cputime if sum_cputime > cputime. Use cmpxchg
183245 * to avoid race conditions with concurrent updates to cputime.
@@ -193,29 +255,56 @@ retry: · 9 unmodified lines
193255 }
194256 }
195257
196−static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic, struct task_cputime *sum)
258+static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic,
259+ struct task_cputime *sum)
197260 {
198261 __update_gt_cputime(&cputime_atomic->utime, sum->utime);
199262 __update_gt_cputime(&cputime_atomic->stime, sum->stime);
200263 __update_gt_cputime(&cputime_atomic->sum_exec_runtime, sum->sum_exec_runtime);
201264 }
202265
203−/* Sample task_cputime_atomic values in "atomic_timers", store results in "times". */
204−static inline void sample_cputime_atomic(struct task_cputime *times,
205− struct task_cputime_atomic *atomic_times)
266+/**
267+ * thread_group_sample_cputime - Sample cputime for a given task
268+ * @tsk: Task for which cputime needs to be started
269+ * @iimes: Storage for time samples
270+ *
271+ * Called from sys_getitimer() to calculate the expiry time of an active
272+ * timer. That means group cputime accounting is already active. Called
273+ * with task sighand lock held.
274+ *
275+ * Updates @times with an uptodate sample of the thread group cputimes.
276+ */
277+void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples)
206278 {
207− times->utime = atomic64_read(&atomic_times->utime);
208− times->stime = atomic64_read(&atomic_times->stime);
209− times->sum_exec_runtime = atomic64_read(&atomic_times->sum_exec_runtime);
279+ struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
280+ struct posix_cputimers *pct = &tsk->signal->posix_cputimers;
281+
282+ WARN_ON_ONCE(!pct->timers_active);
283+
284+ proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
210285 }
211286
212−void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times)
287+/**
288+ * thread_group_start_cputime - Start cputime and return a sample
289+ * @tsk: Task for which cputime needs to be started
290+ * @samples: Storage for time samples
291+ *
292+ * The thread group cputime accouting is avoided when there are no posix
293+ * CPU timers armed. Before starting a timer it's required to check whether
294+ * the time accounting is active. If not, a full update of the atomic
295+ * accounting store needs to be done and the accounting enabled.
296+ *
297+ * Updates @times with an uptodate sample of the thread group cputimes.
298+ */
299+static void thread_group_start_cputime(struct task_struct *tsk, u64 *samples)
213300 {
214301 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
215− struct task_cputime sum;
302+ struct posix_cputimers *pct = &tsk->signal->posix_cputimers;
216303
217304 /* Check if cputimer isn't running. This is accessed without locking. */
218− if (!READ_ONCE(cputimer->running)) {
305+ if (!READ_ONCE(pct->timers_active)) {
306+ struct task_cputime sum;
307+
219308 /*
220309 * The POSIX timer interface allows for absolute time expiry
221310 * values through the TIMER_ABSTIME flag, therefore we have
@@ -225,94 +314,69 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) · 3 unmodified lines
225314 update_gt_cputime(&cputimer->cputime_atomic, &sum);
226315
227316 /*
228− * We're setting cputimer->running without a lock. Ensure
229− * this only gets written to in one operation. We set
230− * running after update_gt_cputime() as a small optimization,
231− * but barriers are not required because update_gt_cputime()
317+ * We're setting timers_active without a lock. Ensure this
318+ * only gets written to in one operation. We set it after
319+ * update_gt_cputime() as a small optimization, but
320+ * barriers are not required because update_gt_cputime()
232321 * can handle concurrent updates.
233322 */
234− WRITE_ONCE(cputimer->running, true);
323+ WRITE_ONCE(pct->timers_active, true);
235324 }
236− sample_cputime_atomic(times, &cputimer->cputime_atomic);
325+ proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
237326 }
238327
239−/*
240− * Sample a process (thread group) clock for the given group_leader task.
241− * Must be called with task sighand lock held for safe while_each_thread()
242− * traversal.
243− */
244−static int cpu_clock_sample_group(const clockid_t which_clock,
245− struct task_struct *p,
246− u64 *sample)
328+static void __thread_group_cputime(struct task_struct *tsk, u64 *samples)
247329 {
248− struct task_cputime cputime;
330+ struct task_cputime ct;
249331
250− switch (CPUCLOCK_WHICH(which_clock)) {
251− default:
252− return -EINVAL;
253− case CPUCLOCK_PROF:
254− thread_group_cputime(p, &cputime);
255− *sample = cputime.utime + cputime.stime;
256− break;
257− case CPUCLOCK_VIRT:
258− thread_group_cputime(p, &cputime);
259− *sample = cputime.utime;
260− break;
261− case CPUCLOCK_SCHED:
262− thread_group_cputime(p, &cputime);
263− *sample = cputime.sum_exec_runtime;
264− break;
265− }
266− return 0;
332+ thread_group_cputime(tsk, &ct);
333+ store_samples(samples, ct.stime, ct.utime, ct.sum_exec_runtime);
267334 }
268335
269−static int posix_cpu_clock_get_task(struct task_struct *tsk,
270− const clockid_t which_clock,
271− struct timespec64 *tp)
336+/*
337+ * Sample a process (thread group) clock for the given task clkid. If the
338+ * group's cputime accounting is already enabled, read the atomic
339+ * store. Otherwise a full update is required. Task's sighand lock must be
340+ * held to protect the task traversal on a full update. clkid is already
341+ * validated.
342+ */
343+static u64 cpu_clock_sample_group(const clockid_t clkid, struct task_struct *p,
344+ bool start)
272345 {
273− int err = -EINVAL;
274− u64 rtn;
346+ struct thread_group_cputimer *cputimer = &p->signal->cputimer;
347+ struct posix_cputimers *pct = &p->signal->posix_cputimers;
348+ u64 samples[CPUCLOCK_MAX];
275349
276− if (CPUCLOCK_PERTHREAD(which_clock)) {
277− if (same_thread_group(tsk, current))
278− err = cpu_clock_sample(which_clock, tsk, &rtn);
350+ if (!READ_ONCE(pct->timers_active)) {
351+ if (start)
352+ thread_group_start_cputime(p, samples);
353+ else
354+ __thread_group_cputime(p, samples);
279355 } else {
280− if (tsk == current || thread_group_leader(tsk))
281− err = cpu_clock_sample_group(which_clock, tsk, &rtn);
356+ proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
282357 }
283358
284− if (!err)
285− *tp = ns_to_timespec64(rtn);
286−
287− return err;
359+ return samples[clkid];
288360 }
289361
290−
291−static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec64 *tp)
362+static int posix_cpu_clock_get(const clockid_t clock, struct timespec64 *tp)
292363 {
293− const pid_t pid = CPUCLOCK_PID(which_clock);
294− int err = -EINVAL;
364+ const clockid_t clkid = CPUCLOCK_WHICH(clock);
365+ struct task_struct *tsk;
366+ u64 t;
295367
296− if (pid == 0) {
297− /*
298− * Special case constant value for our own clocks.
299− * We don't have to do any lookup to find ourselves.
300− */
301− err = posix_cpu_clock_get_task(current, which_clock, tp);
302− } else {
303− /*
304− * Find the given PID, and validate that the caller
305− * should be able to see it.
306− */
307− struct task_struct *p;
308− rcu_read_lock();
309− p = find_task_by_vpid(pid);
310− if (p)
311− err = posix_cpu_clock_get_task(p, which_clock, tp);
312− rcu_read_unlock();
313− }
368+ tsk = get_task_for_clock_get(clock);
369+ if (!tsk)
370+ return -EINVAL;
314371
315− return err;
372+ if (CPUCLOCK_PERTHREAD(clock))
373+ t = cpu_clock_sample(clkid, tsk);
374+ else
375+ t = cpu_clock_sample_group(clkid, tsk, false);
376+ put_task_struct(tsk);
377+
378+ *tp = ns_to_timespec64(t);
379+ return 0;
316380 }
317381
318382 /*
@@ -322,44 +386,15 @@ static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec64 *t · 3 unmodified lines
322386 */
323387 static int posix_cpu_timer_create(struct k_itimer *new_timer)
324388 {
325− int ret = 0;
326− const pid_t pid = CPUCLOCK_PID(new_timer->it_clock);
327− struct task_struct *p;
389+ struct task_struct *p = get_task_for_clock(new_timer->it_clock);
328390
329− if (CPUCLOCK_WHICH(new_timer->it_clock) >= CPUCLOCK_MAX)
391+ if (!p)
330392 return -EINVAL;
331393
332394 new_timer->kclock = &clock_posix_cpu;
333−
334− INIT_LIST_HEAD(&new_timer->it.cpu.entry);
335−
336− rcu_read_lock();
337− if (CPUCLOCK_PERTHREAD(new_timer->it_clock)) {
338− if (pid == 0) {
339− p = current;
340− } else {
341− p = find_task_by_vpid(pid);
342− if (p && !same_thread_group(p, current))
343− p = NULL;
344− }
345− } else {
346− if (pid == 0) {
347− p = current->group_leader;
348− } else {
349− p = find_task_by_vpid(pid);
350− if (p && !has_group_leader_pid(p))
351− p = NULL;
352− }
353− }
395+ timerqueue_init(&new_timer->it.cpu.node);
354396 new_timer->it.cpu.task = p;
355− if (p) {
356− get_task_struct(p);
357− } else {
358− ret = -EINVAL;
359− }
360− rcu_read_unlock();
361−
362− return ret;
397+ return 0;
363398 }
364399
365400 /*
@@ -370,12 +405,14 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) · 4 unmodified lines
370405 */
371406 static int posix_cpu_timer_del(struct k_itimer *timer)
372407 {
373− int ret = 0;
374− unsigned long flags;
408+ struct cpu_timer *ctmr = &timer->it.cpu;
409+ struct task_struct *p = ctmr->task;
375410 struct sighand_struct *sighand;
376− struct task_struct *p = timer->it.cpu.task;
411+ unsigned long flags;
412+ int ret = 0;
377413
378− WARN_ON_ONCE(p == NULL);
414+ if (WARN_ON_ONCE(!p))
415+ return -EINVAL;
379416
380417 /*
381418 * Protect against sighand release/switch in exit/exec and process/
@@ -384,15 +421,15 @@ static int posix_cpu_timer_del(struct k_itimer *timer) · 2 unmodified lines
384421 sighand = lock_task_sighand(p, &flags);
385422 if (unlikely(sighand == NULL)) {
386423 /*
387− * We raced with the reaping of the task.
388− * The deletion should have cleared us off the list.
424+ * This raced with the reaping of the task. The exit cleanup
425+ * should have removed this timer from the timer queue.
389426 */
390− WARN_ON_ONCE(!list_empty(&timer->it.cpu.entry));
427+ WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node));
391428 } else {
392429 if (timer->it.cpu.firing)
393430 ret = TIMER_RETRY;
394431 else
395− list_del(&timer->it.cpu.entry);
432+ cpu_timer_dequeue(ctmr);
396433
397434 unlock_task_sighand(p, &flags);
398435 }
@@ -403,25 +440,30 @@ static int posix_cpu_timer_del(struct k_itimer *timer) · 4 unmodified lines
403440 return ret;
404441 }
405442
406−static void cleanup_timers_list(struct list_head *head)
443+static void cleanup_timerqueue(struct timerqueue_head *head)
407444 {
408− struct cpu_timer_list *timer, *next;
445+ struct timerqueue_node *node;
446+ struct cpu_timer *ctmr;
409447
410− list_for_each_entry_safe(timer, next, head, entry)
411− list_del_init(&timer->entry);
448+ while ((node = timerqueue_getnext(head))) {
449+ timerqueue_del(head, node);
450+ ctmr = container_of(node, struct cpu_timer, node);
451+ ctmr->head = NULL;
452+ }
412453 }
413454
414455 /*
415− * Clean out CPU timers still ticking when a thread exited. The task
416− * pointer is cleared, and the expiry time is replaced with the residual
417− * time for later timer_gettime calls to return.
456+ * Clean out CPU timers which are still armed when a thread exits. The
457+ * timers are only removed from the list. No other updates are done. The
458+ * corresponding posix timers are still accessible, but cannot be rearmed.
459+ *
418460 * This must be called with the siglock held.
419461 */
420−static void cleanup_timers(struct list_head *head)
462+static void cleanup_timers(struct posix_cputimers *pct)
421463 {
422− cleanup_timers_list(head);
423− cleanup_timers_list(++head);
424− cleanup_timers_list(++head);
464+ cleanup_timerqueue(&pct->bases[CPUCLOCK_PROF].tqhead);
465+ cleanup_timerqueue(&pct->bases[CPUCLOCK_VIRT].tqhead);
466+ cleanup_timerqueue(&pct->bases[CPUCLOCK_SCHED].tqhead);
425467 }
426468
427469 /*
@@ -431,16 +473,11 @@ static void cleanup_timers(struct list_head *head) · 3 unmodified lines
431473 */
432474 void posix_cpu_timers_exit(struct task_struct *tsk)
433475 {
434− cleanup_timers(tsk->cpu_timers);
476+ cleanup_timers(&tsk->posix_cputimers);
435477 }
436478 void posix_cpu_timers_exit_group(struct task_struct *tsk)
437479 {
438− cleanup_timers(tsk->signal->cpu_timers);
439−}
440−
441−static inline int expires_gt(u64 expires, u64 new_exp)
442−{
443− return expires == 0 || expires > new_exp;
480+ cleanup_timers(&tsk->signal->posix_cputimers);
444481 }
445482
446483 /*
@@ -449,58 +486,33 @@ static inline int expires_gt(u64 expires, u64 new_exp) · 2 unmodified lines
449486 */
450487 static void arm_timer(struct k_itimer *timer)
451488 {
452− struct task_struct *p = timer->it.cpu.task;
453− struct list_head *head, *listpos;
454− struct task_cputime *cputime_expires;
455− struct cpu_timer_list *const nt = &timer->it.cpu;
456− struct cpu_timer_list *next;
457−
458− if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
459− head = p->cpu_timers;
460− cputime_expires = &p->cputime_expires;
461− } else {
462− head = p->signal->cpu_timers;
463− cputime_expires = &p->signal->cputime_expires;
464− }
465− head += CPUCLOCK_WHICH(timer->it_clock);
466−
467− listpos = head;
468− list_for_each_entry(next, head, entry) {
469− if (nt->expires < next->expires)
470− break;
471− listpos = &next->entry;
472− }
473− list_add(&nt->entry, listpos);
474−
475− if (listpos == head) {
476− u64 exp = nt->expires;
489+ int clkidx = CPUCLOCK_WHICH(timer->it_clock);
490+ struct cpu_timer *ctmr = &timer->it.cpu;
491+ u64 newexp = cpu_timer_getexpires(ctmr);
492+ struct task_struct *p = ctmr->task;
493+ struct posix_cputimer_base *base;
494+
495+ if (CPUCLOCK_PERTHREAD(timer->it_clock))
496+ base = p->posix_cputimers.bases + clkidx;
497+ else
498+ base = p->signal->posix_cputimers.bases + clkidx;
499+
500+ if (!cpu_timer_enqueue(&base->tqhead, ctmr))
501+ return;
477502
478− /*
479− * We are the new earliest-expiring POSIX 1.b timer, hence
480− * need to update expiration cache. Take into account that
481− * for process timers we share expiration cache with itimers
482− * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME.
483− */
503+ /*
504+ * We are the new earliest-expiring POSIX 1.b timer, hence
505+ * need to update expiration cache. Take into account that
506+ * for process timers we share expiration cache with itimers
507+ * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME.
508+ */
509+ if (newexp < base->nextevt)
510+ base->nextevt = newexp;
484511
485− switch (CPUCLOCK_WHICH(timer->it_clock)) {
486− case CPUCLOCK_PROF:
487− if (expires_gt(cputime_expires->prof_exp, exp))
488− cputime_expires->prof_exp = exp;
489− break;
490− case CPUCLOCK_VIRT:
491− if (expires_gt(cputime_expires->virt_exp, exp))
492− cputime_expires->virt_exp = exp;
493− break;
494− case CPUCLOCK_SCHED:
495− if (expires_gt(cputime_expires->sched_exp, exp))
496− cputime_expires->sched_exp = exp;
497− break;
498− }
499− if (CPUCLOCK_PERTHREAD(timer->it_clock))
500− tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER);
501− else
502− tick_dep_set_signal(p->signal, TICK_DEP_BIT_POSIX_TIMER);
503− }
512+ if (CPUCLOCK_PERTHREAD(timer->it_clock))
513+ tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER);
514+ else
515+ tick_dep_set_signal(p->signal, TICK_DEP_BIT_POSIX_TIMER);
504516 }
505517
506518 /*
@@ -508,24 +520,26 @@ static void arm_timer(struct k_itimer *timer) · 1 unmodified line
508520 */
509521 static void cpu_timer_fire(struct k_itimer *timer)
510522 {
523+ struct cpu_timer *ctmr = &timer->it.cpu;
524+
511525 if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) {
512526 /*
513527 * User don't want any signal.
514528 */
515− timer->it.cpu.expires = 0;
529+ cpu_timer_setexpires(ctmr, 0);
516530 } else if (unlikely(timer->sigq == NULL)) {
517531 /*
518532 * This a special case for clock_nanosleep,
519533 * not a normal timer from sys_timer_create.
520534 */
521535 wake_up_process(timer->it_process);
522− timer->it.cpu.expires = 0;
536+ cpu_timer_setexpires(ctmr, 0);
523537 } else if (!timer->it_interval) {
524538 /*
525539 * One-shot timer. Clear it as soon as it's fired.
526540 */
527541 posix_timer_event(timer, 0);
528− timer->it.cpu.expires = 0;
542+ cpu_timer_setexpires(ctmr, 0);
529543 } else if (posix_timer_event(timer, ++timer->it_requeue_pending)) {
530544 /*
531545 * The signal did not get queued because the signal
@@ -538,33 +552,6 @@ static void cpu_timer_fire(struct k_itimer *timer) · 6 unmodified lines
538552 }
539553 }
540554
541−/*
542− * Sample a process (thread group) timer for the given group_leader task.
543− * Must be called with task sighand lock held for safe while_each_thread()
544− * traversal.
545− */
546−static int cpu_timer_sample_group(const clockid_t which_clock,
547− struct task_struct *p, u64 *sample)
548−{
549− struct task_cputime cputime;
550−
551− thread_group_cputimer(p, &cputime);
552− switch (CPUCLOCK_WHICH(which_clock)) {
553− default:
554− return -EINVAL;
555− case CPUCLOCK_PROF:
556− *sample = cputime.utime + cputime.stime;
557− break;
558− case CPUCLOCK_VIRT:
559− *sample = cputime.utime;
560− break;
561− case CPUCLOCK_SCHED:
562− *sample = cputime.sum_exec_runtime;
563− break;
564− }
565− return 0;
566−}
567−
568555 /*
569556 * Guts of sys_timer_settime for CPU timers.
570557 * This is called with the timer locked and interrupts disabled.
@@ -574,13 +561,16 @@ static int cpu_timer_sample_group(const clockid_t which_clock, · 3 unmodified lines
574561 static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
575562 struct itimerspec64 *new, struct itimerspec64 *old)
576563 {
577− unsigned long flags;
578− struct sighand_struct *sighand;
579− struct task_struct *p = timer->it.cpu.task;
564+ clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
580565 u64 old_expires, new_expires, old_incr, val;
581− int ret;
566+ struct cpu_timer *ctmr = &timer->it.cpu;
567+ struct task_struct *p = ctmr->task;
568+ struct sighand_struct *sighand;
569+ unsigned long flags;
570+ int ret = 0;
582571
583− WARN_ON_ONCE(p == NULL);
572+ if (WARN_ON_ONCE(!p))
573+ return -EINVAL;
584574
585575 /*
586576 * Use the to_ktime conversion because that clamps the maximum
@@ -597,22 +587,21 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, · 10 unmodified lines
597587 * If p has just been reaped, we can no
598588 * longer get any information about it at all.
599589 */
600− if (unlikely(sighand == NULL)) {
590+ if (unlikely(sighand == NULL))
601591 return -ESRCH;
602− }
603592
604593 /*
605594 * Disarm any old timer after extracting its expiry time.
606595 */
607−
608− ret = 0;
609596 old_incr = timer->it_interval;
610− old_expires = timer->it.cpu.expires;
597+ old_expires = cpu_timer_getexpires(ctmr);
598+
611599 if (unlikely(timer->it.cpu.firing)) {
612600 timer->it.cpu.firing = -1;
613601 ret = TIMER_RETRY;
614− } else
615− list_del_init(&timer->it.cpu.entry);
602+ } else {
603+ cpu_timer_dequeue(ctmr);
604+ }
616605
617606 /*
618607 * We need to sample the current value to convert the new
@@ -622,11 +611,10 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, · 3 unmodified lines
622611 * times (in arm_timer). With an absolute time, we must
623612 * check if it's already passed. In short, we need a sample.
624613 */
625− if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
626− cpu_clock_sample(timer->it_clock, p, &val);
627− } else {
628− cpu_timer_sample_group(timer->it_clock, p, &val);
629− }
614+ if (CPUCLOCK_PERTHREAD(timer->it_clock))
615+ val = cpu_clock_sample(clkid, p);
616+ else
617+ val = cpu_clock_sample_group(clkid, p, true);
630618
631619 if (old) {
632620 if (old_expires == 0) {
@@ -634,18 +622,16 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, · 1 unmodified line
634622 old->it_value.tv_nsec = 0;
635623 } else {
636624 /*
637− * Update the timer in case it has
638− * overrun already. If it has,
639− * we'll report it as having overrun
640− * and with the next reloaded timer
641− * already ticking, though we are
642− * swallowing that pending
643− * notification here to install the
644− * new setting.
625+ * Update the timer in case it has overrun already.
626+ * If it has, we'll report it as having overrun and
627+ * with the next reloaded timer already ticking,
628+ * though we are swallowing that pending
629+ * notification here to install the new setting.
645630 */
646− bump_cpu_timer(timer, val);
647− if (val < timer->it.cpu.expires) {
648− old_expires = timer->it.cpu.expires - val;
631+ u64 exp = bump_cpu_timer(timer, val);
632+
633+ if (val < exp) {
634+ old_expires = exp - val;
649635 old->it_value = ns_to_timespec64(old_expires);
650636 } else {
651637 old->it_value.tv_nsec = 1;
@@ -674,7 +660,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, · 22 unmodified lines
674660 * For a timer with no notification action, we don't actually
675661 * arm the timer (we'll just fake it for timer_gettime).
676662 */
677− timer->it.cpu.expires = new_expires;
663+ cpu_timer_setexpires(ctmr, new_expires);
678664 if (new_expires != 0 && val < new_expires) {
679665 arm_timer(timer);
680666 }
@@ -715,24 +701,27 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, · 34 unmodified lines
715701
716702 static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp)
717703 {
718− u64 now;
719− struct task_struct *p = timer->it.cpu.task;
704+ clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
705+ struct cpu_timer *ctmr = &timer->it.cpu;
706+ u64 now, expires = cpu_timer_getexpires(ctmr);
707+ struct task_struct *p = ctmr->task;
720708
721− WARN_ON_ONCE(p == NULL);
709+ if (WARN_ON_ONCE(!p))
710+ return;
722711
723712 /*
724713 * Easy part: convert the reload time.
725714 */
726715 itp->it_interval = ktime_to_timespec64(timer->it_interval);
727716
728− if (!timer->it.cpu.expires)
717+ if (!expires)
729718 return;
730719
731720 /*
732721 * Sample the clock to take the difference with the expiry time.
733722 */
734723 if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
735− cpu_clock_sample(timer->it_clock, p, &now);
724+ now = cpu_clock_sample(clkid, p);
736725 } else {
737726 struct sighand_struct *sighand;
738727 unsigned long flags;
@@ -747,18 +736,18 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp · 8 unmodified lines
747736 /*
748737 * The process has been reaped.
749738 * We can't even collect a sample any more.
750− * Call the timer disarmed, nothing else to do.
739+ * Disarm the timer, nothing else to do.
751740 */
752− timer->it.cpu.expires = 0;
741+ cpu_timer_setexpires(ctmr, 0);
753742 return;
754743 } else {
755− cpu_timer_sample_group(timer->it_clock, p, &now);
744+ now = cpu_clock_sample_group(clkid, p, false);
756745 unlock_task_sighand(p, &flags);
757746 }
758747 }
759748
760− if (now < timer->it.cpu.expires) {
761− itp->it_value = ns_to_timespec64(timer->it.cpu.expires - now);
749+ if (now < expires) {
750+ itp->it_value = ns_to_timespec64(expires - now);
762751 } else {
763752 /*
764753 * The timer should have expired already, but the firing
@@ -769,26 +758,42 @@ static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp · 4 unmodified lines
769758 }
770759 }
771760
772−static unsigned long long
773−check_timers_list(struct list_head *timers,
774− struct list_head *firing,
775− unsigned long long curr)
776−{
777− int maxfire = 20;
761+#define MAX_COLLECTED 20
778762
779− while (!list_empty(timers)) {
780− struct cpu_timer_list *t;
763+static u64 collect_timerqueue(struct timerqueue_head *head,
764+ struct list_head *firing, u64 now)
765+{
766+ struct timerqueue_node *next;
767+ int i = 0;
768+
769+ while ((next = timerqueue_getnext(head))) {
770+ struct cpu_timer *ctmr;
771+ u64 expires;
772+
773+ ctmr = container_of(next, struct cpu_timer, node);
774+ expires = cpu_timer_getexpires(ctmr);
775+ /* Limit the number of timers to expire at once */
776+ if (++i == MAX_COLLECTED || now < expires)
777+ return expires;
778+
779+ ctmr->firing = 1;
780+ cpu_timer_dequeue(ctmr);
781+ list_add_tail(&ctmr->elist, firing);
782+ }
781783
782− t = list_first_entry(timers, struct cpu_timer_list, entry);
784+ return U64_MAX;
785+}
783786
784− if (!--maxfire || curr < t->expires)
785− return t->expires;
787+static void collect_posix_cputimers(struct posix_cputimers *pct, u64 *samples,
788+ struct list_head *firing)
789+{
790+ struct posix_cputimer_base *base = pct->bases;
791+ int i;
786792
787− t->firing = 1;
788− list_move_tail(&t->entry, firing);
793+ for (i = 0; i < CPUCLOCK_MAX; i++, base++) {
794+ base->nextevt = collect_timerqueue(&base->tqhead, firing,
795+ samples[i]);
789796 }
790−
791− return 0;
792797 }
793798
794799 static inline void check_dl_overrun(struct task_struct *tsk)
@@ -799,6 +804,20 @@ static inline void check_dl_overrun(struct task_struct *tsk) · 4 unmodified lines
799804 }
800805 }
801806
807+static bool check_rlimit(u64 time, u64 limit, int signo, bool rt, bool hard)
808+{
809+ if (time < limit)
810+ return false;
811+
812+ if (print_fatal_signals) {
813+ pr_info("%s Watchdog Timeout (%s): %s[%d]\n",
814+ rt ? "RT" : "CPU", hard ? "hard" : "soft",
815+ current->comm, task_pid_nr(current));
816+ }
817+ __group_send_sig_info(signo, SEND_SIG_PRIV, current);
818+ return true;
819+}
820+
802821 /*
803822 * Check for any per-thread CPU timers that have fired and move them off
804823 * the tsk->cpu_timers[N] list onto the firing list. Here we update the
@@ -807,76 +826,50 @@ static inline void check_dl_overrun(struct task_struct *tsk) · 2 unmodified lines
807826 static void check_thread_timers(struct task_struct *tsk,
808827 struct list_head *firing)
809828 {
810− struct list_head *timers = tsk->cpu_timers;
811− struct task_cputime *tsk_expires = &tsk->cputime_expires;
812− u64 expires;
829+ struct posix_cputimers *pct = &tsk->posix_cputimers;
830+ u64 samples[CPUCLOCK_MAX];
813831 unsigned long soft;
814832
815833 if (dl_task(tsk))
816834 check_dl_overrun(tsk);
817835
818− /*
819− * If cputime_expires is zero, then there are no active
820− * per thread CPU timers.
821− */
822− if (task_cputime_zero(&tsk->cputime_expires))
836+ if (expiry_cache_is_inactive(pct))
823837 return;
824838
825− expires = check_timers_list(timers, firing, prof_ticks(tsk));
826− tsk_expires->prof_exp = expires;
827−
828− expires = check_timers_list(++timers, firing, virt_ticks(tsk));
829− tsk_expires->virt_exp = expires;
830−
831− tsk_expires->sched_exp = check_timers_list(++timers, firing,
832− tsk->se.sum_exec_runtime);
839+ task_sample_cputime(tsk, samples);
840+ collect_posix_cputimers(pct, samples, firing);
833841
834842 /*
835843 * Check for the special case thread timers.
836844 */
837845 soft = task_rlimit(tsk, RLIMIT_RTTIME);
838846 if (soft != RLIM_INFINITY) {
847+ /* Task RT timeout is accounted in jiffies. RTTIME is usec */
848+ unsigned long rttime = tsk->rt.timeout * (USEC_PER_SEC / HZ);
839849 unsigned long hard = task_rlimit_max(tsk, RLIMIT_RTTIME);
840850
851+ /* At the hard limit, send SIGKILL. No further action. */
841852 if (hard != RLIM_INFINITY &&
842− tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) {
843− /*
844− * At the hard limit, we just die.
845− * No need to calculate anything else now.
846− */
847− if (print_fatal_signals) {
848− pr_info("CPU Watchdog Timeout (hard): %s[%d]\n",
849− tsk->comm, task_pid_nr(tsk));
850− }
851− __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk);
853+ check_rlimit(rttime, hard, SIGKILL, true, true))
852854 return;
853− }
854− if (tsk->rt.timeout > DIV_ROUND_UP(soft, USEC_PER_SEC/HZ)) {
855− /*
856− * At the soft limit, send a SIGXCPU every second.
857− */
858− if (soft < hard) {
859− soft += USEC_PER_SEC;
860− tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur =
861− soft;
862− }
863− if (print_fatal_signals) {
864− pr_info("RT Watchdog Timeout (soft): %s[%d]\n",
865− tsk->comm, task_pid_nr(tsk));
866− }
867− __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
855+
856+ /* At the soft limit, send a SIGXCPU every second */
857+ if (check_rlimit(rttime, soft, SIGXCPU, true, false)) {
858+ soft += USEC_PER_SEC;
859+ tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur = soft;
868860 }
869861 }
870− if (task_cputime_zero(tsk_expires))
862+
863+ if (expiry_cache_is_inactive(pct))
871864 tick_dep_clear_task(tsk, TICK_DEP_BIT_POSIX_TIMER);
872865 }
873866
874867 static inline void stop_process_timers(struct signal_struct *sig)
875868 {
876− struct thread_group_cputimer *cputimer = &sig->cputimer;
869+ struct posix_cputimers *pct = &sig->posix_cputimers;
877870
878− /* Turn off cputimer->running. This is done without locking. */
879− WRITE_ONCE(cputimer->running, false);
871+ /* Turn off the active flag. This is done without locking. */
872+ WRITE_ONCE(pct->timers_active, false);
880873 tick_dep_clear_signal(sig, TICK_DEP_BIT_POSIX_TIMER);
881874 }
882875
@@ -898,7 +891,7 @@ static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, · 15 unmodified lines
898891 __group_send_sig_info(signo, SEND_SIG_PRIV, tsk);
899892 }
900893
901− if (it->expires && (!*expires || it->expires < *expires))
894+ if (it->expires && it->expires < *expires)
902895 *expires = it->expires;
903896 }
904897
@@ -911,87 +904,69 @@ static void check_process_timers(struct task_struct *tsk, · 6 unmodified lines
911904 struct list_head *firing)
912905 {
913906 struct signal_struct *const sig = tsk->signal;
914− u64 utime, ptime, virt_expires, prof_expires;
915− u64 sum_sched_runtime, sched_expires;
916− struct list_head *timers = sig->cpu_timers;
917− struct task_cputime cputime;
907+ struct posix_cputimers *pct = &sig->posix_cputimers;
908+ u64 samples[CPUCLOCK_MAX];
918909 unsigned long soft;
919910
920911 /*
921− * If cputimer is not running, then there are no active
922− * process wide timers (POSIX 1.b, itimers, RLIMIT_CPU).
912+ * If there are no active process wide timers (POSIX 1.b, itimers,
913+ * RLIMIT_CPU) nothing to check. Also skip the process wide timer
914+ * processing when there is already another task handling them.
923915 */
924− if (!READ_ONCE(tsk->signal->cputimer.running))
916+ if (!READ_ONCE(pct->timers_active) || pct->expiry_active)
925917 return;
926918
927− /*
919+ /*
928920 * Signify that a thread is checking for process timers.
929921 * Write access to this field is protected by the sighand lock.
930922 */
931− sig->cputimer.checking_timer = true;
923+ pct->expiry_active = true;
932924
933925 /*
934− * Collect the current process totals.
926+ * Collect the current process totals. Group accounting is active
927+ * so the sample can be taken directly.
935928 */
936− thread_group_cputimer(tsk, &cputime);
937− utime = cputime.utime;
938− ptime = utime + cputime.stime;
939− sum_sched_runtime = cputime.sum_exec_runtime;
940−
941− prof_expires = check_timers_list(timers, firing, ptime);
942− virt_expires = check_timers_list(++timers, firing, utime);
943− sched_expires = check_timers_list(++timers, firing, sum_sched_runtime);
929+ proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, samples);
930+ collect_posix_cputimers(pct, samples, firing);
944931
945932 /*
946933 * Check for the special case process timers.
947934 */
948− check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], &prof_expires, ptime,
949− SIGPROF);
950− check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_expires, utime,
951− SIGVTALRM);
935+ check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF],
936+ &pct->bases[CPUCLOCK_PROF].nextevt,
937+ samples[CPUCLOCK_PROF], SIGPROF);
938+ check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT],
939+ &pct->bases[CPUCLOCK_VIRT].nextevt,
940+ samples[CPUCLOCK_VIRT], SIGVTALRM);
941+
952942 soft = task_rlimit(tsk, RLIMIT_CPU);
953943 if (soft != RLIM_INFINITY) {
954− unsigned long psecs = div_u64(ptime, NSEC_PER_SEC);
944+ /* RLIMIT_CPU is in seconds. Samples are nanoseconds */
955945 unsigned long hard = task_rlimit_max(tsk, RLIMIT_CPU);
956− u64 x;
957− if (psecs >= hard) {
958− /*
959− * At the hard limit, we just die.
960− * No need to calculate anything else now.
961− */
962− if (print_fatal_signals) {
963− pr_info("RT Watchdog Timeout (hard): %s[%d]\n",
964− tsk->comm, task_pid_nr(tsk));
965− }
966− __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk);
946+ u64 ptime = samples[CPUCLOCK_PROF];
947+ u64 softns = (u64)soft * NSEC_PER_SEC;
948+ u64 hardns = (u64)hard * NSEC_PER_SEC;
949+
950+ /* At the hard limit, send SIGKILL. No further action. */
951+ if (hard != RLIM_INFINITY &&
952+ check_rlimit(ptime, hardns, SIGKILL, false, true))
967953 return;
954+
955+ /* At the soft limit, send a SIGXCPU every second */
956+ if (check_rlimit(ptime, softns, SIGXCPU, false, false)) {
957+ sig->rlim[RLIMIT_CPU].rlim_cur = soft + 1;
958+ softns += NSEC_PER_SEC;
968959 }
969− if (psecs >= soft) {
970− /*
971− * At the soft limit, send a SIGXCPU every second.
972− */
973− if (print_fatal_signals) {
974− pr_info("CPU Watchdog Timeout (soft): %s[%d]\n",
975− tsk->comm, task_pid_nr(tsk));
976− }
977− __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
978− if (soft < hard) {
979− soft++;
980− sig->rlim[RLIMIT_CPU].rlim_cur = soft;
981− }
982− }
983− x = soft * NSEC_PER_SEC;
984− if (!prof_expires || x < prof_expires)
985− prof_expires = x;
960+
961+ /* Update the expiry cache */
962+ if (softns < pct->bases[CPUCLOCK_PROF].nextevt)
963+ pct->bases[CPUCLOCK_PROF].nextevt = softns;
986964 }
987965
988− sig->cputime_expires.prof_exp = prof_expires;
989− sig->cputime_expires.virt_exp = virt_expires;
990− sig->cputime_expires.sched_exp = sched_expires;
991− if (task_cputime_zero(&sig->cputime_expires))
966+ if (expiry_cache_is_inactive(pct))
992967 stop_process_timers(sig);
993968
994− sig->cputimer.checking_timer = false;
969+ pct->expiry_active = false;
995970 }
996971
997972 /*
@@ -1000,18 +975,21 @@ static void check_process_timers(struct task_struct *tsk, · 2 unmodified lines
1000975 */
1001976 static void posix_cpu_timer_rearm(struct k_itimer *timer)
1002977 {
978+ clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
979+ struct cpu_timer *ctmr = &timer->it.cpu;
980+ struct task_struct *p = ctmr->task;
1003981 struct sighand_struct *sighand;
1004982 unsigned long flags;
1005− struct task_struct *p = timer->it.cpu.task;
1006983 u64 now;
1007984
1008− WARN_ON_ONCE(p == NULL);
985+ if (WARN_ON_ONCE(!p))
986+ return;
1009987
1010988 /*
1011989 * Fetch the current sample and update the timer's expiry time.
1012990 */
1013991 if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
1014− cpu_clock_sample(timer->it_clock, p, &now);
992+ now = cpu_clock_sample(clkid, p);
1015993 bump_cpu_timer(timer, now);
1016994 if (unlikely(p->exit_state))
1017995 return;
@@ -1031,13 +1009,13 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer) · 13 unmodified lines
10311009 * The process has been reaped.
10321010 * We can't even collect a sample any more.
10331011 */
1034− timer->it.cpu.expires = 0;
1012+ cpu_timer_setexpires(ctmr, 0);
10351013 return;
10361014 } else if (unlikely(p->exit_state) && thread_group_empty(p)) {
10371015 /* If the process is dying, no need to rearm */
10381016 goto unlock;
10391017 }
1040− cpu_timer_sample_group(timer->it_clock, p, &now);
1018+ now = cpu_clock_sample_group(clkid, p, true);
10411019 bump_cpu_timer(timer, now);
10421020 /* Leave the sighand locked for the call below. */
10431021 }
@@ -1051,26 +1029,24 @@ unlock: · 7 unmodified lines
10511029 }
10521030
10531031 /**
1054− * task_cputime_expired - Compare two task_cputime entities.
1032+ * task_cputimers_expired - Check whether posix CPU timers are expired
10551033 *
1056− * @sample: The task_cputime structure to be checked for expiration.
1057− * @expires: Expiration times, against which @sample will be checked.
1034+ * @samples: Array of current samples for the CPUCLOCK clocks
1035+ * @pct: Pointer to a posix_cputimers container
10581036 *
1059− * Checks @sample against @expires to see if any field of @sample has expired.
1060− * Returns true if any field of the former is greater than the corresponding
1061− * field of the latter if the latter field is set. Otherwise returns false.
1037+ * Returns true if any member of @samples is greater than the corresponding
1038+ * member of @pct->bases[CLK].nextevt. False otherwise
10621039 */
1063−static inline int task_cputime_expired(const struct task_cputime *sample,
1064− const struct task_cputime *expires)
1040+static inline bool
1041+task_cputimers_expired(const u64 *sample, struct posix_cputimers *pct)
10651042 {
1066− if (expires->utime && sample->utime >= expires->utime)
1067− return 1;
1068− if (expires->stime && sample->utime + sample->stime >= expires->stime)
1069− return 1;
1070− if (expires->sum_exec_runtime != 0 &&
1071− sample->sum_exec_runtime >= expires->sum_exec_runtime)
1072− return 1;
1073− return 0;
1043+ int i;
1044+
1045+ for (i = 0; i < CPUCLOCK_MAX; i++) {
1046+ if (sample[i] >= pct->bases[i].nextevt)
1047+ return true;
1048+ }
1049+ return false;
10741050 }
10751051
10761052 /**
@@ -1083,48 +1059,50 @@ static inline int task_cputime_expired(const struct task_cputime *sample, · 6 unmodified lines
10831059 * timers and compare them with the corresponding expiration times. Return
10841060 * true if a timer has expired, else return false.
10851061 */
1086−static inline int fastpath_timer_check(struct task_struct *tsk)
1062+static inline bool fastpath_timer_check(struct task_struct *tsk)
10871063 {
1064+ struct posix_cputimers *pct = &tsk->posix_cputimers;
10881065 struct signal_struct *sig;
10891066
1090− if (!task_cputime_zero(&tsk->cputime_expires)) {
1091− struct task_cputime task_sample;
1067+ if (!expiry_cache_is_inactive(pct)) {
1068+ u64 samples[CPUCLOCK_MAX];
10921069
1093− task_cputime(tsk, &task_sample.utime, &task_sample.stime);
1094− task_sample.sum_exec_runtime = tsk->se.sum_exec_runtime;
1095− if (task_cputime_expired(&task_sample, &tsk->cputime_expires))
1096− return 1;
1070+ task_sample_cputime(tsk, samples);
1071+ if (task_cputimers_expired(samples, pct))
1072+ return true;
10971073 }
10981074
10991075 sig = tsk->signal;
1076+ pct = &sig->posix_cputimers;
11001077 /*
1101− * Check if thread group timers expired when the cputimer is
1102− * running and no other thread in the group is already checking
1103− * for thread group cputimers. These fields are read without the
1104− * sighand lock. However, this is fine because this is meant to
1105− * be a fastpath heuristic to determine whether we should try to
1106− * acquire the sighand lock to check/handle timers.
1078+ * Check if thread group timers expired when timers are active and
1079+ * no other thread in the group is already handling expiry for
1080+ * thread group cputimers. These fields are read without the
1081+ * sighand lock. However, this is fine because this is meant to be
1082+ * a fastpath heuristic to determine whether we should try to
1083+ * acquire the sighand lock to handle timer expiry.
11071084 *
1108− * In the worst case scenario, if 'running' or 'checking_timer' gets
1109− * set but the current thread doesn't see the change yet, we'll wait
1110− * until the next thread in the group gets a scheduler interrupt to
1111− * handle the timer. This isn't an issue in practice because these
1112− * types of delays with signals actually getting sent are expected.
1085+ * In the worst case scenario, if concurrently timers_active is set
1086+ * or expiry_active is cleared, but the current thread doesn't see
1087+ * the change yet, the timer checks are delayed until the next
1088+ * thread in the group gets a scheduler interrupt to handle the
1089+ * timer. This isn't an issue in practice because these types of
1090+ * delays with signals actually getting sent are expected.
11131091 */
1114− if (READ_ONCE(sig->cputimer.running) &&
1115− !READ_ONCE(sig->cputimer.checking_timer)) {
1116− struct task_cputime group_sample;
1092+ if (READ_ONCE(pct->timers_active) && !READ_ONCE(pct->expiry_active)) {
1093+ u64 samples[CPUCLOCK_MAX];
11171094
1118− sample_cputime_atomic(&group_sample, &sig->cputimer.cputime_atomic);
1095+ proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic,
1096+ samples);
11191097
1120− if (task_cputime_expired(&group_sample, &sig->cputime_expires))
1121− return 1;
1098+ if (task_cputimers_expired(samples, pct))
1099+ return true;
11221100 }
11231101
11241102 if (dl_task(tsk) && tsk->dl.dl_overrun)
1125− return 1;
1103+ return true;
11261104
1127− return 0;
1105+ return false;
11281106 }
11291107
11301108 /*
@@ -1132,11 +1110,12 @@ static inline int fastpath_timer_check(struct task_struct *tsk) · 1 unmodified line
11321110 * already updated our counts. We need to check if any timers fire now.
11331111 * Interrupts are disabled.
11341112 */
1135−void run_posix_cpu_timers(struct task_struct *tsk)
1113+void run_posix_cpu_timers(void)
11361114 {
1137− LIST_HEAD(firing);
1115+ struct task_struct *tsk = current;
11381116 struct k_itimer *timer, *next;
11391117 unsigned long flags;
1118+ LIST_HEAD(firing);
11401119
11411120 lockdep_assert_irqs_disabled();
11421121
@@ -1174,11 +1153,11 @@ void run_posix_cpu_timers(struct task_struct *tsk) · 31 unmodified lines
11741153 * each timer's lock before clearing its firing flag, so no
11751154 * timer call will interfere.
11761155 */
1177− list_for_each_entry_safe(timer, next, &firing, it.cpu.entry) {
1156+ list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) {
11781157 int cpu_firing;
11791158
11801159 spin_lock(&timer->it_lock);
1181− list_del_init(&timer->it.cpu.entry);
1160+ list_del_init(&timer->it.cpu.elist);
11821161 cpu_firing = timer->it.cpu.firing;
11831162 timer->it.cpu.firing = 0;
11841163 /*
@@ -1196,16 +1175,18 @@ void run_posix_cpu_timers(struct task_struct *tsk) · 11 unmodified lines
11961175 * Set one of the process-wide special case CPU timers or RLIMIT_CPU.
11971176 * The tsk->sighand->siglock must be held by the caller.
11981177 */
1199−void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx,
1178+void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid,
12001179 u64 *newval, u64 *oldval)
12011180 {
1202− u64 now;
1203− int ret;
1181+ u64 now, *nextevt;
1182+
1183+ if (WARN_ON_ONCE(clkid >= CPUCLOCK_SCHED))
1184+ return;
12041185
1205− WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED);
1206− ret = cpu_timer_sample_group(clock_idx, tsk, &now);
1186+ nextevt = &tsk->signal->posix_cputimers.bases[clkid].nextevt;
1187+ now = cpu_clock_sample_group(clkid, tsk, true);
12071188
1208− if (oldval && ret != -EINVAL) {
1189+ if (oldval) {
12091190 /*
12101191 * We are setting itimer. The *oldval is absolute and we update
12111192 * it to be relative, *newval argument is relative and we update
@@ -1226,19 +1207,11 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, · 14 unmodified lines
12261207 }
12271208
12281209 /*
1229− * Update expiration cache if we are the earliest timer, or eventually
1230− * RLIMIT_CPU limit is earlier than prof_exp cpu timer expire.
1210+ * Update expiration cache if this is the earliest timer. CPUCLOCK_PROF
1211+ * expiry cache is also used by RLIMIT_CPU!.
12311212 */
1232− switch (clock_idx) {
1233− case CPUCLOCK_PROF:
1234− if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval))
1235− tsk->signal->cputime_expires.prof_exp = *newval;
1236− break;
1237− case CPUCLOCK_VIRT:
1238− if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval))
1239− tsk->signal->cputime_expires.virt_exp = *newval;
1240− break;
1241− }
1213+ if (*newval < *nextevt)
1214+ *nextevt = *newval;
12421215
12431216 tick_dep_set_signal(tsk->signal, TICK_DEP_BIT_POSIX_TIMER);
12441217 }
@@ -1260,6 +1233,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, · 15 unmodified lines
12601233 timer.it_overrun = -1;
12611234 error = posix_cpu_timer_create(&timer);
12621235 timer.it_process = current;
1236+
12631237 if (!error) {
12641238 static struct itimerspec64 zero_it;
12651239 struct restart_block *restart;
@@ -1275,7 +1249,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, · 9 unmodified lines
12751249 }
12761250
12771251 while (!signal_pending(current)) {
1278− if (timer.it.cpu.expires == 0) {
1252+ if (!cpu_timer_getexpires(&timer.it.cpu)) {
12791253 /*
12801254 * Our timer fired and was reset, below
12811255 * deletion can not fail.
@@ -1297,7 +1271,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags, · 15 unmodified lines
12971271 /*
12981272 * We were interrupted by a signal.
12991273 */
1300− expires = timer.it.cpu.expires;
1274+ expires = cpu_timer_getexpires(&timer.it.cpu);
13011275 error = posix_cpu_timer_set(&timer, 0, &zero_it, &it);
13021276 if (!error) {
13031277 /*
kernel/time/posix-timers.c+50−11
@@ -442,7 +442,7 @@ static struct k_itimer * alloc_posix_timer(void) · 441 unmodified lines
442442
443443 static void k_itimer_rcu_free(struct rcu_head *head)
444444 {
445− struct k_itimer *tmr = container_of(head, struct k_itimer, it.rcu);
445+ struct k_itimer *tmr = container_of(head, struct k_itimer, rcu);
446446
447447 kmem_cache_free(posix_timers_cache, tmr);
448448 }
@@ -459,7 +459,7 @@ static void release_posix_timer(struct k_itimer *tmr, int it_id_set) · 10 unmodified lines
459459 }
460460 put_pid(tmr->it_pid);
461461 sigqueue_free(tmr->sigq);
462− call_rcu(&tmr->it.rcu, k_itimer_rcu_free);
462+ call_rcu(&tmr->rcu, k_itimer_rcu_free);
463463 }
464464
465465 static int common_timer_create(struct k_itimer *new_timer)
@@ -805,6 +805,35 @@ static int common_hrtimer_try_to_cancel(struct k_itimer *timr) · 339 unmodified lines
805805 return hrtimer_try_to_cancel(&timr->it.real.timer);
806806 }
807807
808+static void common_timer_wait_running(struct k_itimer *timer)
809+{
810+ hrtimer_cancel_wait_running(&timer->it.real.timer);
811+}
812+
813+/*
814+ * On PREEMPT_RT this prevent priority inversion against softirq kthread in
815+ * case it gets preempted while executing a timer callback. See comments in
816+ * hrtimer_cancel_wait_running. For PREEMPT_RT=n this just results in a
817+ * cpu_relax().
818+ */
819+static struct k_itimer *timer_wait_running(struct k_itimer *timer,
820+ unsigned long *flags)
821+{
822+ const struct k_clock *kc = READ_ONCE(timer->kclock);
823+ timer_t timer_id = READ_ONCE(timer->it_id);
824+
825+ /* Prevent kfree(timer) after dropping the lock */
826+ rcu_read_lock();
827+ unlock_timer(timer, *flags);
828+
829+ if (!WARN_ON_ONCE(!kc->timer_wait_running))
830+ kc->timer_wait_running(timer);
831+
832+ rcu_read_unlock();
833+ /* Relock the timer. It might be not longer hashed. */
834+ return lock_timer(timer_id, flags);
835+}
836+
808837 /* Set a POSIX.1b interval timer. */
809838 int common_timer_set(struct k_itimer *timr, int flags,
810839 struct itimerspec64 *new_setting,
@@ -844,13 +873,13 @@ int common_timer_set(struct k_itimer *timr, int flags, · 33 unmodified lines
844873 return 0;
845874 }
846875
847−static int do_timer_settime(timer_t timer_id, int flags,
876+static int do_timer_settime(timer_t timer_id, int tmr_flags,
848877 struct itimerspec64 *new_spec64,
849878 struct itimerspec64 *old_spec64)
850879 {
851880 const struct k_clock *kc;
852881 struct k_itimer *timr;
853− unsigned long flag;
882+ unsigned long flags;
854883 int error = 0;
855884
856885 if (!timespec64_valid(&new_spec64->it_interval) ||
@@ -859,8 +888,9 @@ static int do_timer_settime(timer_t timer_id, int flags, · 2 unmodified lines
859888
860889 if (old_spec64)
861890 memset(old_spec64, 0, sizeof(*old_spec64));
891+
892+ timr = lock_timer(timer_id, &flags);
862893 retry:
863− timr = lock_timer(timer_id, &flag);
864894 if (!timr)
865895 return -EINVAL;
866896
@@ -868,13 +898,16 @@ retry: · 1 unmodified line
868898 if (WARN_ON_ONCE(!kc || !kc->timer_set))
869899 error = -EINVAL;
870900 else
871− error = kc->timer_set(timr, flags, new_spec64, old_spec64);
901+ error = kc->timer_set(timr, tmr_flags, new_spec64, old_spec64);
872902
873− unlock_timer(timr, flag);
874903 if (error == TIMER_RETRY) {
875− old_spec64 = NULL; // We already got the old time...
904+ // We already got the old time...
905+ old_spec64 = NULL;
906+ /* Unlocks and relocks the timer if it still exists */
907+ timr = timer_wait_running(timr, &flags);
876908 goto retry;
877909 }
910+ unlock_timer(timr, flags);
878911
879912 return error;
880913 }
@@ -951,13 +984,15 @@ SYSCALL_DEFINE1(timer_delete, timer_t, timer_id) · 70 unmodified lines
951984 struct k_itimer *timer;
952985 unsigned long flags;
953986
954−retry_delete:
955987 timer = lock_timer(timer_id, &flags);
988+
989+retry_delete:
956990 if (!timer)
957991 return -EINVAL;
958992
959− if (timer_delete_hook(timer) == TIMER_RETRY) {
960− unlock_timer(timer, flags);
993+ if (unlikely(timer_delete_hook(timer) == TIMER_RETRY)) {
994+ /* Unlocks and relocks the timer if it still exists */
995+ timer = timer_wait_running(timer, &flags);
961996 goto retry_delete;
962997 }
963998
@@ -1238,6 +1273,7 @@ static const struct k_clock clock_realtime = { · 274 unmodified lines
12381273 .timer_forward = common_hrtimer_forward,
12391274 .timer_remaining = common_hrtimer_remaining,
12401275 .timer_try_to_cancel = common_hrtimer_try_to_cancel,
1276+ .timer_wait_running = common_timer_wait_running,
12411277 .timer_arm = common_hrtimer_arm,
12421278 };
12431279
@@ -1253,6 +1289,7 @@ static const struct k_clock clock_monotonic = { · 9 unmodified lines
12531289 .timer_forward = common_hrtimer_forward,
12541290 .timer_remaining = common_hrtimer_remaining,
12551291 .timer_try_to_cancel = common_hrtimer_try_to_cancel,
1292+ .timer_wait_running = common_timer_wait_running,
12561293 .timer_arm = common_hrtimer_arm,
12571294 };
12581295
@@ -1283,6 +1320,7 @@ static const struct k_clock clock_tai = { · 24 unmodified lines
12831320 .timer_forward = common_hrtimer_forward,
12841321 .timer_remaining = common_hrtimer_remaining,
12851322 .timer_try_to_cancel = common_hrtimer_try_to_cancel,
1323+ .timer_wait_running = common_timer_wait_running,
12861324 .timer_arm = common_hrtimer_arm,
12871325 };
12881326
@@ -1298,6 +1336,7 @@ static const struct k_clock clock_boottime = { · 9 unmodified lines
12981336 .timer_forward = common_hrtimer_forward,
12991337 .timer_remaining = common_hrtimer_remaining,
13001338 .timer_try_to_cancel = common_hrtimer_try_to_cancel,
1339+ .timer_wait_running = common_timer_wait_running,
13011340 .timer_arm = common_hrtimer_arm,
13021341 };
13031342
kernel/time/posix-timers.h+1−0
@@ -24,6 +24,7 @@ struct k_clock { · 23 unmodified lines
2424 int (*timer_try_to_cancel)(struct k_itimer *timr);
2525 void (*timer_arm)(struct k_itimer *timr, ktime_t expires,
2626 bool absolute, bool sigev_none);
27+ void (*timer_wait_running)(struct k_itimer *timr);
2728 };
2829
2930 extern const struct k_clock clock_posix_cpu;
kernel/time/tick-broadcast-hrtimer.c+9−4
@@ -59,11 +59,16 @@ static int bc_set_next(ktime_t expires, struct clock_event_device *bc) · 58 unmodified lines
5959 * hrtimer_{start/cancel} functions call into tracing,
6060 * calls to these functions must be bound within RCU_NONIDLE.
6161 */
62− RCU_NONIDLE({
62+ RCU_NONIDLE(
63+ {
6364 bc_moved = hrtimer_try_to_cancel(&bctimer) >= 0;
64− if (bc_moved)
65+ if (bc_moved) {
6566 hrtimer_start(&bctimer, expires,
66− HRTIMER_MODE_ABS_PINNED);});
67+ HRTIMER_MODE_ABS_PINNED_HARD);
68+ }
69+ }
70+ );
71+
6772 if (bc_moved) {
6873 /* Bind the "device" to the cpu */
6974 bc->bound_on = smp_processor_id();
@@ -104,7 +109,7 @@ static enum hrtimer_restart bc_handler(struct hrtimer *t) · 34 unmodified lines
104109
105110 void tick_setup_hrtimer_broadcast(void)
106111 {
107− hrtimer_init(&bctimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
112+ hrtimer_init(&bctimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
108113 bctimer.function = bc_handler;
109114 clockevents_register_device(&ce_broadcast_hrtimer);
110115 }
kernel/time/tick-sched.c+10−7
@@ -634,10 +634,12 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) · 633 unmodified lines
634634 /* Forward the time to expire in the future */
635635 hrtimer_forward(&ts->sched_timer, now, tick_period);
636636
637− if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
638− hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
639− else
637+ if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
638+ hrtimer_start_expires(&ts->sched_timer,
639+ HRTIMER_MODE_ABS_PINNED_HARD);
640+ } else {
640641 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
642+ }
641643
642644 /*
643645 * Reset to make sure next tick stop doesn't get fooled by past
@@ -802,7 +804,8 @@ static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu) · 158 unmodified lines
802804 }
803805
804806 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
805− hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);
807+ hrtimer_start(&ts->sched_timer, tick,
808+ HRTIMER_MODE_ABS_PINNED_HARD);
806809 } else {
807810 hrtimer_set_expires(&ts->sched_timer, tick);
808811 tick_program_event(tick, 1);
@@ -1230,7 +1233,7 @@ static void tick_nohz_switch_to_nohz(void) · 421 unmodified lines
12301233 * Recycle the hrtimer in ts, so we can share the
12311234 * hrtimer_forward with the highres code.
12321235 */
1233− hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1236+ hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
12341237 /* Get the next period */
12351238 next = tick_init_jiffy_update();
12361239
@@ -1327,7 +1330,7 @@ void tick_setup_sched_timer(void) · 90 unmodified lines
13271330 /*
13281331 * Emulate tick processing via per-CPU hrtimers:
13291332 */
1330− hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1333+ hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
13311334 ts->sched_timer.function = tick_sched_timer;
13321335
13331336 /* Get the next period (per-CPU) */
@@ -1342,7 +1345,7 @@ void tick_setup_sched_timer(void) · 8 unmodified lines
13421345 }
13431346
13441347 hrtimer_forward(&ts->sched_timer, now, tick_period);
1345− hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);
1348+ hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
13461349 tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
13471350 }
13481351 #endif /* HIGH_RES_TIMERS */
kernel/time/timer.c+96−9
@@ -196,6 +196,10 @@ EXPORT_SYMBOL(jiffies_64); · 195 unmodified lines
196196 struct timer_base {
197197 raw_spinlock_t lock;
198198 struct timer_list *running_timer;
199+#ifdef CONFIG_PREEMPT_RT
200+ spinlock_t expiry_lock;
201+ atomic_t timer_waiters;
202+#endif
199203 unsigned long clk;
200204 unsigned long next_expiry;
201205 unsigned int cpu;
@@ -1227,7 +1231,78 @@ int try_to_del_timer_sync(struct timer_list *timer) · 1025 unmodified lines
12271231 }
12281232 EXPORT_SYMBOL(try_to_del_timer_sync);
12291233
1230−#ifdef CONFIG_SMP
1234+#ifdef CONFIG_PREEMPT_RT
1235+static __init void timer_base_init_expiry_lock(struct timer_base *base)
1236+{
1237+ spin_lock_init(&base->expiry_lock);
1238+}
1239+
1240+static inline void timer_base_lock_expiry(struct timer_base *base)
1241+{
1242+ spin_lock(&base->expiry_lock);
1243+}
1244+
1245+static inline void timer_base_unlock_expiry(struct timer_base *base)
1246+{
1247+ spin_unlock(&base->expiry_lock);
1248+}
1249+
1250+/*
1251+ * The counterpart to del_timer_wait_running().
1252+ *
1253+ * If there is a waiter for base->expiry_lock, then it was waiting for the
1254+ * timer callback to finish. Drop expiry_lock and reaquire it. That allows
1255+ * the waiter to acquire the lock and make progress.
1256+ */
1257+static void timer_sync_wait_running(struct timer_base *base)
1258+{
1259+ if (atomic_read(&base->timer_waiters)) {
1260+ spin_unlock(&base->expiry_lock);
1261+ spin_lock(&base->expiry_lock);
1262+ }
1263+}
1264+
1265+/*
1266+ * This function is called on PREEMPT_RT kernels when the fast path
1267+ * deletion of a timer failed because the timer callback function was
1268+ * running.
1269+ *
1270+ * This prevents priority inversion, if the softirq thread on a remote CPU
1271+ * got preempted, and it prevents a life lock when the task which tries to
1272+ * delete a timer preempted the softirq thread running the timer callback
1273+ * function.
1274+ */
1275+static void del_timer_wait_running(struct timer_list *timer)
1276+{
1277+ u32 tf;
1278+
1279+ tf = READ_ONCE(timer->flags);
1280+ if (!(tf & TIMER_MIGRATING)) {
1281+ struct timer_base *base = get_timer_base(tf);
1282+
1283+ /*
1284+ * Mark the base as contended and grab the expiry lock,
1285+ * which is held by the softirq across the timer
1286+ * callback. Drop the lock immediately so the softirq can
1287+ * expire the next timer. In theory the timer could already
1288+ * be running again, but that's more than unlikely and just
1289+ * causes another wait loop.
1290+ */
1291+ atomic_inc(&base->timer_waiters);
1292+ spin_lock_bh(&base->expiry_lock);
1293+ atomic_dec(&base->timer_waiters);
1294+ spin_unlock_bh(&base->expiry_lock);
1295+ }
1296+}
1297+#else
1298+static inline void timer_base_init_expiry_lock(struct timer_base *base) { }
1299+static inline void timer_base_lock_expiry(struct timer_base *base) { }
1300+static inline void timer_base_unlock_expiry(struct timer_base *base) { }
1301+static inline void timer_sync_wait_running(struct timer_base *base) { }
1302+static inline void del_timer_wait_running(struct timer_list *timer) { }
1303+#endif
1304+
1305+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
12311306 /**
12321307 * del_timer_sync - deactivate a timer and wait for the handler to finish.
12331308 * @timer: the timer to be deactivated
@@ -1266,6 +1341,8 @@ EXPORT_SYMBOL(try_to_del_timer_sync); · 32 unmodified lines
12661341 */
12671342 int del_timer_sync(struct timer_list *timer)
12681343 {
1344+ int ret;
1345+
12691346 #ifdef CONFIG_LOCKDEP
12701347 unsigned long flags;
12711348
@@ -1283,12 +1360,17 @@ int del_timer_sync(struct timer_list *timer) · 11 unmodified lines
12831360 * could lead to deadlock.
12841361 */
12851362 WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));
1286− for (;;) {
1287− int ret = try_to_del_timer_sync(timer);
1288− if (ret >= 0)
1289− return ret;
1290− cpu_relax();
1291− }
1363+
1364+ do {
1365+ ret = try_to_del_timer_sync(timer);
1366+
1367+ if (unlikely(ret < 0)) {
1368+ del_timer_wait_running(timer);
1369+ cpu_relax();
1370+ }
1371+ } while (ret < 0);
1372+
1373+ return ret;
12921374 }
12931375 EXPORT_SYMBOL(del_timer_sync);
12941376 #endif
@@ -1360,10 +1442,13 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head) · 65 unmodified lines
13601442 if (timer->flags & TIMER_IRQSAFE) {
13611443 raw_spin_unlock(&base->lock);
13621444 call_timer_fn(timer, fn, baseclk);
1445+ base->running_timer = NULL;
13631446 raw_spin_lock(&base->lock);
13641447 } else {
13651448 raw_spin_unlock_irq(&base->lock);
13661449 call_timer_fn(timer, fn, baseclk);
1450+ base->running_timer = NULL;
1451+ timer_sync_wait_running(base);
13671452 raw_spin_lock_irq(&base->lock);
13681453 }
13691454 }
@@ -1643,7 +1728,7 @@ void update_process_times(int user_tick) · 273 unmodified lines
16431728 #endif
16441729 scheduler_tick();
16451730 if (IS_ENABLED(CONFIG_POSIX_TIMERS))
1646− run_posix_cpu_timers(p);
1731+ run_posix_cpu_timers();
16471732 }
16481733
16491734 /**
@@ -1658,6 +1743,7 @@ static inline void __run_timers(struct timer_base *base) · 8 unmodified lines
16581743 if (!time_after_eq(jiffies, base->clk))
16591744 return;
16601745
1746+ timer_base_lock_expiry(base);
16611747 raw_spin_lock_irq(&base->lock);
16621748
16631749 /*
@@ -1684,8 +1770,8 @@ static inline void __run_timers(struct timer_base *base) · 20 unmodified lines
16841770 while (levels--)
16851771 expire_timers(base, heads + levels);
16861772 }
1687− base->running_timer = NULL;
16881773 raw_spin_unlock_irq(&base->lock);
1774+ timer_base_unlock_expiry(base);
16891775 }
16901776
16911777 /*
@@ -1930,6 +2016,7 @@ static void __init init_timer_cpu(int cpu) · 238 unmodified lines
19302016 base->cpu = cpu;
19312017 raw_spin_lock_init(&base->lock);
19322018 base->clk = jiffies;
2019+ timer_base_init_expiry_lock(base);
19332020 }
19342021 }
19352022
kernel/watchdog.c+2−2
@@ -490,10 +490,10 @@ static void watchdog_enable(unsigned int cpu) · 489 unmodified lines
490490 * Start the timer first to prevent the NMI watchdog triggering
491491 * before the timer has a chance to fire.
492492 */
493− hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
493+ hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
494494 hrtimer->function = watchdog_timer_fn;
495495 hrtimer_start(hrtimer, ns_to_ktime(sample_period),
496− HRTIMER_MODE_REL_PINNED);
496+ HRTIMER_MODE_REL_PINNED_HARD);
497497
498498 /* Initialize timestamp */
499499 __touch_watchdog();
lib/timerqueue.c+12−18
@@ -26,9 +26,10 @@ · 25 unmodified lines
2626 */
2727 bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node)
2828 {
29− struct rb_node **p = &head->head.rb_node;
29+ struct rb_node **p = &head->rb_root.rb_root.rb_node;
3030 struct rb_node *parent = NULL;
31− struct timerqueue_node *ptr;
31+ struct timerqueue_node *ptr;
32+ bool leftmost = true;
3233
3334 /* Make sure we don't add nodes that are already added */
3435 WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node));
@@ -36,19 +37,17 @@ bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node) · 1 unmodified line
3637 while (*p) {
3738 parent = *p;
3839 ptr = rb_entry(parent, struct timerqueue_node, node);
39− if (node->expires < ptr->expires)
40+ if (node->expires < ptr->expires) {
4041 p = &(*p)->rb_left;
41− else
42+ } else {
4243 p = &(*p)->rb_right;
44+ leftmost = false;
45+ }
4346 }
4447 rb_link_node(&node->node, parent, p);
45− rb_insert_color(&node->node, &head->head);
48+ rb_insert_color_cached(&node->node, &head->rb_root, leftmost);
4649
47− if (!head->next || node->expires < head->next->expires) {
48− head->next = node;
49− return true;
50− }
51− return false;
50+ return leftmost;
5251 }
5352 EXPORT_SYMBOL_GPL(timerqueue_add);
5453
@@ -65,15 +64,10 @@ bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node) · 10 unmodified lines
6564 {
6665 WARN_ON_ONCE(RB_EMPTY_NODE(&node->node));
6766
68− /* update next pointer */
69− if (head->next == node) {
70− struct rb_node *rbn = rb_next(&node->node);
71−
72− head->next = rb_entry_safe(rbn, struct timerqueue_node, node);
73− }
74− rb_erase(&node->node, &head->head);
67+ rb_erase_cached(&node->node, &head->rb_root);
7568 RB_CLEAR_NODE(&node->node);
76− return head->next != NULL;
69+
70+ return !RB_EMPTY_ROOT(&head->rb_root.rb_root);
7771 }
7872 EXPORT_SYMBOL_GPL(timerqueue_del);
7973
net/core/pktgen.c+2−4
@@ -2156,7 +2156,7 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) · 2155 unmodified lines
21562156 s64 remaining;
21572157 struct hrtimer_sleeper t;
21582158
2159− hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2159+ hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
21602160 hrtimer_set_expires(&t.timer, spin_until);
21612161
21622162 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
@@ -2170,11 +2170,9 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) · 7 unmodified lines
21702170 end_time = ktime_get();
21712171 } while (ktime_compare(end_time, spin_until) < 0);
21722172 } else {
2173− /* see do_nanosleep */
2174− hrtimer_init_sleeper(&t, current);
21752173 do {
21762174 set_current_state(TASK_INTERRUPTIBLE);
2177− hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS);
2175+ hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS);
21782176
21792177 if (likely(t.task))
21802178 schedule();

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t