Pull core timer updates from Thomas Gleixner: "Timers and timekeeping updates: - A large overhaul of the posix CPU timer code which is a preparation for moving the CPU timer expiry out into task work so it can be properly accounted on the task/process. An update to the bogus permission checks will come later during the merge window as feedback was not complete before heading of for travel. - Switch the timerqueue code to use cached rbtrees and get rid of the homebrewn caching of the leftmost node. - Consolidate hrtimer_init() + hrtimer_init_sleeper() calls into a single function - Implement the separation of hrtimers to be forced to expire in hard interrupt context even when PREEMPT_RT is enabled and mark the affected timers accordingly. - Implement a mechanism for hrtimers and the timer wheel to protect RT against priority inversion and live lock issues when a (hr)timer which should be canceled is currently executing the callback. Instead of infinitely spinning, the task which tries to cancel the timer blocks on a per cpu base expiry lock which is held and released by the (hr)timer expiry code. - Enable the Hyper-V TSC page based sched_clock for Hyper-V guests resulting in faster access to timekeeping functions. - Updates to various clocksource/clockevent drivers and their device tree bindings. - The usual small improvements all over the place" * 'timers-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (101 commits) posix-cpu-timers: Fix permission check regression posix-cpu-timers: Always clear head pointer on dequeue hrtimer: Add a missing bracket and hide `migration_base' on !SMP posix-cpu-timers: Make expiry_active check actually work correctly posix-timers: Unbreak CONFIG_POSIX_TIMERS=n build tick: Mark sched_timer to expire in hard interrupt context hrtimer: Add kernel doc annotation for HRTIMER_MODE_HARD x86/hyperv: Hide pv_ops access for CONFIG_PARAVIRT=n posix-cpu-timers: Utilize timerqueue for storage posix-cpu-timers: Move state tracking to struct posix_cputimers posix-cpu-timers: Deduplicate rlimit handling posix-cpu-timers: Remove pointless comparisons posix-cpu-timers: Get rid of 64bit divisions posix-cpu-timers: Consolidate timer expiry further posix-cpu-timers: Get rid of zero checks rlimit: Rewrite non-sensical RLIMIT_CPU comment posix-cpu-timers: Respect INFINITY for hard RTTIME limit posix-cpu-timers: Switch thread group sampling to array posix-cpu-timers: Restructure expiry array posix-cpu-timers: Remove cputime_expires ...
Showing the changes to include/linux (11 of 61 files). Show all
Unified · Split
| @@ -5,7 +5,8 @@ · 4 unmodified lines | |||
| 5 | #include <linux/time.h> | 5 | #include <linux/time.h> |
| 6 | #include <linux/hrtimer.h> | 6 | #include <linux/hrtimer.h> |
| 7 | #include <linux/timerqueue.h> | 7 | #include <linux/timerqueue.h> |
| 8 | #include <linux/rtc.h> | 8 | |
| 9 | struct rtc_device; | ||
| 9 | 10 | ||
| 10 | enum alarmtimer_type { | 11 | enum alarmtimer_type { |
| 11 | ALARM_REALTIME, | 12 | ALARM_REALTIME, |
| @@ -32,12 +32,15 @@ struct hrtimer_cpu_base; · 31 unmodified lines | |||
| 32 | * when starting the timer) | 32 | * when starting the timer) |
| 33 | * HRTIMER_MODE_SOFT - Timer callback function will be executed in | 33 | * HRTIMER_MODE_SOFT - Timer callback function will be executed in |
| 34 | * soft irq context | 34 | * soft irq context |
| 35 | * HRTIMER_MODE_HARD - Timer callback function will be executed in | ||
| 36 | * hard irq context even on PREEMPT_RT. | ||
| 35 | */ | 37 | */ |
| 36 | enum hrtimer_mode { | 38 | enum hrtimer_mode { |
| 37 | HRTIMER_MODE_ABS = 0x00, | 39 | HRTIMER_MODE_ABS = 0x00, |
| 38 | HRTIMER_MODE_REL = 0x01, | 40 | HRTIMER_MODE_REL = 0x01, |
| 39 | HRTIMER_MODE_PINNED = 0x02, | 41 | HRTIMER_MODE_PINNED = 0x02, |
| 40 | HRTIMER_MODE_SOFT = 0x04, | 42 | HRTIMER_MODE_SOFT = 0x04, |
| 43 | HRTIMER_MODE_HARD = 0x08, | ||
| 41 | 44 | ||
| 42 | HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED, | 45 | HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED, |
| 43 | HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED, | 46 | HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED, |
| @@ -48,6 +51,11 @@ enum hrtimer_mode { · 4 unmodified lines | |||
| 48 | HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT, | 51 | HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT, |
| 49 | HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT, | 52 | HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT, |
| 50 | 53 | ||
| 54 | HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD, | ||
| 55 | HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD, | ||
| 56 | |||
| 57 | HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD, | ||
| 58 | HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD, | ||
| 51 | }; | 59 | }; |
| 52 | 60 | ||
| 53 | /* | 61 | /* |
| @@ -101,6 +109,8 @@ enum hrtimer_restart { · 47 unmodified lines | |||
| 101 | * @state: state information (See bit values above) | 109 | * @state: state information (See bit values above) |
| 102 | * @is_rel: Set if the timer was armed relative | 110 | * @is_rel: Set if the timer was armed relative |
| 103 | * @is_soft: Set if hrtimer will be expired in soft interrupt context. | 111 | * @is_soft: Set if hrtimer will be expired in soft interrupt context. |
| 112 | * @is_hard: Set if hrtimer will be expired in hard interrupt context | ||
| 113 | * even on RT. | ||
| 104 | * | 114 | * |
| 105 | * The hrtimer structure must be initialized by hrtimer_init() | 115 | * The hrtimer structure must be initialized by hrtimer_init() |
| 106 | */ | 116 | */ |
| @@ -112,6 +122,7 @@ struct hrtimer { · 5 unmodified lines | |||
| 112 | u8 state; | 122 | u8 state; |
| 113 | u8 is_rel; | 123 | u8 is_rel; |
| 114 | u8 is_soft; | 124 | u8 is_soft; |
| 125 | u8 is_hard; | ||
| 115 | }; | 126 | }; |
| 116 | 127 | ||
| 117 | /** | 128 | /** |
| @@ -183,6 +194,10 @@ enum hrtimer_base_type { · 65 unmodified lines | |||
| 183 | * @nr_retries: Total number of hrtimer interrupt retries | 194 | * @nr_retries: Total number of hrtimer interrupt retries |
| 184 | * @nr_hangs: Total number of hrtimer interrupt hangs | 195 | * @nr_hangs: Total number of hrtimer interrupt hangs |
| 185 | * @max_hang_time: Maximum time spent in hrtimer_interrupt | 196 | * @max_hang_time: Maximum time spent in hrtimer_interrupt |
| 197 | * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are | ||
| 198 | * expired | ||
| 199 | * @timer_waiters: A hrtimer_cancel() invocation waits for the timer | ||
| 200 | * callback to finish. | ||
| 186 | * @expires_next: absolute time of the next event, is required for remote | 201 | * @expires_next: absolute time of the next event, is required for remote |
| 187 | * hrtimer enqueue; it is the total first expiry time (hard | 202 | * hrtimer enqueue; it is the total first expiry time (hard |
| 188 | * and soft hrtimer are taken into account) | 203 | * and soft hrtimer are taken into account) |
| @@ -209,6 +224,10 @@ struct hrtimer_cpu_base { · 20 unmodified lines | |||
| 209 | unsigned short nr_retries; | 224 | unsigned short nr_retries; |
| 210 | unsigned short nr_hangs; | 225 | unsigned short nr_hangs; |
| 211 | unsigned int max_hang_time; | 226 | unsigned int max_hang_time; |
| 227 | #endif | ||
| 228 | #ifdef CONFIG_PREEMPT_RT | ||
| 229 | spinlock_t softirq_expiry_lock; | ||
| 230 | atomic_t timer_waiters; | ||
| 212 | #endif | 231 | #endif |
| 213 | ktime_t expires_next; | 232 | ktime_t expires_next; |
| 214 | struct hrtimer *next_timer; | 233 | struct hrtimer *next_timer; |
| @@ -341,16 +360,29 @@ extern void hrtimers_resume(void); · 126 unmodified lines | |||
| 341 | 360 | ||
| 342 | DECLARE_PER_CPU(struct tick_device, tick_cpu_device); | 361 | DECLARE_PER_CPU(struct tick_device, tick_cpu_device); |
| 343 | 362 | ||
| 363 | #ifdef CONFIG_PREEMPT_RT | ||
| 364 | void hrtimer_cancel_wait_running(const struct hrtimer *timer); | ||
| 365 | #else | ||
| 366 | static inline void hrtimer_cancel_wait_running(struct hrtimer *timer) | ||
| 367 | { | ||
| 368 | cpu_relax(); | ||
| 369 | } | ||
| 370 | #endif | ||
| 344 | 371 | ||
| 345 | /* Exported timer functions: */ | 372 | /* Exported timer functions: */ |
| 346 | 373 | ||
| 347 | /* Initialize timers: */ | 374 | /* Initialize timers: */ |
| 348 | extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, | 375 | extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, |
| 349 | enum hrtimer_mode mode); | 376 | enum hrtimer_mode mode); |
| 377 | extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id, | ||
| 378 | enum hrtimer_mode mode); | ||
| 350 | 379 | ||
| 351 | #ifdef CONFIG_DEBUG_OBJECTS_TIMERS | 380 | #ifdef CONFIG_DEBUG_OBJECTS_TIMERS |
| 352 | extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock, | 381 | extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock, |
| 353 | enum hrtimer_mode mode); | 382 | enum hrtimer_mode mode); |
| 383 | extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, | ||
| 384 | clockid_t clock_id, | ||
| 385 | enum hrtimer_mode mode); | ||
| 354 | 386 | ||
| 355 | extern void destroy_hrtimer_on_stack(struct hrtimer *timer); | 387 | extern void destroy_hrtimer_on_stack(struct hrtimer *timer); |
| 356 | #else | 388 | #else |
| @@ -360,6 +392,14 @@ static inline void hrtimer_init_on_stack(struct hrtimer *timer, · 3 unmodified lines | |||
| 360 | { | 392 | { |
| 361 | hrtimer_init(timer, which_clock, mode); | 393 | hrtimer_init(timer, which_clock, mode); |
| 362 | } | 394 | } |
| 395 | |||
| 396 | static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl, | ||
| 397 | clockid_t clock_id, | ||
| 398 | enum hrtimer_mode mode) | ||
| 399 | { | ||
| 400 | hrtimer_init_sleeper(sl, clock_id, mode); | ||
| 401 | } | ||
| 402 | |||
| 363 | static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { } | 403 | static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { } |
| 364 | #endif | 404 | #endif |
| 365 | 405 | ||
| @@ -395,6 +435,9 @@ static inline void hrtimer_start_expires(struct hrtimer *timer, · 29 unmodified lines | |||
| 395 | hrtimer_start_range_ns(timer, soft, delta, mode); | 435 | hrtimer_start_range_ns(timer, soft, delta, mode); |
| 396 | } | 436 | } |
| 397 | 437 | ||
| 438 | void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, | ||
| 439 | enum hrtimer_mode mode); | ||
| 440 | |||
| 398 | static inline void hrtimer_restart(struct hrtimer *timer) | 441 | static inline void hrtimer_restart(struct hrtimer *timer) |
| 399 | { | 442 | { |
| 400 | hrtimer_start_expires(timer, HRTIMER_MODE_ABS); | 443 | hrtimer_start_expires(timer, HRTIMER_MODE_ABS); |
| @@ -463,11 +506,8 @@ extern long hrtimer_nanosleep(const struct timespec64 *rqtp, · 62 unmodified lines | |||
| 463 | const enum hrtimer_mode mode, | 506 | const enum hrtimer_mode mode, |
| 464 | const clockid_t clockid); | 507 | const clockid_t clockid); |
| 465 | 508 | ||
| 466 | extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, | ||
| 467 | struct task_struct *tsk); | ||
| 468 | |||
| 469 | extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta, | 509 | extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta, |
| 470 | const enum hrtimer_mode mode); | 510 | const enum hrtimer_mode mode); |
| 471 | extern int schedule_hrtimeout_range_clock(ktime_t *expires, | 511 | extern int schedule_hrtimeout_range_clock(ktime_t *expires, |
| 472 | u64 delta, | 512 | u64 delta, |
| 473 | const enum hrtimer_mode mode, | 513 | const enum hrtimer_mode mode, |
| @@ -36,17 +36,6 @@ extern struct cred init_cred; · 35 unmodified lines | |||
| 36 | #define INIT_PREV_CPUTIME(x) | 36 | #define INIT_PREV_CPUTIME(x) |
| 37 | #endif | 37 | #endif |
| 38 | 38 | ||
| 39 | #ifdef CONFIG_POSIX_TIMERS | ||
| 40 | #define INIT_CPU_TIMERS(s) \ | ||
| 41 | .cpu_timers = { \ | ||
| 42 | LIST_HEAD_INIT(s.cpu_timers[0]), \ | ||
| 43 | LIST_HEAD_INIT(s.cpu_timers[1]), \ | ||
| 44 | LIST_HEAD_INIT(s.cpu_timers[2]), \ | ||
| 45 | }, | ||
| 46 | #else | ||
| 47 | #define INIT_CPU_TIMERS(s) | ||
| 48 | #endif | ||
| 49 | |||
| 50 | #define INIT_TASK_COMM "swapper" | 39 | #define INIT_TASK_COMM "swapper" |
| 51 | 40 | ||
| 52 | /* Attach to the init_task data structure for proper alignment */ | 41 | /* Attach to the init_task data structure for proper alignment */ |
| @@ -4,18 +4,11 @@ · 3 unmodified lines | |||
| 4 | 4 | ||
| 5 | #include <linux/spinlock.h> | 5 | #include <linux/spinlock.h> |
| 6 | #include <linux/list.h> | 6 | #include <linux/list.h> |
| 7 | #include <linux/sched.h> | ||
| 8 | #include <linux/timex.h> | ||
| 9 | #include <linux/alarmtimer.h> | 7 | #include <linux/alarmtimer.h> |
| 8 | #include <linux/timerqueue.h> | ||
| 10 | 9 | ||
| 11 | struct siginfo; | 10 | struct kernel_siginfo; |
| 12 | 11 | struct task_struct; | |
| 13 | struct cpu_timer_list { | ||
| 14 | struct list_head entry; | ||
| 15 | u64 expires; | ||
| 16 | struct task_struct *task; | ||
| 17 | int firing; | ||
| 18 | }; | ||
| 19 | 12 | ||
| 20 | /* | 13 | /* |
| 21 | * Bit fields within a clockid: | 14 | * Bit fields within a clockid: |
| @@ -63,6 +56,115 @@ static inline int clockid_to_fd(const clockid_t clk) · 41 unmodified lines | |||
| 63 | return ~(clk >> 3); | 56 | return ~(clk >> 3); |
| 64 | } | 57 | } |
| 65 | 58 | ||
| 59 | #ifdef CONFIG_POSIX_TIMERS | ||
| 60 | |||
| 61 | /** | ||
| 62 | * cpu_timer - Posix CPU timer representation for k_itimer | ||
| 63 | * @node: timerqueue node to queue in the task/sig | ||
| 64 | * @head: timerqueue head on which this timer is queued | ||
| 65 | * @task: Pointer to target task | ||
| 66 | * @elist: List head for the expiry list | ||
| 67 | * @firing: Timer is currently firing | ||
| 68 | */ | ||
| 69 | struct cpu_timer { | ||
| 70 | struct timerqueue_node node; | ||
| 71 | struct timerqueue_head *head; | ||
| 72 | struct task_struct *task; | ||
| 73 | struct list_head elist; | ||
| 74 | int firing; | ||
| 75 | }; | ||
| 76 | |||
| 77 | static inline bool cpu_timer_enqueue(struct timerqueue_head *head, | ||
| 78 | struct cpu_timer *ctmr) | ||
| 79 | { | ||
| 80 | ctmr->head = head; | ||
| 81 | return timerqueue_add(head, &ctmr->node); | ||
| 82 | } | ||
| 83 | |||
| 84 | static inline void cpu_timer_dequeue(struct cpu_timer *ctmr) | ||
| 85 | { | ||
| 86 | if (ctmr->head) { | ||
| 87 | timerqueue_del(ctmr->head, &ctmr->node); | ||
| 88 | ctmr->head = NULL; | ||
| 89 | } | ||
| 90 | } | ||
| 91 | |||
| 92 | static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr) | ||
| 93 | { | ||
| 94 | return ctmr->node.expires; | ||
| 95 | } | ||
| 96 | |||
| 97 | static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp) | ||
| 98 | { | ||
| 99 | ctmr->node.expires = exp; | ||
| 100 | } | ||
| 101 | |||
| 102 | /** | ||
| 103 | * posix_cputimer_base - Container per posix CPU clock | ||
| 104 | * @nextevt: Earliest-expiration cache | ||
| 105 | * @tqhead: timerqueue head for cpu_timers | ||
| 106 | */ | ||
| 107 | struct posix_cputimer_base { | ||
| 108 | u64 nextevt; | ||
| 109 | struct timerqueue_head tqhead; | ||
| 110 | }; | ||
| 111 | |||
| 112 | /** | ||
| 113 | * posix_cputimers - Container for posix CPU timer related data | ||
| 114 | * @bases: Base container for posix CPU clocks | ||
| 115 | * @timers_active: Timers are queued. | ||
| 116 | * @expiry_active: Timer expiry is active. Used for | ||
| 117 | * process wide timers to avoid multiple | ||
| 118 | * task trying to handle expiry concurrently | ||
| 119 | * | ||
| 120 | * Used in task_struct and signal_struct | ||
| 121 | */ | ||
| 122 | struct posix_cputimers { | ||
| 123 | struct posix_cputimer_base bases[CPUCLOCK_MAX]; | ||
| 124 | unsigned int timers_active; | ||
| 125 | unsigned int expiry_active; | ||
| 126 | }; | ||
| 127 | |||
| 128 | static inline void posix_cputimers_init(struct posix_cputimers *pct) | ||
| 129 | { | ||
| 130 | memset(pct, 0, sizeof(*pct)); | ||
| 131 | pct->bases[0].nextevt = U64_MAX; | ||
| 132 | pct->bases[1].nextevt = U64_MAX; | ||
| 133 | pct->bases[2].nextevt = U64_MAX; | ||
| 134 | } | ||
| 135 | |||
| 136 | void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit); | ||
| 137 | |||
| 138 | static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct, | ||
| 139 | u64 runtime) | ||
| 140 | { | ||
| 141 | pct->bases[CPUCLOCK_SCHED].nextevt = runtime; | ||
| 142 | } | ||
| 143 | |||
| 144 | /* Init task static initializer */ | ||
| 145 | #define INIT_CPU_TIMERBASE(b) { \ | ||
| 146 | .nextevt = U64_MAX, \ | ||
| 147 | } | ||
| 148 | |||
| 149 | #define INIT_CPU_TIMERBASES(b) { \ | ||
| 150 | INIT_CPU_TIMERBASE(b[0]), \ | ||
| 151 | INIT_CPU_TIMERBASE(b[1]), \ | ||
| 152 | INIT_CPU_TIMERBASE(b[2]), \ | ||
| 153 | } | ||
| 154 | |||
| 155 | #define INIT_CPU_TIMERS(s) \ | ||
| 156 | .posix_cputimers = { \ | ||
| 157 | .bases = INIT_CPU_TIMERBASES(s.posix_cputimers.bases), \ | ||
| 158 | }, | ||
| 159 | #else | ||
| 160 | struct posix_cputimers { }; | ||
| 161 | struct cpu_timer { }; | ||
| 162 | #define INIT_CPU_TIMERS(s) | ||
| 163 | static inline void posix_cputimers_init(struct posix_cputimers *pct) { } | ||
| 164 | static inline void posix_cputimers_group_init(struct posix_cputimers *pct, | ||
| 165 | u64 cpu_limit) { } | ||
| 166 | #endif | ||
| 167 | |||
| 66 | #define REQUEUE_PENDING 1 | 168 | #define REQUEUE_PENDING 1 |
| 67 | 169 | ||
| 68 | /** | 170 | /** |
| @@ -85,7 +187,8 @@ static inline int clockid_to_fd(const clockid_t clk) · 16 unmodified lines | |||
| 85 | * @it_process: The task to wakeup on clock_nanosleep (CPU timers) | 187 | * @it_process: The task to wakeup on clock_nanosleep (CPU timers) |
| 86 | * @sigq: Pointer to preallocated sigqueue | 188 | * @sigq: Pointer to preallocated sigqueue |
| 87 | * @it: Union representing the various posix timer type | 189 | * @it: Union representing the various posix timer type |
| 88 | * internals. Also used for rcu freeing the timer. | 190 | * internals. |
| 191 | * @rcu: RCU head for freeing the timer. | ||
| 89 | */ | 192 | */ |
| 90 | struct k_itimer { | 193 | struct k_itimer { |
| 91 | struct list_head list; | 194 | struct list_head list; |
| @@ -110,15 +213,15 @@ struct k_itimer { · 18 unmodified lines | |||
| 110 | struct { | 213 | struct { |
| 111 | struct hrtimer timer; | 214 | struct hrtimer timer; |
| 112 | } real; | 215 | } real; |
| 113 | struct cpu_timer_list cpu; | 216 | struct cpu_timer cpu; |
| 114 | struct { | 217 | struct { |
| 115 | struct alarm alarmtimer; | 218 | struct alarm alarmtimer; |
| 116 | } alarm; | 219 | } alarm; |
| 117 | struct rcu_head rcu; | ||
| 118 | } it; | 220 | } it; |
| 221 | struct rcu_head rcu; | ||
| 119 | }; | 222 | }; |
| 120 | 223 | ||
| 121 | void run_posix_cpu_timers(struct task_struct *task); | 224 | void run_posix_cpu_timers(void); |
| 122 | void posix_cpu_timers_exit(struct task_struct *task); | 225 | void posix_cpu_timers_exit(struct task_struct *task); |
| 123 | void posix_cpu_timers_exit_group(struct task_struct *task); | 226 | void posix_cpu_timers_exit_group(struct task_struct *task); |
| 124 | void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx, | 227 | void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx, |
| @@ -25,9 +25,11 @@ · 24 unmodified lines | |||
| 25 | #include <linux/resource.h> | 25 | #include <linux/resource.h> |
| 26 | #include <linux/latencytop.h> | 26 | #include <linux/latencytop.h> |
| 27 | #include <linux/sched/prio.h> | 27 | #include <linux/sched/prio.h> |
| 28 | #include <linux/sched/types.h> | ||
| 28 | #include <linux/signal_types.h> | 29 | #include <linux/signal_types.h> |
| 29 | #include <linux/mm_types_task.h> | 30 | #include <linux/mm_types_task.h> |
| 30 | #include <linux/task_io_accounting.h> | 31 | #include <linux/task_io_accounting.h> |
| 32 | #include <linux/posix-timers.h> | ||
| 31 | #include <linux/rseq.h> | 33 | #include <linux/rseq.h> |
| 32 | 34 | ||
| 33 | /* task_struct member predeclarations (sorted alphabetically): */ | 35 | /* task_struct member predeclarations (sorted alphabetically): */ |
| @@ -244,27 +246,6 @@ struct prev_cputime { · 210 unmodified lines | |||
| 244 | #endif | 246 | #endif |
| 245 | }; | 247 | }; |
| 246 | 248 | ||
| 247 | /** | ||
| 248 | * struct task_cputime - collected CPU time counts | ||
| 249 | * @utime: time spent in user mode, in nanoseconds | ||
| 250 | * @stime: time spent in kernel mode, in nanoseconds | ||
| 251 | * @sum_exec_runtime: total time spent on the CPU, in nanoseconds | ||
| 252 | * | ||
| 253 | * This structure groups together three kinds of CPU time that are tracked for | ||
| 254 | * threads and thread groups. Most things considering CPU time want to group | ||
| 255 | * these counts together and treat all three of them in parallel. | ||
| 256 | */ | ||
| 257 | struct task_cputime { | ||
| 258 | u64 utime; | ||
| 259 | u64 stime; | ||
| 260 | unsigned long long sum_exec_runtime; | ||
| 261 | }; | ||
| 262 | |||
| 263 | /* Alternate field names when used on cache expirations: */ | ||
| 264 | #define virt_exp utime | ||
| 265 | #define prof_exp stime | ||
| 266 | #define sched_exp sum_exec_runtime | ||
| 267 | |||
| 268 | enum vtime_state { | 249 | enum vtime_state { |
| 269 | /* Task is sleeping or running in a CPU with VTIME inactive: */ | 250 | /* Task is sleeping or running in a CPU with VTIME inactive: */ |
| 270 | VTIME_INACTIVE = 0, | 251 | VTIME_INACTIVE = 0, |
| @@ -881,10 +862,8 @@ struct task_struct { · 610 unmodified lines | |||
| 881 | unsigned long min_flt; | 862 | unsigned long min_flt; |
| 882 | unsigned long maj_flt; | 863 | unsigned long maj_flt; |
| 883 | 864 | ||
| 884 | #ifdef CONFIG_POSIX_TIMERS | 865 | /* Empty if CONFIG_POSIX_CPUTIMERS=n */ |
| 885 | struct task_cputime cputime_expires; | 866 | struct posix_cputimers posix_cputimers; |
| 886 | struct list_head cpu_timers[3]; | ||
| 887 | #endif | ||
| 888 | 867 | ||
| 889 | /* Process credentials: */ | 868 | /* Process credentials: */ |
| 890 | 869 | ||
| @@ -61,8 +61,7 @@ extern void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, · 60 unmodified lines | |||
| 61 | * Thread group CPU time accounting. | 61 | * Thread group CPU time accounting. |
| 62 | */ | 62 | */ |
| 63 | void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); | 63 | void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); |
| 64 | void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); | 64 | void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples); |
| 65 | |||
| 66 | 65 | ||
| 67 | /* | 66 | /* |
| 68 | * The following are functions that support scheduler-internal time accounting. | 67 | * The following are functions that support scheduler-internal time accounting. |
| @@ -71,7 +70,7 @@ void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); · 2 unmodified lines | |||
| 71 | */ | 70 | */ |
| 72 | 71 | ||
| 73 | /** | 72 | /** |
| 74 | * get_running_cputimer - return &tsk->signal->cputimer if cputimer is running | 73 | * get_running_cputimer - return &tsk->signal->cputimer if cputimers are active |
| 75 | * | 74 | * |
| 76 | * @tsk: Pointer to target task. | 75 | * @tsk: Pointer to target task. |
| 77 | */ | 76 | */ |
| @@ -81,8 +80,11 @@ struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk) · 3 unmodified lines | |||
| 81 | { | 80 | { |
| 82 | struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; | 81 | struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; |
| 83 | 82 | ||
| 84 | /* Check if cputimer isn't running. This is accessed without locking. */ | 83 | /* |
| 85 | if (!READ_ONCE(cputimer->running)) | 84 | * Check whether posix CPU timers are active. If not the thread |
| 85 | * group accounting is not active either. Lockless check. | ||
| 86 | */ | ||
| 87 | if (!READ_ONCE(tsk->signal->posix_cputimers.timers_active)) | ||
| 86 | return NULL; | 88 | return NULL; |
| 87 | 89 | ||
| 88 | /* | 90 | /* |
| @@ -9,6 +9,7 @@ · 8 unmodified lines | |||
| 9 | #include <linux/sched/task.h> | 9 | #include <linux/sched/task.h> |
| 10 | #include <linux/cred.h> | 10 | #include <linux/cred.h> |
| 11 | #include <linux/refcount.h> | 11 | #include <linux/refcount.h> |
| 12 | #include <linux/posix-timers.h> | ||
| 12 | 13 | ||
| 13 | /* | 14 | /* |
| 14 | * Types defining task->signal and task->sighand and APIs using them: | 15 | * Types defining task->signal and task->sighand and APIs using them: |
| @@ -56,18 +57,12 @@ struct task_cputime_atomic { · 41 unmodified lines | |||
| 56 | /** | 57 | /** |
| 57 | * struct thread_group_cputimer - thread group interval timer counts | 58 | * struct thread_group_cputimer - thread group interval timer counts |
| 58 | * @cputime_atomic: atomic thread group interval timers. | 59 | * @cputime_atomic: atomic thread group interval timers. |
| 59 | * @running: true when there are timers running and | ||
| 60 | * @cputime_atomic receives updates. | ||
| 61 | * @checking_timer: true when a thread in the group is in the | ||
| 62 | * process of checking for thread group timers. | ||
| 63 | * | 60 | * |
| 64 | * This structure contains the version of task_cputime, above, that is | 61 | * This structure contains the version of task_cputime, above, that is |
| 65 | * used for thread group CPU timer calculations. | 62 | * used for thread group CPU timer calculations. |
| 66 | */ | 63 | */ |
| 67 | struct thread_group_cputimer { | 64 | struct thread_group_cputimer { |
| 68 | struct task_cputime_atomic cputime_atomic; | 65 | struct task_cputime_atomic cputime_atomic; |
| 69 | bool running; | ||
| 70 | bool checking_timer; | ||
| 71 | }; | 66 | }; |
| 72 | 67 | ||
| 73 | struct multiprocess_signals { | 68 | struct multiprocess_signals { |
| @@ -148,12 +143,9 @@ struct signal_struct { · 74 unmodified lines | |||
| 148 | */ | 143 | */ |
| 149 | struct thread_group_cputimer cputimer; | 144 | struct thread_group_cputimer cputimer; |
| 150 | 145 | ||
| 151 | /* Earliest-expiration cache. */ | ||
| 152 | struct task_cputime cputime_expires; | ||
| 153 | |||
| 154 | struct list_head cpu_timers[3]; | ||
| 155 | |||
| 156 | #endif | 146 | #endif |
| 147 | /* Empty if CONFIG_POSIX_TIMERS=n */ | ||
| 148 | struct posix_cputimers posix_cputimers; | ||
| 157 | 149 | ||
| 158 | /* PID/PID hash table linkage. */ | 150 | /* PID/PID hash table linkage. */ |
| 159 | struct pid *pids[PIDTYPE_MAX]; | 151 | struct pid *pids[PIDTYPE_MAX]; |
| @@ -0,0 +1,23 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 2 | #ifndef _LINUX_SCHED_TYPES_H | ||
| 3 | #define _LINUX_SCHED_TYPES_H | ||
| 4 | |||
| 5 | #include <linux/types.h> | ||
| 6 | |||
| 7 | /** | ||
| 8 | * struct task_cputime - collected CPU time counts | ||
| 9 | * @stime: time spent in kernel mode, in nanoseconds | ||
| 10 | * @utime: time spent in user mode, in nanoseconds | ||
| 11 | * @sum_exec_runtime: total time spent on the CPU, in nanoseconds | ||
| 12 | * | ||
| 13 | * This structure groups together three kinds of CPU time that are tracked for | ||
| 14 | * threads and thread groups. Most things considering CPU time want to group | ||
| 15 | * these counts together and treat all three of them in parallel. | ||
| 16 | */ | ||
| 17 | struct task_cputime { | ||
| 18 | u64 stime; | ||
| 19 | u64 utime; | ||
| 20 | unsigned long long sum_exec_runtime; | ||
| 21 | }; | ||
| 22 | |||
| 23 | #endif | ||
| @@ -183,7 +183,7 @@ extern void add_timer(struct timer_list *timer); · 182 unmodified lines | |||
| 183 | 183 | ||
| 184 | extern int try_to_del_timer_sync(struct timer_list *timer); | 184 | extern int try_to_del_timer_sync(struct timer_list *timer); |
| 185 | 185 | ||
| 186 | #ifdef CONFIG_SMP | 186 | #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT) |
| 187 | extern int del_timer_sync(struct timer_list *timer); | 187 | extern int del_timer_sync(struct timer_list *timer); |
| 188 | #else | 188 | #else |
| 189 | # define del_timer_sync(t) del_timer(t) | 189 | # define del_timer_sync(t) del_timer(t) |
| @@ -12,8 +12,7 @@ struct timerqueue_node { · 11 unmodified lines | |||
| 12 | }; | 12 | }; |
| 13 | 13 | ||
| 14 | struct timerqueue_head { | 14 | struct timerqueue_head { |
| 15 | struct rb_root head; | 15 | struct rb_root_cached rb_root; |
| 16 | struct timerqueue_node *next; | ||
| 17 | }; | 16 | }; |
| 18 | 17 | ||
| 19 | 18 | ||
| @@ -29,13 +28,14 @@ extern struct timerqueue_node *timerqueue_iterate_next( · 9 unmodified lines | |||
| 29 | * | 28 | * |
| 30 | * @head: head of timerqueue | 29 | * @head: head of timerqueue |
| 31 | * | 30 | * |
| 32 | * Returns a pointer to the timer node that has the | 31 | * Returns a pointer to the timer node that has the earliest expiration time. |
| 33 | * earliest expiration time. | ||
| 34 | */ | 32 | */ |
| 35 | static inline | 33 | static inline |
| 36 | struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head) | 34 | struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head) |
| 37 | { | 35 | { |
| 38 | return head->next; | 36 | struct rb_node *leftmost = rb_first_cached(&head->rb_root); |
| 37 | |||
| 38 | return rb_entry(leftmost, struct timerqueue_node, node); | ||
| 39 | } | 39 | } |
| 40 | 40 | ||
| 41 | static inline void timerqueue_init(struct timerqueue_node *node) | 41 | static inline void timerqueue_init(struct timerqueue_node *node) |
| @@ -43,9 +43,18 @@ static inline void timerqueue_init(struct timerqueue_node *node) · 1 unmodified line | |||
| 43 | RB_CLEAR_NODE(&node->node); | 43 | RB_CLEAR_NODE(&node->node); |
| 44 | } | 44 | } |
| 45 | 45 | ||
| 46 | static inline bool timerqueue_node_queued(struct timerqueue_node *node) | ||
| 47 | { | ||
| 48 | return !RB_EMPTY_NODE(&node->node); | ||
| 49 | } | ||
| 50 | |||
| 51 | static inline bool timerqueue_node_expires(struct timerqueue_node *node) | ||
| 52 | { | ||
| 53 | return node->expires; | ||
| 54 | } | ||
| 55 | |||
| 46 | static inline void timerqueue_init_head(struct timerqueue_head *head) | 56 | static inline void timerqueue_init_head(struct timerqueue_head *head) |
| 47 | { | 57 | { |
| 48 | head->head = RB_ROOT; | 58 | head->rb_root = RB_ROOT_CACHED; |
| 49 | head->next = NULL; | ||
| 50 | } | 59 | } |
| 51 | #endif /* _LINUX_TIMERQUEUE_H */ | 60 | #endif /* _LINUX_TIMERQUEUE_H */ |
| @@ -501,8 +501,8 @@ do { \ · 500 unmodified lines | |||
| 501 | int __ret = 0; \ | 501 | int __ret = 0; \ |
| 502 | struct hrtimer_sleeper __t; \ | 502 | struct hrtimer_sleeper __t; \ |
| 503 | \ | 503 | \ |
| 504 | hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); \ | 504 | hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \ |
| 505 | hrtimer_init_sleeper(&__t, current); \ | 505 | HRTIMER_MODE_REL); \ |
| 506 | if ((timeout) != KTIME_MAX) \ | 506 | if ((timeout) != KTIME_MAX) \ |
| 507 | hrtimer_start_range_ns(&__t.timer, timeout, \ | 507 | hrtimer_start_range_ns(&__t.timer, timeout, \ |
| 508 | current->timer_slack_ns, \ | 508 | current->timer_slack_ns, \ |