Pull in tip/sched/core to resolve two merge conflicts: - 96fd6c65efc6 ("sched: Factor out update_other_load_avgs() from __update_blocked_others()") 5d871a63997f ("sched/fair: Move effective_cpu_util() and effective_cpu_util() in fair.c") A simple context conflict. The former added __update_blocked_others() in the same #ifdef CONFIG_SMP block that effective_cpu_util() and sched_cpu_util() are in and the latter moved those functions to fair.c. This makes __update_blocked_others() more out of place. Will follow up with a patch to relocate. - 96fd6c65efc6 ("sched: Factor out update_other_load_avgs() from __update_blocked_others()") 84d265281d6c ("sched/pelt: Use rq_clock_task() for hw_pressure") The former factored out the body of __update_blocked_others() into update_other_load_avgs(). The latter changed how update_hw_load_avg() is called in the body. Resolved by applying the change to update_other_load_avgs() instead.
Unified · Split
| @@ -749,21 +749,19 @@ Appendix A. Test suite · 748 unmodified lines | |||
| 749 | of the command line options. Please refer to rt-app documentation for more | 749 | of the command line options. Please refer to rt-app documentation for more |
| 750 | details (`<rt-app-sources>/doc/*.json`). | 750 | details (`<rt-app-sources>/doc/*.json`). |
| 751 | 751 | ||
| 752 | The second testing application is a modification of schedtool, called | 752 | The second testing application is done using chrt which has support |
| 753 | schedtool-dl, which can be used to setup SCHED_DEADLINE parameters for a | 753 | for SCHED_DEADLINE. |
| 754 | certain pid/application. schedtool-dl is available at: | ||
| 755 | https://github.com/scheduler-tools/schedtool-dl.git. | ||
| 756 | 754 | ||
| 757 | The usage is straightforward:: | 755 | The usage is straightforward:: |
| 758 | 756 | ||
| 759 | # schedtool -E -t 10000000:100000000 -e ./my_cpuhog_app | 757 | # chrt -d -T 10000000 -D 100000000 0 ./my_cpuhog_app |
| 760 | 758 | ||
| 761 | With this, my_cpuhog_app is put to run inside a SCHED_DEADLINE reservation | 759 | With this, my_cpuhog_app is put to run inside a SCHED_DEADLINE reservation |
| 762 | of 10ms every 100ms (note that parameters are expressed in microseconds). | 760 | of 10ms every 100ms (note that parameters are expressed in nanoseconds). |
| 763 | You can also use schedtool to create a reservation for an already running | 761 | You can also use chrt to create a reservation for an already running |
| 764 | application, given that you know its pid:: | 762 | application, given that you know its pid:: |
| 765 | 763 | ||
| 766 | # schedtool -E -t 10000000:100000000 my_app_pid | 764 | # chrt -d -T 10000000 -D 100000000 -p 0 my_app_pid |
| 767 | 765 | ||
| 768 | Appendix B. Minimal main() | 766 | Appendix B. Minimal main() |
| 769 | ========================== | 767 | ========================== |
| @@ -224,9 +224,9 @@ static void __init cppc_freq_invariance_init(void) · 223 unmodified lines | |||
| 224 | * Fake (unused) bandwidth; workaround to "fix" | 224 | * Fake (unused) bandwidth; workaround to "fix" |
| 225 | * priority inheritance. | 225 | * priority inheritance. |
| 226 | */ | 226 | */ |
| 227 | .sched_runtime = 1000000, | 227 | .sched_runtime = NSEC_PER_MSEC, |
| 228 | .sched_deadline = 10000000, | 228 | .sched_deadline = 10 * NSEC_PER_MSEC, |
| 229 | .sched_period = 10000000, | 229 | .sched_period = 10 * NSEC_PER_MSEC, |
| 230 | }; | 230 | }; |
| 231 | int ret; | 231 | int ret; |
| 232 | 232 | ||
| @@ -58,9 +58,9 @@ · 57 unmodified lines | |||
| 58 | * | 58 | * |
| 59 | * This is reflected by the following fields of the sched_attr structure: | 59 | * This is reflected by the following fields of the sched_attr structure: |
| 60 | * | 60 | * |
| 61 | * @sched_deadline representative of the task's deadline | 61 | * @sched_deadline representative of the task's deadline in nanoseconds |
| 62 | * @sched_runtime representative of the task's runtime | 62 | * @sched_runtime representative of the task's runtime in nanoseconds |
| 63 | * @sched_period representative of the task's period | 63 | * @sched_period representative of the task's period in nanoseconds |
| 64 | * | 64 | * |
| 65 | * Given this task model, there are a multiplicity of scheduling algorithms | 65 | * Given this task model, there are a multiplicity of scheduling algorithms |
| 66 | * and policies, that can be used to ensure all the tasks will make their | 66 | * and policies, that can be used to ensure all the tasks will make their |
| @@ -845,8 +845,16 @@ repeat: · 844 unmodified lines | |||
| 845 | * event only cares about the address. | 845 | * event only cares about the address. |
| 846 | */ | 846 | */ |
| 847 | trace_sched_kthread_work_execute_end(work, func); | 847 | trace_sched_kthread_work_execute_end(work, func); |
| 848 | } else if (!freezing(current)) | 848 | } else if (!freezing(current)) { |
| 849 | schedule(); | 849 | schedule(); |
| 850 | } else { | ||
| 851 | /* | ||
| 852 | * Handle the case where the current remains | ||
| 853 | * TASK_INTERRUPTIBLE. try_to_freeze() expects | ||
| 854 | * the current to be TASK_RUNNING. | ||
| 855 | */ | ||
| 856 | __set_current_state(TASK_RUNNING); | ||
| 857 | } | ||
| 850 | 858 | ||
| 851 | try_to_freeze(); | 859 | try_to_freeze(); |
| 852 | cond_resched(); | 860 | cond_resched(); |
| @@ -267,6 +267,9 @@ static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) · 266 unmodified lines | |||
| 267 | 267 | ||
| 268 | void sched_core_enqueue(struct rq *rq, struct task_struct *p) | 268 | void sched_core_enqueue(struct rq *rq, struct task_struct *p) |
| 269 | { | 269 | { |
| 270 | if (p->se.sched_delayed) | ||
| 271 | return; | ||
| 272 | |||
| 270 | rq->core->core_task_seq++; | 273 | rq->core->core_task_seq++; |
| 271 | 274 | ||
| 272 | if (!p->core_cookie) | 275 | if (!p->core_cookie) |
| @@ -277,6 +280,9 @@ void sched_core_enqueue(struct rq *rq, struct task_struct *p) · 4 unmodified lines | |||
| 277 | 280 | ||
| 278 | void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) | 281 | void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) |
| 279 | { | 282 | { |
| 283 | if (p->se.sched_delayed) | ||
| 284 | return; | ||
| 285 | |||
| 280 | rq->core->core_task_seq++; | 286 | rq->core->core_task_seq++; |
| 281 | 287 | ||
| 282 | if (sched_core_enqueued(p)) { | 288 | if (sched_core_enqueued(p)) { |
| @@ -6477,19 +6483,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) · 6194 unmodified lines | |||
| 6477 | * Constants for the sched_mode argument of __schedule(). | 6483 | * Constants for the sched_mode argument of __schedule(). |
| 6478 | * | 6484 | * |
| 6479 | * The mode argument allows RT enabled kernels to differentiate a | 6485 | * The mode argument allows RT enabled kernels to differentiate a |
| 6480 | * preemption from blocking on an 'sleeping' spin/rwlock. Note that | 6486 | * preemption from blocking on an 'sleeping' spin/rwlock. |
| 6481 | * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to | ||
| 6482 | * optimize the AND operation out and just check for zero. | ||
| 6483 | */ | 6487 | */ |
| 6484 | #define SM_NONE 0x0 | 6488 | #define SM_IDLE (-1) |
| 6485 | #define SM_PREEMPT 0x1 | 6489 | #define SM_NONE 0 |
| 6486 | #define SM_RTLOCK_WAIT 0x2 | 6490 | #define SM_PREEMPT 1 |
| 6487 | 6491 | #define SM_RTLOCK_WAIT 2 | |
| 6488 | #ifndef CONFIG_PREEMPT_RT | ||
| 6489 | # define SM_MASK_PREEMPT (~0U) | ||
| 6490 | #else | ||
| 6491 | # define SM_MASK_PREEMPT SM_PREEMPT | ||
| 6492 | #endif | ||
| 6493 | 6492 | ||
| 6494 | /* | 6493 | /* |
| 6495 | * __schedule() is the main scheduler function. | 6494 | * __schedule() is the main scheduler function. |
| @@ -6530,9 +6529,14 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) · 34 unmodified lines | |||
| 6530 | * | 6529 | * |
| 6531 | * WARNING: must be called with preemption disabled! | 6530 | * WARNING: must be called with preemption disabled! |
| 6532 | */ | 6531 | */ |
| 6533 | static void __sched notrace __schedule(unsigned int sched_mode) | 6532 | static void __sched notrace __schedule(int sched_mode) |
| 6534 | { | 6533 | { |
| 6535 | struct task_struct *prev, *next; | 6534 | struct task_struct *prev, *next; |
| 6535 | /* | ||
| 6536 | * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted | ||
| 6537 | * as a preemption by schedule_debug() and RCU. | ||
| 6538 | */ | ||
| 6539 | bool preempt = sched_mode > SM_NONE; | ||
| 6536 | unsigned long *switch_count; | 6540 | unsigned long *switch_count; |
| 6537 | unsigned long prev_state; | 6541 | unsigned long prev_state; |
| 6538 | struct rq_flags rf; | 6542 | struct rq_flags rf; |
| @@ -6543,13 +6547,13 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 4 unmodified lines | |||
| 6543 | rq = cpu_rq(cpu); | 6547 | rq = cpu_rq(cpu); |
| 6544 | prev = rq->curr; | 6548 | prev = rq->curr; |
| 6545 | 6549 | ||
| 6546 | schedule_debug(prev, !!sched_mode); | 6550 | schedule_debug(prev, preempt); |
| 6547 | 6551 | ||
| 6548 | if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) | 6552 | if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) |
| 6549 | hrtick_clear(rq); | 6553 | hrtick_clear(rq); |
| 6550 | 6554 | ||
| 6551 | local_irq_disable(); | 6555 | local_irq_disable(); |
| 6552 | rcu_note_context_switch(!!sched_mode); | 6556 | rcu_note_context_switch(preempt); |
| 6553 | 6557 | ||
| 6554 | /* | 6558 | /* |
| 6555 | * Make sure that signal_pending_state()->signal_pending() below | 6559 | * Make sure that signal_pending_state()->signal_pending() below |
| @@ -6578,12 +6582,20 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 22 unmodified lines | |||
| 6578 | 6582 | ||
| 6579 | switch_count = &prev->nivcsw; | 6583 | switch_count = &prev->nivcsw; |
| 6580 | 6584 | ||
| 6585 | /* Task state changes only considers SM_PREEMPT as preemption */ | ||
| 6586 | preempt = sched_mode == SM_PREEMPT; | ||
| 6587 | |||
| 6581 | /* | 6588 | /* |
| 6582 | * We must load prev->state once (task_struct::state is volatile), such | 6589 | * We must load prev->state once (task_struct::state is volatile), such |
| 6583 | * that we form a control dependency vs deactivate_task() below. | 6590 | * that we form a control dependency vs deactivate_task() below. |
| 6584 | */ | 6591 | */ |
| 6585 | prev_state = READ_ONCE(prev->__state); | 6592 | prev_state = READ_ONCE(prev->__state); |
| 6586 | if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) { | 6593 | if (sched_mode == SM_IDLE) { |
| 6594 | if (!rq->nr_running) { | ||
| 6595 | next = prev; | ||
| 6596 | goto picked; | ||
| 6597 | } | ||
| 6598 | } else if (!preempt && prev_state) { | ||
| 6587 | if (signal_pending_state(prev_state, prev)) { | 6599 | if (signal_pending_state(prev_state, prev)) { |
| 6588 | WRITE_ONCE(prev->__state, TASK_RUNNING); | 6600 | WRITE_ONCE(prev->__state, TASK_RUNNING); |
| 6589 | } else { | 6601 | } else { |
| @@ -6614,6 +6626,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 24 unmodified lines | |||
| 6614 | } | 6626 | } |
| 6615 | 6627 | ||
| 6616 | next = pick_next_task(rq, prev, &rf); | 6628 | next = pick_next_task(rq, prev, &rf); |
| 6629 | picked: | ||
| 6617 | clear_tsk_need_resched(prev); | 6630 | clear_tsk_need_resched(prev); |
| 6618 | clear_preempt_need_resched(); | 6631 | clear_preempt_need_resched(); |
| 6619 | #ifdef CONFIG_SCHED_DEBUG | 6632 | #ifdef CONFIG_SCHED_DEBUG |
| @@ -6655,7 +6668,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 35 unmodified lines | |||
| 6655 | psi_account_irqtime(rq, prev, next); | 6668 | psi_account_irqtime(rq, prev, next); |
| 6656 | psi_sched_switch(prev, next, !task_on_rq_queued(prev)); | 6669 | psi_sched_switch(prev, next, !task_on_rq_queued(prev)); |
| 6657 | 6670 | ||
| 6658 | trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state); | 6671 | trace_sched_switch(preempt, prev, next, prev_state); |
| 6659 | 6672 | ||
| 6660 | /* Also unlocks the rq: */ | 6673 | /* Also unlocks the rq: */ |
| 6661 | rq = context_switch(rq, prev, next, &rf); | 6674 | rq = context_switch(rq, prev, next, &rf); |
| @@ -6731,7 +6744,7 @@ static void sched_update_worker(struct task_struct *tsk) · 69 unmodified lines | |||
| 6731 | } | 6744 | } |
| 6732 | } | 6745 | } |
| 6733 | 6746 | ||
| 6734 | static __always_inline void __schedule_loop(unsigned int sched_mode) | 6747 | static __always_inline void __schedule_loop(int sched_mode) |
| 6735 | { | 6748 | { |
| 6736 | do { | 6749 | do { |
| 6737 | preempt_disable(); | 6750 | preempt_disable(); |
| @@ -6776,7 +6789,7 @@ void __sched schedule_idle(void) · 38 unmodified lines | |||
| 6776 | */ | 6789 | */ |
| 6777 | WARN_ON_ONCE(current->__state); | 6790 | WARN_ON_ONCE(current->__state); |
| 6778 | do { | 6791 | do { |
| 6779 | __schedule(SM_NONE); | 6792 | __schedule(SM_IDLE); |
| 6780 | } while (need_resched()); | 6793 | } while (need_resched()); |
| 6781 | } | 6794 | } |
| 6782 | 6795 | ||
| @@ -662,9 +662,9 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy) · 661 unmodified lines | |||
| 662 | * Fake (unused) bandwidth; workaround to "fix" | 662 | * Fake (unused) bandwidth; workaround to "fix" |
| 663 | * priority inheritance. | 663 | * priority inheritance. |
| 664 | */ | 664 | */ |
| 665 | .sched_runtime = 1000000, | 665 | .sched_runtime = NSEC_PER_MSEC, |
| 666 | .sched_deadline = 10000000, | 666 | .sched_deadline = 10 * NSEC_PER_MSEC, |
| 667 | .sched_period = 10000000, | 667 | .sched_period = 10 * NSEC_PER_MSEC, |
| 668 | }; | 668 | }; |
| 669 | struct cpufreq_policy *policy = sg_policy->policy; | 669 | struct cpufreq_policy *policy = sg_policy->policy; |
| 670 | int ret; | 670 | int ret; |
| @@ -739,7 +739,7 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) · 738 unmodified lines | |||
| 739 | else | 739 | else |
| 740 | SEQ_printf(m, " %c", task_state_to_char(p)); | 740 | SEQ_printf(m, " %c", task_state_to_char(p)); |
| 741 | 741 | ||
| 742 | SEQ_printf(m, "%15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", | 742 | SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", |
| 743 | p->comm, task_pid_nr(p), | 743 | p->comm, task_pid_nr(p), |
| 744 | SPLIT_NS(p->se.vruntime), | 744 | SPLIT_NS(p->se.vruntime), |
| 745 | entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N', | 745 | entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N', |
| @@ -750,17 +750,16 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) · 4 unmodified lines | |||
| 750 | (long long)(p->nvcsw + p->nivcsw), | 750 | (long long)(p->nvcsw + p->nivcsw), |
| 751 | p->prio); | 751 | p->prio); |
| 752 | 752 | ||
| 753 | SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld %9lld.%06ld", | 753 | SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld", |
| 754 | SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)), | 754 | SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)), |
| 755 | SPLIT_NS(p->se.sum_exec_runtime), | ||
| 756 | SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)), | 755 | SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)), |
| 757 | SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime))); | 756 | SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime))); |
| 758 | 757 | ||
| 759 | #ifdef CONFIG_NUMA_BALANCING | 758 | #ifdef CONFIG_NUMA_BALANCING |
| 760 | SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p)); | 759 | SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p)); |
| 761 | #endif | 760 | #endif |
| 762 | #ifdef CONFIG_CGROUP_SCHED | 761 | #ifdef CONFIG_CGROUP_SCHED |
| 763 | SEQ_printf_task_group_path(m, task_group(p), " %s") | 762 | SEQ_printf_task_group_path(m, task_group(p), " %s") |
| 764 | #endif | 763 | #endif |
| 765 | 764 | ||
| 766 | SEQ_printf(m, "\n"); | 765 | SEQ_printf(m, "\n"); |
| @@ -772,10 +771,26 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) · 5 unmodified lines | |||
| 772 | 771 | ||
| 773 | SEQ_printf(m, "\n"); | 772 | SEQ_printf(m, "\n"); |
| 774 | SEQ_printf(m, "runnable tasks:\n"); | 773 | SEQ_printf(m, "runnable tasks:\n"); |
| 775 | SEQ_printf(m, " S task PID tree-key switches prio" | 774 | SEQ_printf(m, " S task PID vruntime eligible " |
| 776 | " wait-time sum-exec sum-sleep\n"); | 775 | "deadline slice sum-exec switches " |
| 776 | "prio wait-time sum-sleep sum-block" | ||
| 777 | #ifdef CONFIG_NUMA_BALANCING | ||
| 778 | " node group-id" | ||
| 779 | #endif | ||
| 780 | #ifdef CONFIG_CGROUP_SCHED | ||
| 781 | " group-path" | ||
| 782 | #endif | ||
| 783 | "\n"); | ||
| 777 | SEQ_printf(m, "-------------------------------------------------------" | 784 | SEQ_printf(m, "-------------------------------------------------------" |
| 778 | "------------------------------------------------------\n"); | 785 | "------------------------------------------------------" |
| 786 | "------------------------------------------------------" | ||
| 787 | #ifdef CONFIG_NUMA_BALANCING | ||
| 788 | "--------------" | ||
| 789 | #endif | ||
| 790 | #ifdef CONFIG_CGROUP_SCHED | ||
| 791 | "--------------" | ||
| 792 | #endif | ||
| 793 | "\n"); | ||
| 779 | 794 | ||
| 780 | rcu_read_lock(); | 795 | rcu_read_lock(); |
| 781 | for_each_process_thread(g, p) { | 796 | for_each_process_thread(g, p) { |
| @@ -6949,18 +6949,19 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 6948 unmodified lines | |||
| 6949 | int rq_h_nr_running = rq->cfs.h_nr_running; | 6949 | int rq_h_nr_running = rq->cfs.h_nr_running; |
| 6950 | u64 slice = 0; | 6950 | u64 slice = 0; |
| 6951 | 6951 | ||
| 6952 | if (flags & ENQUEUE_DELAYED) { | ||
| 6953 | requeue_delayed_entity(se); | ||
| 6954 | return; | ||
| 6955 | } | ||
| 6956 | |||
| 6957 | /* | 6952 | /* |
| 6958 | * The code below (indirectly) updates schedutil which looks at | 6953 | * The code below (indirectly) updates schedutil which looks at |
| 6959 | * the cfs_rq utilization to select a frequency. | 6954 | * the cfs_rq utilization to select a frequency. |
| 6960 | * Let's add the task's estimated utilization to the cfs_rq's | 6955 | * Let's add the task's estimated utilization to the cfs_rq's |
| 6961 | * estimated utilization, before we update schedutil. | 6956 | * estimated utilization, before we update schedutil. |
| 6962 | */ | 6957 | */ |
| 6963 | util_est_enqueue(&rq->cfs, p); | 6958 | if (!(p->se.sched_delayed && (task_on_rq_migrating(p) || (flags & ENQUEUE_RESTORE)))) |
| 6959 | util_est_enqueue(&rq->cfs, p); | ||
| 6960 | |||
| 6961 | if (flags & ENQUEUE_DELAYED) { | ||
| 6962 | requeue_delayed_entity(se); | ||
| 6963 | return; | ||
| 6964 | } | ||
| 6964 | 6965 | ||
| 6965 | /* | 6966 | /* |
| 6966 | * If in_iowait is set, the code below may not trigger any cpufreq | 6967 | * If in_iowait is set, the code below may not trigger any cpufreq |
| @@ -7178,7 +7179,8 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 211 unmodified lines | |||
| 7178 | */ | 7179 | */ |
| 7179 | static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) | 7180 | static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) |
| 7180 | { | 7181 | { |
| 7181 | util_est_dequeue(&rq->cfs, p); | 7182 | if (!(p->se.sched_delayed && (task_on_rq_migrating(p) || (flags & DEQUEUE_SAVE)))) |
| 7183 | util_est_dequeue(&rq->cfs, p); | ||
| 7182 | 7184 | ||
| 7183 | if (dequeue_entities(rq, &p->se, flags) < 0) { | 7185 | if (dequeue_entities(rq, &p->se, flags) < 0) { |
| 7184 | util_est_update(&rq->cfs, p, DEQUEUE_SLEEP); | 7186 | util_est_update(&rq->cfs, p, DEQUEUE_SLEEP); |
| @@ -8085,6 +8087,105 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) · 900 unmodified lines | |||
| 8085 | return cpu_util(cpu, p, -1, 0); | 8087 | return cpu_util(cpu, p, -1, 0); |
| 8086 | } | 8088 | } |
| 8087 | 8089 | ||
| 8090 | /* | ||
| 8091 | * This function computes an effective utilization for the given CPU, to be | ||
| 8092 | * used for frequency selection given the linear relation: f = u * f_max. | ||
| 8093 | * | ||
| 8094 | * The scheduler tracks the following metrics: | ||
| 8095 | * | ||
| 8096 | * cpu_util_{cfs,rt,dl,irq}() | ||
| 8097 | * cpu_bw_dl() | ||
| 8098 | * | ||
| 8099 | * Where the cfs,rt and dl util numbers are tracked with the same metric and | ||
| 8100 | * synchronized windows and are thus directly comparable. | ||
| 8101 | * | ||
| 8102 | * The cfs,rt,dl utilization are the running times measured with rq->clock_task | ||
| 8103 | * which excludes things like IRQ and steal-time. These latter are then accrued | ||
| 8104 | * in the IRQ utilization. | ||
| 8105 | * | ||
| 8106 | * The DL bandwidth number OTOH is not a measured metric but a value computed | ||
| 8107 | * based on the task model parameters and gives the minimal utilization | ||
| 8108 | * required to meet deadlines. | ||
| 8109 | */ | ||
| 8110 | unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, | ||
| 8111 | unsigned long *min, | ||
| 8112 | unsigned long *max) | ||
| 8113 | { | ||
| 8114 | unsigned long util, irq, scale; | ||
| 8115 | struct rq *rq = cpu_rq(cpu); | ||
| 8116 | |||
| 8117 | scale = arch_scale_cpu_capacity(cpu); | ||
| 8118 | |||
| 8119 | /* | ||
| 8120 | * Early check to see if IRQ/steal time saturates the CPU, can be | ||
| 8121 | * because of inaccuracies in how we track these -- see | ||
| 8122 | * update_irq_load_avg(). | ||
| 8123 | */ | ||
| 8124 | irq = cpu_util_irq(rq); | ||
| 8125 | if (unlikely(irq >= scale)) { | ||
| 8126 | if (min) | ||
| 8127 | *min = scale; | ||
| 8128 | if (max) | ||
| 8129 | *max = scale; | ||
| 8130 | return scale; | ||
| 8131 | } | ||
| 8132 | |||
| 8133 | if (min) { | ||
| 8134 | /* | ||
| 8135 | * The minimum utilization returns the highest level between: | ||
| 8136 | * - the computed DL bandwidth needed with the IRQ pressure which | ||
| 8137 | * steals time to the deadline task. | ||
| 8138 | * - The minimum performance requirement for CFS and/or RT. | ||
| 8139 | */ | ||
| 8140 | *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN)); | ||
| 8141 | |||
| 8142 | /* | ||
| 8143 | * When an RT task is runnable and uclamp is not used, we must | ||
| 8144 | * ensure that the task will run at maximum compute capacity. | ||
| 8145 | */ | ||
| 8146 | if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt)) | ||
| 8147 | *min = max(*min, scale); | ||
| 8148 | } | ||
| 8149 | |||
| 8150 | /* | ||
| 8151 | * Because the time spend on RT/DL tasks is visible as 'lost' time to | ||
| 8152 | * CFS tasks and we use the same metric to track the effective | ||
| 8153 | * utilization (PELT windows are synchronized) we can directly add them | ||
| 8154 | * to obtain the CPU's actual utilization. | ||
| 8155 | */ | ||
| 8156 | util = util_cfs + cpu_util_rt(rq); | ||
| 8157 | util += cpu_util_dl(rq); | ||
| 8158 | |||
| 8159 | /* | ||
| 8160 | * The maximum hint is a soft bandwidth requirement, which can be lower | ||
| 8161 | * than the actual utilization because of uclamp_max requirements. | ||
| 8162 | */ | ||
| 8163 | if (max) | ||
| 8164 | *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX)); | ||
| 8165 | |||
| 8166 | if (util >= scale) | ||
| 8167 | return scale; | ||
| 8168 | |||
| 8169 | /* | ||
| 8170 | * There is still idle time; further improve the number by using the | ||
| 8171 | * IRQ metric. Because IRQ/steal time is hidden from the task clock we | ||
| 8172 | * need to scale the task numbers: | ||
| 8173 | * | ||
| 8174 | * max - irq | ||
| 8175 | * U' = irq + --------- * U | ||
| 8176 | * max | ||
| 8177 | */ | ||
| 8178 | util = scale_irq_capacity(util, irq, scale); | ||
| 8179 | util += irq; | ||
| 8180 | |||
| 8181 | return min(scale, util); | ||
| 8182 | } | ||
| 8183 | |||
| 8184 | unsigned long sched_cpu_util(int cpu) | ||
| 8185 | { | ||
| 8186 | return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL); | ||
| 8187 | } | ||
| 8188 | |||
| 8088 | /* | 8189 | /* |
| 8089 | * energy_env - Utilization landscape for energy estimation. | 8190 | * energy_env - Utilization landscape for energy estimation. |
| 8090 | * @task_busy_time: Utilization contribution by the task for which we test the | 8191 | * @task_busy_time: Utilization contribution by the task for which we test the |
| @@ -272,110 +272,12 @@ bool update_other_load_avgs(struct rq *rq) · 271 unmodified lines | |||
| 272 | 272 | ||
| 273 | lockdep_assert_rq_held(rq); | 273 | lockdep_assert_rq_held(rq); |
| 274 | 274 | ||
| 275 | /* hw_pressure doesn't care about invariance */ | ||
| 275 | return update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | | 276 | return update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) | |
| 276 | update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | | 277 | update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) | |
| 277 | update_hw_load_avg(now, rq, hw_pressure) | | 278 | update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure) | |
| 278 | update_irq_load_avg(rq, 0); | 279 | update_irq_load_avg(rq, 0); |
| 279 | } | 280 | } |
| 280 | |||
| 281 | /* | ||
| 282 | * This function computes an effective utilization for the given CPU, to be | ||
| 283 | * used for frequency selection given the linear relation: f = u * f_max. | ||
| 284 | * | ||
| 285 | * The scheduler tracks the following metrics: | ||
| 286 | * | ||
| 287 | * cpu_util_{cfs,rt,dl,irq}() | ||
| 288 | * cpu_bw_dl() | ||
| 289 | * | ||
| 290 | * Where the cfs,rt and dl util numbers are tracked with the same metric and | ||
| 291 | * synchronized windows and are thus directly comparable. | ||
| 292 | * | ||
| 293 | * The cfs,rt,dl utilization are the running times measured with rq->clock_task | ||
| 294 | * which excludes things like IRQ and steal-time. These latter are then accrued | ||
| 295 | * in the IRQ utilization. | ||
| 296 | * | ||
| 297 | * The DL bandwidth number OTOH is not a measured metric but a value computed | ||
| 298 | * based on the task model parameters and gives the minimal utilization | ||
| 299 | * required to meet deadlines. | ||
| 300 | */ | ||
| 301 | unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, | ||
| 302 | unsigned long *min, | ||
| 303 | unsigned long *max) | ||
| 304 | { | ||
| 305 | unsigned long util, irq, scale; | ||
| 306 | struct rq *rq = cpu_rq(cpu); | ||
| 307 | |||
| 308 | scale = arch_scale_cpu_capacity(cpu); | ||
| 309 | |||
| 310 | /* | ||
| 311 | * Early check to see if IRQ/steal time saturates the CPU, can be | ||
| 312 | * because of inaccuracies in how we track these -- see | ||
| 313 | * update_irq_load_avg(). | ||
| 314 | */ | ||
| 315 | irq = cpu_util_irq(rq); | ||
| 316 | if (unlikely(irq >= scale)) { | ||
| 317 | if (min) | ||
| 318 | *min = scale; | ||
| 319 | if (max) | ||
| 320 | *max = scale; | ||
| 321 | return scale; | ||
| 322 | } | ||
| 323 | |||
| 324 | if (min) { | ||
| 325 | /* | ||
| 326 | * The minimum utilization returns the highest level between: | ||
| 327 | * - the computed DL bandwidth needed with the IRQ pressure which | ||
| 328 | * steals time to the deadline task. | ||
| 329 | * - The minimum performance requirement for CFS and/or RT. | ||
| 330 | */ | ||
| 331 | *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN)); | ||
| 332 | |||
| 333 | /* | ||
| 334 | * When an RT task is runnable and uclamp is not used, we must | ||
| 335 | * ensure that the task will run at maximum compute capacity. | ||
| 336 | */ | ||
| 337 | if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt)) | ||
| 338 | *min = max(*min, scale); | ||
| 339 | } | ||
| 340 | |||
| 341 | /* | ||
| 342 | * Because the time spend on RT/DL tasks is visible as 'lost' time to | ||
| 343 | * CFS tasks and we use the same metric to track the effective | ||
| 344 | * utilization (PELT windows are synchronized) we can directly add them | ||
| 345 | * to obtain the CPU's actual utilization. | ||
| 346 | */ | ||
| 347 | util = util_cfs + cpu_util_rt(rq); | ||
| 348 | util += cpu_util_dl(rq); | ||
| 349 | |||
| 350 | /* | ||
| 351 | * The maximum hint is a soft bandwidth requirement, which can be lower | ||
| 352 | * than the actual utilization because of uclamp_max requirements. | ||
| 353 | */ | ||
| 354 | if (max) | ||
| 355 | *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX)); | ||
| 356 | |||
| 357 | if (util >= scale) | ||
| 358 | return scale; | ||
| 359 | |||
| 360 | /* | ||
| 361 | * There is still idle time; further improve the number by using the | ||
| 362 | * IRQ metric. Because IRQ/steal time is hidden from the task clock we | ||
| 363 | * need to scale the task numbers: | ||
| 364 | * | ||
| 365 | * max - irq | ||
| 366 | * U' = irq + --------- * U | ||
| 367 | * max | ||
| 368 | */ | ||
| 369 | util = scale_irq_capacity(util, irq, scale); | ||
| 370 | util += irq; | ||
| 371 | |||
| 372 | return min(scale, util); | ||
| 373 | } | ||
| 374 | |||
| 375 | unsigned long sched_cpu_util(int cpu) | ||
| 376 | { | ||
| 377 | return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL); | ||
| 378 | } | ||
| 379 | #endif /* CONFIG_SMP */ | 281 | #endif /* CONFIG_SMP */ |
| 380 | 282 | ||
| 381 | /** | 283 | /** |