History

Merge branch 'tip/sched/core' into sched_ext/for-6.12

Pull in tip/sched/core to resolve two merge conflicts: - 96fd6c65efc6 ("sched: Factor out update_other_load_avgs() from __update_blocked_others()") 5d871a63997f ("sched/fair: Move effective_cpu_util() and effective_cpu_util() in fair.c") A simple context conflict. The former added __update_blocked_others() in the same #ifdef CONFIG_SMP block that effective_cpu_util() and sched_cpu_util() are in and the latter moved those functions to fair.c. This makes __update_blocked_others() more out of place. Will follow up with a patch to relocate. - 96fd6c65efc6 ("sched: Factor out update_other_load_avgs() from __update_blocked_others()") 84d265281d6c ("sched/pelt: Use rq_clock_task() for hw_pressure") The former factored out the body of __update_blocked_others() into update_other_load_avgs(). The latter changed how update_hw_load_avg() is called in the body. Resolved by applying the change to update_other_load_avgs() instead.

Tejun Heo committed 2 years ago

Commit 0b1777f2 parents 513ed0c bc9057d9 files · +189 −152Browse files

Unified · Split

9 files
  1. Documentation/scheduler/sched-deadline.rst +6 −8
  2. drivers/cpufreq/cppc_cpufreq.c +3 −3
  3. include/uapi/linux/sched/types.h +3 −3
  4. kernel/kthread.c +9 −1
  5. kernel/sched/core.c +32 −19
  6. kernel/sched/cpufreq_schedutil.c +3 −3
  7. kernel/sched/debug.c +23 −8
  8. kernel/sched/fair.c +108 −7
  9. kernel/sched/syscalls.c +2 −100
Documentation/scheduler/sched-deadline.rst+6−8
@@ -749,21 +749,19 @@ Appendix A. Test suite · 748 unmodified lines
749 of the command line options. Please refer to rt-app documentation for more749 of the command line options. Please refer to rt-app documentation for more
750 details (`<rt-app-sources>/doc/*.json`).750 details (`<rt-app-sources>/doc/*.json`).
751 751
752 The second testing application is a modification of schedtool, called752 The second testing application is done using chrt which has support
753 schedtool-dl, which can be used to setup SCHED_DEADLINE parameters for a753 for SCHED_DEADLINE.
754 certain pid/application. schedtool-dl is available at:
755 https://github.com/scheduler-tools/schedtool-dl.git.
756 754
757 The usage is straightforward::755 The usage is straightforward::
758 756
759 # schedtool -E -t 10000000:100000000 -e ./my_cpuhog_app757 # chrt -d -T 10000000 -D 100000000 0 ./my_cpuhog_app
760 758
761 With this, my_cpuhog_app is put to run inside a SCHED_DEADLINE reservation759 With this, my_cpuhog_app is put to run inside a SCHED_DEADLINE reservation
762 of 10ms every 100ms (note that parameters are expressed in microseconds).760 of 10ms every 100ms (note that parameters are expressed in nanoseconds).
763 You can also use schedtool to create a reservation for an already running761 You can also use chrt to create a reservation for an already running
764 application, given that you know its pid::762 application, given that you know its pid::
765 763
766 # schedtool -E -t 10000000:100000000 my_app_pid764 # chrt -d -T 10000000 -D 100000000 -p 0 my_app_pid
767 765
768Appendix B. Minimal main()766Appendix B. Minimal main()
769==========================767==========================
drivers/cpufreq/cppc_cpufreq.c+3−3
@@ -224,9 +224,9 @@ static void __init cppc_freq_invariance_init(void) · 223 unmodified lines
224 * Fake (unused) bandwidth; workaround to "fix"224 * Fake (unused) bandwidth; workaround to "fix"
225 * priority inheritance.225 * priority inheritance.
226 */226 */
227 .sched_runtime = 1000000,227 .sched_runtime = NSEC_PER_MSEC,
228 .sched_deadline = 10000000,228 .sched_deadline = 10 * NSEC_PER_MSEC,
229 .sched_period = 10000000,229 .sched_period = 10 * NSEC_PER_MSEC,
230 };230 };
231 int ret;231 int ret;
232 232
include/uapi/linux/sched/types.h+3−3
@@ -58,9 +58,9 @@ · 57 unmodified lines
58 *58 *
59 * This is reflected by the following fields of the sched_attr structure:59 * This is reflected by the following fields of the sched_attr structure:
60 *60 *
61 * @sched_deadline representative of the task's deadline61 * @sched_deadline representative of the task's deadline in nanoseconds
62 * @sched_runtime representative of the task's runtime62 * @sched_runtime representative of the task's runtime in nanoseconds
63 * @sched_period representative of the task's period63 * @sched_period representative of the task's period in nanoseconds
64 *64 *
65 * Given this task model, there are a multiplicity of scheduling algorithms65 * Given this task model, there are a multiplicity of scheduling algorithms
66 * and policies, that can be used to ensure all the tasks will make their66 * and policies, that can be used to ensure all the tasks will make their
kernel/kthread.c+9−1
@@ -845,8 +845,16 @@ repeat: · 844 unmodified lines
845 * event only cares about the address.845 * event only cares about the address.
846 */846 */
847 trace_sched_kthread_work_execute_end(work, func);847 trace_sched_kthread_work_execute_end(work, func);
848 } else if (!freezing(current))848 } else if (!freezing(current)) {
849 schedule();849 schedule();
850 } else {
851 /*
852 * Handle the case where the current remains
853 * TASK_INTERRUPTIBLE. try_to_freeze() expects
854 * the current to be TASK_RUNNING.
855 */
856 __set_current_state(TASK_RUNNING);
857 }
850 858
851 try_to_freeze();859 try_to_freeze();
852 cond_resched();860 cond_resched();
kernel/sched/core.c+32−19
@@ -267,6 +267,9 @@ static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) · 266 unmodified lines
267 267
268void sched_core_enqueue(struct rq *rq, struct task_struct *p)268void sched_core_enqueue(struct rq *rq, struct task_struct *p)
269{269{
270 if (p->se.sched_delayed)
271 return;
272
270 rq->core->core_task_seq++;273 rq->core->core_task_seq++;
271 274
272 if (!p->core_cookie)275 if (!p->core_cookie)
@@ -277,6 +280,9 @@ void sched_core_enqueue(struct rq *rq, struct task_struct *p) · 4 unmodified lines
277 280
278void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags)281void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags)
279{282{
283 if (p->se.sched_delayed)
284 return;
285
280 rq->core->core_task_seq++;286 rq->core->core_task_seq++;
281 287
282 if (sched_core_enqueued(p)) {288 if (sched_core_enqueued(p)) {
@@ -6477,19 +6483,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) · 6194 unmodified lines
6477 * Constants for the sched_mode argument of __schedule().6483 * Constants for the sched_mode argument of __schedule().
6478 *6484 *
6479 * The mode argument allows RT enabled kernels to differentiate a6485 * The mode argument allows RT enabled kernels to differentiate a
6480 * preemption from blocking on an 'sleeping' spin/rwlock. Note that6486 * preemption from blocking on an 'sleeping' spin/rwlock.
6481 * SM_MASK_PREEMPT for !RT has all bits set, which allows the compiler to
6482 * optimize the AND operation out and just check for zero.
6483 */6487 */
6484#define SM_NONE 0x06488#define SM_IDLE (-1)
6485#define SM_PREEMPT 0x16489#define SM_NONE 0
6486#define SM_RTLOCK_WAIT 0x26490#define SM_PREEMPT 1
6487 6491#define SM_RTLOCK_WAIT 2
6488#ifndef CONFIG_PREEMPT_RT
6489# define SM_MASK_PREEMPT (~0U)
6490#else
6491# define SM_MASK_PREEMPT SM_PREEMPT
6492#endif
6493 6492
6494/*6493/*
6495 * __schedule() is the main scheduler function.6494 * __schedule() is the main scheduler function.
@@ -6530,9 +6529,14 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) · 34 unmodified lines
6530 *6529 *
6531 * WARNING: must be called with preemption disabled!6530 * WARNING: must be called with preemption disabled!
6532 */6531 */
6533static void __sched notrace __schedule(unsigned int sched_mode)6532static void __sched notrace __schedule(int sched_mode)
6534{6533{
6535 struct task_struct *prev, *next;6534 struct task_struct *prev, *next;
6535 /*
6536 * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted
6537 * as a preemption by schedule_debug() and RCU.
6538 */
6539 bool preempt = sched_mode > SM_NONE;
6536 unsigned long *switch_count;6540 unsigned long *switch_count;
6537 unsigned long prev_state;6541 unsigned long prev_state;
6538 struct rq_flags rf;6542 struct rq_flags rf;
@@ -6543,13 +6547,13 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 4 unmodified lines
6543 rq = cpu_rq(cpu);6547 rq = cpu_rq(cpu);
6544 prev = rq->curr;6548 prev = rq->curr;
6545 6549
6546 schedule_debug(prev, !!sched_mode);6550 schedule_debug(prev, preempt);
6547 6551
6548 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))6552 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))
6549 hrtick_clear(rq);6553 hrtick_clear(rq);
6550 6554
6551 local_irq_disable();6555 local_irq_disable();
6552 rcu_note_context_switch(!!sched_mode);6556 rcu_note_context_switch(preempt);
6553 6557
6554 /*6558 /*
6555 * Make sure that signal_pending_state()->signal_pending() below6559 * Make sure that signal_pending_state()->signal_pending() below
@@ -6578,12 +6582,20 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 22 unmodified lines
6578 6582
6579 switch_count = &prev->nivcsw;6583 switch_count = &prev->nivcsw;
6580 6584
6585 /* Task state changes only considers SM_PREEMPT as preemption */
6586 preempt = sched_mode == SM_PREEMPT;
6587
6581 /*6588 /*
6582 * We must load prev->state once (task_struct::state is volatile), such6589 * We must load prev->state once (task_struct::state is volatile), such
6583 * that we form a control dependency vs deactivate_task() below.6590 * that we form a control dependency vs deactivate_task() below.
6584 */6591 */
6585 prev_state = READ_ONCE(prev->__state);6592 prev_state = READ_ONCE(prev->__state);
6586 if (!(sched_mode & SM_MASK_PREEMPT) && prev_state) {6593 if (sched_mode == SM_IDLE) {
6594 if (!rq->nr_running) {
6595 next = prev;
6596 goto picked;
6597 }
6598 } else if (!preempt && prev_state) {
6587 if (signal_pending_state(prev_state, prev)) {6599 if (signal_pending_state(prev_state, prev)) {
6588 WRITE_ONCE(prev->__state, TASK_RUNNING);6600 WRITE_ONCE(prev->__state, TASK_RUNNING);
6589 } else {6601 } else {
@@ -6614,6 +6626,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 24 unmodified lines
6614 }6626 }
6615 6627
6616 next = pick_next_task(rq, prev, &rf);6628 next = pick_next_task(rq, prev, &rf);
6629picked:
6617 clear_tsk_need_resched(prev);6630 clear_tsk_need_resched(prev);
6618 clear_preempt_need_resched();6631 clear_preempt_need_resched();
6619#ifdef CONFIG_SCHED_DEBUG6632#ifdef CONFIG_SCHED_DEBUG
@@ -6655,7 +6668,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) · 35 unmodified lines
6655 psi_account_irqtime(rq, prev, next);6668 psi_account_irqtime(rq, prev, next);
6656 psi_sched_switch(prev, next, !task_on_rq_queued(prev));6669 psi_sched_switch(prev, next, !task_on_rq_queued(prev));
6657 6670
6658 trace_sched_switch(sched_mode & SM_MASK_PREEMPT, prev, next, prev_state);6671 trace_sched_switch(preempt, prev, next, prev_state);
6659 6672
6660 /* Also unlocks the rq: */6673 /* Also unlocks the rq: */
6661 rq = context_switch(rq, prev, next, &rf);6674 rq = context_switch(rq, prev, next, &rf);
@@ -6731,7 +6744,7 @@ static void sched_update_worker(struct task_struct *tsk) · 69 unmodified lines
6731 }6744 }
6732}6745}
6733 6746
6734static __always_inline void __schedule_loop(unsigned int sched_mode)6747static __always_inline void __schedule_loop(int sched_mode)
6735{6748{
6736 do {6749 do {
6737 preempt_disable();6750 preempt_disable();
@@ -6776,7 +6789,7 @@ void __sched schedule_idle(void) · 38 unmodified lines
6776 */6789 */
6777 WARN_ON_ONCE(current->__state);6790 WARN_ON_ONCE(current->__state);
6778 do {6791 do {
6779 __schedule(SM_NONE);6792 __schedule(SM_IDLE);
6780 } while (need_resched());6793 } while (need_resched());
6781}6794}
6782 6795
kernel/sched/cpufreq_schedutil.c+3−3
@@ -662,9 +662,9 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy) · 661 unmodified lines
662 * Fake (unused) bandwidth; workaround to "fix"662 * Fake (unused) bandwidth; workaround to "fix"
663 * priority inheritance.663 * priority inheritance.
664 */664 */
665 .sched_runtime = 1000000,665 .sched_runtime = NSEC_PER_MSEC,
666 .sched_deadline = 10000000,666 .sched_deadline = 10 * NSEC_PER_MSEC,
667 .sched_period = 10000000,667 .sched_period = 10 * NSEC_PER_MSEC,
668 };668 };
669 struct cpufreq_policy *policy = sg_policy->policy;669 struct cpufreq_policy *policy = sg_policy->policy;
670 int ret;670 int ret;
kernel/sched/debug.c+23−8
@@ -739,7 +739,7 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) · 738 unmodified lines
739 else739 else
740 SEQ_printf(m, " %c", task_state_to_char(p));740 SEQ_printf(m, " %c", task_state_to_char(p));
741 741
742 SEQ_printf(m, "%15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ",742 SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ",
743 p->comm, task_pid_nr(p),743 p->comm, task_pid_nr(p),
744 SPLIT_NS(p->se.vruntime),744 SPLIT_NS(p->se.vruntime),
745 entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N',745 entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N',
@@ -750,17 +750,16 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) · 4 unmodified lines
750 (long long)(p->nvcsw + p->nivcsw),750 (long long)(p->nvcsw + p->nivcsw),
751 p->prio);751 p->prio);
752 752
753 SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld %9lld.%06ld",753 SEQ_printf(m, "%9lld.%06ld %9lld.%06ld %9lld.%06ld",
754 SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)),754 SPLIT_NS(schedstat_val_or_zero(p->stats.wait_sum)),
755 SPLIT_NS(p->se.sum_exec_runtime),
756 SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),755 SPLIT_NS(schedstat_val_or_zero(p->stats.sum_sleep_runtime)),
757 SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));756 SPLIT_NS(schedstat_val_or_zero(p->stats.sum_block_runtime)));
758 757
759#ifdef CONFIG_NUMA_BALANCING758#ifdef CONFIG_NUMA_BALANCING
760 SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));759 SEQ_printf(m, " %d %d", task_node(p), task_numa_group_id(p));
761#endif760#endif
762#ifdef CONFIG_CGROUP_SCHED761#ifdef CONFIG_CGROUP_SCHED
763 SEQ_printf_task_group_path(m, task_group(p), " %s")762 SEQ_printf_task_group_path(m, task_group(p), " %s")
764#endif763#endif
765 764
766 SEQ_printf(m, "\n");765 SEQ_printf(m, "\n");
@@ -772,10 +771,26 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) · 5 unmodified lines
772 771
773 SEQ_printf(m, "\n");772 SEQ_printf(m, "\n");
774 SEQ_printf(m, "runnable tasks:\n");773 SEQ_printf(m, "runnable tasks:\n");
775 SEQ_printf(m, " S task PID tree-key switches prio"774 SEQ_printf(m, " S task PID vruntime eligible "
776 " wait-time sum-exec sum-sleep\n");775 "deadline slice sum-exec switches "
776 "prio wait-time sum-sleep sum-block"
777#ifdef CONFIG_NUMA_BALANCING
778 " node group-id"
779#endif
780#ifdef CONFIG_CGROUP_SCHED
781 " group-path"
782#endif
783 "\n");
777 SEQ_printf(m, "-------------------------------------------------------"784 SEQ_printf(m, "-------------------------------------------------------"
778 "------------------------------------------------------\n");785 "------------------------------------------------------"
786 "------------------------------------------------------"
787#ifdef CONFIG_NUMA_BALANCING
788 "--------------"
789#endif
790#ifdef CONFIG_CGROUP_SCHED
791 "--------------"
792#endif
793 "\n");
779 794
780 rcu_read_lock();795 rcu_read_lock();
781 for_each_process_thread(g, p) {796 for_each_process_thread(g, p) {
kernel/sched/fair.c+108−7
@@ -6949,18 +6949,19 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 6948 unmodified lines
6949 int rq_h_nr_running = rq->cfs.h_nr_running;6949 int rq_h_nr_running = rq->cfs.h_nr_running;
6950 u64 slice = 0;6950 u64 slice = 0;
6951 6951
6952 if (flags & ENQUEUE_DELAYED) {
6953 requeue_delayed_entity(se);
6954 return;
6955 }
6956
6957 /*6952 /*
6958 * The code below (indirectly) updates schedutil which looks at6953 * The code below (indirectly) updates schedutil which looks at
6959 * the cfs_rq utilization to select a frequency.6954 * the cfs_rq utilization to select a frequency.
6960 * Let's add the task's estimated utilization to the cfs_rq's6955 * Let's add the task's estimated utilization to the cfs_rq's
6961 * estimated utilization, before we update schedutil.6956 * estimated utilization, before we update schedutil.
6962 */6957 */
6963 util_est_enqueue(&rq->cfs, p);6958 if (!(p->se.sched_delayed && (task_on_rq_migrating(p) || (flags & ENQUEUE_RESTORE))))
6959 util_est_enqueue(&rq->cfs, p);
6960
6961 if (flags & ENQUEUE_DELAYED) {
6962 requeue_delayed_entity(se);
6963 return;
6964 }
6964 6965
6965 /*6966 /*
6966 * If in_iowait is set, the code below may not trigger any cpufreq6967 * If in_iowait is set, the code below may not trigger any cpufreq
@@ -7178,7 +7179,8 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 211 unmodified lines
7178 */7179 */
7179static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)7180static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
7180{7181{
7181 util_est_dequeue(&rq->cfs, p);7182 if (!(p->se.sched_delayed && (task_on_rq_migrating(p) || (flags & DEQUEUE_SAVE))))
7183 util_est_dequeue(&rq->cfs, p);
7182 7184
7183 if (dequeue_entities(rq, &p->se, flags) < 0) {7185 if (dequeue_entities(rq, &p->se, flags) < 0) {
7184 util_est_update(&rq->cfs, p, DEQUEUE_SLEEP);7186 util_est_update(&rq->cfs, p, DEQUEUE_SLEEP);
@@ -8085,6 +8087,105 @@ static unsigned long cpu_util_without(int cpu, struct task_struct *p) · 900 unmodified lines
8085 return cpu_util(cpu, p, -1, 0);8087 return cpu_util(cpu, p, -1, 0);
8086}8088}
8087 8089
8090/*
8091 * This function computes an effective utilization for the given CPU, to be
8092 * used for frequency selection given the linear relation: f = u * f_max.
8093 *
8094 * The scheduler tracks the following metrics:
8095 *
8096 * cpu_util_{cfs,rt,dl,irq}()
8097 * cpu_bw_dl()
8098 *
8099 * Where the cfs,rt and dl util numbers are tracked with the same metric and
8100 * synchronized windows and are thus directly comparable.
8101 *
8102 * The cfs,rt,dl utilization are the running times measured with rq->clock_task
8103 * which excludes things like IRQ and steal-time. These latter are then accrued
8104 * in the IRQ utilization.
8105 *
8106 * The DL bandwidth number OTOH is not a measured metric but a value computed
8107 * based on the task model parameters and gives the minimal utilization
8108 * required to meet deadlines.
8109 */
8110unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
8111 unsigned long *min,
8112 unsigned long *max)
8113{
8114 unsigned long util, irq, scale;
8115 struct rq *rq = cpu_rq(cpu);
8116
8117 scale = arch_scale_cpu_capacity(cpu);
8118
8119 /*
8120 * Early check to see if IRQ/steal time saturates the CPU, can be
8121 * because of inaccuracies in how we track these -- see
8122 * update_irq_load_avg().
8123 */
8124 irq = cpu_util_irq(rq);
8125 if (unlikely(irq >= scale)) {
8126 if (min)
8127 *min = scale;
8128 if (max)
8129 *max = scale;
8130 return scale;
8131 }
8132
8133 if (min) {
8134 /*
8135 * The minimum utilization returns the highest level between:
8136 * - the computed DL bandwidth needed with the IRQ pressure which
8137 * steals time to the deadline task.
8138 * - The minimum performance requirement for CFS and/or RT.
8139 */
8140 *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN));
8141
8142 /*
8143 * When an RT task is runnable and uclamp is not used, we must
8144 * ensure that the task will run at maximum compute capacity.
8145 */
8146 if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt))
8147 *min = max(*min, scale);
8148 }
8149
8150 /*
8151 * Because the time spend on RT/DL tasks is visible as 'lost' time to
8152 * CFS tasks and we use the same metric to track the effective
8153 * utilization (PELT windows are synchronized) we can directly add them
8154 * to obtain the CPU's actual utilization.
8155 */
8156 util = util_cfs + cpu_util_rt(rq);
8157 util += cpu_util_dl(rq);
8158
8159 /*
8160 * The maximum hint is a soft bandwidth requirement, which can be lower
8161 * than the actual utilization because of uclamp_max requirements.
8162 */
8163 if (max)
8164 *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX));
8165
8166 if (util >= scale)
8167 return scale;
8168
8169 /*
8170 * There is still idle time; further improve the number by using the
8171 * IRQ metric. Because IRQ/steal time is hidden from the task clock we
8172 * need to scale the task numbers:
8173 *
8174 * max - irq
8175 * U' = irq + --------- * U
8176 * max
8177 */
8178 util = scale_irq_capacity(util, irq, scale);
8179 util += irq;
8180
8181 return min(scale, util);
8182}
8183
8184unsigned long sched_cpu_util(int cpu)
8185{
8186 return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
8187}
8188
8088/*8189/*
8089 * energy_env - Utilization landscape for energy estimation.8190 * energy_env - Utilization landscape for energy estimation.
8090 * @task_busy_time: Utilization contribution by the task for which we test the8191 * @task_busy_time: Utilization contribution by the task for which we test the
kernel/sched/syscalls.c+2−100
@@ -272,110 +272,12 @@ bool update_other_load_avgs(struct rq *rq) · 271 unmodified lines
272 272
273 lockdep_assert_rq_held(rq);273 lockdep_assert_rq_held(rq);
274 274
275 /* hw_pressure doesn't care about invariance */
275 return update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) |276 return update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) |
276 update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) |277 update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) |
277 update_hw_load_avg(now, rq, hw_pressure) |278 update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure) |
278 update_irq_load_avg(rq, 0);279 update_irq_load_avg(rq, 0);
279}280}
280
281/*
282 * This function computes an effective utilization for the given CPU, to be
283 * used for frequency selection given the linear relation: f = u * f_max.
284 *
285 * The scheduler tracks the following metrics:
286 *
287 * cpu_util_{cfs,rt,dl,irq}()
288 * cpu_bw_dl()
289 *
290 * Where the cfs,rt and dl util numbers are tracked with the same metric and
291 * synchronized windows and are thus directly comparable.
292 *
293 * The cfs,rt,dl utilization are the running times measured with rq->clock_task
294 * which excludes things like IRQ and steal-time. These latter are then accrued
295 * in the IRQ utilization.
296 *
297 * The DL bandwidth number OTOH is not a measured metric but a value computed
298 * based on the task model parameters and gives the minimal utilization
299 * required to meet deadlines.
300 */
301unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
302 unsigned long *min,
303 unsigned long *max)
304{
305 unsigned long util, irq, scale;
306 struct rq *rq = cpu_rq(cpu);
307
308 scale = arch_scale_cpu_capacity(cpu);
309
310 /*
311 * Early check to see if IRQ/steal time saturates the CPU, can be
312 * because of inaccuracies in how we track these -- see
313 * update_irq_load_avg().
314 */
315 irq = cpu_util_irq(rq);
316 if (unlikely(irq >= scale)) {
317 if (min)
318 *min = scale;
319 if (max)
320 *max = scale;
321 return scale;
322 }
323
324 if (min) {
325 /*
326 * The minimum utilization returns the highest level between:
327 * - the computed DL bandwidth needed with the IRQ pressure which
328 * steals time to the deadline task.
329 * - The minimum performance requirement for CFS and/or RT.
330 */
331 *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN));
332
333 /*
334 * When an RT task is runnable and uclamp is not used, we must
335 * ensure that the task will run at maximum compute capacity.
336 */
337 if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt))
338 *min = max(*min, scale);
339 }
340
341 /*
342 * Because the time spend on RT/DL tasks is visible as 'lost' time to
343 * CFS tasks and we use the same metric to track the effective
344 * utilization (PELT windows are synchronized) we can directly add them
345 * to obtain the CPU's actual utilization.
346 */
347 util = util_cfs + cpu_util_rt(rq);
348 util += cpu_util_dl(rq);
349
350 /*
351 * The maximum hint is a soft bandwidth requirement, which can be lower
352 * than the actual utilization because of uclamp_max requirements.
353 */
354 if (max)
355 *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX));
356
357 if (util >= scale)
358 return scale;
359
360 /*
361 * There is still idle time; further improve the number by using the
362 * IRQ metric. Because IRQ/steal time is hidden from the task clock we
363 * need to scale the task numbers:
364 *
365 * max - irq
366 * U' = irq + --------- * U
367 * max
368 */
369 util = scale_irq_capacity(util, irq, scale);
370 util += irq;
371
372 return min(scale, util);
373}
374
375unsigned long sched_cpu_util(int cpu)
376{
377 return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
378}
379#endif /* CONFIG_SMP */281#endif /* CONFIG_SMP */
380 282
381/**283/**

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t