Instead of passing util and max between functions while computing the utilization and capacity, store the former in struct sg_cpu (along with the latter and bw_dl). This will allow the current utilization value to be compared with the one obtained previously (which is requisite for some code changes to follow this one), but also it causes the code to look slightly more consistent and cleaner.
Unified · Split
| @@ -53,6 +53,7 @@ struct sugov_cpu { · 52 unmodified lines | |||
| 53 | unsigned int iowait_boost; | 53 | unsigned int iowait_boost; |
| 54 | u64 last_update; | 54 | u64 last_update; |
| 55 | 55 | ||
| 56 | unsigned long util; | ||
| 56 | unsigned long bw_dl; | 57 | unsigned long bw_dl; |
| 57 | unsigned long max; | 58 | unsigned long max; |
| 58 | 59 | ||
| @@ -276,16 +277,15 @@ unsigned long schedutil_cpu_util(int cpu, unsigned long util_cfs, · 217 unmodified lines | |||
| 276 | return min(max, util); | 277 | return min(max, util); |
| 277 | } | 278 | } |
| 278 | 279 | ||
| 279 | static unsigned long sugov_get_util(struct sugov_cpu *sg_cpu) | 280 | static void sugov_get_util(struct sugov_cpu *sg_cpu) |
| 280 | { | 281 | { |
| 281 | struct rq *rq = cpu_rq(sg_cpu->cpu); | 282 | struct rq *rq = cpu_rq(sg_cpu->cpu); |
| 282 | unsigned long util = cpu_util_cfs(rq); | ||
| 283 | unsigned long max = arch_scale_cpu_capacity(sg_cpu->cpu); | 283 | unsigned long max = arch_scale_cpu_capacity(sg_cpu->cpu); |
| 284 | 284 | ||
| 285 | sg_cpu->max = max; | 285 | sg_cpu->max = max; |
| 286 | sg_cpu->bw_dl = cpu_bw_dl(rq); | 286 | sg_cpu->bw_dl = cpu_bw_dl(rq); |
| 287 | 287 | sg_cpu->util = schedutil_cpu_util(sg_cpu->cpu, cpu_util_cfs(rq), max, | |
| 288 | return schedutil_cpu_util(sg_cpu->cpu, util, max, FREQUENCY_UTIL, NULL); | 288 | FREQUENCY_UTIL, NULL); |
| 289 | } | 289 | } |
| 290 | 290 | ||
| 291 | /** | 291 | /** |
| @@ -362,8 +362,6 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, · 70 unmodified lines | |||
| 362 | * sugov_iowait_apply() - Apply the IO boost to a CPU. | 362 | * sugov_iowait_apply() - Apply the IO boost to a CPU. |
| 363 | * @sg_cpu: the sugov data for the cpu to boost | 363 | * @sg_cpu: the sugov data for the cpu to boost |
| 364 | * @time: the update time from the caller | 364 | * @time: the update time from the caller |
| 365 | * @util: the utilization to (eventually) boost | ||
| 366 | * @max: the maximum value the utilization can be boosted to | ||
| 367 | * | 365 | * |
| 368 | * A CPU running a task which woken up after an IO operation can have its | 366 | * A CPU running a task which woken up after an IO operation can have its |
| 369 | * utilization boosted to speed up the completion of those IO operations. | 367 | * utilization boosted to speed up the completion of those IO operations. |
| @@ -377,18 +375,17 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, · 7 unmodified lines | |||
| 377 | * This mechanism is designed to boost high frequently IO waiting tasks, while | 375 | * This mechanism is designed to boost high frequently IO waiting tasks, while |
| 378 | * being more conservative on tasks which does sporadic IO operations. | 376 | * being more conservative on tasks which does sporadic IO operations. |
| 379 | */ | 377 | */ |
| 380 | static unsigned long sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time, | 378 | static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time) |
| 381 | unsigned long util, unsigned long max) | ||
| 382 | { | 379 | { |
| 383 | unsigned long boost; | 380 | unsigned long boost; |
| 384 | 381 | ||
| 385 | /* No boost currently required */ | 382 | /* No boost currently required */ |
| 386 | if (!sg_cpu->iowait_boost) | 383 | if (!sg_cpu->iowait_boost) |
| 387 | return util; | 384 | return; |
| 388 | 385 | ||
| 389 | /* Reset boost if the CPU appears to have been idle enough */ | 386 | /* Reset boost if the CPU appears to have been idle enough */ |
| 390 | if (sugov_iowait_reset(sg_cpu, time, false)) | 387 | if (sugov_iowait_reset(sg_cpu, time, false)) |
| 391 | return util; | 388 | return; |
| 392 | 389 | ||
| 393 | if (!sg_cpu->iowait_boost_pending) { | 390 | if (!sg_cpu->iowait_boost_pending) { |
| 394 | /* | 391 | /* |
| @@ -397,18 +394,19 @@ static unsigned long sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time, · 2 unmodified lines | |||
| 397 | sg_cpu->iowait_boost >>= 1; | 394 | sg_cpu->iowait_boost >>= 1; |
| 398 | if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { | 395 | if (sg_cpu->iowait_boost < IOWAIT_BOOST_MIN) { |
| 399 | sg_cpu->iowait_boost = 0; | 396 | sg_cpu->iowait_boost = 0; |
| 400 | return util; | 397 | return; |
| 401 | } | 398 | } |
| 402 | } | 399 | } |
| 403 | 400 | ||
| 404 | sg_cpu->iowait_boost_pending = false; | 401 | sg_cpu->iowait_boost_pending = false; |
| 405 | 402 | ||
| 406 | /* | 403 | /* |
| 407 | * @util is already in capacity scale; convert iowait_boost | 404 | * sg_cpu->util is already in capacity scale; convert iowait_boost |
| 408 | * into the same scale so we can compare. | 405 | * into the same scale so we can compare. |
| 409 | */ | 406 | */ |
| 410 | boost = (sg_cpu->iowait_boost * max) >> SCHED_CAPACITY_SHIFT; | 407 | boost = (sg_cpu->iowait_boost * sg_cpu->max) >> SCHED_CAPACITY_SHIFT; |
| 411 | return max(boost, util); | 408 | if (sg_cpu->util < boost) |
| 409 | sg_cpu->util = boost; | ||
| 412 | } | 410 | } |
| 413 | 411 | ||
| 414 | #ifdef CONFIG_NO_HZ_COMMON | 412 | #ifdef CONFIG_NO_HZ_COMMON |
| @@ -439,9 +437,8 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, · 24 unmodified lines | |||
| 439 | { | 437 | { |
| 440 | struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); | 438 | struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); |
| 441 | struct sugov_policy *sg_policy = sg_cpu->sg_policy; | 439 | struct sugov_policy *sg_policy = sg_cpu->sg_policy; |
| 442 | unsigned long util, max; | ||
| 443 | unsigned int next_f; | ||
| 444 | unsigned int cached_freq = sg_policy->cached_raw_freq; | 440 | unsigned int cached_freq = sg_policy->cached_raw_freq; |
| 441 | unsigned int next_f; | ||
| 445 | 442 | ||
| 446 | sugov_iowait_boost(sg_cpu, time, flags); | 443 | sugov_iowait_boost(sg_cpu, time, flags); |
| 447 | sg_cpu->last_update = time; | 444 | sg_cpu->last_update = time; |
| @@ -451,10 +448,10 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, · 3 unmodified lines | |||
| 451 | if (!sugov_should_update_freq(sg_policy, time)) | 448 | if (!sugov_should_update_freq(sg_policy, time)) |
| 452 | return; | 449 | return; |
| 453 | 450 | ||
| 454 | util = sugov_get_util(sg_cpu); | 451 | sugov_get_util(sg_cpu); |
| 455 | max = sg_cpu->max; | 452 | sugov_iowait_apply(sg_cpu, time); |
| 456 | util = sugov_iowait_apply(sg_cpu, time, util, max); | 453 | |
| 457 | next_f = get_next_freq(sg_policy, util, max); | 454 | next_f = get_next_freq(sg_policy, sg_cpu->util, sg_cpu->max); |
| 458 | /* | 455 | /* |
| 459 | * Do not reduce the frequency if the CPU has not been idle | 456 | * Do not reduce the frequency if the CPU has not been idle |
| 460 | * recently, as the reduction is likely to be premature then. | 457 | * recently, as the reduction is likely to be premature then. |
| @@ -491,9 +488,10 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) · 30 unmodified lines | |||
| 491 | struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j); | 488 | struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j); |
| 492 | unsigned long j_util, j_max; | 489 | unsigned long j_util, j_max; |
| 493 | 490 | ||
| 494 | j_util = sugov_get_util(j_sg_cpu); | 491 | sugov_get_util(j_sg_cpu); |
| 492 | sugov_iowait_apply(j_sg_cpu, time); | ||
| 493 | j_util = j_sg_cpu->util; | ||
| 495 | j_max = j_sg_cpu->max; | 494 | j_max = j_sg_cpu->max; |
| 496 | j_util = sugov_iowait_apply(j_sg_cpu, time, j_util, j_max); | ||
| 497 | 495 | ||
| 498 | if (j_util * max > j_max * util) { | 496 | if (j_util * max > j_max * util) { |
| 499 | util = j_util; | 497 | util = j_util; |