History

sched/uclamp: Add a new sysctl to control RT default boost value

RT tasks by default run at the highest capacity/performance level. When uclamp is selected this default behavior is retained by enforcing the requested uclamp.min (p->uclamp_req[UCLAMP_MIN]) of the RT tasks to be uclamp_none(UCLAMP_MAX), which is SCHED_CAPACITY_SCALE; the maximum value. This is also referred to as 'the default boost value of RT tasks'. See commit 1a00d999971c ("sched/uclamp: Set default clamps for RT tasks"). On battery powered devices, it is desired to control this default (currently hardcoded) behavior at runtime to reduce energy consumed by RT tasks. For example, a mobile device manufacturer where big.LITTLE architecture is dominant, the performance of the little cores varies across SoCs, and on high end ones the big cores could be too power hungry. Given the diversity of SoCs, the new knob allows manufactures to tune the best performance/power for RT tasks for the particular hardware they run on. They could opt to further tune the value when the user selects a different power saving mode or when the device is actively charging. The runtime aspect of it further helps in creating a single kernel image that can be run on multiple devices that require different tuning. Keep in mind that a lot of RT tasks in the system are created by the kernel. On Android for instance I can see over 50 RT tasks, only a handful of which created by the Android framework. To control the default behavior globally by system admins and device integrator, introduce the new sysctl_sched_uclamp_util_min_rt_default to change the default boost value of the RT tasks. I anticipate this to be mostly in the form of modifying the init script of a particular device. To avoid polluting the fast path with unnecessary code, the approach taken is to synchronously do the update by traversing all the existing tasks in the system. This could race with a concurrent fork(), which is dealt with by introducing sched_post_fork() function which will ensure the racy fork will get the right update applied. Tested on Juno-r2 in combination with the RT capacity awareness [1]. By default an RT task will go to the highest capacity CPU and run at the maximum frequency, which is particularly energy inefficient on high end mobile devices because the biggest core[s] are 'huge' and power hungry. With this patch the RT task can be controlled to run anywhere by default, and doesn't cause the frequency to be maximum all the time. Yet any task that really needs to be boosted can easily escape this default behavior by modifying its requested uclamp.min value (p->uclamp_req[UCLAMP_MIN]) via sched_setattr() syscall. [1] 804d402fb6f6: ("sched/rt: Make RT capacity-aware")

Qais Yousef committed 6 years ago

Commit 13685c41 parent e65855a6 files · +131 −8Browse files

Unified · Split

6 files
  1. include/linux/sched.h +8 −2
  2. include/linux/sched/sysctl.h +1 −0
  3. include/linux/sched/task.h +1 −0
  4. kernel/fork.c +1 −0
  5. kernel/sched/core.c +113 −6
  6. kernel/sysctl.c +7 −0
include/linux/sched.h+8−2
@@ -686,9 +686,15 @@ struct task_struct { · 685 unmodified lines
686 struct sched_dl_entity dl;686 struct sched_dl_entity dl;
687 687
688#ifdef CONFIG_UCLAMP_TASK688#ifdef CONFIG_UCLAMP_TASK
689 /* Clamp values requested for a scheduling entity */689 /*
690 * Clamp values requested for a scheduling entity.
691 * Must be updated with task_rq_lock() held.
692 */
690 struct uclamp_se uclamp_req[UCLAMP_CNT];693 struct uclamp_se uclamp_req[UCLAMP_CNT];
691 /* Effective clamp values used for a scheduling entity */694 /*
695 * Effective clamp values used for a scheduling entity.
696 * Must be updated with task_rq_lock() held.
697 */
692 struct uclamp_se uclamp[UCLAMP_CNT];698 struct uclamp_se uclamp[UCLAMP_CNT];
693#endif699#endif
694 700
include/linux/sched/sysctl.h+1−0
@@ -67,6 +67,7 @@ extern unsigned int sysctl_sched_dl_period_min; · 66 unmodified lines
67#ifdef CONFIG_UCLAMP_TASK67#ifdef CONFIG_UCLAMP_TASK
68extern unsigned int sysctl_sched_uclamp_util_min;68extern unsigned int sysctl_sched_uclamp_util_min;
69extern unsigned int sysctl_sched_uclamp_util_max;69extern unsigned int sysctl_sched_uclamp_util_max;
70extern unsigned int sysctl_sched_uclamp_util_min_rt_default;
70#endif71#endif
71 72
72#ifdef CONFIG_CFS_BANDWIDTH73#ifdef CONFIG_CFS_BANDWIDTH
include/linux/sched/task.h+1−0
@@ -55,6 +55,7 @@ extern asmlinkage void schedule_tail(struct task_struct *prev); · 54 unmodified lines
55extern void init_idle(struct task_struct *idle, int cpu);55extern void init_idle(struct task_struct *idle, int cpu);
56 56
57extern int sched_fork(unsigned long clone_flags, struct task_struct *p);57extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
58extern void sched_post_fork(struct task_struct *p);
58extern void sched_dead(struct task_struct *p);59extern void sched_dead(struct task_struct *p);
59 60
60void __noreturn do_task_dead(void);61void __noreturn do_task_dead(void);
kernel/fork.c+1−0
@@ -2304,6 +2304,7 @@ static __latent_entropy struct task_struct *copy_process( · 2303 unmodified lines
2304 write_unlock_irq(&tasklist_lock);2304 write_unlock_irq(&tasklist_lock);
2305 2305
2306 proc_fork_connector(p);2306 proc_fork_connector(p);
2307 sched_post_fork(p);
2307 cgroup_post_fork(p, args);2308 cgroup_post_fork(p, args);
2308 perf_event_fork(p);2309 perf_event_fork(p);
2309 2310
kernel/sched/core.c+113−6
@@ -889,6 +889,23 @@ unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; · 888 unmodified lines
889/* Max allowed maximum utilization */889/* Max allowed maximum utilization */
890unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;890unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;
891 891
892/*
893 * By default RT tasks run at the maximum performance point/capacity of the
894 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to
895 * SCHED_CAPACITY_SCALE.
896 *
897 * This knob allows admins to change the default behavior when uclamp is being
898 * used. In battery powered devices, particularly, running at the maximum
899 * capacity and frequency will increase energy consumption and shorten the
900 * battery life.
901 *
902 * This knob only affects RT tasks that their uclamp_se->user_defined == false.
903 *
904 * This knob will not override the system default sched_util_clamp_min defined
905 * above.
906 */
907unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE;
908
892/* All clamps are required to be less or equal than these values */909/* All clamps are required to be less or equal than these values */
893static struct uclamp_se uclamp_default[UCLAMP_CNT];910static struct uclamp_se uclamp_default[UCLAMP_CNT];
894 911
@@ -991,6 +1008,64 @@ unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, · 96 unmodified lines
991 return uclamp_idle_value(rq, clamp_id, clamp_value);1008 return uclamp_idle_value(rq, clamp_id, clamp_value);
992}1009}
993 1010
1011static void __uclamp_update_util_min_rt_default(struct task_struct *p)
1012{
1013 unsigned int default_util_min;
1014 struct uclamp_se *uc_se;
1015
1016 lockdep_assert_held(&p->pi_lock);
1017
1018 uc_se = &p->uclamp_req[UCLAMP_MIN];
1019
1020 /* Only sync if user didn't override the default */
1021 if (uc_se->user_defined)
1022 return;
1023
1024 default_util_min = sysctl_sched_uclamp_util_min_rt_default;
1025 uclamp_se_set(uc_se, default_util_min, false);
1026}
1027
1028static void uclamp_update_util_min_rt_default(struct task_struct *p)
1029{
1030 struct rq_flags rf;
1031 struct rq *rq;
1032
1033 if (!rt_task(p))
1034 return;
1035
1036 /* Protect updates to p->uclamp_* */
1037 rq = task_rq_lock(p, &rf);
1038 __uclamp_update_util_min_rt_default(p);
1039 task_rq_unlock(rq, p, &rf);
1040}
1041
1042static void uclamp_sync_util_min_rt_default(void)
1043{
1044 struct task_struct *g, *p;
1045
1046 /*
1047 * copy_process() sysctl_uclamp
1048 * uclamp_min_rt = X;
1049 * write_lock(&tasklist_lock) read_lock(&tasklist_lock)
1050 * // link thread smp_mb__after_spinlock()
1051 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock);
1052 * sched_post_fork() for_each_process_thread()
1053 * __uclamp_sync_rt() __uclamp_sync_rt()
1054 *
1055 * Ensures that either sched_post_fork() will observe the new
1056 * uclamp_min_rt or for_each_process_thread() will observe the new
1057 * task.
1058 */
1059 read_lock(&tasklist_lock);
1060 smp_mb__after_spinlock();
1061 read_unlock(&tasklist_lock);
1062
1063 rcu_read_lock();
1064 for_each_process_thread(g, p)
1065 uclamp_update_util_min_rt_default(p);
1066 rcu_read_unlock();
1067}
1068
994static inline struct uclamp_se1069static inline struct uclamp_se
995uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id)1070uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id)
996{1071{
@@ -1278,12 +1353,13 @@ int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, · 281 unmodified lines
1278 void *buffer, size_t *lenp, loff_t *ppos)1353 void *buffer, size_t *lenp, loff_t *ppos)
1279{1354{
1280 bool update_root_tg = false;1355 bool update_root_tg = false;
1281 int old_min, old_max;1356 int old_min, old_max, old_min_rt;
1282 int result;1357 int result;
1283 1358
1284 mutex_lock(&uclamp_mutex);1359 mutex_lock(&uclamp_mutex);
1285 old_min = sysctl_sched_uclamp_util_min;1360 old_min = sysctl_sched_uclamp_util_min;
1286 old_max = sysctl_sched_uclamp_util_max;1361 old_max = sysctl_sched_uclamp_util_max;
1362 old_min_rt = sysctl_sched_uclamp_util_min_rt_default;
1287 1363
1288 result = proc_dointvec(table, write, buffer, lenp, ppos);1364 result = proc_dointvec(table, write, buffer, lenp, ppos);
1289 if (result)1365 if (result)
@@ -1292,7 +1368,9 @@ int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, · 2 unmodified lines
1292 goto done;1368 goto done;
1293 1369
1294 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||1370 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||
1295 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE) {1371 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE ||
1372 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) {
1373
1296 result = -EINVAL;1374 result = -EINVAL;
1297 goto undo;1375 goto undo;
1298 }1376 }
@@ -1313,6 +1391,11 @@ int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, · 14 unmodified lines
1313 uclamp_update_root_tg();1391 uclamp_update_root_tg();
1314 }1392 }
1315 1393
1394 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) {
1395 static_branch_enable(&sched_uclamp_used);
1396 uclamp_sync_util_min_rt_default();
1397 }
1398
1316 /*1399 /*
1317 * We update all RUNNABLE tasks only when task groups are in use.1400 * We update all RUNNABLE tasks only when task groups are in use.
1318 * Otherwise, keep it simple and do just a lazy update at each next1401 * Otherwise, keep it simple and do just a lazy update at each next
@@ -1324,6 +1407,7 @@ int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, · 5 unmodified lines
1324undo:1407undo:
1325 sysctl_sched_uclamp_util_min = old_min;1408 sysctl_sched_uclamp_util_min = old_min;
1326 sysctl_sched_uclamp_util_max = old_max;1409 sysctl_sched_uclamp_util_max = old_max;
1410 sysctl_sched_uclamp_util_min_rt_default = old_min_rt;
1327done:1411done:
1328 mutex_unlock(&uclamp_mutex);1412 mutex_unlock(&uclamp_mutex);
1329 1413
@@ -1369,17 +1453,20 @@ static void __setscheduler_uclamp(struct task_struct *p, · 39 unmodified lines
1369 */1453 */
1370 for_each_clamp_id(clamp_id) {1454 for_each_clamp_id(clamp_id) {
1371 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id];1455 struct uclamp_se *uc_se = &p->uclamp_req[clamp_id];
1372 unsigned int clamp_value = uclamp_none(clamp_id);
1373 1456
1374 /* Keep using defined clamps across class changes */1457 /* Keep using defined clamps across class changes */
1375 if (uc_se->user_defined)1458 if (uc_se->user_defined)
1376 continue;1459 continue;
1377 1460
1378 /* By default, RT tasks always get 100% boost */1461 /*
1462 * RT by default have a 100% boost value that could be modified
1463 * at runtime.
1464 */
1379 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))1465 if (unlikely(rt_task(p) && clamp_id == UCLAMP_MIN))
1380 clamp_value = uclamp_none(UCLAMP_MAX);1466 __uclamp_update_util_min_rt_default(p);
1467 else
1468 uclamp_se_set(uc_se, uclamp_none(clamp_id), false);
1381 1469
1382 uclamp_se_set(uc_se, clamp_value, false);
1383 }1470 }
1384 1471
1385 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)))1472 if (likely(!(attr->sched_flags & SCHED_FLAG_UTIL_CLAMP)))
@@ -1400,6 +1487,10 @@ static void uclamp_fork(struct task_struct *p) · 14 unmodified lines
1400{1487{
1401 enum uclamp_id clamp_id;1488 enum uclamp_id clamp_id;
1402 1489
1490 /*
1491 * We don't need to hold task_rq_lock() when updating p->uclamp_* here
1492 * as the task is still at its early fork stages.
1493 */
1403 for_each_clamp_id(clamp_id)1494 for_each_clamp_id(clamp_id)
1404 p->uclamp[clamp_id].active = false;1495 p->uclamp[clamp_id].active = false;
1405 1496
@@ -1412,6 +1503,11 @@ static void uclamp_fork(struct task_struct *p) · 6 unmodified lines
1412 }1503 }
1413}1504}
1414 1505
1506static void uclamp_post_fork(struct task_struct *p)
1507{
1508 uclamp_update_util_min_rt_default(p);
1509}
1510
1415static void __init init_uclamp_rq(struct rq *rq)1511static void __init init_uclamp_rq(struct rq *rq)
1416{1512{
1417 enum uclamp_id clamp_id;1513 enum uclamp_id clamp_id;
@@ -1462,6 +1558,7 @@ static inline int uclamp_validate(struct task_struct *p, · 44 unmodified lines
1462static void __setscheduler_uclamp(struct task_struct *p,1558static void __setscheduler_uclamp(struct task_struct *p,
1463 const struct sched_attr *attr) { }1559 const struct sched_attr *attr) { }
1464static inline void uclamp_fork(struct task_struct *p) { }1560static inline void uclamp_fork(struct task_struct *p) { }
1561static inline void uclamp_post_fork(struct task_struct *p) { }
1465static inline void init_uclamp(void) { }1562static inline void init_uclamp(void) { }
1466#endif /* CONFIG_UCLAMP_TASK */1563#endif /* CONFIG_UCLAMP_TASK */
1467 1564
@@ -3205,6 +3302,11 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) · 1737 unmodified lines
3205 return 0;3302 return 0;
3206}3303}
3207 3304
3305void sched_post_fork(struct task_struct *p)
3306{
3307 uclamp_post_fork(p);
3308}
3309
3208unsigned long to_ratio(u64 period, u64 runtime)3310unsigned long to_ratio(u64 period, u64 runtime)
3209{3311{
3210 if (runtime == RUNTIME_INF)3312 if (runtime == RUNTIME_INF)
@@ -5724,6 +5826,11 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, · 2513 unmodified lines
5724 kattr.sched_nice = task_nice(p);5826 kattr.sched_nice = task_nice(p);
5725 5827
5726#ifdef CONFIG_UCLAMP_TASK5828#ifdef CONFIG_UCLAMP_TASK
5829 /*
5830 * This could race with another potential updater, but this is fine
5831 * because it'll correctly read the old or the new value. We don't need
5832 * to guarantee who wins the race as long as it doesn't return garbage.
5833 */
5727 kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value;5834 kattr.sched_util_min = p->uclamp_req[UCLAMP_MIN].value;
5728 kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value;5835 kattr.sched_util_max = p->uclamp_req[UCLAMP_MAX].value;
5729#endif5836#endif
kernel/sysctl.c+7−0
@@ -1815,6 +1815,13 @@ static struct ctl_table kern_table[] = { · 1814 unmodified lines
1815 .mode = 0644,1815 .mode = 0644,
1816 .proc_handler = sysctl_sched_uclamp_handler,1816 .proc_handler = sysctl_sched_uclamp_handler,
1817 },1817 },
1818 {
1819 .procname = "sched_util_clamp_min_rt_default",
1820 .data = &sysctl_sched_uclamp_util_min_rt_default,
1821 .maxlen = sizeof(unsigned int),
1822 .mode = 0644,
1823 .proc_handler = sysctl_sched_uclamp_handler,
1824 },
1818#endif1825#endif
1819#ifdef CONFIG_SCHED_AUTOGROUP1826#ifdef CONFIG_SCHED_AUTOGROUP
1820 {1827 {