| @@ -43,6 +43,28 @@ static inline int on_dl_rq(struct sched_dl_entity *dl_se) · 42 unmodified lines |
| 43 | return !RB_EMPTY_NODE(&dl_se->rb_node); | 43 | return !RB_EMPTY_NODE(&dl_se->rb_node); |
| 44 | } | 44 | } |
| 45 | | 45 | |
| | 46 | #ifdef CONFIG_RT_MUTEXES |
| | 47 | static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) |
| | 48 | { |
| | 49 | return dl_se->pi_se; |
| | 50 | } |
| | 51 | |
| | 52 | static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) |
| | 53 | { |
| | 54 | return pi_of(dl_se) != dl_se; |
| | 55 | } |
| | 56 | #else |
| | 57 | static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) |
| | 58 | { |
| | 59 | return dl_se; |
| | 60 | } |
| | 61 | |
| | 62 | static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) |
| | 63 | { |
| | 64 | return false; |
| | 65 | } |
| | 66 | #endif |
| | 67 | |
| 46 | #ifdef CONFIG_SMP | 68 | #ifdef CONFIG_SMP |
| 47 | static inline struct dl_bw *dl_bw_of(int i) | 69 | static inline struct dl_bw *dl_bw_of(int i) |
| 48 | { | 70 | { |
| @@ -698,7 +720,7 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) · 649 unmodified lines |
| 698 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); | 720 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); |
| 699 | struct rq *rq = rq_of_dl_rq(dl_rq); | 721 | struct rq *rq = rq_of_dl_rq(dl_rq); |
| 700 | | 722 | |
| 701 | WARN_ON(dl_se->dl_boosted); | 723 | WARN_ON(is_dl_boosted(dl_se)); |
| 702 | WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); | 724 | WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); |
| 703 | | 725 | |
| 704 | /* | 726 | /* |
| @@ -736,21 +758,20 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) · 31 unmodified lines |
| 736 | * could happen are, typically, a entity voluntarily trying to overcome its | 758 | * could happen are, typically, a entity voluntarily trying to overcome its |
| 737 | * runtime, or it just underestimated it during sched_setattr(). | 759 | * runtime, or it just underestimated it during sched_setattr(). |
| 738 | */ | 760 | */ |
| 739 | static void replenish_dl_entity(struct sched_dl_entity *dl_se, | 761 | static void replenish_dl_entity(struct sched_dl_entity *dl_se) |
| 740 | struct sched_dl_entity *pi_se) | | |
| 741 | { | 762 | { |
| 742 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); | 763 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); |
| 743 | struct rq *rq = rq_of_dl_rq(dl_rq); | 764 | struct rq *rq = rq_of_dl_rq(dl_rq); |
| 744 | | 765 | |
| 745 | BUG_ON(pi_se->dl_runtime <= 0); | 766 | BUG_ON(pi_of(dl_se)->dl_runtime <= 0); |
| 746 | | 767 | |
| 747 | /* | 768 | /* |
| 748 | * This could be the case for a !-dl task that is boosted. | 769 | * This could be the case for a !-dl task that is boosted. |
| 749 | * Just go with full inherited parameters. | 770 | * Just go with full inherited parameters. |
| 750 | */ | 771 | */ |
| 751 | if (dl_se->dl_deadline == 0) { | 772 | if (dl_se->dl_deadline == 0) { |
| 752 | dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; | 773 | dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; |
| 753 | dl_se->runtime = pi_se->dl_runtime; | 774 | dl_se->runtime = pi_of(dl_se)->dl_runtime; |
| 754 | } | 775 | } |
| 755 | | 776 | |
| 756 | if (dl_se->dl_yielded && dl_se->runtime > 0) | 777 | if (dl_se->dl_yielded && dl_se->runtime > 0) |
| @@ -763,8 +784,8 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se, · 6 unmodified lines |
| 763 | * arbitrary large. | 784 | * arbitrary large. |
| 764 | */ | 785 | */ |
| 765 | while (dl_se->runtime <= 0) { | 786 | while (dl_se->runtime <= 0) { |
| 766 | dl_se->deadline += pi_se->dl_period; | 787 | dl_se->deadline += pi_of(dl_se)->dl_period; |
| 767 | dl_se->runtime += pi_se->dl_runtime; | 788 | dl_se->runtime += pi_of(dl_se)->dl_runtime; |
| 768 | } | 789 | } |
| 769 | | 790 | |
| 770 | /* | 791 | /* |
| @@ -778,8 +799,8 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se, · 7 unmodified lines |
| 778 | */ | 799 | */ |
| 779 | if (dl_time_before(dl_se->deadline, rq_clock(rq))) { | 800 | if (dl_time_before(dl_se->deadline, rq_clock(rq))) { |
| 780 | printk_deferred_once("sched: DL replenish lagged too much\n"); | 801 | printk_deferred_once("sched: DL replenish lagged too much\n"); |
| 781 | dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; | 802 | dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; |
| 782 | dl_se->runtime = pi_se->dl_runtime; | 803 | dl_se->runtime = pi_of(dl_se)->dl_runtime; |
| 783 | } | 804 | } |
| 784 | | 805 | |
| 785 | if (dl_se->dl_yielded) | 806 | if (dl_se->dl_yielded) |
| @@ -812,8 +833,7 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se, · 26 unmodified lines |
| 812 | * task with deadline equal to period this is the same of using | 833 | * task with deadline equal to period this is the same of using |
| 813 | * dl_period instead of dl_deadline in the equation above. | 834 | * dl_period instead of dl_deadline in the equation above. |
| 814 | */ | 835 | */ |
| 815 | static bool dl_entity_overflow(struct sched_dl_entity *dl_se, | 836 | static bool dl_entity_overflow(struct sched_dl_entity *dl_se, u64 t) |
| 816 | struct sched_dl_entity *pi_se, u64 t) | | |
| 817 | { | 837 | { |
| 818 | u64 left, right; | 838 | u64 left, right; |
| 819 | | 839 | |
| @@ -835,9 +855,9 @@ static bool dl_entity_overflow(struct sched_dl_entity *dl_se, · 15 unmodified lines |
| 835 | * of anything below microseconds resolution is actually fiction | 855 | * of anything below microseconds resolution is actually fiction |
| 836 | * (but still we want to give the user that illusion >;). | 856 | * (but still we want to give the user that illusion >;). |
| 837 | */ | 857 | */ |
| 838 | left = (pi_se->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); | 858 | left = (pi_of(dl_se)->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); |
| 839 | right = ((dl_se->deadline - t) >> DL_SCALE) * | 859 | right = ((dl_se->deadline - t) >> DL_SCALE) * |
| 840 | (pi_se->dl_runtime >> DL_SCALE); | 860 | (pi_of(dl_se)->dl_runtime >> DL_SCALE); |
| 841 | | 861 | |
| 842 | return dl_time_before(right, left); | 862 | return dl_time_before(right, left); |
| 843 | } | 863 | } |
| @@ -922,24 +942,23 @@ static inline bool dl_is_implicit(struct sched_dl_entity *dl_se) · 78 unmodified lines |
| 922 | * Please refer to the comments update_dl_revised_wakeup() function to find | 942 | * Please refer to the comments update_dl_revised_wakeup() function to find |
| 923 | * more about the Revised CBS rule. | 943 | * more about the Revised CBS rule. |
| 924 | */ | 944 | */ |
| 925 | static void update_dl_entity(struct sched_dl_entity *dl_se, | 945 | static void update_dl_entity(struct sched_dl_entity *dl_se) |
| 926 | struct sched_dl_entity *pi_se) | | |
| 927 | { | 946 | { |
| 928 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); | 947 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); |
| 929 | struct rq *rq = rq_of_dl_rq(dl_rq); | 948 | struct rq *rq = rq_of_dl_rq(dl_rq); |
| 930 | | 949 | |
| 931 | if (dl_time_before(dl_se->deadline, rq_clock(rq)) || | 950 | if (dl_time_before(dl_se->deadline, rq_clock(rq)) || |
| 932 | dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) { | 951 | dl_entity_overflow(dl_se, rq_clock(rq))) { |
| 933 | | 952 | |
| 934 | if (unlikely(!dl_is_implicit(dl_se) && | 953 | if (unlikely(!dl_is_implicit(dl_se) && |
| 935 | !dl_time_before(dl_se->deadline, rq_clock(rq)) && | 954 | !dl_time_before(dl_se->deadline, rq_clock(rq)) && |
| 936 | !dl_se->dl_boosted)){ | 955 | !is_dl_boosted(dl_se))) { |
| 937 | update_dl_revised_wakeup(dl_se, rq); | 956 | update_dl_revised_wakeup(dl_se, rq); |
| 938 | return; | 957 | return; |
| 939 | } | 958 | } |
| 940 | | 959 | |
| 941 | dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; | 960 | dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; |
| 942 | dl_se->runtime = pi_se->dl_runtime; | 961 | dl_se->runtime = pi_of(dl_se)->dl_runtime; |
| 943 | } | 962 | } |
| 944 | } | 963 | } |
| 945 | | 964 | |
| @@ -1038,7 +1057,7 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) · 92 unmodified lines |
| 1038 | * The task might have been boosted by someone else and might be in the | 1057 | * The task might have been boosted by someone else and might be in the |
| 1039 | * boosting/deboosting path, its not throttled. | 1058 | * boosting/deboosting path, its not throttled. |
| 1040 | */ | 1059 | */ |
| 1041 | if (dl_se->dl_boosted) | 1060 | if (is_dl_boosted(dl_se)) |
| 1042 | goto unlock; | 1061 | goto unlock; |
| 1043 | | 1062 | |
| 1044 | /* | 1063 | /* |
| @@ -1066,7 +1085,7 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) · 21 unmodified lines |
| 1066 | * but do not enqueue -- wait for our wakeup to do that. | 1085 | * but do not enqueue -- wait for our wakeup to do that. |
| 1067 | */ | 1086 | */ |
| 1068 | if (!task_on_rq_queued(p)) { | 1087 | if (!task_on_rq_queued(p)) { |
| 1069 | replenish_dl_entity(dl_se, dl_se); | 1088 | replenish_dl_entity(dl_se); |
| 1070 | goto unlock; | 1089 | goto unlock; |
| 1071 | } | 1090 | } |
| 1072 | | 1091 | |
| @@ -1156,7 +1175,7 @@ static inline void dl_check_constrained_dl(struct sched_dl_entity *dl_se) · 83 unmodified lines |
| 1156 | | 1175 | |
| 1157 | if (dl_time_before(dl_se->deadline, rq_clock(rq)) && | 1176 | if (dl_time_before(dl_se->deadline, rq_clock(rq)) && |
| 1158 | dl_time_before(rq_clock(rq), dl_next_period(dl_se))) { | 1177 | dl_time_before(rq_clock(rq), dl_next_period(dl_se))) { |
| 1159 | if (unlikely(dl_se->dl_boosted || !start_dl_timer(p))) | 1178 | if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(p))) |
| 1160 | return; | 1179 | return; |
| 1161 | dl_se->dl_throttled = 1; | 1180 | dl_se->dl_throttled = 1; |
| 1162 | if (dl_se->runtime > 0) | 1181 | if (dl_se->runtime > 0) |
| @@ -1287,7 +1306,7 @@ throttle: · 124 unmodified lines |
| 1287 | dl_se->dl_overrun = 1; | 1306 | dl_se->dl_overrun = 1; |
| 1288 | | 1307 | |
| 1289 | __dequeue_task_dl(rq, curr, 0); | 1308 | __dequeue_task_dl(rq, curr, 0); |
| 1290 | if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr))) | 1309 | if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(curr))) |
| 1291 | enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); | 1310 | enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); |
| 1292 | | 1311 | |
| 1293 | if (!is_leftmost(curr, &rq->dl)) | 1312 | if (!is_leftmost(curr, &rq->dl)) |
| @@ -1481,8 +1500,7 @@ static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) · 187 unmodified lines |
| 1481 | } | 1500 | } |
| 1482 | | 1501 | |
| 1483 | static void | 1502 | static void |
| 1484 | enqueue_dl_entity(struct sched_dl_entity *dl_se, | 1503 | enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) |
| 1485 | struct sched_dl_entity *pi_se, int flags) | | |
| 1486 | { | 1504 | { |
| 1487 | BUG_ON(on_dl_rq(dl_se)); | 1505 | BUG_ON(on_dl_rq(dl_se)); |
| 1488 | | 1506 | |
| @@ -1493,9 +1511,9 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, · 4 unmodified lines |
| 1493 | */ | 1511 | */ |
| 1494 | if (flags & ENQUEUE_WAKEUP) { | 1512 | if (flags & ENQUEUE_WAKEUP) { |
| 1495 | task_contending(dl_se, flags); | 1513 | task_contending(dl_se, flags); |
| 1496 | update_dl_entity(dl_se, pi_se); | 1514 | update_dl_entity(dl_se); |
| 1497 | } else if (flags & ENQUEUE_REPLENISH) { | 1515 | } else if (flags & ENQUEUE_REPLENISH) { |
| 1498 | replenish_dl_entity(dl_se, pi_se); | 1516 | replenish_dl_entity(dl_se); |
| 1499 | } else if ((flags & ENQUEUE_RESTORE) && | 1517 | } else if ((flags & ENQUEUE_RESTORE) && |
| 1500 | dl_time_before(dl_se->deadline, | 1518 | dl_time_before(dl_se->deadline, |
| 1501 | rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) { | 1519 | rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) { |
| @@ -1512,19 +1530,7 @@ static void dequeue_dl_entity(struct sched_dl_entity *dl_se) · 10 unmodified lines |
| 1512 | | 1530 | |
| 1513 | static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) | 1531 | static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) |
| 1514 | { | 1532 | { |
| 1515 | struct task_struct *pi_task = rt_mutex_get_top_task(p); | 1533 | if (is_dl_boosted(&p->dl)) { |
| 1516 | struct sched_dl_entity *pi_se = &p->dl; | | |
| 1517 | | | |
| 1518 | /* | | |
| 1519 | * Use the scheduling parameters of the top pi-waiter task if: | | |
| 1520 | * - we have a top pi-waiter which is a SCHED_DEADLINE task AND | | |
| 1521 | * - our dl_boosted is set (i.e. the pi-waiter's (absolute) deadline is | | |
| 1522 | * smaller than our deadline OR we are a !SCHED_DEADLINE task getting | | |
| 1523 | * boosted due to a SCHED_DEADLINE pi-waiter). | | |
| 1524 | * Otherwise we keep our runtime and deadline. | | |
| 1525 | */ | | |
| 1526 | if (pi_task && dl_prio(pi_task->normal_prio) && p->dl.dl_boosted) { | | |
| 1527 | pi_se = &pi_task->dl; | | |
| 1528 | /* | 1534 | /* |
| 1529 | * Because of delays in the detection of the overrun of a | 1535 | * Because of delays in the detection of the overrun of a |
| 1530 | * thread's runtime, it might be the case that a thread | 1536 | * thread's runtime, it might be the case that a thread |
| @@ -1557,7 +1563,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) · 26 unmodified lines |
| 1557 | * the throttle. | 1563 | * the throttle. |
| 1558 | */ | 1564 | */ |
| 1559 | p->dl.dl_throttled = 0; | 1565 | p->dl.dl_throttled = 0; |
| 1560 | BUG_ON(!p->dl.dl_boosted || flags != ENQUEUE_REPLENISH); | 1566 | BUG_ON(!is_dl_boosted(&p->dl) || flags != ENQUEUE_REPLENISH); |
| 1561 | return; | 1567 | return; |
| 1562 | } | 1568 | } |
| 1563 | | 1569 | |
| @@ -1594,7 +1600,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) · 30 unmodified lines |
| 1594 | return; | 1600 | return; |
| 1595 | } | 1601 | } |
| 1596 | | 1602 | |
| 1597 | enqueue_dl_entity(&p->dl, pi_se, flags); | 1603 | enqueue_dl_entity(&p->dl, flags); |
| 1598 | | 1604 | |
| 1599 | if (!task_current(rq, p) && p->nr_cpus_allowed > 1) | 1605 | if (!task_current(rq, p) && p->nr_cpus_allowed > 1) |
| 1600 | enqueue_pushable_dl_task(rq, p); | 1606 | enqueue_pushable_dl_task(rq, p); |
| @@ -2787,11 +2793,14 @@ void __dl_clear_params(struct task_struct *p) · 1186 unmodified lines |
| 2787 | dl_se->dl_bw = 0; | 2793 | dl_se->dl_bw = 0; |
| 2788 | dl_se->dl_density = 0; | 2794 | dl_se->dl_density = 0; |
| 2789 | | 2795 | |
| 2790 | dl_se->dl_boosted = 0; | | |
| 2791 | dl_se->dl_throttled = 0; | 2796 | dl_se->dl_throttled = 0; |
| 2792 | dl_se->dl_yielded = 0; | 2797 | dl_se->dl_yielded = 0; |
| 2793 | dl_se->dl_non_contending = 0; | 2798 | dl_se->dl_non_contending = 0; |
| 2794 | dl_se->dl_overrun = 0; | 2799 | dl_se->dl_overrun = 0; |
| | 2800 | |
| | 2801 | #ifdef CONFIG_RT_MUTEXES |
| | 2802 | dl_se->pi_se = dl_se; |
| | 2803 | #endif |
| 2795 | } | 2804 | } |
| 2796 | | 2805 | |
| 2797 | bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr) | 2806 | bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr) |