| @@ -43,6 +43,28 @@ static inline int on_dl_rq(struct sched_dl_entity *dl_se) · 42 unmodified lines |
| 43 | 43 | return !RB_EMPTY_NODE(&dl_se->rb_node); |
| 44 | 44 | } |
| 45 | 45 | |
| 46 | +#ifdef CONFIG_RT_MUTEXES |
| 47 | +static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) |
| 48 | +{ |
| 49 | + return dl_se->pi_se; |
| 50 | +} |
| 51 | + |
| 52 | +static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) |
| 53 | +{ |
| 54 | + return pi_of(dl_se) != dl_se; |
| 55 | +} |
| 56 | +#else |
| 57 | +static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) |
| 58 | +{ |
| 59 | + return dl_se; |
| 60 | +} |
| 61 | + |
| 62 | +static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) |
| 63 | +{ |
| 64 | + return false; |
| 65 | +} |
| 66 | +#endif |
| 67 | + |
| 46 | 68 | #ifdef CONFIG_SMP |
| 47 | 69 | static inline struct dl_bw *dl_bw_of(int i) |
| 48 | 70 | { |
| @@ -698,7 +720,7 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) · 649 unmodified lines |
| 698 | 720 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); |
| 699 | 721 | struct rq *rq = rq_of_dl_rq(dl_rq); |
| 700 | 722 | |
| 701 | | − WARN_ON(dl_se->dl_boosted); |
| 723 | + WARN_ON(is_dl_boosted(dl_se)); |
| 702 | 724 | WARN_ON(dl_time_before(rq_clock(rq), dl_se->deadline)); |
| 703 | 725 | |
| 704 | 726 | /* |
| @@ -736,21 +758,20 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) · 31 unmodified lines |
| 736 | 758 | * could happen are, typically, a entity voluntarily trying to overcome its |
| 737 | 759 | * runtime, or it just underestimated it during sched_setattr(). |
| 738 | 760 | */ |
| 739 | | −static void replenish_dl_entity(struct sched_dl_entity *dl_se, |
| 740 | | − struct sched_dl_entity *pi_se) |
| 761 | +static void replenish_dl_entity(struct sched_dl_entity *dl_se) |
| 741 | 762 | { |
| 742 | 763 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); |
| 743 | 764 | struct rq *rq = rq_of_dl_rq(dl_rq); |
| 744 | 765 | |
| 745 | | − BUG_ON(pi_se->dl_runtime <= 0); |
| 766 | + BUG_ON(pi_of(dl_se)->dl_runtime <= 0); |
| 746 | 767 | |
| 747 | 768 | /* |
| 748 | 769 | * This could be the case for a !-dl task that is boosted. |
| 749 | 770 | * Just go with full inherited parameters. |
| 750 | 771 | */ |
| 751 | 772 | if (dl_se->dl_deadline == 0) { |
| 752 | | − dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; |
| 753 | | − dl_se->runtime = pi_se->dl_runtime; |
| 773 | + dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; |
| 774 | + dl_se->runtime = pi_of(dl_se)->dl_runtime; |
| 754 | 775 | } |
| 755 | 776 | |
| 756 | 777 | if (dl_se->dl_yielded && dl_se->runtime > 0) |
| @@ -763,8 +784,8 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se, · 6 unmodified lines |
| 763 | 784 | * arbitrary large. |
| 764 | 785 | */ |
| 765 | 786 | while (dl_se->runtime <= 0) { |
| 766 | | − dl_se->deadline += pi_se->dl_period; |
| 767 | | − dl_se->runtime += pi_se->dl_runtime; |
| 787 | + dl_se->deadline += pi_of(dl_se)->dl_period; |
| 788 | + dl_se->runtime += pi_of(dl_se)->dl_runtime; |
| 768 | 789 | } |
| 769 | 790 | |
| 770 | 791 | /* |
| @@ -778,8 +799,8 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se, · 7 unmodified lines |
| 778 | 799 | */ |
| 779 | 800 | if (dl_time_before(dl_se->deadline, rq_clock(rq))) { |
| 780 | 801 | printk_deferred_once("sched: DL replenish lagged too much\n"); |
| 781 | | − dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; |
| 782 | | − dl_se->runtime = pi_se->dl_runtime; |
| 802 | + dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; |
| 803 | + dl_se->runtime = pi_of(dl_se)->dl_runtime; |
| 783 | 804 | } |
| 784 | 805 | |
| 785 | 806 | if (dl_se->dl_yielded) |
| @@ -812,8 +833,7 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se, · 26 unmodified lines |
| 812 | 833 | * task with deadline equal to period this is the same of using |
| 813 | 834 | * dl_period instead of dl_deadline in the equation above. |
| 814 | 835 | */ |
| 815 | | −static bool dl_entity_overflow(struct sched_dl_entity *dl_se, |
| 816 | | − struct sched_dl_entity *pi_se, u64 t) |
| 836 | +static bool dl_entity_overflow(struct sched_dl_entity *dl_se, u64 t) |
| 817 | 837 | { |
| 818 | 838 | u64 left, right; |
| 819 | 839 | |
| @@ -835,9 +855,9 @@ static bool dl_entity_overflow(struct sched_dl_entity *dl_se, · 15 unmodified lines |
| 835 | 855 | * of anything below microseconds resolution is actually fiction |
| 836 | 856 | * (but still we want to give the user that illusion >;). |
| 837 | 857 | */ |
| 838 | | − left = (pi_se->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); |
| 858 | + left = (pi_of(dl_se)->dl_deadline >> DL_SCALE) * (dl_se->runtime >> DL_SCALE); |
| 839 | 859 | right = ((dl_se->deadline - t) >> DL_SCALE) * |
| 840 | | − (pi_se->dl_runtime >> DL_SCALE); |
| 860 | + (pi_of(dl_se)->dl_runtime >> DL_SCALE); |
| 841 | 861 | |
| 842 | 862 | return dl_time_before(right, left); |
| 843 | 863 | } |
| @@ -922,24 +942,23 @@ static inline bool dl_is_implicit(struct sched_dl_entity *dl_se) · 78 unmodified lines |
| 922 | 942 | * Please refer to the comments update_dl_revised_wakeup() function to find |
| 923 | 943 | * more about the Revised CBS rule. |
| 924 | 944 | */ |
| 925 | | −static void update_dl_entity(struct sched_dl_entity *dl_se, |
| 926 | | − struct sched_dl_entity *pi_se) |
| 945 | +static void update_dl_entity(struct sched_dl_entity *dl_se) |
| 927 | 946 | { |
| 928 | 947 | struct dl_rq *dl_rq = dl_rq_of_se(dl_se); |
| 929 | 948 | struct rq *rq = rq_of_dl_rq(dl_rq); |
| 930 | 949 | |
| 931 | 950 | if (dl_time_before(dl_se->deadline, rq_clock(rq)) || |
| 932 | | − dl_entity_overflow(dl_se, pi_se, rq_clock(rq))) { |
| 951 | + dl_entity_overflow(dl_se, rq_clock(rq))) { |
| 933 | 952 | |
| 934 | 953 | if (unlikely(!dl_is_implicit(dl_se) && |
| 935 | 954 | !dl_time_before(dl_se->deadline, rq_clock(rq)) && |
| 936 | | − !dl_se->dl_boosted)){ |
| 955 | + !is_dl_boosted(dl_se))) { |
| 937 | 956 | update_dl_revised_wakeup(dl_se, rq); |
| 938 | 957 | return; |
| 939 | 958 | } |
| 940 | 959 | |
| 941 | | − dl_se->deadline = rq_clock(rq) + pi_se->dl_deadline; |
| 942 | | − dl_se->runtime = pi_se->dl_runtime; |
| 960 | + dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; |
| 961 | + dl_se->runtime = pi_of(dl_se)->dl_runtime; |
| 943 | 962 | } |
| 944 | 963 | } |
| 945 | 964 | |
| @@ -1038,7 +1057,7 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) · 92 unmodified lines |
| 1038 | 1057 | * The task might have been boosted by someone else and might be in the |
| 1039 | 1058 | * boosting/deboosting path, its not throttled. |
| 1040 | 1059 | */ |
| 1041 | | − if (dl_se->dl_boosted) |
| 1060 | + if (is_dl_boosted(dl_se)) |
| 1042 | 1061 | goto unlock; |
| 1043 | 1062 | |
| 1044 | 1063 | /* |
| @@ -1066,7 +1085,7 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) · 21 unmodified lines |
| 1066 | 1085 | * but do not enqueue -- wait for our wakeup to do that. |
| 1067 | 1086 | */ |
| 1068 | 1087 | if (!task_on_rq_queued(p)) { |
| 1069 | | − replenish_dl_entity(dl_se, dl_se); |
| 1088 | + replenish_dl_entity(dl_se); |
| 1070 | 1089 | goto unlock; |
| 1071 | 1090 | } |
| 1072 | 1091 | |
| @@ -1156,7 +1175,7 @@ static inline void dl_check_constrained_dl(struct sched_dl_entity *dl_se) · 83 unmodified lines |
| 1156 | 1175 | |
| 1157 | 1176 | if (dl_time_before(dl_se->deadline, rq_clock(rq)) && |
| 1158 | 1177 | dl_time_before(rq_clock(rq), dl_next_period(dl_se))) { |
| 1159 | | − if (unlikely(dl_se->dl_boosted || !start_dl_timer(p))) |
| 1178 | + if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(p))) |
| 1160 | 1179 | return; |
| 1161 | 1180 | dl_se->dl_throttled = 1; |
| 1162 | 1181 | if (dl_se->runtime > 0) |
| @@ -1287,7 +1306,7 @@ throttle: · 124 unmodified lines |
| 1287 | 1306 | dl_se->dl_overrun = 1; |
| 1288 | 1307 | |
| 1289 | 1308 | __dequeue_task_dl(rq, curr, 0); |
| 1290 | | − if (unlikely(dl_se->dl_boosted || !start_dl_timer(curr))) |
| 1309 | + if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(curr))) |
| 1291 | 1310 | enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH); |
| 1292 | 1311 | |
| 1293 | 1312 | if (!is_leftmost(curr, &rq->dl)) |
| @@ -1481,8 +1500,7 @@ static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) · 187 unmodified lines |
| 1481 | 1500 | } |
| 1482 | 1501 | |
| 1483 | 1502 | static void |
| 1484 | | −enqueue_dl_entity(struct sched_dl_entity *dl_se, |
| 1485 | | − struct sched_dl_entity *pi_se, int flags) |
| 1503 | +enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) |
| 1486 | 1504 | { |
| 1487 | 1505 | BUG_ON(on_dl_rq(dl_se)); |
| 1488 | 1506 | |
| @@ -1493,9 +1511,9 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, · 4 unmodified lines |
| 1493 | 1511 | */ |
| 1494 | 1512 | if (flags & ENQUEUE_WAKEUP) { |
| 1495 | 1513 | task_contending(dl_se, flags); |
| 1496 | | − update_dl_entity(dl_se, pi_se); |
| 1514 | + update_dl_entity(dl_se); |
| 1497 | 1515 | } else if (flags & ENQUEUE_REPLENISH) { |
| 1498 | | − replenish_dl_entity(dl_se, pi_se); |
| 1516 | + replenish_dl_entity(dl_se); |
| 1499 | 1517 | } else if ((flags & ENQUEUE_RESTORE) && |
| 1500 | 1518 | dl_time_before(dl_se->deadline, |
| 1501 | 1519 | rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) { |
| @@ -1512,19 +1530,7 @@ static void dequeue_dl_entity(struct sched_dl_entity *dl_se) · 10 unmodified lines |
| 1512 | 1530 | |
| 1513 | 1531 | static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) |
| 1514 | 1532 | { |
| 1515 | | − struct task_struct *pi_task = rt_mutex_get_top_task(p); |
| 1516 | | − struct sched_dl_entity *pi_se = &p->dl; |
| 1517 | | − |
| 1518 | | − /* |
| 1519 | | − * Use the scheduling parameters of the top pi-waiter task if: |
| 1520 | | − * - we have a top pi-waiter which is a SCHED_DEADLINE task AND |
| 1521 | | − * - our dl_boosted is set (i.e. the pi-waiter's (absolute) deadline is |
| 1522 | | − * smaller than our deadline OR we are a !SCHED_DEADLINE task getting |
| 1523 | | − * boosted due to a SCHED_DEADLINE pi-waiter). |
| 1524 | | − * Otherwise we keep our runtime and deadline. |
| 1525 | | − */ |
| 1526 | | − if (pi_task && dl_prio(pi_task->normal_prio) && p->dl.dl_boosted) { |
| 1527 | | − pi_se = &pi_task->dl; |
| 1533 | + if (is_dl_boosted(&p->dl)) { |
| 1528 | 1534 | /* |
| 1529 | 1535 | * Because of delays in the detection of the overrun of a |
| 1530 | 1536 | * thread's runtime, it might be the case that a thread |
| @@ -1557,7 +1563,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) · 26 unmodified lines |
| 1557 | 1563 | * the throttle. |
| 1558 | 1564 | */ |
| 1559 | 1565 | p->dl.dl_throttled = 0; |
| 1560 | | − BUG_ON(!p->dl.dl_boosted || flags != ENQUEUE_REPLENISH); |
| 1566 | + BUG_ON(!is_dl_boosted(&p->dl) || flags != ENQUEUE_REPLENISH); |
| 1561 | 1567 | return; |
| 1562 | 1568 | } |
| 1563 | 1569 | |
| @@ -1594,7 +1600,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) · 30 unmodified lines |
| 1594 | 1600 | return; |
| 1595 | 1601 | } |
| 1596 | 1602 | |
| 1597 | | − enqueue_dl_entity(&p->dl, pi_se, flags); |
| 1603 | + enqueue_dl_entity(&p->dl, flags); |
| 1598 | 1604 | |
| 1599 | 1605 | if (!task_current(rq, p) && p->nr_cpus_allowed > 1) |
| 1600 | 1606 | enqueue_pushable_dl_task(rq, p); |
| @@ -2787,11 +2793,14 @@ void __dl_clear_params(struct task_struct *p) · 1186 unmodified lines |
| 2787 | 2793 | dl_se->dl_bw = 0; |
| 2788 | 2794 | dl_se->dl_density = 0; |
| 2789 | 2795 | |
| 2790 | | − dl_se->dl_boosted = 0; |
| 2791 | 2796 | dl_se->dl_throttled = 0; |
| 2792 | 2797 | dl_se->dl_yielded = 0; |
| 2793 | 2798 | dl_se->dl_non_contending = 0; |
| 2794 | 2799 | dl_se->dl_overrun = 0; |
| 2800 | + |
| 2801 | +#ifdef CONFIG_RT_MUTEXES |
| 2802 | + dl_se->pi_se = dl_se; |
| 2803 | +#endif |
| 2795 | 2804 | } |
| 2796 | 2805 | |
| 2797 | 2806 | bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr) |