When a task with a shorter slice is enqueued, we protect the running task which has a longer slice until it becomes ineligible instead of a full slice in order to speedup the switch to other tasks until the task with the shortest slice is scheduled. This helps to the task to not wait too many full slices before running.
Unified · Split
| @@ -777,6 +777,67 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) · 776 unmodified lines | |||
| 777 | return cfs_rq->zero_vruntime; | 777 | return cfs_rq->zero_vruntime; |
| 778 | } | 778 | } |
| 779 | 779 | ||
| 780 | /* | ||
| 781 | * \Sum (v_i - v0)*w_i | ||
| 782 | * V = ------------------- + v0 | ||
| 783 | * \Sum w_i | ||
| 784 | * | ||
| 785 | * Let W = \Sum w_i, and move v_j such that 'v_j == V', thus: | ||
| 786 | * | ||
| 787 | * V = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 | ||
| 788 | * | ||
| 789 | * v_j = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 | ||
| 790 | * | ||
| 791 | * v_j = 1/W * (v_j - v0)*w_j + 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 | ||
| 792 | * | ||
| 793 | * v_j - 1/W * (v_j - v0)*w_j = 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 | ||
| 794 | * | ||
| 795 | * v_j*W - (v_j - v0)*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W | ||
| 796 | * | ||
| 797 | * v_j*(W - w_j) + v0*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W | ||
| 798 | * | ||
| 799 | * v_j*(W - w_j) = \Sum_i!=j (v_i - v0)*w_i + v0*(W - w_j) | ||
| 800 | * | ||
| 801 | * \Sum_i!=j (v_i - v0)*w_i | ||
| 802 | * v_j = ------------------------ + v0 | ||
| 803 | * W - w_j | ||
| 804 | * | ||
| 805 | * When v_j happens to be curr, then '\Sum_i!=j (v_i - v0)*w_i' | ||
| 806 | * is cfs_rq->sum_w_runtime, and 'W - w_j' is cfs_rq->sum_weight, since curr | ||
| 807 | * is not included in the sum. | ||
| 808 | */ | ||
| 809 | static u64 ineligible_vruntime(struct cfs_rq *cfs_rq) | ||
| 810 | { | ||
| 811 | struct sched_entity *curr = cfs_rq->curr; | ||
| 812 | long weight = cfs_rq->sum_weight; | ||
| 813 | s64 delta = 0; | ||
| 814 | |||
| 815 | if (curr && !curr->on_rq) | ||
| 816 | curr = NULL; | ||
| 817 | |||
| 818 | /* | ||
| 819 | * This is called from set_next_task_fair(.first=true) / | ||
| 820 | * set_protect_slice() so curr had better be set and on_rq. | ||
| 821 | */ | ||
| 822 | WARN_ON_ONCE(!curr); | ||
| 823 | |||
| 824 | if (weight) { | ||
| 825 | s64 runtime = cfs_rq->sum_w_vruntime; | ||
| 826 | |||
| 827 | /* | ||
| 828 | * Do not add @curr to obtain the effective '- w_j' terms. | ||
| 829 | */ | ||
| 830 | |||
| 831 | /* sign flips effective floor / ceiling */ | ||
| 832 | if (runtime < 0) | ||
| 833 | runtime -= (weight - 1); | ||
| 834 | |||
| 835 | delta = div64_long(runtime, weight); | ||
| 836 | } | ||
| 837 | |||
| 838 | return cfs_rq->zero_vruntime + delta + 1; | ||
| 839 | } | ||
| 840 | |||
| 780 | static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); | 841 | static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); |
| 781 | 842 | ||
| 782 | /* | 843 | /* |
| @@ -1058,8 +1119,14 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity · 275 unmodified lines | |||
| 1058 | slice = cfs_rq_min_slice(cfs_rq); | 1119 | slice = cfs_rq_min_slice(cfs_rq); |
| 1059 | 1120 | ||
| 1060 | slice = min(slice, se->slice); | 1121 | slice = min(slice, se->slice); |
| 1061 | if (slice != se->slice) | 1122 | |
| 1062 | vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); | 1123 | /* If there are shorter slices than se's one */ |
| 1124 | if (slice != se->slice) { | ||
| 1125 | if (sched_feat(PREEMPT_SHORT)) | ||
| 1126 | vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); | ||
| 1127 | else | ||
| 1128 | vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); | ||
| 1129 | } | ||
| 1063 | 1130 | ||
| 1064 | se->vprot = vprot; | 1131 | se->vprot = vprot; |
| 1065 | } | 1132 | } |