Routine conversion of rcu_read_lock(), spin_lock*, and rq_lock usage within the cfs bandwidth controller to use class guards. Only notable changes are: - Checking for "cfs_rq->runtime_remaining <= 0" instead of the inverse to spot a throttle and break early. This also saves the need for extra indentation in the unthrottle case. - Reordering of list_del_rcu() against throttled_clock indicator update in unthrottle_cfs_rq(). Both are done with "cfs_b->lock" held after the "cfs_rq->throttled" is cleared which make the reordering safe against concurrent list modifications. No functional changes intended.
Unified · Split
| @@ -5035,13 +5035,13 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) · 5034 unmodified lines | |||
| 5035 | */ | 5035 | */ |
| 5036 | rq_clock_start_loop_update(rq); | 5036 | rq_clock_start_loop_update(rq); |
| 5037 | 5037 | ||
| 5038 | rcu_read_lock(); | 5038 | guard(rcu)(); |
| 5039 | |||
| 5039 | list_for_each_entry_rcu(tg, &task_groups, list) { | 5040 | list_for_each_entry_rcu(tg, &task_groups, list) { |
| 5040 | struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); | 5041 | struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); |
| 5041 | 5042 | ||
| 5042 | clear_tg_load_avg(cfs_rq); | 5043 | clear_tg_load_avg(cfs_rq); |
| 5043 | } | 5044 | } |
| 5044 | rcu_read_unlock(); | ||
| 5045 | 5045 | ||
| 5046 | rq_clock_stop_loop_update(rq); | 5046 | rq_clock_stop_loop_update(rq); |
| 5047 | } | 5047 | } |
| @@ -6540,13 +6540,10 @@ static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, · 1492 unmodified lines | |||
| 6540 | static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) | 6540 | static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
| 6541 | { | 6541 | { |
| 6542 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); | 6542 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); |
| 6543 | int ret; | ||
| 6544 | 6543 | ||
| 6545 | raw_spin_lock(&cfs_b->lock); | 6544 | guard(raw_spinlock)(&cfs_b->lock); |
| 6546 | ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); | ||
| 6547 | raw_spin_unlock(&cfs_b->lock); | ||
| 6548 | 6545 | ||
| 6549 | return ret; | 6546 | return __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); |
| 6550 | } | 6547 | } |
| 6551 | 6548 | ||
| 6552 | static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) | 6549 | static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) |
| @@ -6835,33 +6832,32 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 282 unmodified lines | |||
| 6835 | { | 6832 | { |
| 6836 | struct rq *rq = rq_of(cfs_rq); | 6833 | struct rq *rq = rq_of(cfs_rq); |
| 6837 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); | 6834 | struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); |
| 6838 | int dequeue = 1; | ||
| 6839 | 6835 | ||
| 6840 | raw_spin_lock(&cfs_b->lock); | 6836 | scoped_guard(raw_spinlock, &cfs_b->lock) { |
| 6841 | /* This will start the period timer if necessary */ | ||
| 6842 | if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { | ||
| 6843 | /* | 6837 | /* |
| 6844 | * We have raced with bandwidth becoming available, and if we | 6838 | * Check if We have raced with bandwidth becoming available. If |
| 6845 | * actually throttled the timer might not unthrottle us for an | 6839 | * we actually throttled the timer might not unthrottle us for |
| 6846 | * entire period. We additionally needed to make sure that any | 6840 | * an entire period. We additionally needed to make sure that |
| 6847 | * subsequent check_cfs_rq_runtime calls agree not to throttle | 6841 | * any subsequent check_cfs_rq_runtime calls agree not to |
| 6848 | * us, as we may commit to do cfs put_prev+pick_next, so we ask | 6842 | * throttle us, as we may commit to do cfs put_prev+pick_next, |
| 6849 | * for 1ns of runtime rather than just check cfs_b. | 6843 | * so we ask for 1ns of runtime rather than just check cfs_b. |
| 6844 | * | ||
| 6845 | * This will start the period timer if necessary. | ||
| 6846 | */ | ||
| 6847 | if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) | ||
| 6848 | return false; | ||
| 6849 | |||
| 6850 | /* | ||
| 6851 | * No bandwidth available; Add ourselves on the list to be | ||
| 6852 | * unthrottled later. | ||
| 6850 | */ | 6853 | */ |
| 6851 | dequeue = 0; | ||
| 6852 | } else { | ||
| 6853 | list_add_tail_rcu(&cfs_rq->throttled_list, | 6854 | list_add_tail_rcu(&cfs_rq->throttled_list, |
| 6854 | &cfs_b->throttled_cfs_rq); | 6855 | &cfs_b->throttled_cfs_rq); |
| 6855 | } | 6856 | } |
| 6856 | raw_spin_unlock(&cfs_b->lock); | ||
| 6857 | |||
| 6858 | if (!dequeue) | ||
| 6859 | return false; /* Throttle no longer required. */ | ||
| 6860 | 6857 | ||
| 6861 | /* freeze hierarchy runnable averages while throttled */ | 6858 | /* freeze hierarchy runnable averages while throttled */ |
| 6862 | rcu_read_lock(); | 6859 | scoped_guard(rcu) |
| 6863 | walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); | 6860 | walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); |
| 6864 | rcu_read_unlock(); | ||
| 6865 | 6861 | ||
| 6866 | /* | 6862 | /* |
| 6867 | * Note: distribution will already see us throttled via the | 6863 | * Note: distribution will already see us throttled via the |
| @@ -6894,13 +6890,15 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 26 unmodified lines | |||
| 6894 | 6890 | ||
| 6895 | update_rq_clock(rq); | 6891 | update_rq_clock(rq); |
| 6896 | 6892 | ||
| 6897 | raw_spin_lock(&cfs_b->lock); | 6893 | scoped_guard(raw_spinlock, &cfs_b->lock) { |
| 6898 | if (cfs_rq->throttled_clock) { | 6894 | list_del_rcu(&cfs_rq->throttled_list); |
| 6895 | |||
| 6896 | if (!cfs_rq->throttled_clock) | ||
| 6897 | break; | ||
| 6898 | |||
| 6899 | cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; | 6899 | cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; |
| 6900 | cfs_rq->throttled_clock = 0; | 6900 | cfs_rq->throttled_clock = 0; |
| 6901 | } | 6901 | } |
| 6902 | list_del_rcu(&cfs_rq->throttled_list); | ||
| 6903 | raw_spin_unlock(&cfs_b->lock); | ||
| 6904 | 6902 | ||
| 6905 | /* update hierarchical throttle state */ | 6903 | /* update hierarchical throttle state */ |
| 6906 | walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); | 6904 | walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); |
| @@ -6929,9 +6927,8 @@ static void __cfsb_csd_unthrottle(void *arg) · 22 unmodified lines | |||
| 6929 | { | 6927 | { |
| 6930 | struct cfs_rq *cursor, *tmp; | 6928 | struct cfs_rq *cursor, *tmp; |
| 6931 | struct rq *rq = arg; | 6929 | struct rq *rq = arg; |
| 6932 | struct rq_flags rf; | ||
| 6933 | 6930 | ||
| 6934 | rq_lock(rq, &rf); | 6931 | guard(rq_lock)(rq); |
| 6935 | 6932 | ||
| 6936 | /* | 6933 | /* |
| 6937 | * Iterating over the list can trigger several call to | 6934 | * Iterating over the list can trigger several call to |
| @@ -6948,7 +6945,7 @@ static void __cfsb_csd_unthrottle(void *arg) · 10 unmodified lines | |||
| 6948 | * race with group being freed in the window between removing it | 6945 | * race with group being freed in the window between removing it |
| 6949 | * from the list and advancing to the next entry in the list. | 6946 | * from the list and advancing to the next entry in the list. |
| 6950 | */ | 6947 | */ |
| 6951 | rcu_read_lock(); | 6948 | guard(rcu)(); |
| 6952 | 6949 | ||
| 6953 | list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, | 6950 | list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, |
| 6954 | throttled_csd_list) { | 6951 | throttled_csd_list) { |
| @@ -6958,10 +6955,7 @@ static void __cfsb_csd_unthrottle(void *arg) · 3 unmodified lines | |||
| 6958 | unthrottle_cfs_rq(cursor); | 6955 | unthrottle_cfs_rq(cursor); |
| 6959 | } | 6956 | } |
| 6960 | 6957 | ||
| 6961 | rcu_read_unlock(); | ||
| 6962 | |||
| 6963 | rq_clock_stop_loop_update(rq); | 6958 | rq_clock_stop_loop_update(rq); |
| 6964 | rq_unlock(rq, &rf); | ||
| 6965 | } | 6959 | } |
| 6966 | 6960 | ||
| 6967 | static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) | 6961 | static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) |
| @@ -7001,11 +6995,11 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) · 33 unmodified lines | |||
| 7001 | u64 runtime, remaining = 1; | 6995 | u64 runtime, remaining = 1; |
| 7002 | bool throttled = false; | 6996 | bool throttled = false; |
| 7003 | struct cfs_rq *cfs_rq, *tmp; | 6997 | struct cfs_rq *cfs_rq, *tmp; |
| 7004 | struct rq_flags rf; | ||
| 7005 | struct rq *rq; | 6998 | struct rq *rq; |
| 7006 | LIST_HEAD(local_unthrottle); | 6999 | LIST_HEAD(local_unthrottle); |
| 7007 | 7000 | ||
| 7008 | rcu_read_lock(); | 7001 | guard(rcu)(); |
| 7002 | |||
| 7009 | list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, | 7003 | list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, |
| 7010 | throttled_list) { | 7004 | throttled_list) { |
| 7011 | rq = rq_of(cfs_rq); | 7005 | rq = rq_of(cfs_rq); |
| @@ -7015,65 +7009,63 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) · 3 unmodified lines | |||
| 7015 | break; | 7009 | break; |
| 7016 | } | 7010 | } |
| 7017 | 7011 | ||
| 7018 | rq_lock_irqsave(rq, &rf); | 7012 | guard(rq_lock_irqsave)(rq); |
| 7013 | |||
| 7019 | if (!cfs_rq_throttled(cfs_rq)) | 7014 | if (!cfs_rq_throttled(cfs_rq)) |
| 7020 | goto next; | 7015 | continue; |
| 7021 | 7016 | ||
| 7022 | /* Already queued for async unthrottle */ | 7017 | /* Already queued for async unthrottle */ |
| 7023 | if (!list_empty(&cfs_rq->throttled_csd_list)) | 7018 | if (!list_empty(&cfs_rq->throttled_csd_list)) |
| 7024 | goto next; | 7019 | continue; |
| 7025 | 7020 | ||
| 7026 | /* By the above checks, this should never be true */ | 7021 | /* By the above checks, this should never be true */ |
| 7027 | WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); | 7022 | WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); |
| 7028 | 7023 | ||
| 7029 | raw_spin_lock(&cfs_b->lock); | 7024 | scoped_guard(raw_spinlock, &cfs_b->lock) { |
| 7030 | runtime = -cfs_rq->runtime_remaining + 1; | 7025 | runtime = -cfs_rq->runtime_remaining + 1; |
| 7031 | if (runtime > cfs_b->runtime) | 7026 | if (runtime > cfs_b->runtime) |
| 7032 | runtime = cfs_b->runtime; | 7027 | runtime = cfs_b->runtime; |
| 7033 | cfs_b->runtime -= runtime; | 7028 | cfs_b->runtime -= runtime; |
| 7034 | remaining = cfs_b->runtime; | 7029 | remaining = cfs_b->runtime; |
| 7035 | raw_spin_unlock(&cfs_b->lock); | 7030 | } |
| 7036 | 7031 | ||
| 7037 | cfs_rq->runtime_remaining += runtime; | 7032 | cfs_rq->runtime_remaining += runtime; |
| 7038 | 7033 | ||
| 7039 | /* we check whether we're throttled above */ | 7034 | /* |
| 7040 | if (cfs_rq->runtime_remaining > 0) { | 7035 | * Ran out of bandwidth during distribution! |
| 7041 | if (cpu_of(rq) != this_cpu) { | 7036 | * Indicate throttled entities and break early. |
| 7042 | unthrottle_cfs_rq_async(cfs_rq); | 7037 | */ |
| 7043 | } else { | 7038 | if (cfs_rq->runtime_remaining <= 0) { |
| 7044 | /* | ||
| 7045 | * We currently only expect to be unthrottling | ||
| 7046 | * a single cfs_rq locally. | ||
| 7047 | */ | ||
| 7048 | WARN_ON_ONCE(!list_empty(&local_unthrottle)); | ||
| 7049 | list_add_tail(&cfs_rq->throttled_csd_list, | ||
| 7050 | &local_unthrottle); | ||
| 7051 | } | ||
| 7052 | } else { | ||
| 7053 | throttled = true; | 7039 | throttled = true; |
| 7040 | break; | ||
| 7054 | } | 7041 | } |
| 7055 | 7042 | ||
| 7056 | next: | 7043 | /* we check whether we're throttled above */ |
| 7057 | rq_unlock_irqrestore(rq, &rf); | 7044 | if (cpu_of(rq) != this_cpu) { |
| 7045 | unthrottle_cfs_rq_async(cfs_rq); | ||
| 7046 | continue; | ||
| 7047 | } | ||
| 7048 | |||
| 7049 | /* | ||
| 7050 | * We currently only expect to be unthrottling | ||
| 7051 | * a single cfs_rq locally. | ||
| 7052 | */ | ||
| 7053 | WARN_ON_ONCE(!list_empty(&local_unthrottle)); | ||
| 7054 | list_add_tail(&cfs_rq->throttled_csd_list, &local_unthrottle); | ||
| 7058 | } | 7055 | } |
| 7059 | 7056 | ||
| 7060 | list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, | 7057 | list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, |
| 7061 | throttled_csd_list) { | 7058 | throttled_csd_list) { |
| 7062 | struct rq *rq = rq_of(cfs_rq); | 7059 | struct rq *rq = rq_of(cfs_rq); |
| 7063 | 7060 | ||
| 7064 | rq_lock_irqsave(rq, &rf); | 7061 | guard(rq_lock_irqsave)(rq); |
| 7065 | 7062 | ||
| 7066 | list_del_init(&cfs_rq->throttled_csd_list); | 7063 | list_del_init(&cfs_rq->throttled_csd_list); |
| 7067 | |||
| 7068 | if (cfs_rq_throttled(cfs_rq)) | 7064 | if (cfs_rq_throttled(cfs_rq)) |
| 7069 | unthrottle_cfs_rq(cfs_rq); | 7065 | unthrottle_cfs_rq(cfs_rq); |
| 7070 | |||
| 7071 | rq_unlock_irqrestore(rq, &rf); | ||
| 7072 | } | 7066 | } |
| 7073 | WARN_ON_ONCE(!list_empty(&local_unthrottle)); | 7067 | WARN_ON_ONCE(!list_empty(&local_unthrottle)); |
| 7074 | 7068 | ||
| 7075 | rcu_read_unlock(); | ||
| 7076 | |||
| 7077 | return throttled; | 7069 | return throttled; |
| 7078 | } | 7070 | } |
| 7079 | 7071 | ||
| @@ -7196,7 +7188,8 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) · 116 unmodified lines | |||
| 7196 | if (slack_runtime <= 0) | 7188 | if (slack_runtime <= 0) |
| 7197 | return; | 7189 | return; |
| 7198 | 7190 | ||
| 7199 | raw_spin_lock(&cfs_b->lock); | 7191 | guard(raw_spinlock)(&cfs_b->lock); |
| 7192 | |||
| 7200 | if (cfs_b->quota != RUNTIME_INF) { | 7193 | if (cfs_b->quota != RUNTIME_INF) { |
| 7201 | cfs_b->runtime += slack_runtime; | 7194 | cfs_b->runtime += slack_runtime; |
| 7202 | 7195 | ||
| @@ -7205,7 +7198,6 @@ static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) · 2 unmodified lines | |||
| 7205 | !list_empty(&cfs_b->throttled_cfs_rq)) | 7198 | !list_empty(&cfs_b->throttled_cfs_rq)) |
| 7206 | start_cfs_slack_bandwidth(cfs_b); | 7199 | start_cfs_slack_bandwidth(cfs_b); |
| 7207 | } | 7200 | } |
| 7208 | raw_spin_unlock(&cfs_b->lock); | ||
| 7209 | 7201 | ||
| 7210 | /* even if it's not valid for return we don't want to try again */ | 7202 | /* even if it's not valid for return we don't want to try again */ |
| 7211 | cfs_rq->runtime_remaining -= slack_runtime; | 7203 | cfs_rq->runtime_remaining -= slack_runtime; |
| @@ -7228,25 +7220,21 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) · 16 unmodified lines | |||
| 7228 | */ | 7220 | */ |
| 7229 | static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) | 7221 | static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) |
| 7230 | { | 7222 | { |
| 7231 | u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); | ||
| 7232 | unsigned long flags; | ||
| 7233 | |||
| 7234 | /* confirm we're still not at a refresh boundary */ | 7223 | /* confirm we're still not at a refresh boundary */ |
| 7235 | raw_spin_lock_irqsave(&cfs_b->lock, flags); | 7224 | scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { |
| 7236 | cfs_b->slack_started = false; | 7225 | u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); |
| 7237 | 7226 | ||
| 7238 | if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { | 7227 | cfs_b->slack_started = false; |
| 7239 | raw_spin_unlock_irqrestore(&cfs_b->lock, flags); | ||
| 7240 | return; | ||
| 7241 | } | ||
| 7242 | 7228 | ||
| 7243 | if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) | 7229 | if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) |
| 7244 | runtime = cfs_b->runtime; | 7230 | return; |
| 7245 | 7231 | ||
| 7246 | raw_spin_unlock_irqrestore(&cfs_b->lock, flags); | 7232 | if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) |
| 7233 | runtime = cfs_b->runtime; | ||
| 7247 | 7234 | ||
| 7248 | if (!runtime) | 7235 | if (!runtime) |
| 7249 | return; | 7236 | return; |
| 7237 | } | ||
| 7250 | 7238 | ||
| 7251 | distribute_cfs_runtime(cfs_b); | 7239 | distribute_cfs_runtime(cfs_b); |
| 7252 | } | 7240 | } |
| @@ -7335,18 +7323,18 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) · 82 unmodified lines | |||
| 7335 | { | 7323 | { |
| 7336 | struct cfs_bandwidth *cfs_b = | 7324 | struct cfs_bandwidth *cfs_b = |
| 7337 | container_of(timer, struct cfs_bandwidth, period_timer); | 7325 | container_of(timer, struct cfs_bandwidth, period_timer); |
| 7338 | unsigned long flags; | ||
| 7339 | int overrun; | 7326 | int overrun; |
| 7340 | int idle = 0; | 7327 | int idle = 0; |
| 7341 | int count = 0; | 7328 | int count = 0; |
| 7342 | 7329 | ||
| 7343 | raw_spin_lock_irqsave(&cfs_b->lock, flags); | 7330 | CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); |
| 7331 | |||
| 7344 | for (;;) { | 7332 | for (;;) { |
| 7345 | overrun = hrtimer_forward_now(timer, cfs_b->period); | 7333 | overrun = hrtimer_forward_now(timer, cfs_b->period); |
| 7346 | if (!overrun) | 7334 | if (!overrun) |
| 7347 | break; | 7335 | break; |
| 7348 | 7336 | ||
| 7349 | idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); | 7337 | idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags); |
| 7350 | 7338 | ||
| 7351 | if (++count > 3) { | 7339 | if (++count > 3) { |
| 7352 | u64 new, old = ktime_to_ns(cfs_b->period); | 7340 | u64 new, old = ktime_to_ns(cfs_b->period); |
| @@ -7379,11 +7367,13 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) · 26 unmodified lines | |||
| 7379 | count = 0; | 7367 | count = 0; |
| 7380 | } | 7368 | } |
| 7381 | } | 7369 | } |
| 7382 | if (idle) | 7370 | |
| 7371 | if (idle) { | ||
| 7383 | cfs_b->period_active = 0; | 7372 | cfs_b->period_active = 0; |
| 7384 | raw_spin_unlock_irqrestore(&cfs_b->lock, flags); | 7373 | return HRTIMER_NORESTART; |
| 7374 | } | ||
| 7385 | 7375 | ||
| 7386 | return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; | 7376 | return HRTIMER_RESTART; |
| 7387 | } | 7377 | } |
| 7388 | 7378 | ||
| 7389 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) | 7379 | void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) |
| @@ -7450,14 +7440,12 @@ static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) · 60 unmodified lines | |||
| 7450 | */ | 7440 | */ |
| 7451 | for_each_possible_cpu(i) { | 7441 | for_each_possible_cpu(i) { |
| 7452 | struct rq *rq = cpu_rq(i); | 7442 | struct rq *rq = cpu_rq(i); |
| 7453 | unsigned long flags; | ||
| 7454 | 7443 | ||
| 7455 | if (list_empty(&rq->cfsb_csd_list)) | 7444 | if (list_empty(&rq->cfsb_csd_list)) |
| 7456 | continue; | 7445 | continue; |
| 7457 | 7446 | ||
| 7458 | local_irq_save(flags); | 7447 | scoped_guard(irqsave) |
| 7459 | __cfsb_csd_unthrottle(rq); | 7448 | __cfsb_csd_unthrottle(rq); |
| 7460 | local_irq_restore(flags); | ||
| 7461 | } | 7449 | } |
| 7462 | } | 7450 | } |
| 7463 | 7451 | ||
| @@ -7475,16 +7463,15 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq) · 11 unmodified lines | |||
| 7475 | 7463 | ||
| 7476 | lockdep_assert_rq_held(rq); | 7464 | lockdep_assert_rq_held(rq); |
| 7477 | 7465 | ||
| 7478 | rcu_read_lock(); | 7466 | guard(rcu)(); |
| 7467 | |||
| 7479 | list_for_each_entry_rcu(tg, &task_groups, list) { | 7468 | list_for_each_entry_rcu(tg, &task_groups, list) { |
| 7480 | struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; | 7469 | struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; |
| 7481 | struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); | 7470 | struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); |
| 7482 | 7471 | ||
| 7483 | raw_spin_lock(&cfs_b->lock); | 7472 | scoped_guard(raw_spinlock, &cfs_b->lock) |
| 7484 | cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; | 7473 | cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; |
| 7485 | raw_spin_unlock(&cfs_b->lock); | ||
| 7486 | } | 7474 | } |
| 7487 | rcu_read_unlock(); | ||
| 7488 | } | 7475 | } |
| 7489 | 7476 | ||
| 7490 | /* cpu offline callback */ | 7477 | /* cpu offline callback */ |
| @@ -7505,7 +7492,8 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) · 14 unmodified lines | |||
| 7505 | */ | 7492 | */ |
| 7506 | rq_clock_start_loop_update(rq); | 7493 | rq_clock_start_loop_update(rq); |
| 7507 | 7494 | ||
| 7508 | rcu_read_lock(); | 7495 | guard(rcu)(); |
| 7496 | |||
| 7509 | list_for_each_entry_rcu(tg, &task_groups, list) { | 7497 | list_for_each_entry_rcu(tg, &task_groups, list) { |
| 7510 | struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); | 7498 | struct cfs_rq *cfs_rq = tg_cfs_rq(tg, cpu_of(rq)); |
| 7511 | 7499 | ||
| @@ -7528,7 +7516,6 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) · 16 unmodified lines | |||
| 7528 | cfs_rq->runtime_remaining = 1; | 7516 | cfs_rq->runtime_remaining = 1; |
| 7529 | unthrottle_cfs_rq(cfs_rq); | 7517 | unthrottle_cfs_rq(cfs_rq); |
| 7530 | } | 7518 | } |
| 7531 | rcu_read_unlock(); | ||
| 7532 | 7519 | ||
| 7533 | rq_clock_stop_loop_update(rq); | 7520 | rq_clock_stop_loop_update(rq); |
| 7534 | } | 7521 | } |