/* * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR * policies) */ #ifdef CONFIG_SMP static inline int rt_overloaded(struct rq *rq){ return atomic_read(&rq->rd->rto_count);} static inline void rt_set_overload(struct rq *rq){ cpu_set(rq->cpu, rq->rd->rto_mask); /* * Make sure the mask is visible before we set * the overload count. That is checked to determine * if we should look at the mask. It would be a shame * if we looked at the mask, but the mask was not * updated yet. */ wmb(); atomic_inc(&rq->rd->rto_count);} static inline void rt_clear_overload(struct rq *rq){ /* the order here really doesn't matter */ atomic_dec(&rq->rd->rto_count); cpu_clear(rq->cpu, rq->rd->rto_mask);} static void update_rt_migration(struct rq *rq){ if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) { if (!rq->rt.overloaded) { rt_set_overload(rq); rq->rt.overloaded = 1; } } else if (rq->rt.overloaded) { rt_clear_overload(rq); rq->rt.overloaded = 0; }}#endif /* CONFIG_SMP */ static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se){ return container_of(rt_se, struct task_struct, rt);} static inline int on_rt_rq(struct sched_rt_entity *rt_se){ return !list_empty(&rt_se->run_list);} #ifdef CONFIG_RT_GROUP_SCHED static inline u64 sched_rt_runtime(struct rt_rq *rt_rq){ if (!rt_rq->tg) return RUNTIME_INF; return rt_rq->rt_runtime;} static inline u64 sched_rt_period(struct rt_rq *rt_rq){ return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);} #define for_each_leaf_rt_rq(rt_rq, rq) \ list_for_each_entry(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){ return rt_rq->rq;} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){ return rt_se->rt_rq;} #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = rt_se->parent) static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se){ return rt_se->my_q;} static void enqueue_rt_entity(struct sched_rt_entity *rt_se);static void dequeue_rt_entity(struct sched_rt_entity *rt_se); static void sched_rt_rq_enqueue(struct rt_rq *rt_rq){ struct sched_rt_entity *rt_se = rt_rq->rt_se; if (rt_se && !on_rt_rq(rt_se) && rt_rq->rt_nr_running) { struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; enqueue_rt_entity(rt_se); if (rt_rq->highest_prio < curr->prio) resched_task(curr); }} static void sched_rt_rq_dequeue(struct rt_rq *rt_rq){ struct sched_rt_entity *rt_se = rt_rq->rt_se; if (rt_se && on_rt_rq(rt_se)) dequeue_rt_entity(rt_se);} static inline int rt_rq_throttled(struct rt_rq *rt_rq){ return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;} static int rt_se_boosted(struct sched_rt_entity *rt_se){ struct rt_rq *rt_rq = group_rt_rq(rt_se); struct task_struct *p; if (rt_rq) return !!rt_rq->rt_nr_boosted; p = rt_task_of(rt_se); return p->prio != p->normal_prio;} #ifdef CONFIG_SMPstatic inline cpumask_t sched_rt_period_mask(void){ return cpu_rq(smp_processor_id())->rd->span;}#elsestatic inline cpumask_t sched_rt_period_mask(void){ return cpu_online_map;}#endif static inlinestruct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu){ return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];} static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq){ return &rt_rq->tg->rt_bandwidth;} #else static inline u64 sched_rt_runtime(struct rt_rq *rt_rq){ return rt_rq->rt_runtime;} static inline u64 sched_rt_period(struct rt_rq *rt_rq){ return ktime_to_ns(def_rt_bandwidth.rt_period);} #define for_each_leaf_rt_rq(rt_rq, rq) \ for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL) static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){ return container_of(rt_rq, struct rq, rt);} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){ struct task_struct *p = rt_task_of(rt_se); struct rq *rq = task_rq(p); return &rq->rt;} #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = NULL) static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se){ return NULL;} static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq){} static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq){} static inline int rt_rq_throttled(struct rt_rq *rt_rq){ return rt_rq->rt_throttled;} static inline cpumask_t sched_rt_period_mask(void){ return cpu_online_map;} static inlinestruct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu){ return &cpu_rq(cpu)->rt;} static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq){ return &def_rt_bandwidth;} #endif static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun){ int i, idle = 1; cpumask_t span; if (rt_b->rt_runtime == RUNTIME_INF) return 1; span = sched_rt_period_mask(); for_each_cpu_mask(i, span) { int enqueue = 0; struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); struct rq *rq = rq_of_rt_rq(rt_rq); spin_lock(&rq->lock); if (rt_rq->rt_time) { u64 runtime; spin_lock(&rt_rq->rt_runtime_lock); runtime = rt_rq->rt_runtime; rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime); if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { rt_rq->rt_throttled = 0; enqueue = 1; } if (rt_rq->rt_time || rt_rq->rt_nr_running) idle = 0; spin_unlock(&rt_rq->rt_runtime_lock); } else if (rt_rq->rt_nr_running) idle = 0; if (enqueue) sched_rt_rq_enqueue(rt_rq); spin_unlock(&rq->lock); } return idle;} #ifdef CONFIG_SMPstatic int balance_runtime(struct rt_rq *rt_rq){ struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); struct root_domain *rd = cpu_rq(smp_processor_id())->rd; int i, weight, more = 0; u64 rt_period; weight = cpus_weight(rd->span); spin_lock(&rt_b->rt_runtime_lock); rt_period = ktime_to_ns(rt_b->rt_period); for_each_cpu_mask(i, rd->span) { struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); s64 diff; if (iter == rt_rq) continue; spin_lock(&iter->rt_runtime_lock); diff = iter->rt_runtime - iter->rt_time; if (diff > 0) { do_div(diff, weight); if (rt_rq->rt_runtime + diff > rt_period) diff = rt_period - rt_rq->rt_runtime; iter->rt_runtime -= diff; rt_rq->rt_runtime += diff; more = 1; if (rt_rq->rt_runtime == rt_period) { spin_unlock(&iter->rt_runtime_lock); break; } } spin_unlock(&iter->rt_runtime_lock); } spin_unlock(&rt_b->rt_runtime_lock); return more;}#endif static inline int rt_se_prio(struct sched_rt_entity *rt_se){#ifdef CONFIG_RT_GROUP_SCHED struct rt_rq *rt_rq = group_rt_rq(rt_se); if (rt_rq) return rt_rq->highest_prio;#endif return rt_task_of(rt_se)->prio;} static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq){ u64 runtime = sched_rt_runtime(rt_rq); if (runtime == RUNTIME_INF) return 0; if (rt_rq->rt_throttled) return rt_rq_throttled(rt_rq); if (sched_rt_runtime(rt_rq) >= sched_rt_period(rt_rq)) return 0; #ifdef CONFIG_SMP if (rt_rq->rt_time > runtime) { int more; spin_unlock(&rt_rq->rt_runtime_lock); more = balance_runtime(rt_rq); spin_lock(&rt_rq->rt_runtime_lock); if (more) runtime = sched_rt_runtime(rt_rq); }#endif if (rt_rq->rt_time > runtime) { rt_rq->rt_throttled = 1; if (rt_rq_throttled(rt_rq)) { sched_rt_rq_dequeue(rt_rq); return 1; } } return 0;} /* * Update the current task's runtime statistics. Skip current tasks that * are not in our scheduling class. */static void update_curr_rt(struct rq *rq){ struct task_struct *curr = rq->curr; struct sched_rt_entity *rt_se = &curr->rt; struct rt_rq *rt_rq = rt_rq_of_se(rt_se); u64 delta_exec; if (!task_has_rt_policy(curr)) return; delta_exec = rq->clock - curr->se.exec_start; if (unlikely((s64)delta_exec < 0)) delta_exec = 0; schedstat_set(curr->se.exec_max, max(curr->se.exec_max, delta_exec)); curr->se.sum_exec_runtime += delta_exec; curr->se.exec_start = rq->clock; cpuacct_charge(curr, delta_exec); for_each_sched_rt_entity(rt_se) { rt_rq = rt_rq_of_se(rt_se); spin_lock(&rt_rq->rt_runtime_lock); rt_rq->rt_time += delta_exec; if (sched_rt_runtime_exceeded(rt_rq)) resched_task(curr); spin_unlock(&rt_rq->rt_runtime_lock); }} static inlinevoid inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){ WARN_ON(!rt_prio(rt_se_prio(rt_se))); rt_rq->rt_nr_running++;#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED if (rt_se_prio(rt_se) < rt_rq->highest_prio) rt_rq->highest_prio = rt_se_prio(rt_se);#endif#ifdef CONFIG_SMP if (rt_se->nr_cpus_allowed > 1) { struct rq *rq = rq_of_rt_rq(rt_rq); rq->rt.rt_nr_migratory++; } update_rt_migration(rq_of_rt_rq(rt_rq));#endif#ifdef CONFIG_RT_GROUP_SCHED if (rt_se_boosted(rt_se)) rt_rq->rt_nr_boosted++; if (rt_rq->tg) start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);#else start_rt_bandwidth(&def_rt_bandwidth);#endif} static inlinevoid dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){ WARN_ON(!rt_prio(rt_se_prio(rt_se))); WARN_ON(!rt_rq->rt_nr_running); rt_rq->rt_nr_running--;#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED if (rt_rq->rt_nr_running) { struct rt_prio_array *array; WARN_ON(rt_se_prio(rt_se) < rt_rq->highest_prio); if (rt_se_prio(rt_se) == rt_rq->highest_prio) { /* recalculate */ array = &rt_rq->active; rt_rq->highest_prio = sched_find_first_bit(array->bitmap); } /* otherwise leave rq->highest prio alone */ } else rt_rq->highest_prio = MAX_RT_PRIO;#endif#ifdef CONFIG_SMP if (rt_se->nr_cpus_allowed > 1) { struct rq *rq = rq_of_rt_rq(rt_rq); rq->rt.rt_nr_migratory--; } update_rt_migration(rq_of_rt_rq(rt_rq));#endif /* CONFIG_SMP */#ifdef CONFIG_RT_GROUP_SCHED if (rt_se_boosted(rt_se)) rt_rq->rt_nr_boosted--; WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);#endif} static void __enqueue_rt_entity(struct sched_rt_entity *rt_se){ struct rt_rq *rt_rq = rt_rq_of_se(rt_se); struct rt_prio_array *array = &rt_rq->active; struct rt_rq *group_rq = group_rt_rq(rt_se); /* * Don't enqueue the group if its throttled, or when empty. * The latter is a consequence of the former when a child group * get throttled and the current group doesn't have any other * active members. */ if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) return; list_add_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se)); __set_bit(rt_se_prio(rt_se), array->bitmap); inc_rt_tasks(rt_se, rt_rq);} static void __dequeue_rt_entity(struct sched_rt_entity *rt_se){ struct rt_rq *rt_rq = rt_rq_of_se(rt_se); struct rt_prio_array *array = &rt_rq->active; list_del_init(&rt_se->run_list); if (list_empty(array->queue + rt_se_prio(rt_se))) __clear_bit(rt_se_prio(rt_se), array->bitmap); dec_rt_tasks(rt_se, rt_rq);} /* * Because the prio of an upper entry depends on the lower * entries, we must remove entries top - down. */static void dequeue_rt_stack(struct sched_rt_entity *rt_se){ struct sched_rt_entity *back = NULL; for_each_sched_rt_entity(rt_se) { rt_se->back = back; back = rt_se; } for (rt_se = back; rt_se; rt_se = rt_se->back) { if (on_rt_rq(rt_se))Lines 1–500 of 1352. Load more
/* * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR * policies) */ #ifdef CONFIG_SMP static inline int rt_overloaded(struct rq *rq) { return atomic_read(&rq->rd->rto_count); } static inline void rt_set_overload(struct rq *rq) { cpu_set(rq->cpu, rq->rd->rto_mask); /* * Make sure the mask is visible before we set * the overload count. That is checked to determine * if we should look at the mask. It would be a shame * if we looked at the mask, but the mask was not * updated yet. */ wmb(); atomic_inc(&rq->rd->rto_count); } static inline void rt_clear_overload(struct rq *rq) { /* the order here really doesn't matter */ atomic_dec(&rq->rd->rto_count); cpu_clear(rq->cpu, rq->rd->rto_mask); } static void update_rt_migration(struct rq *rq) { if (rq->rt.rt_nr_migratory && (rq->rt.rt_nr_running > 1)) { if (!rq->rt.overloaded) { rt_set_overload(rq); rq->rt.overloaded = 1; } } else if (rq->rt.overloaded) { rt_clear_overload(rq); rq->rt.overloaded = 0; } } #endif /* CONFIG_SMP */ static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se) { return container_of(rt_se, struct task_struct, rt); } static inline int on_rt_rq(struct sched_rt_entity *rt_se) { return !list_empty(&rt_se->run_list); } #ifdef CONFIG_RT_GROUP_SCHED static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) { if (!rt_rq->tg) return RUNTIME_INF; return rt_rq->rt_runtime; } static inline u64 sched_rt_period(struct rt_rq *rt_rq) { return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period); } #define for_each_leaf_rt_rq(rt_rq, rq) \ list_for_each_entry(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) { return rt_rq->rq; } static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) { return rt_se->rt_rq; } #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = rt_se->parent) static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) { return rt_se->my_q; } static void enqueue_rt_entity(struct sched_rt_entity *rt_se); static void dequeue_rt_entity(struct sched_rt_entity *rt_se); static void sched_rt_rq_enqueue(struct rt_rq *rt_rq) { struct sched_rt_entity *rt_se = rt_rq->rt_se; if (rt_se && !on_rt_rq(rt_se) && rt_rq->rt_nr_running) { struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr; enqueue_rt_entity(rt_se); if (rt_rq->highest_prio < curr->prio) resched_task(curr); } } static void sched_rt_rq_dequeue(struct rt_rq *rt_rq) { struct sched_rt_entity *rt_se = rt_rq->rt_se; if (rt_se && on_rt_rq(rt_se)) dequeue_rt_entity(rt_se); } static inline int rt_rq_throttled(struct rt_rq *rt_rq) { return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted; } static int rt_se_boosted(struct sched_rt_entity *rt_se) { struct rt_rq *rt_rq = group_rt_rq(rt_se); struct task_struct *p; if (rt_rq) return !!rt_rq->rt_nr_boosted; p = rt_task_of(rt_se); return p->prio != p->normal_prio; } #ifdef CONFIG_SMP static inline cpumask_t sched_rt_period_mask(void) { return cpu_rq(smp_processor_id())->rd->span; } #else static inline cpumask_t sched_rt_period_mask(void) { return cpu_online_map; } #endif static inline struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) { return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu]; } static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) { return &rt_rq->tg->rt_bandwidth; } #else static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) { return rt_rq->rt_runtime; } static inline u64 sched_rt_period(struct rt_rq *rt_rq) { return ktime_to_ns(def_rt_bandwidth.rt_period); } #define for_each_leaf_rt_rq(rt_rq, rq) \ for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL) static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) { return container_of(rt_rq, struct rq, rt); } static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) { struct task_struct *p = rt_task_of(rt_se); struct rq *rq = task_rq(p); return &rq->rt; } #define for_each_sched_rt_entity(rt_se) \ for (; rt_se; rt_se = NULL) static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se) { return NULL; } static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq) { } static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq) { } static inline int rt_rq_throttled(struct rt_rq *rt_rq) { return rt_rq->rt_throttled; } static inline cpumask_t sched_rt_period_mask(void) { return cpu_online_map; } static inline struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu) { return &cpu_rq(cpu)->rt; } static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq) { return &def_rt_bandwidth; } #endif static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) { int i, idle = 1; cpumask_t span; if (rt_b->rt_runtime == RUNTIME_INF) return 1; span = sched_rt_period_mask(); for_each_cpu_mask(i, span) { int enqueue = 0; struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); struct rq *rq = rq_of_rt_rq(rt_rq); spin_lock(&rq->lock); if (rt_rq->rt_time) { u64 runtime; spin_lock(&rt_rq->rt_runtime_lock); runtime = rt_rq->rt_runtime; rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime); if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { rt_rq->rt_throttled = 0; enqueue = 1; } if (rt_rq->rt_time || rt_rq->rt_nr_running) idle = 0; spin_unlock(&rt_rq->rt_runtime_lock); } else if (rt_rq->rt_nr_running) idle = 0; if (enqueue) sched_rt_rq_enqueue(rt_rq); spin_unlock(&rq->lock); } return idle; } #ifdef CONFIG_SMP static int balance_runtime(struct rt_rq *rt_rq) { struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq); struct root_domain *rd = cpu_rq(smp_processor_id())->rd; int i, weight, more = 0; u64 rt_period; weight = cpus_weight(rd->span); spin_lock(&rt_b->rt_runtime_lock); rt_period = ktime_to_ns(rt_b->rt_period); for_each_cpu_mask(i, rd->span) { struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); s64 diff; if (iter == rt_rq) continue; spin_lock(&iter->rt_runtime_lock); diff = iter->rt_runtime - iter->rt_time; if (diff > 0) { do_div(diff, weight); if (rt_rq->rt_runtime + diff > rt_period) diff = rt_period - rt_rq->rt_runtime; iter->rt_runtime -= diff; rt_rq->rt_runtime += diff; more = 1; if (rt_rq->rt_runtime == rt_period) { spin_unlock(&iter->rt_runtime_lock); break; } } spin_unlock(&iter->rt_runtime_lock); } spin_unlock(&rt_b->rt_runtime_lock); return more; } #endif static inline int rt_se_prio(struct sched_rt_entity *rt_se) { #ifdef CONFIG_RT_GROUP_SCHED struct rt_rq *rt_rq = group_rt_rq(rt_se); if (rt_rq) return rt_rq->highest_prio; #endif return rt_task_of(rt_se)->prio; } static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) { u64 runtime = sched_rt_runtime(rt_rq); if (runtime == RUNTIME_INF) return 0; if (rt_rq->rt_throttled) return rt_rq_throttled(rt_rq); if (sched_rt_runtime(rt_rq) >= sched_rt_period(rt_rq)) return 0; #ifdef CONFIG_SMP if (rt_rq->rt_time > runtime) { int more; spin_unlock(&rt_rq->rt_runtime_lock); more = balance_runtime(rt_rq); spin_lock(&rt_rq->rt_runtime_lock); if (more) runtime = sched_rt_runtime(rt_rq); } #endif if (rt_rq->rt_time > runtime) { rt_rq->rt_throttled = 1; if (rt_rq_throttled(rt_rq)) { sched_rt_rq_dequeue(rt_rq); return 1; } } return 0; } /* * Update the current task's runtime statistics. Skip current tasks that * are not in our scheduling class. */ static void update_curr_rt(struct rq *rq) { struct task_struct *curr = rq->curr; struct sched_rt_entity *rt_se = &curr->rt; struct rt_rq *rt_rq = rt_rq_of_se(rt_se); u64 delta_exec; if (!task_has_rt_policy(curr)) return; delta_exec = rq->clock - curr->se.exec_start; if (unlikely((s64)delta_exec < 0)) delta_exec = 0; schedstat_set(curr->se.exec_max, max(curr->se.exec_max, delta_exec)); curr->se.sum_exec_runtime += delta_exec; curr->se.exec_start = rq->clock; cpuacct_charge(curr, delta_exec); for_each_sched_rt_entity(rt_se) { rt_rq = rt_rq_of_se(rt_se); spin_lock(&rt_rq->rt_runtime_lock); rt_rq->rt_time += delta_exec; if (sched_rt_runtime_exceeded(rt_rq)) resched_task(curr); spin_unlock(&rt_rq->rt_runtime_lock); } } static inline void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { WARN_ON(!rt_prio(rt_se_prio(rt_se))); rt_rq->rt_nr_running++; #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED if (rt_se_prio(rt_se) < rt_rq->highest_prio) rt_rq->highest_prio = rt_se_prio(rt_se); #endif #ifdef CONFIG_SMP if (rt_se->nr_cpus_allowed > 1) { struct rq *rq = rq_of_rt_rq(rt_rq); rq->rt.rt_nr_migratory++; } update_rt_migration(rq_of_rt_rq(rt_rq)); #endif #ifdef CONFIG_RT_GROUP_SCHED if (rt_se_boosted(rt_se)) rt_rq->rt_nr_boosted++; if (rt_rq->tg) start_rt_bandwidth(&rt_rq->tg->rt_bandwidth); #else start_rt_bandwidth(&def_rt_bandwidth); #endif } static inline void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) { WARN_ON(!rt_prio(rt_se_prio(rt_se))); WARN_ON(!rt_rq->rt_nr_running); rt_rq->rt_nr_running--; #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED if (rt_rq->rt_nr_running) { struct rt_prio_array *array; WARN_ON(rt_se_prio(rt_se) < rt_rq->highest_prio); if (rt_se_prio(rt_se) == rt_rq->highest_prio) { /* recalculate */ array = &rt_rq->active; rt_rq->highest_prio = sched_find_first_bit(array->bitmap); } /* otherwise leave rq->highest prio alone */ } else rt_rq->highest_prio = MAX_RT_PRIO; #endif #ifdef CONFIG_SMP if (rt_se->nr_cpus_allowed > 1) { struct rq *rq = rq_of_rt_rq(rt_rq); rq->rt.rt_nr_migratory--; } update_rt_migration(rq_of_rt_rq(rt_rq)); #endif /* CONFIG_SMP */ #ifdef CONFIG_RT_GROUP_SCHED if (rt_se_boosted(rt_se)) rt_rq->rt_nr_boosted--; WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted); #endif } static void __enqueue_rt_entity(struct sched_rt_entity *rt_se) { struct rt_rq *rt_rq = rt_rq_of_se(rt_se); struct rt_prio_array *array = &rt_rq->active; struct rt_rq *group_rq = group_rt_rq(rt_se); /* * Don't enqueue the group if its throttled, or when empty. * The latter is a consequence of the former when a child group * get throttled and the current group doesn't have any other * active members. */ if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) return; list_add_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se)); __set_bit(rt_se_prio(rt_se), array->bitmap); inc_rt_tasks(rt_se, rt_rq); } static void __dequeue_rt_entity(struct sched_rt_entity *rt_se) { struct rt_rq *rt_rq = rt_rq_of_se(rt_se); struct rt_prio_array *array = &rt_rq->active; list_del_init(&rt_se->run_list); if (list_empty(array->queue + rt_se_prio(rt_se))) __clear_bit(rt_se_prio(rt_se), array->bitmap); dec_rt_tasks(rt_se, rt_rq); } /* * Because the prio of an upper entry depends on the lower * entries, we must remove entries top - down. */ static void dequeue_rt_stack(struct sched_rt_entity *rt_se) { struct sched_rt_entity *back = NULL; for_each_sched_rt_entity(rt_se) { rt_se->back = back; back = rt_se; } for (rt_se = back; rt_se; rt_se = rt_se->back) { if (on_rt_rq(rt_se))