/* * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR * policies) */ #ifdef CONFIG_RT_GROUP_SCHED #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se){#ifdef CONFIG_SCHED_DEBUG	WARN_ON_ONCE(!rt_entity_is_task(rt_se));#endif	return container_of(rt_se, struct task_struct, rt);} static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){	return rt_rq->rq;} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){	return rt_se->rt_rq;} #else /* CONFIG_RT_GROUP_SCHED */ #define rt_entity_is_task(rt_se) (1) static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se){	return container_of(rt_se, struct task_struct, rt);} static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){	return container_of(rt_rq, struct rq, rt);} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){	struct task_struct *p = rt_task_of(rt_se);	struct rq *rq = task_rq(p); 	return &rq->rt;} #endif /* CONFIG_RT_GROUP_SCHED */ #ifdef CONFIG_SMP static inline int rt_overloaded(struct rq *rq){	return atomic_read(&rq->rd->rto_count);} static inline void rt_set_overload(struct rq *rq){	if (!rq->online)		return; 	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);	/*	 * Make sure the mask is visible before we set	 * the overload count. That is checked to determine	 * if we should look at the mask. It would be a shame	 * if we looked at the mask, but the mask was not	 * updated yet.	 */	wmb();	atomic_inc(&rq->rd->rto_count);} static inline void rt_clear_overload(struct rq *rq){	if (!rq->online)		return; 	/* the order here really doesn't matter */	atomic_dec(&rq->rd->rto_count);	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);} static void update_rt_migration(struct rt_rq *rt_rq){	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {		if (!rt_rq->overloaded) {			rt_set_overload(rq_of_rt_rq(rt_rq));			rt_rq->overloaded = 1;		}	} else if (rt_rq->overloaded) {		rt_clear_overload(rq_of_rt_rq(rt_rq));		rt_rq->overloaded = 0;	}} static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){	if (!rt_entity_is_task(rt_se))		return; 	rt_rq = &rq_of_rt_rq(rt_rq)->rt; 	rt_rq->rt_nr_total++;	if (rt_se->nr_cpus_allowed > 1)		rt_rq->rt_nr_migratory++; 	update_rt_migration(rt_rq);} static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){	if (!rt_entity_is_task(rt_se))		return; 	rt_rq = &rq_of_rt_rq(rt_rq)->rt; 	rt_rq->rt_nr_total--;	if (rt_se->nr_cpus_allowed > 1)		rt_rq->rt_nr_migratory--; 	update_rt_migration(rt_rq);} static inline int has_pushable_tasks(struct rq *rq){	return !plist_head_empty(&rq->rt.pushable_tasks);} static void enqueue_pushable_task(struct rq *rq, struct task_struct *p){	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);	plist_node_init(&p->pushable_tasks, p->prio);	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); 	/* Update the highest prio pushable task */	if (p->prio < rq->rt.highest_prio.next)		rq->rt.highest_prio.next = p->prio;} static void dequeue_pushable_task(struct rq *rq, struct task_struct *p){	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 	/* Update the new highest prio pushable task */	if (has_pushable_tasks(rq)) {		p = plist_first_entry(&rq->rt.pushable_tasks,				      struct task_struct, pushable_tasks);		rq->rt.highest_prio.next = p->prio;	} else		rq->rt.highest_prio.next = MAX_RT_PRIO;} #else static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p){} static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p){} static inlinevoid inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){} static inlinevoid dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){} #endif /* CONFIG_SMP */ static inline int on_rt_rq(struct sched_rt_entity *rt_se){	return !list_empty(&rt_se->run_list);} #ifdef CONFIG_RT_GROUP_SCHED static inline u64 sched_rt_runtime(struct rt_rq *rt_rq){	if (!rt_rq->tg)		return RUNTIME_INF; 	return rt_rq->rt_runtime;} static inline u64 sched_rt_period(struct rt_rq *rt_rq){	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);} typedef struct task_group *rt_rq_iter_t; static inline struct task_group *next_task_group(struct task_group *tg){	do {		tg = list_entry_rcu(tg->list.next,			typeof(struct task_group), list);	} while (&tg->list != &task_groups && task_group_is_autogroup(tg)); 	if (&tg->list == &task_groups)		tg = NULL; 	return tg;} #define for_each_rt_rq(rt_rq, iter, rq)					\	for (iter = container_of(&task_groups, typeof(*iter), list);	\		(iter = next_task_group(iter)) &&			\		(rt_rq = iter->rt_rq[cpu_of(rq)]);) static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq){	list_add_rcu(&rt_rq->leaf_rt_rq_list,			&rq_of_rt_rq(rt_rq)->leaf_rt_rq_list);} static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq){	list_del_rcu(&rt_rq->leaf_rt_rq_list);} #define for_each_leaf_rt_rq(rt_rq, rq) \	list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) #define for_each_sched_rt_entity(rt_se) \	for (; rt_se; rt_se = rt_se->parent) static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se){	return rt_se->my_q;} static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head);static void dequeue_rt_entity(struct sched_rt_entity *rt_se); static void sched_rt_rq_enqueue(struct rt_rq *rt_rq){	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;	struct sched_rt_entity *rt_se; 	int cpu = cpu_of(rq_of_rt_rq(rt_rq)); 	rt_se = rt_rq->tg->rt_se[cpu]; 	if (rt_rq->rt_nr_running) {		if (rt_se && !on_rt_rq(rt_se))			enqueue_rt_entity(rt_se, false);		if (rt_rq->highest_prio.curr < curr->prio)			resched_task(curr);	}} static void sched_rt_rq_dequeue(struct rt_rq *rt_rq){	struct sched_rt_entity *rt_se;	int cpu = cpu_of(rq_of_rt_rq(rt_rq)); 	rt_se = rt_rq->tg->rt_se[cpu]; 	if (rt_se && on_rt_rq(rt_se))		dequeue_rt_entity(rt_se);} static inline int rt_rq_throttled(struct rt_rq *rt_rq){	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;} static int rt_se_boosted(struct sched_rt_entity *rt_se){	struct rt_rq *rt_rq = group_rt_rq(rt_se);	struct task_struct *p; 	if (rt_rq)		return !!rt_rq->rt_nr_boosted; 	p = rt_task_of(rt_se);	return p->prio != p->normal_prio;} #ifdef CONFIG_SMPstatic inline const struct cpumask *sched_rt_period_mask(void){	return cpu_rq(smp_processor_id())->rd->span;}#elsestatic inline const struct cpumask *sched_rt_period_mask(void){	return cpu_online_mask;}#endif static inlinestruct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu){	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];} static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq){	return &rt_rq->tg->rt_bandwidth;} #else /* !CONFIG_RT_GROUP_SCHED */ static inline u64 sched_rt_runtime(struct rt_rq *rt_rq){	return rt_rq->rt_runtime;} static inline u64 sched_rt_period(struct rt_rq *rt_rq){	return ktime_to_ns(def_rt_bandwidth.rt_period);} typedef struct rt_rq *rt_rq_iter_t; #define for_each_rt_rq(rt_rq, iter, rq) \	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL) static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq){} static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq){} #define for_each_leaf_rt_rq(rt_rq, rq) \	for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL) #define for_each_sched_rt_entity(rt_se) \	for (; rt_se; rt_se = NULL) static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se){	return NULL;} static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq){	if (rt_rq->rt_nr_running)		resched_task(rq_of_rt_rq(rt_rq)->curr);} static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq){} static inline int rt_rq_throttled(struct rt_rq *rt_rq){	return rt_rq->rt_throttled;} static inline const struct cpumask *sched_rt_period_mask(void){	return cpu_online_mask;} static inlinestruct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu){	return &cpu_rq(cpu)->rt;} static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq){	return &def_rt_bandwidth;} #endif /* CONFIG_RT_GROUP_SCHED */ #ifdef CONFIG_SMP/* * We ran out of runtime, see if we can borrow some from our neighbours. */static int do_balance_runtime(struct rt_rq *rt_rq){	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);	struct root_domain *rd = cpu_rq(smp_processor_id())->rd;	int i, weight, more = 0;	u64 rt_period; 	weight = cpumask_weight(rd->span); 	raw_spin_lock(&rt_b->rt_runtime_lock);	rt_period = ktime_to_ns(rt_b->rt_period);	for_each_cpu(i, rd->span) {		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);		s64 diff; 		if (iter == rt_rq)			continue; 		raw_spin_lock(&iter->rt_runtime_lock);		/*		 * Either all rqs have inf runtime and there's nothing to steal		 * or __disable_runtime() below sets a specific rq to inf to		 * indicate its been disabled and disalow stealing.		 */		if (iter->rt_runtime == RUNTIME_INF)			goto next; 		/*		 * From runqueues with spare time, take 1/n part of their		 * spare time, but no more than our period.		 */		diff = iter->rt_runtime - iter->rt_time;		if (diff > 0) {			diff = div_u64((u64)diff, weight);			if (rt_rq->rt_runtime + diff > rt_period)				diff = rt_period - rt_rq->rt_runtime;			iter->rt_runtime -= diff;			rt_rq->rt_runtime += diff;			more = 1;			if (rt_rq->rt_runtime == rt_period) {				raw_spin_unlock(&iter->rt_runtime_lock);				break;			}		}next:		raw_spin_unlock(&iter->rt_runtime_lock);	}	raw_spin_unlock(&rt_b->rt_runtime_lock); 	return more;} /* * Ensure this RQ takes back all the runtime it lend to its neighbours. */static void __disable_runtime(struct rq *rq){	struct root_domain *rd = rq->rd;	rt_rq_iter_t iter;	struct rt_rq *rt_rq; 	if (unlikely(!scheduler_running))		return; 	for_each_rt_rq(rt_rq, iter, rq) {		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);		s64 want;		int i; 		raw_spin_lock(&rt_b->rt_runtime_lock);		raw_spin_lock(&rt_rq->rt_runtime_lock);		/*		 * Either we're all inf and nobody needs to borrow, or we're		 * already disabled and thus have nothing to do, or we have		 * exactly the right amount of runtime to take out.		 */		if (rt_rq->rt_runtime == RUNTIME_INF ||				rt_rq->rt_runtime == rt_b->rt_runtime)			goto balanced;		raw_spin_unlock(&rt_rq->rt_runtime_lock); 		/*		 * Calculate the difference between what we started out with		 * and what we current have, that's the amount of runtime		 * we lend and now have to reclaim.		 */		want = rt_b->rt_runtime - rt_rq->rt_runtime; 		/*		 * Greedy reclaim, take back as much as we can.		 */		for_each_cpu(i, rd->span) {			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);			s64 diff; 			/*			 * Can't reclaim from ourselves or disabled runqueues.			 */			if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)				continue; 			raw_spin_lock(&iter->rt_runtime_lock);			if (want > 0) {				diff = min_t(s64, iter->rt_runtime, want);				iter->rt_runtime -= diff;				want -= diff;			} else {				iter->rt_runtime -= want;				want -= want;			}			raw_spin_unlock(&iter->rt_runtime_lock); 			if (!want)				break;		} 		raw_spin_lock(&rt_rq->rt_runtime_lock);

Lines 1–500 of 1848. Load more