/* * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR * policies) */ #include "sched.h" #include <linux/slab.h> int sched_rr_timeslice = RR_TIMESLICE; static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); struct rt_bandwidth def_rt_bandwidth; static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer){	struct rt_bandwidth *rt_b =		container_of(timer, struct rt_bandwidth, rt_period_timer);	ktime_t now;	int overrun;	int idle = 0; 	for (;;) {		now = hrtimer_cb_get_time(timer);		overrun = hrtimer_forward(timer, now, rt_b->rt_period); 		if (!overrun)			break; 		idle = do_sched_rt_period_timer(rt_b, overrun);	} 	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;} void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime){	rt_b->rt_period = ns_to_ktime(period);	rt_b->rt_runtime = runtime; 	raw_spin_lock_init(&rt_b->rt_runtime_lock); 	hrtimer_init(&rt_b->rt_period_timer,			CLOCK_MONOTONIC, HRTIMER_MODE_REL);	rt_b->rt_period_timer.function = sched_rt_period_timer;} static void start_rt_bandwidth(struct rt_bandwidth *rt_b){	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)		return; 	if (hrtimer_active(&rt_b->rt_period_timer))		return; 	raw_spin_lock(&rt_b->rt_runtime_lock);	start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);	raw_spin_unlock(&rt_b->rt_runtime_lock);} void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq){	struct rt_prio_array *array;	int i; 	array = &rt_rq->active;	for (i = 0; i < MAX_RT_PRIO; i++) {		INIT_LIST_HEAD(array->queue + i);		__clear_bit(i, array->bitmap);	}	/* delimiter for bitsearch: */	__set_bit(MAX_RT_PRIO, array->bitmap); #if defined CONFIG_SMP	rt_rq->highest_prio.curr = MAX_RT_PRIO;	rt_rq->highest_prio.next = MAX_RT_PRIO;	rt_rq->rt_nr_migratory = 0;	rt_rq->overloaded = 0;	plist_head_init(&rt_rq->pushable_tasks);#endif 	rt_rq->rt_time = 0;	rt_rq->rt_throttled = 0;	rt_rq->rt_runtime = 0;	raw_spin_lock_init(&rt_rq->rt_runtime_lock);} #ifdef CONFIG_RT_GROUP_SCHEDstatic void destroy_rt_bandwidth(struct rt_bandwidth *rt_b){	hrtimer_cancel(&rt_b->rt_period_timer);} #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se){#ifdef CONFIG_SCHED_DEBUG	WARN_ON_ONCE(!rt_entity_is_task(rt_se));#endif	return container_of(rt_se, struct task_struct, rt);} static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){	return rt_rq->rq;} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){	return rt_se->rt_rq;} void free_rt_sched_group(struct task_group *tg){	int i; 	if (tg->rt_se)		destroy_rt_bandwidth(&tg->rt_bandwidth); 	for_each_possible_cpu(i) {		if (tg->rt_rq)			kfree(tg->rt_rq[i]);		if (tg->rt_se)			kfree(tg->rt_se[i]);	} 	kfree(tg->rt_rq);	kfree(tg->rt_se);} void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,		struct sched_rt_entity *rt_se, int cpu,		struct sched_rt_entity *parent){	struct rq *rq = cpu_rq(cpu); 	rt_rq->highest_prio.curr = MAX_RT_PRIO;	rt_rq->rt_nr_boosted = 0;	rt_rq->rq = rq;	rt_rq->tg = tg; 	tg->rt_rq[cpu] = rt_rq;	tg->rt_se[cpu] = rt_se; 	if (!rt_se)		return; 	if (!parent)		rt_se->rt_rq = &rq->rt;	else		rt_se->rt_rq = parent->my_q; 	rt_se->my_q = rt_rq;	rt_se->parent = parent;	INIT_LIST_HEAD(&rt_se->run_list);} int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent){	struct rt_rq *rt_rq;	struct sched_rt_entity *rt_se;	int i; 	tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);	if (!tg->rt_rq)		goto err;	tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);	if (!tg->rt_se)		goto err; 	init_rt_bandwidth(&tg->rt_bandwidth,			ktime_to_ns(def_rt_bandwidth.rt_period), 0); 	for_each_possible_cpu(i) {		rt_rq = kzalloc_node(sizeof(struct rt_rq),				     GFP_KERNEL, cpu_to_node(i));		if (!rt_rq)			goto err; 		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),				     GFP_KERNEL, cpu_to_node(i));		if (!rt_se)			goto err_free_rq; 		init_rt_rq(rt_rq, cpu_rq(i));		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);	} 	return 1; err_free_rq:	kfree(rt_rq);err:	return 0;} #else /* CONFIG_RT_GROUP_SCHED */ #define rt_entity_is_task(rt_se) (1) static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se){	return container_of(rt_se, struct task_struct, rt);} static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){	return container_of(rt_rq, struct rq, rt);} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){	struct task_struct *p = rt_task_of(rt_se);	struct rq *rq = task_rq(p); 	return &rq->rt;} void free_rt_sched_group(struct task_group *tg) { } int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent){	return 1;}#endif /* CONFIG_RT_GROUP_SCHED */ #ifdef CONFIG_SMP static inline int rt_overloaded(struct rq *rq){	return atomic_read(&rq->rd->rto_count);} static inline void rt_set_overload(struct rq *rq){	if (!rq->online)		return; 	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);	/*	 * Make sure the mask is visible before we set	 * the overload count. That is checked to determine	 * if we should look at the mask. It would be a shame	 * if we looked at the mask, but the mask was not	 * updated yet.	 */	wmb();	atomic_inc(&rq->rd->rto_count);} static inline void rt_clear_overload(struct rq *rq){	if (!rq->online)		return; 	/* the order here really doesn't matter */	atomic_dec(&rq->rd->rto_count);	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);} static void update_rt_migration(struct rt_rq *rt_rq){	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {		if (!rt_rq->overloaded) {			rt_set_overload(rq_of_rt_rq(rt_rq));			rt_rq->overloaded = 1;		}	} else if (rt_rq->overloaded) {		rt_clear_overload(rq_of_rt_rq(rt_rq));		rt_rq->overloaded = 0;	}} static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){	struct task_struct *p; 	if (!rt_entity_is_task(rt_se))		return; 	p = rt_task_of(rt_se);	rt_rq = &rq_of_rt_rq(rt_rq)->rt; 	rt_rq->rt_nr_total++;	if (p->nr_cpus_allowed > 1)		rt_rq->rt_nr_migratory++; 	update_rt_migration(rt_rq);} static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){	struct task_struct *p; 	if (!rt_entity_is_task(rt_se))		return; 	p = rt_task_of(rt_se);	rt_rq = &rq_of_rt_rq(rt_rq)->rt; 	rt_rq->rt_nr_total--;	if (p->nr_cpus_allowed > 1)		rt_rq->rt_nr_migratory--; 	update_rt_migration(rt_rq);} static inline int has_pushable_tasks(struct rq *rq){	return !plist_head_empty(&rq->rt.pushable_tasks);} static void enqueue_pushable_task(struct rq *rq, struct task_struct *p){	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);	plist_node_init(&p->pushable_tasks, p->prio);	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); 	/* Update the highest prio pushable task */	if (p->prio < rq->rt.highest_prio.next)		rq->rt.highest_prio.next = p->prio;} static void dequeue_pushable_task(struct rq *rq, struct task_struct *p){	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 	/* Update the new highest prio pushable task */	if (has_pushable_tasks(rq)) {		p = plist_first_entry(&rq->rt.pushable_tasks,				      struct task_struct, pushable_tasks);		rq->rt.highest_prio.next = p->prio;	} else		rq->rt.highest_prio.next = MAX_RT_PRIO;} #else static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p){} static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p){} static inlinevoid inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){} static inlinevoid dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){} #endif /* CONFIG_SMP */ static inline int on_rt_rq(struct sched_rt_entity *rt_se){	return !list_empty(&rt_se->run_list);} #ifdef CONFIG_RT_GROUP_SCHED static inline u64 sched_rt_runtime(struct rt_rq *rt_rq){	if (!rt_rq->tg)		return RUNTIME_INF; 	return rt_rq->rt_runtime;} static inline u64 sched_rt_period(struct rt_rq *rt_rq){	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);} typedef struct task_group *rt_rq_iter_t; static inline struct task_group *next_task_group(struct task_group *tg){	do {		tg = list_entry_rcu(tg->list.next,			typeof(struct task_group), list);	} while (&tg->list != &task_groups && task_group_is_autogroup(tg)); 	if (&tg->list == &task_groups)		tg = NULL; 	return tg;} #define for_each_rt_rq(rt_rq, iter, rq)					\	for (iter = container_of(&task_groups, typeof(*iter), list);	\		(iter = next_task_group(iter)) &&			\		(rt_rq = iter->rt_rq[cpu_of(rq)]);) static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq){	list_add_rcu(&rt_rq->leaf_rt_rq_list,			&rq_of_rt_rq(rt_rq)->leaf_rt_rq_list);} static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq){	list_del_rcu(&rt_rq->leaf_rt_rq_list);} #define for_each_leaf_rt_rq(rt_rq, rq) \	list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) #define for_each_sched_rt_entity(rt_se) \	for (; rt_se; rt_se = rt_se->parent) static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se){	return rt_se->my_q;} static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head);static void dequeue_rt_entity(struct sched_rt_entity *rt_se); static void sched_rt_rq_enqueue(struct rt_rq *rt_rq){	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;	struct sched_rt_entity *rt_se; 	int cpu = cpu_of(rq_of_rt_rq(rt_rq)); 	rt_se = rt_rq->tg->rt_se[cpu]; 	if (rt_rq->rt_nr_running) {		if (rt_se && !on_rt_rq(rt_se))			enqueue_rt_entity(rt_se, false);		if (rt_rq->highest_prio.curr < curr->prio)			resched_task(curr);	}} static void sched_rt_rq_dequeue(struct rt_rq *rt_rq){	struct sched_rt_entity *rt_se;	int cpu = cpu_of(rq_of_rt_rq(rt_rq)); 	rt_se = rt_rq->tg->rt_se[cpu]; 	if (rt_se && on_rt_rq(rt_se))		dequeue_rt_entity(rt_se);} static inline int rt_rq_throttled(struct rt_rq *rt_rq){	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;} static int rt_se_boosted(struct sched_rt_entity *rt_se){	struct rt_rq *rt_rq = group_rt_rq(rt_se);	struct task_struct *p; 	if (rt_rq)		return !!rt_rq->rt_nr_boosted; 	p = rt_task_of(rt_se);	return p->prio != p->normal_prio;} #ifdef CONFIG_SMPstatic inline const struct cpumask *sched_rt_period_mask(void){	return cpu_rq(smp_processor_id())->rd->span;}#elsestatic inline const struct cpumask *sched_rt_period_mask(void){	return cpu_online_mask;}#endif static inlinestruct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu){	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];} static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq){	return &rt_rq->tg->rt_bandwidth;} #else /* !CONFIG_RT_GROUP_SCHED */ static inline u64 sched_rt_runtime(struct rt_rq *rt_rq){	return rt_rq->rt_runtime;}

Lines 1–500 of 2102. Load more