// SPDX-License-Identifier: GPL-2.0/* * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR * policies) */ int sched_rr_timeslice = RR_TIMESLICE;/* More than 4 hours if BW_SHIFT equals 20. */static const u64 max_rt_runtime = MAX_BW; static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); struct rt_bandwidth def_rt_bandwidth; /* * period over which we measure -rt task CPU usage in us. * default: 1s */unsigned int sysctl_sched_rt_period = 1000000; /* * part of the period that we allow rt tasks to run in us. * default: 0.95s */int sysctl_sched_rt_runtime = 950000; #ifdef CONFIG_SYSCTLstatic int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;static int sched_rt_handler(struct ctl_table *table, int write, void *buffer,		size_t *lenp, loff_t *ppos);static int sched_rr_handler(struct ctl_table *table, int write, void *buffer,		size_t *lenp, loff_t *ppos);static struct ctl_table sched_rt_sysctls[] = {	{		.procname       = "sched_rt_period_us",		.data           = &sysctl_sched_rt_period,		.maxlen         = sizeof(unsigned int),		.mode           = 0644,		.proc_handler   = sched_rt_handler,	},	{		.procname       = "sched_rt_runtime_us",		.data           = &sysctl_sched_rt_runtime,		.maxlen         = sizeof(int),		.mode           = 0644,		.proc_handler   = sched_rt_handler,	},	{		.procname       = "sched_rr_timeslice_ms",		.data           = &sysctl_sched_rr_timeslice,		.maxlen         = sizeof(int),		.mode           = 0644,		.proc_handler   = sched_rr_handler,	},	{}}; static int __init sched_rt_sysctl_init(void){	register_sysctl_init("kernel", sched_rt_sysctls);	return 0;}late_initcall(sched_rt_sysctl_init);#endif static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer){	struct rt_bandwidth *rt_b =		container_of(timer, struct rt_bandwidth, rt_period_timer);	int idle = 0;	int overrun; 	raw_spin_lock(&rt_b->rt_runtime_lock);	for (;;) {		overrun = hrtimer_forward_now(timer, rt_b->rt_period);		if (!overrun)			break; 		raw_spin_unlock(&rt_b->rt_runtime_lock);		idle = do_sched_rt_period_timer(rt_b, overrun);		raw_spin_lock(&rt_b->rt_runtime_lock);	}	if (idle)		rt_b->rt_period_active = 0;	raw_spin_unlock(&rt_b->rt_runtime_lock); 	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;} void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime){	rt_b->rt_period = ns_to_ktime(period);	rt_b->rt_runtime = runtime; 	raw_spin_lock_init(&rt_b->rt_runtime_lock); 	hrtimer_init(&rt_b->rt_period_timer, CLOCK_MONOTONIC,		     HRTIMER_MODE_REL_HARD);	rt_b->rt_period_timer.function = sched_rt_period_timer;} static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b){	raw_spin_lock(&rt_b->rt_runtime_lock);	if (!rt_b->rt_period_active) {		rt_b->rt_period_active = 1;		/*		 * SCHED_DEADLINE updates the bandwidth, as a run away		 * RT task with a DL task could hog a CPU. But DL does		 * not reset the period. If a deadline task was running		 * without an RT task running, it can cause RT tasks to		 * throttle when they start up. Kick the timer right away		 * to update the period.		 */		hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));		hrtimer_start_expires(&rt_b->rt_period_timer,				      HRTIMER_MODE_ABS_PINNED_HARD);	}	raw_spin_unlock(&rt_b->rt_runtime_lock);} static void start_rt_bandwidth(struct rt_bandwidth *rt_b){	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)		return; 	do_start_rt_bandwidth(rt_b);} void init_rt_rq(struct rt_rq *rt_rq){	struct rt_prio_array *array;	int i; 	array = &rt_rq->active;	for (i = 0; i < MAX_RT_PRIO; i++) {		INIT_LIST_HEAD(array->queue + i);		__clear_bit(i, array->bitmap);	}	/* delimiter for bitsearch: */	__set_bit(MAX_RT_PRIO, array->bitmap); #if defined CONFIG_SMP	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;	rt_rq->highest_prio.next = MAX_RT_PRIO-1;	rt_rq->rt_nr_migratory = 0;	rt_rq->overloaded = 0;	plist_head_init(&rt_rq->pushable_tasks);#endif /* CONFIG_SMP */	/* We start is dequeued state, because no RT tasks are queued */	rt_rq->rt_queued = 0; 	rt_rq->rt_time = 0;	rt_rq->rt_throttled = 0;	rt_rq->rt_runtime = 0;	raw_spin_lock_init(&rt_rq->rt_runtime_lock);} #ifdef CONFIG_RT_GROUP_SCHEDstatic void destroy_rt_bandwidth(struct rt_bandwidth *rt_b){	hrtimer_cancel(&rt_b->rt_period_timer);} #define rt_entity_is_task(rt_se) (!(rt_se)->my_q) static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se){#ifdef CONFIG_SCHED_DEBUG	WARN_ON_ONCE(!rt_entity_is_task(rt_se));#endif	return container_of(rt_se, struct task_struct, rt);} static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){	return rt_rq->rq;} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){	return rt_se->rt_rq;} static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se){	struct rt_rq *rt_rq = rt_se->rt_rq; 	return rt_rq->rq;} void unregister_rt_sched_group(struct task_group *tg){	if (tg->rt_se)		destroy_rt_bandwidth(&tg->rt_bandwidth); } void free_rt_sched_group(struct task_group *tg){	int i; 	for_each_possible_cpu(i) {		if (tg->rt_rq)			kfree(tg->rt_rq[i]);		if (tg->rt_se)			kfree(tg->rt_se[i]);	} 	kfree(tg->rt_rq);	kfree(tg->rt_se);} void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,		struct sched_rt_entity *rt_se, int cpu,		struct sched_rt_entity *parent){	struct rq *rq = cpu_rq(cpu); 	rt_rq->highest_prio.curr = MAX_RT_PRIO-1;	rt_rq->rt_nr_boosted = 0;	rt_rq->rq = rq;	rt_rq->tg = tg; 	tg->rt_rq[cpu] = rt_rq;	tg->rt_se[cpu] = rt_se; 	if (!rt_se)		return; 	if (!parent)		rt_se->rt_rq = &rq->rt;	else		rt_se->rt_rq = parent->my_q; 	rt_se->my_q = rt_rq;	rt_se->parent = parent;	INIT_LIST_HEAD(&rt_se->run_list);} int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent){	struct rt_rq *rt_rq;	struct sched_rt_entity *rt_se;	int i; 	tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);	if (!tg->rt_rq)		goto err;	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);	if (!tg->rt_se)		goto err; 	init_rt_bandwidth(&tg->rt_bandwidth,			ktime_to_ns(def_rt_bandwidth.rt_period), 0); 	for_each_possible_cpu(i) {		rt_rq = kzalloc_node(sizeof(struct rt_rq),				     GFP_KERNEL, cpu_to_node(i));		if (!rt_rq)			goto err; 		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),				     GFP_KERNEL, cpu_to_node(i));		if (!rt_se)			goto err_free_rq; 		init_rt_rq(rt_rq);		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);	} 	return 1; err_free_rq:	kfree(rt_rq);err:	return 0;} #else /* CONFIG_RT_GROUP_SCHED */ #define rt_entity_is_task(rt_se) (1) static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se){	return container_of(rt_se, struct task_struct, rt);} static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq){	return container_of(rt_rq, struct rq, rt);} static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se){	struct task_struct *p = rt_task_of(rt_se); 	return task_rq(p);} static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se){	struct rq *rq = rq_of_rt_se(rt_se); 	return &rq->rt;} void unregister_rt_sched_group(struct task_group *tg) { } void free_rt_sched_group(struct task_group *tg) { } int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent){	return 1;}#endif /* CONFIG_RT_GROUP_SCHED */ #ifdef CONFIG_SMP static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev){	/* Try to pull RT tasks here if we lower this rq's prio */	return rq->online && rq->rt.highest_prio.curr > prev->prio;} static inline int rt_overloaded(struct rq *rq){	return atomic_read(&rq->rd->rto_count);} static inline void rt_set_overload(struct rq *rq){	if (!rq->online)		return; 	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);	/*	 * Make sure the mask is visible before we set	 * the overload count. That is checked to determine	 * if we should look at the mask. It would be a shame	 * if we looked at the mask, but the mask was not	 * updated yet.	 *	 * Matched by the barrier in pull_rt_task().	 */	smp_wmb();	atomic_inc(&rq->rd->rto_count);} static inline void rt_clear_overload(struct rq *rq){	if (!rq->online)		return; 	/* the order here really doesn't matter */	atomic_dec(&rq->rd->rto_count);	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);} static void update_rt_migration(struct rt_rq *rt_rq){	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {		if (!rt_rq->overloaded) {			rt_set_overload(rq_of_rt_rq(rt_rq));			rt_rq->overloaded = 1;		}	} else if (rt_rq->overloaded) {		rt_clear_overload(rq_of_rt_rq(rt_rq));		rt_rq->overloaded = 0;	}} static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){	struct task_struct *p; 	if (!rt_entity_is_task(rt_se))		return; 	p = rt_task_of(rt_se);	rt_rq = &rq_of_rt_rq(rt_rq)->rt; 	rt_rq->rt_nr_total++;	if (p->nr_cpus_allowed > 1)		rt_rq->rt_nr_migratory++; 	update_rt_migration(rt_rq);} static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){	struct task_struct *p; 	if (!rt_entity_is_task(rt_se))		return; 	p = rt_task_of(rt_se);	rt_rq = &rq_of_rt_rq(rt_rq)->rt; 	rt_rq->rt_nr_total--;	if (p->nr_cpus_allowed > 1)		rt_rq->rt_nr_migratory--; 	update_rt_migration(rt_rq);} static inline int has_pushable_tasks(struct rq *rq){	return !plist_head_empty(&rq->rt.pushable_tasks);} static DEFINE_PER_CPU(struct balance_callback, rt_push_head);static DEFINE_PER_CPU(struct balance_callback, rt_pull_head); static void push_rt_tasks(struct rq *);static void pull_rt_task(struct rq *); static inline void rt_queue_push_tasks(struct rq *rq){	if (!has_pushable_tasks(rq))		return; 	queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);} static inline void rt_queue_pull_task(struct rq *rq){	queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);} static void enqueue_pushable_task(struct rq *rq, struct task_struct *p){	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);	plist_node_init(&p->pushable_tasks, p->prio);	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks); 	/* Update the highest prio pushable task */	if (p->prio < rq->rt.highest_prio.next)		rq->rt.highest_prio.next = p->prio;} static void dequeue_pushable_task(struct rq *rq, struct task_struct *p){	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks); 	/* Update the new highest prio pushable task */	if (has_pushable_tasks(rq)) {		p = plist_first_entry(&rq->rt.pushable_tasks,				      struct task_struct, pushable_tasks);		rq->rt.highest_prio.next = p->prio;	} else {		rq->rt.highest_prio.next = MAX_RT_PRIO-1;	}} #else static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p){} static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p){} static inlinevoid inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){} static inlinevoid dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq){} static inline void rt_queue_push_tasks(struct rq *rq){}#endif /* CONFIG_SMP */ static void enqueue_top_rt_rq(struct rt_rq *rt_rq);static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count); static inline int on_rt_rq(struct sched_rt_entity *rt_se){	return rt_se->on_rq;} #ifdef CONFIG_UCLAMP_TASK/* * Verify the fitness of task @p to run on @cpu taking into account the uclamp * settings. * * This check is only important for heterogeneous systems where uclamp_min value * is higher than the capacity of a @cpu. For non-heterogeneous system this * function will always return true. * * The function will return true if the capacity of the @cpu is >= the * uclamp_min and false otherwise.

Lines 1–500 of 3060. Load more