// SPDX-License-Identifier: GPL-2.0-only/* * Simple CPU accounting cgroup controller */#include <linux/sched/clock.h>#include <linux/sched/cputime.h>#include <linux/tsacct_kern.h>#include "sched.h" #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #include <asm/cputime.h>#endif #ifdef CONFIG_IRQ_TIME_ACCOUNTING DEFINE_STATIC_KEY_FALSE(sched_clock_irqtime); /* * There are no locks covering percpu hardirq/softirq time. * They are only modified in vtime_account, on corresponding CPU * with interrupts disabled. So, writes are safe. * They are read and saved off onto struct rq in update_rq_clock(). * This may result in other CPU reading this CPU's IRQ time and can * race with irq/vtime_account on this CPU. We would either get old * or new value with a side effect of accounting a slice of IRQ time to wrong * task when IRQ is in progress while we read rq->clock. That is a worthy * compromise in place of having locks on each IRQ in account_system_time. */DEFINE_PER_CPU(struct irqtime, cpu_irqtime); void enable_sched_clock_irqtime(void){	static_branch_enable(&sched_clock_irqtime);} void disable_sched_clock_irqtime(void){	if (irqtime_enabled())		static_branch_disable(&sched_clock_irqtime);} static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,				  enum cpu_usage_stat idx){	u64 *cpustat = kcpustat_this_cpu->cpustat; 	u64_stats_update_begin(&irqtime->sync);	cpustat[idx] += delta;	irqtime->total += delta;	if (!kcpustat_idle_dyntick())		irqtime->tick_delta += delta;	u64_stats_update_end(&irqtime->sync);} /* * Called after incrementing preempt_count on {soft,}irq_enter * and before decrementing preempt_count on {soft,}irq_exit. */void irqtime_account_irq(struct task_struct *curr, unsigned int offset){	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);	unsigned int pc;	s64 delta;	int cpu; 	if (!irqtime_enabled())		return; 	cpu = smp_processor_id();	delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;	irqtime->irq_start_time += delta;	pc = irq_count() - offset; 	/*	 * We do not account for softirq time from ksoftirqd here.	 * We want to continue accounting softirq time to ksoftirqd thread	 * in that case, so as not to confuse scheduler with a special task	 * that do not consume any time, but still wants to run.	 */	if (pc & HARDIRQ_MASK)		irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);	else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd())		irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);} static u64 irqtime_tick_accounted(u64 maxtime){	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);	u64 delta; 	delta = min(irqtime->tick_delta, maxtime);	irqtime->tick_delta -= delta; 	return delta;} #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ static u64 irqtime_tick_accounted(u64 dummy){	return 0;} #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ static inline void task_group_account_field(struct task_struct *p, int index,					    u64 tmp){	/*	 * Since all updates are sure to touch the root cgroup, we	 * get ourselves ahead and touch it first. If the root cgroup	 * is the only cgroup, then nothing else should be necessary.	 *	 */	__this_cpu_add(kernel_cpustat.cpustat[index], tmp); 	cgroup_account_cputime_field(p, index, tmp);} /* * Account user CPU time to a process. * @p: the process that the CPU time gets accounted to * @cputime: the CPU time spent in user space since the last update */void account_user_time(struct task_struct *p, u64 cputime){	int index; 	/* Add user time to process. */	p->utime += cputime;	account_group_user_time(p, cputime); 	index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; 	/* Add user time to cpustat. */	task_group_account_field(p, index, cputime); 	/* Account for user time used */	acct_account_cputime(p);} /* * Account guest CPU time to a process. * @p: the process that the CPU time gets accounted to * @cputime: the CPU time spent in virtual machine since the last update */void account_guest_time(struct task_struct *p, u64 cputime){	u64 *cpustat = kcpustat_this_cpu->cpustat; 	/* Add guest time to process. */	p->utime += cputime;	account_group_user_time(p, cputime);	p->gtime += cputime; 	/* Add guest time to cpustat. */	if (task_nice(p) > 0) {		task_group_account_field(p, CPUTIME_NICE, cputime);		cpustat[CPUTIME_GUEST_NICE] += cputime;	} else {		task_group_account_field(p, CPUTIME_USER, cputime);		cpustat[CPUTIME_GUEST] += cputime;	}} /* * Account system CPU time to a process and desired cpustat field * @p: the process that the CPU time gets accounted to * @cputime: the CPU time spent in kernel space since the last update * @index: pointer to cpustat field that has to be updated */void account_system_index_time(struct task_struct *p,			       u64 cputime, enum cpu_usage_stat index){	/* Add system time to process. */	p->stime += cputime;	account_group_system_time(p, cputime); 	/* Add system time to cpustat. */	task_group_account_field(p, index, cputime); 	/* Account for system time used */	acct_account_cputime(p);} /* * Account system CPU time to a process. * @p: the process that the CPU time gets accounted to * @hardirq_offset: the offset to subtract from hardirq_count() * @cputime: the CPU time spent in kernel space since the last update */void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime){	int index; 	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {		account_guest_time(p, cputime);		return;	} 	if (hardirq_count() - hardirq_offset)		index = CPUTIME_IRQ;	else if (in_serving_softirq())		index = CPUTIME_SOFTIRQ;	else		index = CPUTIME_SYSTEM; 	account_system_index_time(p, cputime, index);} /* * Account for involuntary wait time. * @cputime: the CPU time spent in involuntary wait */void account_steal_time(u64 cputime){	u64 *cpustat = kcpustat_this_cpu->cpustat; 	cpustat[CPUTIME_STEAL] += cputime;} /* * Account for idle time. * @cputime: the CPU time spent in idle wait */void account_idle_time(u64 cputime){	u64 *cpustat = kcpustat_this_cpu->cpustat;	struct rq *rq = this_rq(); 	if (atomic_read(&rq->nr_iowait) > 0)		cpustat[CPUTIME_IOWAIT] += cputime;	else		cpustat[CPUTIME_IDLE] += cputime;}  #ifdef CONFIG_SCHED_CORE/* * Account for forceidle time due to core scheduling. * * REQUIRES: schedstat is enabled. */void __account_forceidle_time(struct task_struct *p, u64 delta){	__schedstat_add(p->stats.core_forceidle_sum, delta); 	task_group_account_field(p, CPUTIME_FORCEIDLE, delta);}#endif /* CONFIG_SCHED_CORE */ /* * When a guest is interrupted for a longer amount of time, missed clock * ticks are not redelivered later. Due to that, this function may on * occasion account more time than the calling functions think elapsed. */#ifdef CONFIG_PARAVIRTstruct static_key paravirt_steal_enabled; #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GENstatic u64 native_steal_clock(int cpu){	return 0;} DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);#endif#endif static __always_inline u64 steal_account_process_time(u64 maxtime){#ifdef CONFIG_PARAVIRT	if (static_key_false(&paravirt_steal_enabled)) {		u64 steal; 		steal = paravirt_steal_clock(smp_processor_id());		steal -= this_rq()->prev_steal_time;		steal = min(steal, maxtime);		account_steal_time(steal);		this_rq()->prev_steal_time += steal; 		return steal;	}#endif /* CONFIG_PARAVIRT */	return 0;} /* * Account how much elapsed time was spent in steal, IRQ, or softirq time. */static inline u64 account_other_time(u64 max){	u64 accounted; 	lockdep_assert_irqs_disabled(); 	accounted = steal_account_process_time(max); 	if (accounted < max)		accounted += irqtime_tick_accounted(max - accounted); 	return accounted;} #ifdef CONFIG_64BITstatic inline u64 read_sum_exec_runtime(struct task_struct *t){	return t->se.sum_exec_runtime;}#else /* !CONFIG_64BIT: */static u64 read_sum_exec_runtime(struct task_struct *t){	u64 ns;	struct rq_flags rf;	struct rq *rq; 	rq = task_rq_lock(t, &rf);	ns = t->se.sum_exec_runtime;	task_rq_unlock(rq, t, &rf); 	return ns;}#endif /* !CONFIG_64BIT */ /* * Accumulate raw cputime values of dead tasks (sig->[us]time) and live * tasks (sum on group iteration) belonging to @tsk's group. */void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times){	struct signal_struct *sig = tsk->signal;	struct task_struct *t;	u64 utime, stime; 	/*	 * Update current task runtime to account pending time since last	 * scheduler action or thread_group_cputime() call. This thread group	 * might have other running tasks on different CPUs, but updating	 * their runtime can affect syscall performance, so we skip account	 * those pending times and rely only on values updated on tick or	 * other scheduler action.	 */	if (same_thread_group(current, tsk))		(void) task_sched_runtime(current); 	guard(rcu)();	scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) {		times->utime = sig->utime;		times->stime = sig->stime;		times->sum_exec_runtime = sig->sum_sched_runtime; 		__for_each_thread(sig, t) {			task_cputime(t, &utime, &stime);			times->utime += utime;			times->stime += stime;			times->sum_exec_runtime += read_sum_exec_runtime(t);		}	}} #ifdef CONFIG_IRQ_TIME_ACCOUNTING/* * Account a tick to a process and cpustat * @p: the process that the CPU time gets accounted to * @user_tick: is the tick from userspace * @rq: the pointer to rq * * Tick demultiplexing follows the order * - pending hardirq update * - pending softirq update * - user_time * - idle_time * - system time *   - check for guest_time *   - else account as system_time * * Check for hardirq is done both for system and user time as there is * no timer going off while we are on hardirq and hence we may never get an * opportunity to update it solely in system time. * p->stime and friends are only updated on system time and not on IRQ * softirq as those do not count in task exec_runtime any more. */static void irqtime_account_process_tick(struct task_struct *p, int user_tick,					 int ticks){	u64 other, cputime = TICK_NSEC * ticks; 	/*	 * When returning from idle, many ticks can get accounted at	 * once, including some ticks of steal, IRQ, and softirq time.	 * Subtract those ticks from the amount of time accounted to	 * idle, or potentially user or system time. Due to rounding,	 * other time can exceed ticks occasionally.	 */	other = account_other_time(ULONG_MAX);	if (other >= cputime)		return; 	cputime -= other; 	if (this_cpu_ksoftirqd() == p) {		/*		 * ksoftirqd time do not get accounted in cpu_softirq_time.		 * So, we have to handle it separately here.		 * Also, p->stime needs to be updated for ksoftirqd.		 */		account_system_index_time(p, cputime, CPUTIME_SOFTIRQ);	} else if (user_tick) {		account_user_time(p, cputime);	} else if (p == this_rq()->idle) {		account_idle_time(cputime);	} else if (p->flags & PF_VCPU) { /* System time or guest time */		account_guest_time(p, cputime);	} else {		account_system_index_time(p, cputime, CPUTIME_SYSTEM);	}} #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,						int nr_ticks) { }#endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ #if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE)static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now){	u64 *cpustat = kc->cpustat;	u64 delta, steal, steal_delta;	int iowait; 	if (!kc->idle_elapse)		return; 	iowait = nr_iowait_cpu(smp_processor_id()) > 0;	delta = now - kc->idle_entrytime;	steal = steal_account_process_time(delta); 	/*	 * Record the idle time after substracting the steal time from	 * previous update sequence. Don't substract the steal time from	 * the current update sequence to avoid readers moving backward.	 */	write_seqcount_begin(&kc->idle_sleeptime_seq);	steal_delta = min_t(u64, kc->idle_stealtime[iowait], delta);	delta -= steal_delta;	kc->idle_stealtime[iowait] -= steal_delta; 	if (iowait)		cpustat[CPUTIME_IOWAIT] += delta;	else		cpustat[CPUTIME_IDLE] += delta; 	kc->idle_stealtime[iowait] += steal;	kc->idle_entrytime = now;	kc->idle_elapse = false;	write_seqcount_end(&kc->idle_sleeptime_seq);} static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now){	/* Irqtime accounting might have been enabled in the middle of the IRQ */	if (kc->idle_elapse)		return; 	write_seqcount_begin(&kc->idle_sleeptime_seq);	kc->idle_entrytime = now;	kc->idle_elapse = true;	write_seqcount_end(&kc->idle_sleeptime_seq);} void kcpustat_dyntick_stop(u64 now){	struct kernel_cpustat *kc = kcpustat_this_cpu; 	if (!vtime_generic_enabled_this_cpu()) {		WARN_ON_ONCE(!kc->idle_dyntick);		kcpustat_idle_stop(kc, now);		kc->idle_dyntick = false;		vtime_dyntick_stop();	}} void kcpustat_dyntick_start(u64 now){	struct kernel_cpustat *kc = kcpustat_this_cpu; 	if (!vtime_generic_enabled_this_cpu()) {		vtime_dyntick_start();		kc->idle_dyntick = true;		kcpustat_idle_start(kc, now);	}} void kcpustat_irq_enter(u64 now){	struct kernel_cpustat *kc = kcpustat_this_cpu; 	if (!vtime_generic_enabled_this_cpu() &&	    (irqtime_enabled() || vtime_accounting_enabled_this_cpu()))		kcpustat_idle_stop(kc, now);

Lines 1–500 of 1302. Load more