History

sched/vtime: Consolidate IRQ time accounting

The 3 architectures implementing CONFIG_VIRT_CPU_ACCOUNTING_NATIVE all have their own version of irq time accounting that dispatch the cputime to the appropriate index: hardirq, softirq, system, idle, guest... from an all-in-one function. Instead of having these ad-hoc versions, move the cputime destination dispatch decision to the core code and leave only the actual per-index cputime accounting to the architecture.

Frederic Weisbecker committed 5 years ago

Commit 8a6a5921 parent 2b91ec95 files · +102 −48Browse files

Unified · Split

5 files
  1. arch/ia64/kernel/time.c +15 −5
  2. arch/powerpc/kernel/time.c +40 −16
  3. arch/s390/kernel/vtime.c +32 −13
  4. include/linux/vtime.h +6 −10
  5. kernel/sched/cputime.c +9 −4
arch/ia64/kernel/time.c+15−5
@@ -138,12 +138,8 @@ void vtime_account_kernel(struct task_struct *tsk) · 137 unmodified lines
138 struct thread_info *ti = task_thread_info(tsk);138 struct thread_info *ti = task_thread_info(tsk);
139 __u64 stime = vtime_delta(tsk);139 __u64 stime = vtime_delta(tsk);
140 140
141 if ((tsk->flags & PF_VCPU) && !irq_count())141 if (tsk->flags & PF_VCPU)
142 ti->gtime += stime;142 ti->gtime += stime;
143 else if (hardirq_count())
144 ti->hardirq_time += stime;
145 else if (in_serving_softirq())
146 ti->softirq_time += stime;
147 else143 else
148 ti->stime += stime;144 ti->stime += stime;
149}145}
@@ -156,6 +152,20 @@ void vtime_account_idle(struct task_struct *tsk) · 6 unmodified lines
156 ti->idle_time += vtime_delta(tsk);152 ti->idle_time += vtime_delta(tsk);
157}153}
158 154
155void vtime_account_softirq(struct task_struct *tsk)
156{
157 struct thread_info *ti = task_thread_info(tsk);
158
159 ti->softirq_time += vtime_delta(tsk);
160}
161
162void vtime_account_hardirq(struct task_struct *tsk)
163{
164 struct thread_info *ti = task_thread_info(tsk);
165
166 ti->hardirq_time += vtime_delta(tsk);
167}
168
159#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */169#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
160 170
161static irqreturn_t171static irqreturn_t
arch/powerpc/kernel/time.c+40−16
@@ -311,12 +311,11 @@ static unsigned long vtime_delta_scaled(struct cpu_accounting_data *acct, · 310 unmodified lines
311 return stime_scaled;311 return stime_scaled;
312}312}
313 313
314static unsigned long vtime_delta(struct task_struct *tsk,314static unsigned long vtime_delta(struct cpu_accounting_data *acct,
315 unsigned long *stime_scaled,315 unsigned long *stime_scaled,
316 unsigned long *steal_time)316 unsigned long *steal_time)
317{317{
318 unsigned long now, stime;318 unsigned long now, stime;
319 struct cpu_accounting_data *acct = get_accounting(tsk);
320 319
321 WARN_ON_ONCE(!irqs_disabled());320 WARN_ON_ONCE(!irqs_disabled());
322 321
@@ -331,29 +330,30 @@ static unsigned long vtime_delta(struct task_struct *tsk, · 8 unmodified lines
331 return stime;330 return stime;
332}331}
333 332
333static void vtime_delta_kernel(struct cpu_accounting_data *acct,
334 unsigned long *stime, unsigned long *stime_scaled)
335{
336 unsigned long steal_time;
337
338 *stime = vtime_delta(acct, stime_scaled, &steal_time);
339 *stime -= min(*stime, steal_time);
340 acct->steal_time += steal_time;
341}
342
334void vtime_account_kernel(struct task_struct *tsk)343void vtime_account_kernel(struct task_struct *tsk)
335{344{
336 unsigned long stime, stime_scaled, steal_time;
337 struct cpu_accounting_data *acct = get_accounting(tsk);345 struct cpu_accounting_data *acct = get_accounting(tsk);
346 unsigned long stime, stime_scaled;
338 347
339 stime = vtime_delta(tsk, &stime_scaled, &steal_time);348 vtime_delta_kernel(acct, &stime, &stime_scaled);
340
341 stime -= min(stime, steal_time);
342 acct->steal_time += steal_time;
343 349
344 if ((tsk->flags & PF_VCPU) && !irq_count()) {350 if (tsk->flags & PF_VCPU) {
345 acct->gtime += stime;351 acct->gtime += stime;
346#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME352#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
347 acct->utime_scaled += stime_scaled;353 acct->utime_scaled += stime_scaled;
348#endif354#endif
349 } else {355 } else {
350 if (hardirq_count())356 acct->stime += stime;
351 acct->hardirq_time += stime;
352 else if (in_serving_softirq())
353 acct->softirq_time += stime;
354 else
355 acct->stime += stime;
356
357#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME357#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
358 acct->stime_scaled += stime_scaled;358 acct->stime_scaled += stime_scaled;
359#endif359#endif
@@ -366,10 +366,34 @@ void vtime_account_idle(struct task_struct *tsk) · 6 unmodified lines
366 unsigned long stime, stime_scaled, steal_time;366 unsigned long stime, stime_scaled, steal_time;
367 struct cpu_accounting_data *acct = get_accounting(tsk);367 struct cpu_accounting_data *acct = get_accounting(tsk);
368 368
369 stime = vtime_delta(tsk, &stime_scaled, &steal_time);369 stime = vtime_delta(acct, &stime_scaled, &steal_time);
370 acct->idle_time += stime + steal_time;370 acct->idle_time += stime + steal_time;
371}371}
372 372
373static void vtime_account_irq_field(struct cpu_accounting_data *acct,
374 unsigned long *field)
375{
376 unsigned long stime, stime_scaled;
377
378 vtime_delta_kernel(acct, &stime, &stime_scaled);
379 *field += stime;
380#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
381 acct->stime_scaled += stime_scaled;
382#endif
383}
384
385void vtime_account_softirq(struct task_struct *tsk)
386{
387 struct cpu_accounting_data *acct = get_accounting(tsk);
388 vtime_account_irq_field(acct, &acct->softirq_time);
389}
390
391void vtime_account_hardirq(struct task_struct *tsk)
392{
393 struct cpu_accounting_data *acct = get_accounting(tsk);
394 vtime_account_irq_field(acct, &acct->hardirq_time);
395}
396
373static void vtime_flush_scaled(struct task_struct *tsk,397static void vtime_flush_scaled(struct task_struct *tsk,
374 struct cpu_accounting_data *acct)398 struct cpu_accounting_data *acct)
375{399{
arch/s390/kernel/vtime.c+32−13
@@ -222,31 +222,50 @@ void vtime_flush(struct task_struct *tsk) · 221 unmodified lines
222 S390_lowcore.avg_steal_timer = avg_steal;222 S390_lowcore.avg_steal_timer = avg_steal;
223}223}
224 224
225static u64 vtime_delta(void)
226{
227 u64 timer = S390_lowcore.last_update_timer;
228
229 S390_lowcore.last_update_timer = get_vtimer();
230
231 return timer - S390_lowcore.last_update_timer;
232}
233
225/*234/*
226 * Update process times based on virtual cpu times stored by entry.S235 * Update process times based on virtual cpu times stored by entry.S
227 * to the lowcore fields user_timer, system_timer & steal_clock.236 * to the lowcore fields user_timer, system_timer & steal_clock.
228 */237 */
229void vtime_account_kernel(struct task_struct *tsk)238void vtime_account_kernel(struct task_struct *tsk)
230{239{
231 u64 timer;240 u64 delta = vtime_delta();
232
233 timer = S390_lowcore.last_update_timer;
234 S390_lowcore.last_update_timer = get_vtimer();
235 timer -= S390_lowcore.last_update_timer;
236 241
237 if ((tsk->flags & PF_VCPU) && (irq_count() == 0))242 if (tsk->flags & PF_VCPU)
238 S390_lowcore.guest_timer += timer;243 S390_lowcore.guest_timer += delta;
239 else if (hardirq_count())
240 S390_lowcore.hardirq_timer += timer;
241 else if (in_serving_softirq())
242 S390_lowcore.softirq_timer += timer;
243 else244 else
244 S390_lowcore.system_timer += timer;245 S390_lowcore.system_timer += delta;
245 246
246 virt_timer_forward(timer);247 virt_timer_forward(delta);
247}248}
248EXPORT_SYMBOL_GPL(vtime_account_kernel);249EXPORT_SYMBOL_GPL(vtime_account_kernel);
249 250
251void vtime_account_softirq(struct task_struct *tsk)
252{
253 u64 delta = vtime_delta();
254
255 S390_lowcore.softirq_timer += delta;
256
257 virt_timer_forward(delta);
258}
259
260void vtime_account_hardirq(struct task_struct *tsk)
261{
262 u64 delta = vtime_delta();
263
264 S390_lowcore.hardirq_timer += delta;
265
266 virt_timer_forward(delta);
267}
268
250/*269/*
251 * Sorted add to a list. List is linear searched until first bigger270 * Sorted add to a list. List is linear searched until first bigger
252 * element is found.271 * element is found.
include/linux/vtime.h+6−10
@@ -83,16 +83,12 @@ static inline void vtime_init_idle(struct task_struct *tsk, int cpu) { } · 82 unmodified lines
83#endif83#endif
84 84
85#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE85#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
86extern void vtime_account_irq_enter(struct task_struct *tsk);86extern void vtime_account_irq(struct task_struct *tsk);
87static inline void vtime_account_irq_exit(struct task_struct *tsk)87extern void vtime_account_softirq(struct task_struct *tsk);
88{88extern void vtime_account_hardirq(struct task_struct *tsk);
89 /* On hard|softirq exit we always account to hard|softirq cputime */
90 vtime_account_kernel(tsk);
91}
92extern void vtime_flush(struct task_struct *tsk);89extern void vtime_flush(struct task_struct *tsk);
93#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */90#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
94static inline void vtime_account_irq_enter(struct task_struct *tsk) { }91static inline void vtime_account_irq(struct task_struct *tsk) { }
95static inline void vtime_account_irq_exit(struct task_struct *tsk) { }
96static inline void vtime_flush(struct task_struct *tsk) { }92static inline void vtime_flush(struct task_struct *tsk) { }
97#endif93#endif
98 94
@@ -105,13 +101,13 @@ static inline void irqtime_account_irq(struct task_struct *tsk) { } · 6 unmodified lines
105 101
106static inline void account_irq_enter_time(struct task_struct *tsk)102static inline void account_irq_enter_time(struct task_struct *tsk)
107{103{
108 vtime_account_irq_enter(tsk);104 vtime_account_irq(tsk);
109 irqtime_account_irq(tsk);105 irqtime_account_irq(tsk);
110}106}
111 107
112static inline void account_irq_exit_time(struct task_struct *tsk)108static inline void account_irq_exit_time(struct task_struct *tsk)
113{109{
114 vtime_account_irq_exit(tsk);110 vtime_account_irq(tsk);
115 irqtime_account_irq(tsk);111 irqtime_account_irq(tsk);
116}112}
117 113
kernel/sched/cputime.c+9−4
@@ -417,13 +417,18 @@ void vtime_task_switch(struct task_struct *prev) · 416 unmodified lines
417}417}
418# endif418# endif
419 419
420void vtime_account_irq_enter(struct task_struct *tsk)420void vtime_account_irq(struct task_struct *tsk)
421{421{
422 if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&422 if (hardirq_count()) {
423 !in_interrupt() && is_idle_task(tsk))423 vtime_account_hardirq(tsk);
424 } else if (in_serving_softirq()) {
425 vtime_account_softirq(tsk);
426 } else if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
427 is_idle_task(tsk)) {
424 vtime_account_idle(tsk);428 vtime_account_idle(tsk);
425 else429 } else {
426 vtime_account_kernel(tsk);430 vtime_account_kernel(tsk);
431 }
427}432}
428 433
429void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,434void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t