History

sched/vtime: Consolidate IRQ time accounting

The 3 architectures implementing CONFIG_VIRT_CPU_ACCOUNTING_NATIVE all have their own version of irq time accounting that dispatch the cputime to the appropriate index: hardirq, softirq, system, idle, guest... from an all-in-one function. Instead of having these ad-hoc versions, move the cputime destination dispatch decision to the core code and leave only the actual per-index cputime accounting to the architecture.

Frederic Weisbecker committed 5 years ago

Commit 8a6a5921 parent 2b91ec95 files · +102 −48Browse files

Unified · Split

5 files
  1. arch/ia64/kernel/time.c +15 −5
  2. arch/powerpc/kernel/time.c +40 −16
  3. arch/s390/kernel/vtime.c +32 −13
  4. include/linux/vtime.h +6 −10
  5. kernel/sched/cputime.c +9 −4
arch/ia64/kernel/time.c+15−5
@@ -138,12 +138,8 @@ void vtime_account_kernel(struct task_struct *tsk) · 137 unmodified lines
138138 struct thread_info *ti = task_thread_info(tsk);
139139 __u64 stime = vtime_delta(tsk);
140140
141− if ((tsk->flags & PF_VCPU) && !irq_count())
141+ if (tsk->flags & PF_VCPU)
142142 ti->gtime += stime;
143− else if (hardirq_count())
144− ti->hardirq_time += stime;
145− else if (in_serving_softirq())
146− ti->softirq_time += stime;
147143 else
148144 ti->stime += stime;
149145 }
@@ -156,6 +152,20 @@ void vtime_account_idle(struct task_struct *tsk) · 6 unmodified lines
156152 ti->idle_time += vtime_delta(tsk);
157153 }
158154
155+void vtime_account_softirq(struct task_struct *tsk)
156+{
157+ struct thread_info *ti = task_thread_info(tsk);
158+
159+ ti->softirq_time += vtime_delta(tsk);
160+}
161+
162+void vtime_account_hardirq(struct task_struct *tsk)
163+{
164+ struct thread_info *ti = task_thread_info(tsk);
165+
166+ ti->hardirq_time += vtime_delta(tsk);
167+}
168+
159169 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
160170
161171 static irqreturn_t
arch/powerpc/kernel/time.c+40−16
@@ -311,12 +311,11 @@ static unsigned long vtime_delta_scaled(struct cpu_accounting_data *acct, · 310 unmodified lines
311311 return stime_scaled;
312312 }
313313
314−static unsigned long vtime_delta(struct task_struct *tsk,
314+static unsigned long vtime_delta(struct cpu_accounting_data *acct,
315315 unsigned long *stime_scaled,
316316 unsigned long *steal_time)
317317 {
318318 unsigned long now, stime;
319− struct cpu_accounting_data *acct = get_accounting(tsk);
320319
321320 WARN_ON_ONCE(!irqs_disabled());
322321
@@ -331,29 +330,30 @@ static unsigned long vtime_delta(struct task_struct *tsk, · 8 unmodified lines
331330 return stime;
332331 }
333332
333+static void vtime_delta_kernel(struct cpu_accounting_data *acct,
334+ unsigned long *stime, unsigned long *stime_scaled)
335+{
336+ unsigned long steal_time;
337+
338+ *stime = vtime_delta(acct, stime_scaled, &steal_time);
339+ *stime -= min(*stime, steal_time);
340+ acct->steal_time += steal_time;
341+}
342+
334343 void vtime_account_kernel(struct task_struct *tsk)
335344 {
336− unsigned long stime, stime_scaled, steal_time;
337345 struct cpu_accounting_data *acct = get_accounting(tsk);
346+ unsigned long stime, stime_scaled;
338347
339− stime = vtime_delta(tsk, &stime_scaled, &steal_time);
340−
341− stime -= min(stime, steal_time);
342− acct->steal_time += steal_time;
348+ vtime_delta_kernel(acct, &stime, &stime_scaled);
343349
344− if ((tsk->flags & PF_VCPU) && !irq_count()) {
350+ if (tsk->flags & PF_VCPU) {
345351 acct->gtime += stime;
346352 #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
347353 acct->utime_scaled += stime_scaled;
348354 #endif
349355 } else {
350− if (hardirq_count())
351− acct->hardirq_time += stime;
352− else if (in_serving_softirq())
353− acct->softirq_time += stime;
354− else
355− acct->stime += stime;
356−
356+ acct->stime += stime;
357357 #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
358358 acct->stime_scaled += stime_scaled;
359359 #endif
@@ -366,10 +366,34 @@ void vtime_account_idle(struct task_struct *tsk) · 6 unmodified lines
366366 unsigned long stime, stime_scaled, steal_time;
367367 struct cpu_accounting_data *acct = get_accounting(tsk);
368368
369− stime = vtime_delta(tsk, &stime_scaled, &steal_time);
369+ stime = vtime_delta(acct, &stime_scaled, &steal_time);
370370 acct->idle_time += stime + steal_time;
371371 }
372372
373+static void vtime_account_irq_field(struct cpu_accounting_data *acct,
374+ unsigned long *field)
375+{
376+ unsigned long stime, stime_scaled;
377+
378+ vtime_delta_kernel(acct, &stime, &stime_scaled);
379+ *field += stime;
380+#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
381+ acct->stime_scaled += stime_scaled;
382+#endif
383+}
384+
385+void vtime_account_softirq(struct task_struct *tsk)
386+{
387+ struct cpu_accounting_data *acct = get_accounting(tsk);
388+ vtime_account_irq_field(acct, &acct->softirq_time);
389+}
390+
391+void vtime_account_hardirq(struct task_struct *tsk)
392+{
393+ struct cpu_accounting_data *acct = get_accounting(tsk);
394+ vtime_account_irq_field(acct, &acct->hardirq_time);
395+}
396+
373397 static void vtime_flush_scaled(struct task_struct *tsk,
374398 struct cpu_accounting_data *acct)
375399 {
arch/s390/kernel/vtime.c+32−13
@@ -222,31 +222,50 @@ void vtime_flush(struct task_struct *tsk) · 221 unmodified lines
222222 S390_lowcore.avg_steal_timer = avg_steal;
223223 }
224224
225+static u64 vtime_delta(void)
226+{
227+ u64 timer = S390_lowcore.last_update_timer;
228+
229+ S390_lowcore.last_update_timer = get_vtimer();
230+
231+ return timer - S390_lowcore.last_update_timer;
232+}
233+
225234 /*
226235 * Update process times based on virtual cpu times stored by entry.S
227236 * to the lowcore fields user_timer, system_timer & steal_clock.
228237 */
229238 void vtime_account_kernel(struct task_struct *tsk)
230239 {
231− u64 timer;
232−
233− timer = S390_lowcore.last_update_timer;
234− S390_lowcore.last_update_timer = get_vtimer();
235− timer -= S390_lowcore.last_update_timer;
240+ u64 delta = vtime_delta();
236241
237− if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
238− S390_lowcore.guest_timer += timer;
239− else if (hardirq_count())
240− S390_lowcore.hardirq_timer += timer;
241− else if (in_serving_softirq())
242− S390_lowcore.softirq_timer += timer;
242+ if (tsk->flags & PF_VCPU)
243+ S390_lowcore.guest_timer += delta;
243244 else
244− S390_lowcore.system_timer += timer;
245+ S390_lowcore.system_timer += delta;
245246
246− virt_timer_forward(timer);
247+ virt_timer_forward(delta);
247248 }
248249 EXPORT_SYMBOL_GPL(vtime_account_kernel);
249250
251+void vtime_account_softirq(struct task_struct *tsk)
252+{
253+ u64 delta = vtime_delta();
254+
255+ S390_lowcore.softirq_timer += delta;
256+
257+ virt_timer_forward(delta);
258+}
259+
260+void vtime_account_hardirq(struct task_struct *tsk)
261+{
262+ u64 delta = vtime_delta();
263+
264+ S390_lowcore.hardirq_timer += delta;
265+
266+ virt_timer_forward(delta);
267+}
268+
250269 /*
251270 * Sorted add to a list. List is linear searched until first bigger
252271 * element is found.
include/linux/vtime.h+6−10
@@ -83,16 +83,12 @@ static inline void vtime_init_idle(struct task_struct *tsk, int cpu) { } · 82 unmodified lines
8383 #endif
8484
8585 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
86−extern void vtime_account_irq_enter(struct task_struct *tsk);
87−static inline void vtime_account_irq_exit(struct task_struct *tsk)
88−{
89− /* On hard|softirq exit we always account to hard|softirq cputime */
90− vtime_account_kernel(tsk);
91−}
86+extern void vtime_account_irq(struct task_struct *tsk);
87+extern void vtime_account_softirq(struct task_struct *tsk);
88+extern void vtime_account_hardirq(struct task_struct *tsk);
9289 extern void vtime_flush(struct task_struct *tsk);
9390 #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
94−static inline void vtime_account_irq_enter(struct task_struct *tsk) { }
95−static inline void vtime_account_irq_exit(struct task_struct *tsk) { }
91+static inline void vtime_account_irq(struct task_struct *tsk) { }
9692 static inline void vtime_flush(struct task_struct *tsk) { }
9793 #endif
9894
@@ -105,13 +101,13 @@ static inline void irqtime_account_irq(struct task_struct *tsk) { } · 6 unmodified lines
105101
106102 static inline void account_irq_enter_time(struct task_struct *tsk)
107103 {
108− vtime_account_irq_enter(tsk);
104+ vtime_account_irq(tsk);
109105 irqtime_account_irq(tsk);
110106 }
111107
112108 static inline void account_irq_exit_time(struct task_struct *tsk)
113109 {
114− vtime_account_irq_exit(tsk);
110+ vtime_account_irq(tsk);
115111 irqtime_account_irq(tsk);
116112 }
117113
kernel/sched/cputime.c+9−4
@@ -417,13 +417,18 @@ void vtime_task_switch(struct task_struct *prev) · 416 unmodified lines
417417 }
418418 # endif
419419
420−void vtime_account_irq_enter(struct task_struct *tsk)
420+void vtime_account_irq(struct task_struct *tsk)
421421 {
422− if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
423− !in_interrupt() && is_idle_task(tsk))
422+ if (hardirq_count()) {
423+ vtime_account_hardirq(tsk);
424+ } else if (in_serving_softirq()) {
425+ vtime_account_softirq(tsk);
426+ } else if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
427+ is_idle_task(tsk)) {
424428 vtime_account_idle(tsk);
425− else
429+ } else {
426430 vtime_account_kernel(tsk);
431+ }
427432 }
428433
429434 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t