History

tracing: Have hwlat ts be first instance and record count of instances

The hwlat tracer runs a loop of width time during a given window. It then reports the max latency over a given threshold and records a timestamp. But this timestamp is the time after the width has finished, and not the time it actually triggered. Record the actual time when the latency was greater than the threshold as well as the number of times it was greater in a given width per window.

Steven Rostedt (VMware) committed 6 years ago

Commit b396bfd1 parent 98d54f84 files · +44 −20Browse files

Unified · Split

4 files
  1. Documentation/trace/ftrace.rst +22 −10
  2. kernel/trace/trace_entries.h +3 −1
  3. kernel/trace/trace_hwlat.c +17 −7
  4. kernel/trace/trace_output.c +2 −2
Documentation/trace/ftrace.rst+22−10
@@ -2126,6 +2126,8 @@ periodically make a CPU constantly busy with interrupts disabled. · 2125 unmodified lines
21262126 # cat trace
21272127 # tracer: hwlat
21282128 #
2129+ # entries-in-buffer/entries-written: 13/13 #P:8
2130+ #
21292131 # _-----=> irqs-off
21302132 # / _----=> need-resched
21312133 # | / _---=> hardirq/softirq
@@ -2133,12 +2135,18 @@ periodically make a CPU constantly busy with interrupts disabled. · 1 unmodified line
21332135 # ||| / delay
21342136 # TASK-PID CPU# |||| TIMESTAMP FUNCTION
21352137 # | | | |||| | |
2136− <...>-3638 [001] d... 19452.055471: #1 inner/outer(us): 12/14 ts:1499801089.066141940
2137− <...>-3638 [003] d... 19454.071354: #2 inner/outer(us): 11/9 ts:1499801091.082164365
2138− <...>-3638 [002] dn.. 19461.126852: #3 inner/outer(us): 12/9 ts:1499801098.138150062
2139− <...>-3638 [001] d... 19488.340960: #4 inner/outer(us): 8/12 ts:1499801125.354139633
2140− <...>-3638 [003] d... 19494.388553: #5 inner/outer(us): 8/12 ts:1499801131.402150961
2141− <...>-3638 [003] d... 19501.283419: #6 inner/outer(us): 0/12 ts:1499801138.297435289 nmi-total:4 nmi-count:1
2138+ <...>-1729 [001] d... 678.473449: #1 inner/outer(us): 11/12 ts:1581527483.343962693 count:6
2139+ <...>-1729 [004] d... 689.556542: #2 inner/outer(us): 16/9 ts:1581527494.889008092 count:1
2140+ <...>-1729 [005] d... 714.756290: #3 inner/outer(us): 16/16 ts:1581527519.678961629 count:5
2141+ <...>-1729 [001] d... 718.788247: #4 inner/outer(us): 9/17 ts:1581527523.889012713 count:1
2142+ <...>-1729 [002] d... 719.796341: #5 inner/outer(us): 13/9 ts:1581527524.912872606 count:1
2143+ <...>-1729 [006] d... 844.787091: #6 inner/outer(us): 9/12 ts:1581527649.889048502 count:2
2144+ <...>-1729 [003] d... 849.827033: #7 inner/outer(us): 18/9 ts:1581527654.889013793 count:1
2145+ <...>-1729 [007] d... 853.859002: #8 inner/outer(us): 9/12 ts:1581527658.889065736 count:1
2146+ <...>-1729 [001] d... 855.874978: #9 inner/outer(us): 9/11 ts:1581527660.861991877 count:1
2147+ <...>-1729 [001] d... 863.938932: #10 inner/outer(us): 9/11 ts:1581527668.970010500 count:1 nmi-total:7 nmi-count:1
2148+ <...>-1729 [007] d... 878.050780: #11 inner/outer(us): 9/12 ts:1581527683.385002600 count:1 nmi-total:5 nmi-count:1
2149+ <...>-1729 [007] d... 886.114702: #12 inner/outer(us): 9/12 ts:1581527691.385001600 count:1
21422150
21432151
21442152 The above output is somewhat the same in the header. All events will have
@@ -2148,7 +2156,7 @@ interrupts disabled 'd'. Under the FUNCTION title there is: · 3 unmodified lines
21482156 This is the count of events recorded that were greater than the
21492157 tracing_threshold (See below).
21502158
2151− inner/outer(us): 12/14
2159+ inner/outer(us): 11/11
21522160
21532161 This shows two numbers as "inner latency" and "outer latency". The test
21542162 runs in a loop checking a timestamp twice. The latency detected within
@@ -2156,11 +2164,15 @@ interrupts disabled 'd'. Under the FUNCTION title there is: · 1 unmodified line
21562164 after the previous timestamp and the next timestamp in the loop is
21572165 the "outer latency".
21582166
2159− ts:1499801089.066141940
2167+ ts:1581527483.343962693
2168+
2169+ The absolute timestamp that the first latency was recorded in the window.
2170+
2171+ count:6
21602172
2161− The absolute timestamp that the event happened.
2173+ The number of times a latency was detected during the window.
21622174
2163− nmi-total:4 nmi-count:1
2175+ nmi-total:7 nmi-count:1
21642176
21652177 On architectures that support it, if an NMI comes in during the
21662178 test, the time spent in NMI is reported in "nmi-total" (in
kernel/trace/trace_entries.h+3−1
@@ -325,14 +325,16 @@ FTRACE_ENTRY(hwlat, hwlat_entry, · 324 unmodified lines
325325 __field_desc( long, timestamp, tv_nsec )
326326 __field( unsigned int, nmi_count )
327327 __field( unsigned int, seqnum )
328+ __field( unsigned int, count )
328329 ),
329330
330− F_printk("cnt:%u\tts:%010llu.%010lu\tinner:%llu\touter:%llu\tnmi-ts:%llu\tnmi-count:%u\n",
331+ F_printk("cnt:%u\tts:%010llu.%010lu\tinner:%llu\touter:%llu\tcount:%d\tnmi-ts:%llu\tnmi-count:%u\n",
331332 __entry->seqnum,
332333 __entry->tv_sec,
333334 __entry->tv_nsec,
334335 __entry->duration,
335336 __entry->outer_duration,
337+ __entry->count,
336338 __entry->nmi_total_ts,
337339 __entry->nmi_count)
338340 );
kernel/trace/trace_hwlat.c+17−7
@@ -83,6 +83,7 @@ struct hwlat_sample { · 82 unmodified lines
8383 u64 nmi_total_ts; /* Total time spent in NMIs */
8484 struct timespec64 timestamp; /* wall time */
8585 int nmi_count; /* # NMIs during this sample */
86+ int count; /* # of iteratons over threash */
8687 };
8788
8889 /* keep the global state somewhere. */
@@ -124,6 +125,7 @@ static void trace_hwlat_sample(struct hwlat_sample *sample) · 35 unmodified lines
124125 entry->timestamp = sample->timestamp;
125126 entry->nmi_total_ts = sample->nmi_total_ts;
126127 entry->nmi_count = sample->nmi_count;
128+ entry->count = sample->count;
127129
128130 if (!call_filter_check_discard(call, entry, buffer, event))
129131 trace_buffer_unlock_commit_nostack(buffer, event);
@@ -167,12 +169,14 @@ void trace_hwlat_callback(bool enter) · 37 unmodified lines
167169 static int get_sample(void)
168170 {
169171 struct trace_array *tr = hwlat_trace;
172+ struct hwlat_sample s;
170173 time_type start, t1, t2, last_t2;
171− s64 diff, total, last_total = 0;
174+ s64 diff, outer_diff, total, last_total = 0;
172175 u64 sample = 0;
173176 u64 thresh = tracing_thresh;
174177 u64 outer_sample = 0;
175178 int ret = -1;
179+ unsigned int count = 0;
176180
177181 do_div(thresh, NSEC_PER_USEC); /* modifies interval value */
178182
@@ -186,6 +190,7 @@ static int get_sample(void) · 7 unmodified lines
186190
187191 init_time(last_t2, 0);
188192 start = time_get(); /* start timestamp */
193+ outer_diff = 0;
189194
190195 do {
191196
@@ -194,14 +199,14 @@ static int get_sample(void) · 2 unmodified lines
194199
195200 if (time_u64(last_t2)) {
196201 /* Check the delta from outer loop (t2 to next t1) */
197− diff = time_to_us(time_sub(t1, last_t2));
202+ outer_diff = time_to_us(time_sub(t1, last_t2));
198203 /* This shouldn't happen */
199− if (diff < 0) {
204+ if (outer_diff < 0) {
200205 pr_err(BANNER "time running backwards\n");
201206 goto out;
202207 }
203− if (diff > outer_sample)
204− outer_sample = diff;
208+ if (outer_diff > outer_sample)
209+ outer_sample = outer_diff;
205210 }
206211 last_t2 = t2;
207212
@@ -217,6 +222,12 @@ static int get_sample(void) · 9 unmodified lines
217222 /* This checks the inner loop (t1 to t2) */
218223 diff = time_to_us(time_sub(t2, t1)); /* current diff */
219224
225+ if (diff > thresh || outer_diff > thresh) {
226+ if (!count)
227+ ktime_get_real_ts64(&s.timestamp);
228+ count++;
229+ }
230+
220231 /* This shouldn't happen */
221232 if (diff < 0) {
222233 pr_err(BANNER "time running backwards\n");
@@ -236,7 +247,6 @@ static int get_sample(void) · 13 unmodified lines
236247
237248 /* If we exceed the threshold value, we have found a hardware latency */
238249 if (sample > thresh || outer_sample > thresh) {
239− struct hwlat_sample s;
240250 u64 latency;
241251
242252 ret = 1;
@@ -249,9 +259,9 @@ static int get_sample(void) · 6 unmodified lines
249259 s.seqnum = hwlat_data.count;
250260 s.duration = sample;
251261 s.outer_duration = outer_sample;
252− ktime_get_real_ts64(&s.timestamp);
253262 s.nmi_total_ts = nmi_total_ts;
254263 s.nmi_count = nmi_count;
264+ s.count = count;
255265 trace_hwlat_sample(&s);
256266
257267 latency = max(sample, outer_sample);
kernel/trace/trace_output.c+2−2
@@ -1158,12 +1158,12 @@ trace_hwlat_print(struct trace_iterator *iter, int flags, · 1157 unmodified lines
11581158
11591159 trace_assign_type(field, entry);
11601160
1161− trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%lld.%09ld",
1161+ trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%lld.%09ld count:%d",
11621162 field->seqnum,
11631163 field->duration,
11641164 field->outer_duration,
11651165 (long long)field->timestamp.tv_sec,
1166− field->timestamp.tv_nsec);
1166+ field->timestamp.tv_nsec, field->count);
11671167
11681168 if (field->nmi_count) {
11691169 /*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t