History

tracing: Have hwlat ts be first instance and record count of instances

The hwlat tracer runs a loop of width time during a given window. It then reports the max latency over a given threshold and records a timestamp. But this timestamp is the time after the width has finished, and not the time it actually triggered. Record the actual time when the latency was greater than the threshold as well as the number of times it was greater in a given width per window.

Steven Rostedt (VMware) committed 6 years ago

Commit b396bfd1 parent 98d54f84 files · +44 −20Browse files

Unified · Split

4 files
  1. Documentation/trace/ftrace.rst +22 −10
  2. kernel/trace/trace_entries.h +3 −1
  3. kernel/trace/trace_hwlat.c +17 −7
  4. kernel/trace/trace_output.c +2 −2
Documentation/trace/ftrace.rst+22−10
@@ -2126,6 +2126,8 @@ periodically make a CPU constantly busy with interrupts disabled. · 2125 unmodified lines
2126 # cat trace2126 # cat trace
2127 # tracer: hwlat2127 # tracer: hwlat
2128 #2128 #
2129 # entries-in-buffer/entries-written: 13/13 #P:8
2130 #
2129 # _-----=> irqs-off2131 # _-----=> irqs-off
2130 # / _----=> need-resched2132 # / _----=> need-resched
2131 # | / _---=> hardirq/softirq2133 # | / _---=> hardirq/softirq
@@ -2133,12 +2135,18 @@ periodically make a CPU constantly busy with interrupts disabled. · 1 unmodified line
2133 # ||| / delay2135 # ||| / delay
2134 # TASK-PID CPU# |||| TIMESTAMP FUNCTION2136 # TASK-PID CPU# |||| TIMESTAMP FUNCTION
2135 # | | | |||| | |2137 # | | | |||| | |
2136 <...>-3638 [001] d... 19452.055471: #1 inner/outer(us): 12/14 ts:1499801089.0661419402138 <...>-1729 [001] d... 678.473449: #1 inner/outer(us): 11/12 ts:1581527483.343962693 count:6
2137 <...>-3638 [003] d... 19454.071354: #2 inner/outer(us): 11/9 ts:1499801091.0821643652139 <...>-1729 [004] d... 689.556542: #2 inner/outer(us): 16/9 ts:1581527494.889008092 count:1
2138 <...>-3638 [002] dn.. 19461.126852: #3 inner/outer(us): 12/9 ts:1499801098.1381500622140 <...>-1729 [005] d... 714.756290: #3 inner/outer(us): 16/16 ts:1581527519.678961629 count:5
2139 <...>-3638 [001] d... 19488.340960: #4 inner/outer(us): 8/12 ts:1499801125.3541396332141 <...>-1729 [001] d... 718.788247: #4 inner/outer(us): 9/17 ts:1581527523.889012713 count:1
2140 <...>-3638 [003] d... 19494.388553: #5 inner/outer(us): 8/12 ts:1499801131.4021509612142 <...>-1729 [002] d... 719.796341: #5 inner/outer(us): 13/9 ts:1581527524.912872606 count:1
2141 <...>-3638 [003] d... 19501.283419: #6 inner/outer(us): 0/12 ts:1499801138.297435289 nmi-total:4 nmi-count:12143 <...>-1729 [006] d... 844.787091: #6 inner/outer(us): 9/12 ts:1581527649.889048502 count:2
2144 <...>-1729 [003] d... 849.827033: #7 inner/outer(us): 18/9 ts:1581527654.889013793 count:1
2145 <...>-1729 [007] d... 853.859002: #8 inner/outer(us): 9/12 ts:1581527658.889065736 count:1
2146 <...>-1729 [001] d... 855.874978: #9 inner/outer(us): 9/11 ts:1581527660.861991877 count:1
2147 <...>-1729 [001] d... 863.938932: #10 inner/outer(us): 9/11 ts:1581527668.970010500 count:1 nmi-total:7 nmi-count:1
2148 <...>-1729 [007] d... 878.050780: #11 inner/outer(us): 9/12 ts:1581527683.385002600 count:1 nmi-total:5 nmi-count:1
2149 <...>-1729 [007] d... 886.114702: #12 inner/outer(us): 9/12 ts:1581527691.385001600 count:1
2142 2150
2143 2151
2144The above output is somewhat the same in the header. All events will have2152The above output is somewhat the same in the header. All events will have
@@ -2148,7 +2156,7 @@ interrupts disabled 'd'. Under the FUNCTION title there is: · 3 unmodified lines
2148 This is the count of events recorded that were greater than the2156 This is the count of events recorded that were greater than the
2149 tracing_threshold (See below).2157 tracing_threshold (See below).
2150 2158
2151 inner/outer(us): 12/142159 inner/outer(us): 11/11
2152 2160
2153 This shows two numbers as "inner latency" and "outer latency". The test2161 This shows two numbers as "inner latency" and "outer latency". The test
2154 runs in a loop checking a timestamp twice. The latency detected within2162 runs in a loop checking a timestamp twice. The latency detected within
@@ -2156,11 +2164,15 @@ interrupts disabled 'd'. Under the FUNCTION title there is: · 1 unmodified line
2156 after the previous timestamp and the next timestamp in the loop is2164 after the previous timestamp and the next timestamp in the loop is
2157 the "outer latency".2165 the "outer latency".
2158 2166
2159 ts:1499801089.0661419402167 ts:1581527483.343962693
2168
2169 The absolute timestamp that the first latency was recorded in the window.
2170
2171 count:6
2160 2172
2161 The absolute timestamp that the event happened.2173 The number of times a latency was detected during the window.
2162 2174
2163 nmi-total:4 nmi-count:12175 nmi-total:7 nmi-count:1
2164 2176
2165 On architectures that support it, if an NMI comes in during the2177 On architectures that support it, if an NMI comes in during the
2166 test, the time spent in NMI is reported in "nmi-total" (in2178 test, the time spent in NMI is reported in "nmi-total" (in
kernel/trace/trace_entries.h+3−1
@@ -325,14 +325,16 @@ FTRACE_ENTRY(hwlat, hwlat_entry, · 324 unmodified lines
325 __field_desc( long, timestamp, tv_nsec )325 __field_desc( long, timestamp, tv_nsec )
326 __field( unsigned int, nmi_count )326 __field( unsigned int, nmi_count )
327 __field( unsigned int, seqnum )327 __field( unsigned int, seqnum )
328 __field( unsigned int, count )
328 ),329 ),
329 330
330 F_printk("cnt:%u\tts:%010llu.%010lu\tinner:%llu\touter:%llu\tnmi-ts:%llu\tnmi-count:%u\n",331 F_printk("cnt:%u\tts:%010llu.%010lu\tinner:%llu\touter:%llu\tcount:%d\tnmi-ts:%llu\tnmi-count:%u\n",
331 __entry->seqnum,332 __entry->seqnum,
332 __entry->tv_sec,333 __entry->tv_sec,
333 __entry->tv_nsec,334 __entry->tv_nsec,
334 __entry->duration,335 __entry->duration,
335 __entry->outer_duration,336 __entry->outer_duration,
337 __entry->count,
336 __entry->nmi_total_ts,338 __entry->nmi_total_ts,
337 __entry->nmi_count)339 __entry->nmi_count)
338);340);
kernel/trace/trace_hwlat.c+17−7
@@ -83,6 +83,7 @@ struct hwlat_sample { · 82 unmodified lines
83 u64 nmi_total_ts; /* Total time spent in NMIs */83 u64 nmi_total_ts; /* Total time spent in NMIs */
84 struct timespec64 timestamp; /* wall time */84 struct timespec64 timestamp; /* wall time */
85 int nmi_count; /* # NMIs during this sample */85 int nmi_count; /* # NMIs during this sample */
86 int count; /* # of iteratons over threash */
86};87};
87 88
88/* keep the global state somewhere. */89/* keep the global state somewhere. */
@@ -124,6 +125,7 @@ static void trace_hwlat_sample(struct hwlat_sample *sample) · 35 unmodified lines
124 entry->timestamp = sample->timestamp;125 entry->timestamp = sample->timestamp;
125 entry->nmi_total_ts = sample->nmi_total_ts;126 entry->nmi_total_ts = sample->nmi_total_ts;
126 entry->nmi_count = sample->nmi_count;127 entry->nmi_count = sample->nmi_count;
128 entry->count = sample->count;
127 129
128 if (!call_filter_check_discard(call, entry, buffer, event))130 if (!call_filter_check_discard(call, entry, buffer, event))
129 trace_buffer_unlock_commit_nostack(buffer, event);131 trace_buffer_unlock_commit_nostack(buffer, event);
@@ -167,12 +169,14 @@ void trace_hwlat_callback(bool enter) · 37 unmodified lines
167static int get_sample(void)169static int get_sample(void)
168{170{
169 struct trace_array *tr = hwlat_trace;171 struct trace_array *tr = hwlat_trace;
172 struct hwlat_sample s;
170 time_type start, t1, t2, last_t2;173 time_type start, t1, t2, last_t2;
171 s64 diff, total, last_total = 0;174 s64 diff, outer_diff, total, last_total = 0;
172 u64 sample = 0;175 u64 sample = 0;
173 u64 thresh = tracing_thresh;176 u64 thresh = tracing_thresh;
174 u64 outer_sample = 0;177 u64 outer_sample = 0;
175 int ret = -1;178 int ret = -1;
179 unsigned int count = 0;
176 180
177 do_div(thresh, NSEC_PER_USEC); /* modifies interval value */181 do_div(thresh, NSEC_PER_USEC); /* modifies interval value */
178 182
@@ -186,6 +190,7 @@ static int get_sample(void) · 7 unmodified lines
186 190
187 init_time(last_t2, 0);191 init_time(last_t2, 0);
188 start = time_get(); /* start timestamp */192 start = time_get(); /* start timestamp */
193 outer_diff = 0;
189 194
190 do {195 do {
191 196
@@ -194,14 +199,14 @@ static int get_sample(void) · 2 unmodified lines
194 199
195 if (time_u64(last_t2)) {200 if (time_u64(last_t2)) {
196 /* Check the delta from outer loop (t2 to next t1) */201 /* Check the delta from outer loop (t2 to next t1) */
197 diff = time_to_us(time_sub(t1, last_t2));202 outer_diff = time_to_us(time_sub(t1, last_t2));
198 /* This shouldn't happen */203 /* This shouldn't happen */
199 if (diff < 0) {204 if (outer_diff < 0) {
200 pr_err(BANNER "time running backwards\n");205 pr_err(BANNER "time running backwards\n");
201 goto out;206 goto out;
202 }207 }
203 if (diff > outer_sample)208 if (outer_diff > outer_sample)
204 outer_sample = diff;209 outer_sample = outer_diff;
205 }210 }
206 last_t2 = t2;211 last_t2 = t2;
207 212
@@ -217,6 +222,12 @@ static int get_sample(void) · 9 unmodified lines
217 /* This checks the inner loop (t1 to t2) */222 /* This checks the inner loop (t1 to t2) */
218 diff = time_to_us(time_sub(t2, t1)); /* current diff */223 diff = time_to_us(time_sub(t2, t1)); /* current diff */
219 224
225 if (diff > thresh || outer_diff > thresh) {
226 if (!count)
227 ktime_get_real_ts64(&s.timestamp);
228 count++;
229 }
230
220 /* This shouldn't happen */231 /* This shouldn't happen */
221 if (diff < 0) {232 if (diff < 0) {
222 pr_err(BANNER "time running backwards\n");233 pr_err(BANNER "time running backwards\n");
@@ -236,7 +247,6 @@ static int get_sample(void) · 13 unmodified lines
236 247
237 /* If we exceed the threshold value, we have found a hardware latency */248 /* If we exceed the threshold value, we have found a hardware latency */
238 if (sample > thresh || outer_sample > thresh) {249 if (sample > thresh || outer_sample > thresh) {
239 struct hwlat_sample s;
240 u64 latency;250 u64 latency;
241 251
242 ret = 1;252 ret = 1;
@@ -249,9 +259,9 @@ static int get_sample(void) · 6 unmodified lines
249 s.seqnum = hwlat_data.count;259 s.seqnum = hwlat_data.count;
250 s.duration = sample;260 s.duration = sample;
251 s.outer_duration = outer_sample;261 s.outer_duration = outer_sample;
252 ktime_get_real_ts64(&s.timestamp);
253 s.nmi_total_ts = nmi_total_ts;262 s.nmi_total_ts = nmi_total_ts;
254 s.nmi_count = nmi_count;263 s.nmi_count = nmi_count;
264 s.count = count;
255 trace_hwlat_sample(&s);265 trace_hwlat_sample(&s);
256 266
257 latency = max(sample, outer_sample);267 latency = max(sample, outer_sample);
kernel/trace/trace_output.c+2−2
@@ -1158,12 +1158,12 @@ trace_hwlat_print(struct trace_iterator *iter, int flags, · 1157 unmodified lines
1158 1158
1159 trace_assign_type(field, entry);1159 trace_assign_type(field, entry);
1160 1160
1161 trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%lld.%09ld",1161 trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%lld.%09ld count:%d",
1162 field->seqnum,1162 field->seqnum,
1163 field->duration,1163 field->duration,
1164 field->outer_duration,1164 field->outer_duration,
1165 (long long)field->timestamp.tv_sec,1165 (long long)field->timestamp.tv_sec,
1166 field->timestamp.tv_nsec);1166 field->timestamp.tv_nsec, field->count);
1167 1167
1168 if (field->nmi_count) {1168 if (field->nmi_count) {
1169 /*1169 /*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t