History

tracing: Have hwlat ts be first instance and record count of instances

The hwlat tracer runs a loop of width time during a given window. It then reports the max latency over a given threshold and records a timestamp. But this timestamp is the time after the width has finished, and not the time it actually triggered. Record the actual time when the latency was greater than the threshold as well as the number of times it was greater in a given width per window.

Steven Rostedt (VMware) committed 6 years ago

Commit b396bfd1 parent 98d54f84 files · +44 −20Browse files

Showing the changes to kernel (3 of 4 files). Show all

Unified · Split

3 files
  1. kernel/trace/trace_entries.h +3 −1
  2. kernel/trace/trace_hwlat.c +17 −7
  3. kernel/trace/trace_output.c +2 −2
kernel/trace/trace_entries.h+3−1
@@ -325,14 +325,16 @@ FTRACE_ENTRY(hwlat, hwlat_entry, · 324 unmodified lines
325 __field_desc( long, timestamp, tv_nsec )325 __field_desc( long, timestamp, tv_nsec )
326 __field( unsigned int, nmi_count )326 __field( unsigned int, nmi_count )
327 __field( unsigned int, seqnum )327 __field( unsigned int, seqnum )
328 __field( unsigned int, count )
328 ),329 ),
329 330
330 F_printk("cnt:%u\tts:%010llu.%010lu\tinner:%llu\touter:%llu\tnmi-ts:%llu\tnmi-count:%u\n",331 F_printk("cnt:%u\tts:%010llu.%010lu\tinner:%llu\touter:%llu\tcount:%d\tnmi-ts:%llu\tnmi-count:%u\n",
331 __entry->seqnum,332 __entry->seqnum,
332 __entry->tv_sec,333 __entry->tv_sec,
333 __entry->tv_nsec,334 __entry->tv_nsec,
334 __entry->duration,335 __entry->duration,
335 __entry->outer_duration,336 __entry->outer_duration,
337 __entry->count,
336 __entry->nmi_total_ts,338 __entry->nmi_total_ts,
337 __entry->nmi_count)339 __entry->nmi_count)
338);340);
kernel/trace/trace_hwlat.c+17−7
@@ -83,6 +83,7 @@ struct hwlat_sample { · 82 unmodified lines
83 u64 nmi_total_ts; /* Total time spent in NMIs */83 u64 nmi_total_ts; /* Total time spent in NMIs */
84 struct timespec64 timestamp; /* wall time */84 struct timespec64 timestamp; /* wall time */
85 int nmi_count; /* # NMIs during this sample */85 int nmi_count; /* # NMIs during this sample */
86 int count; /* # of iteratons over threash */
86};87};
87 88
88/* keep the global state somewhere. */89/* keep the global state somewhere. */
@@ -124,6 +125,7 @@ static void trace_hwlat_sample(struct hwlat_sample *sample) · 35 unmodified lines
124 entry->timestamp = sample->timestamp;125 entry->timestamp = sample->timestamp;
125 entry->nmi_total_ts = sample->nmi_total_ts;126 entry->nmi_total_ts = sample->nmi_total_ts;
126 entry->nmi_count = sample->nmi_count;127 entry->nmi_count = sample->nmi_count;
128 entry->count = sample->count;
127 129
128 if (!call_filter_check_discard(call, entry, buffer, event))130 if (!call_filter_check_discard(call, entry, buffer, event))
129 trace_buffer_unlock_commit_nostack(buffer, event);131 trace_buffer_unlock_commit_nostack(buffer, event);
@@ -167,12 +169,14 @@ void trace_hwlat_callback(bool enter) · 37 unmodified lines
167static int get_sample(void)169static int get_sample(void)
168{170{
169 struct trace_array *tr = hwlat_trace;171 struct trace_array *tr = hwlat_trace;
172 struct hwlat_sample s;
170 time_type start, t1, t2, last_t2;173 time_type start, t1, t2, last_t2;
171 s64 diff, total, last_total = 0;174 s64 diff, outer_diff, total, last_total = 0;
172 u64 sample = 0;175 u64 sample = 0;
173 u64 thresh = tracing_thresh;176 u64 thresh = tracing_thresh;
174 u64 outer_sample = 0;177 u64 outer_sample = 0;
175 int ret = -1;178 int ret = -1;
179 unsigned int count = 0;
176 180
177 do_div(thresh, NSEC_PER_USEC); /* modifies interval value */181 do_div(thresh, NSEC_PER_USEC); /* modifies interval value */
178 182
@@ -186,6 +190,7 @@ static int get_sample(void) · 7 unmodified lines
186 190
187 init_time(last_t2, 0);191 init_time(last_t2, 0);
188 start = time_get(); /* start timestamp */192 start = time_get(); /* start timestamp */
193 outer_diff = 0;
189 194
190 do {195 do {
191 196
@@ -194,14 +199,14 @@ static int get_sample(void) · 2 unmodified lines
194 199
195 if (time_u64(last_t2)) {200 if (time_u64(last_t2)) {
196 /* Check the delta from outer loop (t2 to next t1) */201 /* Check the delta from outer loop (t2 to next t1) */
197 diff = time_to_us(time_sub(t1, last_t2));202 outer_diff = time_to_us(time_sub(t1, last_t2));
198 /* This shouldn't happen */203 /* This shouldn't happen */
199 if (diff < 0) {204 if (outer_diff < 0) {
200 pr_err(BANNER "time running backwards\n");205 pr_err(BANNER "time running backwards\n");
201 goto out;206 goto out;
202 }207 }
203 if (diff > outer_sample)208 if (outer_diff > outer_sample)
204 outer_sample = diff;209 outer_sample = outer_diff;
205 }210 }
206 last_t2 = t2;211 last_t2 = t2;
207 212
@@ -217,6 +222,12 @@ static int get_sample(void) · 9 unmodified lines
217 /* This checks the inner loop (t1 to t2) */222 /* This checks the inner loop (t1 to t2) */
218 diff = time_to_us(time_sub(t2, t1)); /* current diff */223 diff = time_to_us(time_sub(t2, t1)); /* current diff */
219 224
225 if (diff > thresh || outer_diff > thresh) {
226 if (!count)
227 ktime_get_real_ts64(&s.timestamp);
228 count++;
229 }
230
220 /* This shouldn't happen */231 /* This shouldn't happen */
221 if (diff < 0) {232 if (diff < 0) {
222 pr_err(BANNER "time running backwards\n");233 pr_err(BANNER "time running backwards\n");
@@ -236,7 +247,6 @@ static int get_sample(void) · 13 unmodified lines
236 247
237 /* If we exceed the threshold value, we have found a hardware latency */248 /* If we exceed the threshold value, we have found a hardware latency */
238 if (sample > thresh || outer_sample > thresh) {249 if (sample > thresh || outer_sample > thresh) {
239 struct hwlat_sample s;
240 u64 latency;250 u64 latency;
241 251
242 ret = 1;252 ret = 1;
@@ -249,9 +259,9 @@ static int get_sample(void) · 6 unmodified lines
249 s.seqnum = hwlat_data.count;259 s.seqnum = hwlat_data.count;
250 s.duration = sample;260 s.duration = sample;
251 s.outer_duration = outer_sample;261 s.outer_duration = outer_sample;
252 ktime_get_real_ts64(&s.timestamp);
253 s.nmi_total_ts = nmi_total_ts;262 s.nmi_total_ts = nmi_total_ts;
254 s.nmi_count = nmi_count;263 s.nmi_count = nmi_count;
264 s.count = count;
255 trace_hwlat_sample(&s);265 trace_hwlat_sample(&s);
256 266
257 latency = max(sample, outer_sample);267 latency = max(sample, outer_sample);
kernel/trace/trace_output.c+2−2
@@ -1158,12 +1158,12 @@ trace_hwlat_print(struct trace_iterator *iter, int flags, · 1157 unmodified lines
1158 1158
1159 trace_assign_type(field, entry);1159 trace_assign_type(field, entry);
1160 1160
1161 trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%lld.%09ld",1161 trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%lld.%09ld count:%d",
1162 field->seqnum,1162 field->seqnum,
1163 field->duration,1163 field->duration,
1164 field->outer_duration,1164 field->outer_duration,
1165 (long long)field->timestamp.tv_sec,1165 (long long)field->timestamp.tv_sec,
1166 field->timestamp.tv_nsec);1166 field->timestamp.tv_nsec, field->count);
1167 1167
1168 if (field->nmi_count) {1168 if (field->nmi_count) {
1169 /*1169 /*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t