Intel PT timestamps are not provided for every branch, let alone every instruction, so there can be many samples with the same timestamp. With per-cpu contexts, decoding is done for each CPU in turn, which can make it difficult to see what is happening on different CPUs at the same time. Currently the interleaving from perf script --itrace=i0ns is quite coarse grained. There are often long stretches executing on one CPU and nothing on another. Some people are interested in seeing what happened on multiple CPUs before a crash to debug races etc. To improve perf script interleaving for parallel execution, the intel-pt-events.py script has been enhanced to enable interleaving the output with the same timestamp from different CPUs. It is understood that interleaving is not perfect or causal. Add parameter --interleave [<n>] to interleave sample output for the same timestamp so that no more than n samples for a CPU are displayed in a row. 'n' defaults to 4. Note this only affects the order of output, and only when the timestamp is the same. Example: $ perf script intel-pt-events.py --insn-trace --interleave 3 ... bash 2267/2267 [004] 9323.692625625 563caa3c86f0 jz 0x563caa3c89c7 run_pending_traps+0x30 (/usr/bin/bash) IPC: 1.52 (38/25) bash 2267/2267 [004] 9323.692625625 563caa3c89c7 movq 0x118(%rsp), %rax run_pending_traps+0x307 (/usr/bin/bash) bash 2267/2267 [004] 9323.692625625 563caa3c89cf subq %fs:0x28, %rax run_pending_traps+0x30f (/usr/bin/bash) bash 2270/2270 [007] 9323.692625625 55dc58cabf02 jz 0x55dc58cabf48 unquoted_glob_pattern_p+0x102 (/usr/bin/bash) IPC: 1.56 (25/16) bash 2270/2270 [007] 9323.692625625 55dc58cabf04 cmp $0x5d, %al unquoted_glob_pattern_p+0x104 (/usr/bin/bash) bash 2270/2270 [007] 9323.692625625 55dc58cabf06 jnz 0x55dc58cabf10 unquoted_glob_pattern_p+0x106 (/usr/bin/bash) bash 2264/2264 [001] 9323.692625625 7fd556a4376c jbe 0x7fd556a43ac8 round_and_return+0x3fc (/usr/lib/x86_64-linux-gnu/libc.so.6) IPC: 4.30 (43/10) bash 2264/2264 [001] 9323.692625625 7fd556a43772 and $0x8, %edx round_and_return+0x402 (/usr/lib/x86_64-linux-gnu/libc.so.6) bash 2264/2264 [001] 9323.692625625 7fd556a43775 jnz 0x7fd556a43ac8 round_and_return+0x405 (/usr/lib/x86_64-linux-gnu/libc.so.6) bash 2267/2267 [004] 9323.692625625 563caa3c89d8 jnz 0x563caa3c8b11 run_pending_traps+0x318 (/usr/bin/bash) bash 2267/2267 [004] 9323.692625625 563caa3c89de add $0x128, %rsp run_pending_traps+0x31e (/usr/bin/bash) bash 2267/2267 [004] 9323.692625625 563caa3c89e5 popq %rbx run_pending_traps+0x325 (/usr/bin/bash) ...
Showing the changes to tools/perf/scripts/python (1 of 2 files). Show all
Unified · Split
| @@ -13,10 +13,12 @@ · 12 unmodified lines | ||
| 13 | 13 | |
| 14 | 14 | from __future__ import print_function |
| 15 | 15 | |
| 16 | +import io | |
| 16 | 17 | import os |
| 17 | 18 | import sys |
| 18 | 19 | import struct |
| 19 | 20 | import argparse |
| 21 | +import contextlib | |
| 20 | 22 | |
| 21 | 23 | from libxed import LibXED |
| 22 | 24 | from ctypes import create_string_buffer, addressof |
| @@ -39,6 +41,11 @@ glb_src = False · 16 unmodified lines | ||
| 39 | 41 | glb_source_file_name = None |
| 40 | 42 | glb_line_number = None |
| 41 | 43 | glb_dso = None |
| 44 | +glb_stash_dict = {} | |
| 45 | +glb_output = None | |
| 46 | +glb_output_pos = 0 | |
| 47 | +glb_cpu = -1 | |
| 48 | +glb_time = 0 | |
| 42 | 49 | |
| 43 | 50 | def get_optional_null(perf_dict, field): |
| 44 | 51 | if field in perf_dict: |
| @@ -70,6 +77,7 @@ def trace_begin(): · 25 unmodified lines | ||
| 70 | 77 | ap.add_argument("--insn-trace", action='store_true') |
| 71 | 78 | ap.add_argument("--src-trace", action='store_true') |
| 72 | 79 | ap.add_argument("--all-switch-events", action='store_true') |
| 80 | + ap.add_argument("--interleave", type=int, nargs='?', const=4, default=0) | |
| 73 | 81 | global glb_args |
| 74 | 82 | global glb_insn |
| 75 | 83 | global glb_src |
| @@ -94,11 +102,39 @@ def trace_begin(): · 18 unmodified lines | ||
| 94 | 102 | perf_set_itrace_options(perf_script_context, itrace) |
| 95 | 103 | |
| 96 | 104 | def trace_end(): |
| 105 | + if glb_args.interleave: | |
| 106 | + flush_stashed_output() | |
| 97 | 107 | print("End") |
| 98 | 108 | |
| 99 | 109 | def trace_unhandled(event_name, context, event_fields_dict): |
| 100 | 110 | print(' '.join(['%s=%s'%(k,str(v))for k,v in sorted(event_fields_dict.items())])) |
| 101 | 111 | |
| 112 | +def stash_output(): | |
| 113 | + global glb_stash_dict | |
| 114 | + global glb_output_pos | |
| 115 | + output_str = glb_output.getvalue()[glb_output_pos:] | |
| 116 | + n = len(output_str) | |
| 117 | + if n: | |
| 118 | + glb_output_pos += n | |
| 119 | + if glb_cpu not in glb_stash_dict: | |
| 120 | + glb_stash_dict[glb_cpu] = [] | |
| 121 | + glb_stash_dict[glb_cpu].append(output_str) | |
| 122 | + | |
| 123 | +def flush_stashed_output(): | |
| 124 | + global glb_stash_dict | |
| 125 | + while glb_stash_dict: | |
| 126 | + cpus = list(glb_stash_dict.keys()) | |
| 127 | + # Output at most glb_args.interleave output strings per cpu | |
| 128 | + for cpu in cpus: | |
| 129 | + items = glb_stash_dict[cpu] | |
| 130 | + countdown = glb_args.interleave | |
| 131 | + while len(items) and countdown: | |
| 132 | + sys.stdout.write(items[0]) | |
| 133 | + del items[0] | |
| 134 | + countdown -= 1 | |
| 135 | + if not items: | |
| 136 | + del glb_stash_dict[cpu] | |
| 137 | + | |
| 102 | 138 | def print_ptwrite(raw_buf): |
| 103 | 139 | data = struct.unpack_from("<IQ", raw_buf) |
| 104 | 140 | flags = data[0] |
| @@ -375,15 +411,40 @@ def do_process_event(param_dict): · 270 unmodified lines | ||
| 375 | 411 | print_common_start(comm, sample, name) |
| 376 | 412 | print_common_ip(param_dict, sample, symbol, dso) |
| 377 | 413 | |
| 414 | +def interleave_events(param_dict): | |
| 415 | + global glb_cpu | |
| 416 | + global glb_time | |
| 417 | + global glb_output | |
| 418 | + global glb_output_pos | |
| 419 | + | |
| 420 | + sample = param_dict["sample"] | |
| 421 | + glb_cpu = sample["cpu"] | |
| 422 | + ts = sample["time"] | |
| 423 | + | |
| 424 | + if glb_time != ts: | |
| 425 | + glb_time = ts | |
| 426 | + flush_stashed_output() | |
| 427 | + | |
| 428 | + glb_output_pos = 0 | |
| 429 | + with contextlib.redirect_stdout(io.StringIO()) as glb_output: | |
| 430 | + do_process_event(param_dict) | |
| 431 | + | |
| 432 | + stash_output() | |
| 433 | + | |
| 378 | 434 | def process_event(param_dict): |
| 379 | 435 | try: |
| 380 | − do_process_event(param_dict) | |
| 436 | + if glb_args.interleave: | |
| 437 | + interleave_events(param_dict) | |
| 438 | + else: | |
| 439 | + do_process_event(param_dict) | |
| 381 | 440 | except broken_pipe_exception: |
| 382 | 441 | # Stop python printing broken pipe errors and traceback |
| 383 | 442 | sys.stdout = open(os.devnull, 'w') |
| 384 | 443 | sys.exit(1) |
| 385 | 444 | |
| 386 | 445 | def auxtrace_error(typ, code, cpu, pid, tid, ip, ts, msg, cpumode, *x): |
| 446 | + if glb_args.interleave: | |
| 447 | + flush_stashed_output() | |
| 387 | 448 | if len(x) >= 2 and x[0]: |
| 388 | 449 | machine_pid = x[0] |
| 389 | 450 | vcpu = x[1] |
| @@ -403,6 +464,8 @@ def auxtrace_error(typ, code, cpu, pid, tid, ip, ts, msg, cpumode, *x): · 13 unmodified lines | ||
| 403 | 464 | sys.exit(1) |
| 404 | 465 | |
| 405 | 466 | def context_switch(ts, cpu, pid, tid, np_pid, np_tid, machine_pid, out, out_preempt, *x): |
| 467 | + if glb_args.interleave: | |
| 468 | + flush_stashed_output() | |
| 406 | 469 | if out: |
| 407 | 470 | out_str = "Switch out " |
| 408 | 471 | else: |