// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira <bristot@kernel.org> */ #define _GNU_SOURCE#include <getopt.h>#include <stdlib.h>#include <string.h>#include <signal.h>#include <unistd.h>#include <stdio.h>#include <time.h>#include <sched.h>#include <pthread.h> #include "utils.h"#include "osnoise.h"#include "timerlat.h"#include "timerlat_aa.h"#include "timerlat_u.h" struct timerlat_hist_params { char *cpus; cpu_set_t monitored_cpus; char *trace_output; char *cgroup_name; unsigned long long runtime; long long stop_us; long long stop_total_us; long long timerlat_period_us; long long print_stack; int sleep_time; int output_divisor; int duration; int set_sched; int dma_latency; int cgroup; int hk_cpus; int no_aa; int dump_tasks; int user_workload; int kernel_workload; int user_hist; cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; char no_irq; char no_thread; char no_header; char no_summary; char no_index; char with_zeros; int bucket_size; int entries; int warmup; int buffer_size; int deepest_idle_state;}; struct timerlat_hist_cpu { int *irq; int *thread; int *user; unsigned long long irq_count; unsigned long long thread_count; unsigned long long user_count; unsigned long long min_irq; unsigned long long sum_irq; unsigned long long max_irq; unsigned long long min_thread; unsigned long long sum_thread; unsigned long long max_thread; unsigned long long min_user; unsigned long long sum_user; unsigned long long max_user;}; struct timerlat_hist_data { struct timerlat_hist_cpu *hist; int entries; int bucket_size; int nr_cpus;}; /* * timerlat_free_histogram - free runtime data */static voidtimerlat_free_histogram(struct timerlat_hist_data *data){ int cpu; /* one histogram for IRQ and one for thread, per CPU */ for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (data->hist[cpu].irq) free(data->hist[cpu].irq); if (data->hist[cpu].thread) free(data->hist[cpu].thread); if (data->hist[cpu].user) free(data->hist[cpu].user); } /* one set of histograms per CPU */ if (data->hist) free(data->hist); free(data);} /* * timerlat_alloc_histogram - alloc runtime data */static struct timerlat_hist_data*timerlat_alloc_histogram(int nr_cpus, int entries, int bucket_size){ struct timerlat_hist_data *data; int cpu; data = calloc(1, sizeof(*data)); if (!data) return NULL; data->entries = entries; data->bucket_size = bucket_size; data->nr_cpus = nr_cpus; /* one set of histograms per CPU */ data->hist = calloc(1, sizeof(*data->hist) * nr_cpus); if (!data->hist) goto cleanup; /* one histogram for IRQ and one for thread, per cpu */ for (cpu = 0; cpu < nr_cpus; cpu++) { data->hist[cpu].irq = calloc(1, sizeof(*data->hist->irq) * (entries + 1)); if (!data->hist[cpu].irq) goto cleanup; data->hist[cpu].thread = calloc(1, sizeof(*data->hist->thread) * (entries + 1)); if (!data->hist[cpu].thread) goto cleanup; data->hist[cpu].user = calloc(1, sizeof(*data->hist->user) * (entries + 1)); if (!data->hist[cpu].user) goto cleanup; } /* set the min to max */ for (cpu = 0; cpu < nr_cpus; cpu++) { data->hist[cpu].min_irq = ~0; data->hist[cpu].min_thread = ~0; data->hist[cpu].min_user = ~0; } return data; cleanup: timerlat_free_histogram(data); return NULL;} /* * timerlat_hist_update - record a new timerlat occurent on cpu, updating data */static voidtimerlat_hist_update(struct osnoise_tool *tool, int cpu, unsigned long long context, unsigned long long latency){ struct timerlat_hist_params *params = tool->params; struct timerlat_hist_data *data = tool->data; int entries = data->entries; int bucket; int *hist; if (params->output_divisor) latency = latency / params->output_divisor; bucket = latency / data->bucket_size; if (!context) { hist = data->hist[cpu].irq; data->hist[cpu].irq_count++; update_min(&data->hist[cpu].min_irq, &latency); update_sum(&data->hist[cpu].sum_irq, &latency); update_max(&data->hist[cpu].max_irq, &latency); } else if (context == 1) { hist = data->hist[cpu].thread; data->hist[cpu].thread_count++; update_min(&data->hist[cpu].min_thread, &latency); update_sum(&data->hist[cpu].sum_thread, &latency); update_max(&data->hist[cpu].max_thread, &latency); } else { /* user */ hist = data->hist[cpu].user; data->hist[cpu].user_count++; update_min(&data->hist[cpu].min_user, &latency); update_sum(&data->hist[cpu].sum_user, &latency); update_max(&data->hist[cpu].max_user, &latency); } if (bucket < entries) hist[bucket]++; else hist[entries]++;} /* * timerlat_hist_handler - this is the handler for timerlat tracer events */static inttimerlat_hist_handler(struct trace_seq *s, struct tep_record *record, struct tep_event *event, void *data){ struct trace_instance *trace = data; unsigned long long context, latency; struct osnoise_tool *tool; int cpu = record->cpu; tool = container_of(trace, struct osnoise_tool, trace); tep_get_field_val(s, event, "context", record, &context, 1); tep_get_field_val(s, event, "timer_latency", record, &latency, 1); timerlat_hist_update(tool, cpu, context, latency); return 0;} /* * timerlat_hist_header - print the header of the tracer to the output */static void timerlat_hist_header(struct osnoise_tool *tool){ struct timerlat_hist_params *params = tool->params; struct timerlat_hist_data *data = tool->data; struct trace_seq *s = tool->trace.seq; char duration[26]; int cpu; if (params->no_header) return; get_duration(tool->start_time, duration, sizeof(duration)); trace_seq_printf(s, "# RTLA timerlat histogram\n"); trace_seq_printf(s, "# Time unit is %s (%s)\n", params->output_divisor == 1 ? "nanoseconds" : "microseconds", params->output_divisor == 1 ? "ns" : "us"); trace_seq_printf(s, "# Duration: %s\n", duration); if (!params->no_index) trace_seq_printf(s, "Index"); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) trace_seq_printf(s, " IRQ-%03d", cpu); if (!params->no_thread) trace_seq_printf(s, " Thr-%03d", cpu); if (params->user_hist) trace_seq_printf(s, " Usr-%03d", cpu); } trace_seq_printf(s, "\n"); trace_seq_do_printf(s); trace_seq_reset(s);} /* * format_summary_value - format a line of summary value (min, max or avg) * of hist data */static void format_summary_value(struct trace_seq *seq, int count, unsigned long long val, bool avg){ if (count) trace_seq_printf(seq, "%9llu ", avg ? val / count : val); else trace_seq_printf(seq, "%9c ", '-');} /* * timerlat_print_summary - print the summary of the hist data to the output */static voidtimerlat_print_summary(struct timerlat_hist_params *params, struct trace_instance *trace, struct timerlat_hist_data *data){ int cpu; if (params->no_summary) return; if (!params->no_index) trace_seq_printf(trace->seq, "count:"); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].irq_count); if (!params->no_thread) trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].thread_count); if (params->user_hist) trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].user_count); } trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "min: "); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) format_summary_value(trace->seq, data->hist[cpu].irq_count, data->hist[cpu].min_irq, false); if (!params->no_thread) format_summary_value(trace->seq, data->hist[cpu].thread_count, data->hist[cpu].min_thread, false); if (params->user_hist) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].min_user, false); } trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "avg: "); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) format_summary_value(trace->seq, data->hist[cpu].irq_count, data->hist[cpu].sum_irq, true); if (!params->no_thread) format_summary_value(trace->seq, data->hist[cpu].thread_count, data->hist[cpu].sum_thread, true); if (params->user_hist) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].sum_user, true); } trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "max: "); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) format_summary_value(trace->seq, data->hist[cpu].irq_count, data->hist[cpu].max_irq, false); if (!params->no_thread) format_summary_value(trace->seq, data->hist[cpu].thread_count, data->hist[cpu].max_thread, false); if (params->user_hist) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].max_user, false); } trace_seq_printf(trace->seq, "\n"); trace_seq_do_printf(trace->seq); trace_seq_reset(trace->seq);} static voidtimerlat_print_stats_all(struct timerlat_hist_params *params, struct trace_instance *trace, struct timerlat_hist_data *data){ struct timerlat_hist_cpu *cpu_data; struct timerlat_hist_cpu sum; int cpu; if (params->no_summary) return; memset(&sum, 0, sizeof(sum)); sum.min_irq = ~0; sum.min_thread = ~0; sum.min_user = ~0; for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; cpu_data = &data->hist[cpu]; sum.irq_count += cpu_data->irq_count; update_min(&sum.min_irq, &cpu_data->min_irq); update_sum(&sum.sum_irq, &cpu_data->sum_irq); update_max(&sum.max_irq, &cpu_data->max_irq); sum.thread_count += cpu_data->thread_count; update_min(&sum.min_thread, &cpu_data->min_thread); update_sum(&sum.sum_thread, &cpu_data->sum_thread); update_max(&sum.max_thread, &cpu_data->max_thread); sum.user_count += cpu_data->user_count; update_min(&sum.min_user, &cpu_data->min_user); update_sum(&sum.sum_user, &cpu_data->sum_user); update_max(&sum.max_user, &cpu_data->max_user); } if (!params->no_index) trace_seq_printf(trace->seq, "ALL: "); if (!params->no_irq) trace_seq_printf(trace->seq, " IRQ"); if (!params->no_thread) trace_seq_printf(trace->seq, " Thr"); if (params->user_hist) trace_seq_printf(trace->seq, " Usr"); trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "count:"); if (!params->no_irq) trace_seq_printf(trace->seq, "%9llu ", sum.irq_count); if (!params->no_thread) trace_seq_printf(trace->seq, "%9llu ", sum.thread_count); if (params->user_hist) trace_seq_printf(trace->seq, "%9llu ", sum.user_count); trace_seq_printf(trace->seq, "\n");Lines 1–500 of 1400. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira <bristot@kernel.org> */ #define _GNU_SOURCE #include <getopt.h> #include <stdlib.h> #include <string.h> #include <signal.h> #include <unistd.h> #include <stdio.h> #include <time.h> #include <sched.h> #include <pthread.h> #include "utils.h" #include "osnoise.h" #include "timerlat.h" #include "timerlat_aa.h" #include "timerlat_u.h" struct timerlat_hist_params { char *cpus; cpu_set_t monitored_cpus; char *trace_output; char *cgroup_name; unsigned long long runtime; long long stop_us; long long stop_total_us; long long timerlat_period_us; long long print_stack; int sleep_time; int output_divisor; int duration; int set_sched; int dma_latency; int cgroup; int hk_cpus; int no_aa; int dump_tasks; int user_workload; int kernel_workload; int user_hist; cpu_set_t hk_cpu_set; struct sched_attr sched_param; struct trace_events *events; char no_irq; char no_thread; char no_header; char no_summary; char no_index; char with_zeros; int bucket_size; int entries; int warmup; int buffer_size; int deepest_idle_state; }; struct timerlat_hist_cpu { int *irq; int *thread; int *user; unsigned long long irq_count; unsigned long long thread_count; unsigned long long user_count; unsigned long long min_irq; unsigned long long sum_irq; unsigned long long max_irq; unsigned long long min_thread; unsigned long long sum_thread; unsigned long long max_thread; unsigned long long min_user; unsigned long long sum_user; unsigned long long max_user; }; struct timerlat_hist_data { struct timerlat_hist_cpu *hist; int entries; int bucket_size; int nr_cpus; }; /* * timerlat_free_histogram - free runtime data */ static void timerlat_free_histogram(struct timerlat_hist_data *data) { int cpu; /* one histogram for IRQ and one for thread, per CPU */ for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (data->hist[cpu].irq) free(data->hist[cpu].irq); if (data->hist[cpu].thread) free(data->hist[cpu].thread); if (data->hist[cpu].user) free(data->hist[cpu].user); } /* one set of histograms per CPU */ if (data->hist) free(data->hist); free(data); } /* * timerlat_alloc_histogram - alloc runtime data */ static struct timerlat_hist_data *timerlat_alloc_histogram(int nr_cpus, int entries, int bucket_size) { struct timerlat_hist_data *data; int cpu; data = calloc(1, sizeof(*data)); if (!data) return NULL; data->entries = entries; data->bucket_size = bucket_size; data->nr_cpus = nr_cpus; /* one set of histograms per CPU */ data->hist = calloc(1, sizeof(*data->hist) * nr_cpus); if (!data->hist) goto cleanup; /* one histogram for IRQ and one for thread, per cpu */ for (cpu = 0; cpu < nr_cpus; cpu++) { data->hist[cpu].irq = calloc(1, sizeof(*data->hist->irq) * (entries + 1)); if (!data->hist[cpu].irq) goto cleanup; data->hist[cpu].thread = calloc(1, sizeof(*data->hist->thread) * (entries + 1)); if (!data->hist[cpu].thread) goto cleanup; data->hist[cpu].user = calloc(1, sizeof(*data->hist->user) * (entries + 1)); if (!data->hist[cpu].user) goto cleanup; } /* set the min to max */ for (cpu = 0; cpu < nr_cpus; cpu++) { data->hist[cpu].min_irq = ~0; data->hist[cpu].min_thread = ~0; data->hist[cpu].min_user = ~0; } return data; cleanup: timerlat_free_histogram(data); return NULL; } /* * timerlat_hist_update - record a new timerlat occurent on cpu, updating data */ static void timerlat_hist_update(struct osnoise_tool *tool, int cpu, unsigned long long context, unsigned long long latency) { struct timerlat_hist_params *params = tool->params; struct timerlat_hist_data *data = tool->data; int entries = data->entries; int bucket; int *hist; if (params->output_divisor) latency = latency / params->output_divisor; bucket = latency / data->bucket_size; if (!context) { hist = data->hist[cpu].irq; data->hist[cpu].irq_count++; update_min(&data->hist[cpu].min_irq, &latency); update_sum(&data->hist[cpu].sum_irq, &latency); update_max(&data->hist[cpu].max_irq, &latency); } else if (context == 1) { hist = data->hist[cpu].thread; data->hist[cpu].thread_count++; update_min(&data->hist[cpu].min_thread, &latency); update_sum(&data->hist[cpu].sum_thread, &latency); update_max(&data->hist[cpu].max_thread, &latency); } else { /* user */ hist = data->hist[cpu].user; data->hist[cpu].user_count++; update_min(&data->hist[cpu].min_user, &latency); update_sum(&data->hist[cpu].sum_user, &latency); update_max(&data->hist[cpu].max_user, &latency); } if (bucket < entries) hist[bucket]++; else hist[entries]++; } /* * timerlat_hist_handler - this is the handler for timerlat tracer events */ static int timerlat_hist_handler(struct trace_seq *s, struct tep_record *record, struct tep_event *event, void *data) { struct trace_instance *trace = data; unsigned long long context, latency; struct osnoise_tool *tool; int cpu = record->cpu; tool = container_of(trace, struct osnoise_tool, trace); tep_get_field_val(s, event, "context", record, &context, 1); tep_get_field_val(s, event, "timer_latency", record, &latency, 1); timerlat_hist_update(tool, cpu, context, latency); return 0; } /* * timerlat_hist_header - print the header of the tracer to the output */ static void timerlat_hist_header(struct osnoise_tool *tool) { struct timerlat_hist_params *params = tool->params; struct timerlat_hist_data *data = tool->data; struct trace_seq *s = tool->trace.seq; char duration[26]; int cpu; if (params->no_header) return; get_duration(tool->start_time, duration, sizeof(duration)); trace_seq_printf(s, "# RTLA timerlat histogram\n"); trace_seq_printf(s, "# Time unit is %s (%s)\n", params->output_divisor == 1 ? "nanoseconds" : "microseconds", params->output_divisor == 1 ? "ns" : "us"); trace_seq_printf(s, "# Duration: %s\n", duration); if (!params->no_index) trace_seq_printf(s, "Index"); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) trace_seq_printf(s, " IRQ-%03d", cpu); if (!params->no_thread) trace_seq_printf(s, " Thr-%03d", cpu); if (params->user_hist) trace_seq_printf(s, " Usr-%03d", cpu); } trace_seq_printf(s, "\n"); trace_seq_do_printf(s); trace_seq_reset(s); } /* * format_summary_value - format a line of summary value (min, max or avg) * of hist data */ static void format_summary_value(struct trace_seq *seq, int count, unsigned long long val, bool avg) { if (count) trace_seq_printf(seq, "%9llu ", avg ? val / count : val); else trace_seq_printf(seq, "%9c ", '-'); } /* * timerlat_print_summary - print the summary of the hist data to the output */ static void timerlat_print_summary(struct timerlat_hist_params *params, struct trace_instance *trace, struct timerlat_hist_data *data) { int cpu; if (params->no_summary) return; if (!params->no_index) trace_seq_printf(trace->seq, "count:"); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].irq_count); if (!params->no_thread) trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].thread_count); if (params->user_hist) trace_seq_printf(trace->seq, "%9llu ", data->hist[cpu].user_count); } trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "min: "); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) format_summary_value(trace->seq, data->hist[cpu].irq_count, data->hist[cpu].min_irq, false); if (!params->no_thread) format_summary_value(trace->seq, data->hist[cpu].thread_count, data->hist[cpu].min_thread, false); if (params->user_hist) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].min_user, false); } trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "avg: "); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) format_summary_value(trace->seq, data->hist[cpu].irq_count, data->hist[cpu].sum_irq, true); if (!params->no_thread) format_summary_value(trace->seq, data->hist[cpu].thread_count, data->hist[cpu].sum_thread, true); if (params->user_hist) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].sum_user, true); } trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "max: "); for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; if (!params->no_irq) format_summary_value(trace->seq, data->hist[cpu].irq_count, data->hist[cpu].max_irq, false); if (!params->no_thread) format_summary_value(trace->seq, data->hist[cpu].thread_count, data->hist[cpu].max_thread, false); if (params->user_hist) format_summary_value(trace->seq, data->hist[cpu].user_count, data->hist[cpu].max_user, false); } trace_seq_printf(trace->seq, "\n"); trace_seq_do_printf(trace->seq); trace_seq_reset(trace->seq); } static void timerlat_print_stats_all(struct timerlat_hist_params *params, struct trace_instance *trace, struct timerlat_hist_data *data) { struct timerlat_hist_cpu *cpu_data; struct timerlat_hist_cpu sum; int cpu; if (params->no_summary) return; memset(&sum, 0, sizeof(sum)); sum.min_irq = ~0; sum.min_thread = ~0; sum.min_user = ~0; for (cpu = 0; cpu < data->nr_cpus; cpu++) { if (params->cpus && !CPU_ISSET(cpu, ¶ms->monitored_cpus)) continue; if (!data->hist[cpu].irq_count && !data->hist[cpu].thread_count) continue; cpu_data = &data->hist[cpu]; sum.irq_count += cpu_data->irq_count; update_min(&sum.min_irq, &cpu_data->min_irq); update_sum(&sum.sum_irq, &cpu_data->sum_irq); update_max(&sum.max_irq, &cpu_data->max_irq); sum.thread_count += cpu_data->thread_count; update_min(&sum.min_thread, &cpu_data->min_thread); update_sum(&sum.sum_thread, &cpu_data->sum_thread); update_max(&sum.max_thread, &cpu_data->max_thread); sum.user_count += cpu_data->user_count; update_min(&sum.min_user, &cpu_data->min_user); update_sum(&sum.sum_user, &cpu_data->sum_user); update_max(&sum.max_user, &cpu_data->max_user); } if (!params->no_index) trace_seq_printf(trace->seq, "ALL: "); if (!params->no_irq) trace_seq_printf(trace->seq, " IRQ"); if (!params->no_thread) trace_seq_printf(trace->seq, " Thr"); if (params->user_hist) trace_seq_printf(trace->seq, " Usr"); trace_seq_printf(trace->seq, "\n"); if (!params->no_index) trace_seq_printf(trace->seq, "count:"); if (!params->no_irq) trace_seq_printf(trace->seq, "%9llu ", sum.irq_count); if (!params->no_thread) trace_seq_printf(trace->seq, "%9llu ", sum.thread_count); if (params->user_hist) trace_seq_printf(trace->seq, "%9llu ", sum.user_count); trace_seq_printf(trace->seq, "\n");