else:		nr_cpus = 1		cpu_time_ranges = [ CPUTimeRange(-1) ] 	if verbosity.debug:		print("nr_cpus", nr_cpus)		print("cnts_cmd", cnts_cmd)		print("times_cmd", times_cmd) 	# Count the number of "double quick" samples per CPU	ProcessCommandOutputLines(cnts_cmd, per_cpu, CountSamplesByCPU, cpu_time_ranges) 	tot = 0	mx = 0	for cpu_time_range in cpu_time_ranges:		cnt = cpu_time_range.sample_cnt		tot += cnt		if cnt > mx:			mx = cnt		if verbosity.debug:			print("cpu:", cpu_time_range.cpu, "sample_cnt", cnt) 	if min_size < 1:		min_size = 1 	if mx < min_size:		# Too little data to be worth splitting		if verbosity.debug:			print("Too little data to split by time")		if nr == 0:			nr = 1		return [ SplitTimeRangesIntoN(time_ranges, nr, min_interval) ] 	if nr:		divisor = nr		min_size = 1	else:		divisor = NumberOfCPUs() 	interval = int(round(tot / divisor, 0))	if interval < min_size:		interval = min_size 	if verbosity.debug:		print("divisor", divisor)		print("min_size", min_size)		print("interval", interval) 	min_time = time_ranges[0][0]	max_time = time_ranges[-1][1] 	for cpu_time_range in cpu_time_ranges:		cnt = cpu_time_range.sample_cnt		if cnt == 0:			cpu_time_range.time_ranges = copy.deepcopy(time_ranges)			continue		# Adjust target interval for CPU to give approximately equal interval sizes		# Determine number of intervals, rounding to nearest integer		n = int(round(cnt / interval, 0))		if n < 1:			n = 1		# Determine interval size, rounding up		d, m = divmod(cnt, n)		if m:			d += 1		cpu_time_range.interval = d		cpu_time_range.interval_remaining = d		cpu_time_range.remaining = cnt		# Init. time ranges for each CPU with the start time		cpu_time_range.time_ranges = [ [min_time, max_time] ] 	# Set time ranges so that the same number of "double quick" samples	# will fall into each time range.	ProcessCommandOutputLines(times_cmd, per_cpu, CalcTimeRangesByCPU, cpu_time_ranges, max_time) 	for cpu_time_range in cpu_time_ranges:		if cpu_time_range.sample_cnt:			IntersectTimeRanges(cpu_time_range.time_ranges, time_ranges) 	return [cpu_time_ranges[cpu].time_ranges for cpu in cpus] def SplitSingleTimeRangeIntoN(time_range, n):	if n <= 1:		return [time_range]	start = time_range[0]	end   = time_range[1]	duration = int((end - start + 1) / n)	if duration < 1:		return [time_range]	time_ranges = []	for i in range(n):		time_ranges.append([start, start + duration - 1])		start += duration	time_ranges[-1][1] = end	return time_ranges def TimeRangeDuration(r):	return r[1] - r[0] + 1 def TotalDuration(time_ranges):	duration = 0	for r in time_ranges:		duration += TimeRangeDuration(r)	return duration def SplitTimeRangesByInterval(time_ranges, interval):	new_ranges = []	for r in time_ranges:		duration = TimeRangeDuration(r)		n = duration / interval		n = int(round(n, 0))		new_ranges += SplitSingleTimeRangeIntoN(r, n)	return new_ranges def SplitTimeRangesIntoN(time_ranges, n, min_interval):	if n <= len(time_ranges):		return time_ranges	duration = TotalDuration(time_ranges)	interval = duration / n	if interval < min_interval:		interval = min_interval	return SplitTimeRangesByInterval(time_ranges, interval) def RecombineTimeRanges(tr):	new_tr = copy.deepcopy(tr)	n = len(new_tr)	i = 1	while i < len(new_tr):		# if prev end + 1 == cur start, combine them		if new_tr[i - 1][1] + 1 == new_tr[i][0]:			new_tr[i][0] = new_tr[i - 1][0]			del new_tr[i - 1]		else:			i += 1	return new_tr def OpenTimeRangeEnds(time_ranges, min_time, max_time):	if time_ranges[0][0] <= min_time:		time_ranges[0][0] = None	if time_ranges[-1][1] >= max_time:		time_ranges[-1][1] = None def BadTimeStr(time_str):	raise Exception(f"perf command bad time option: '{time_str}'\nCheck also 'time of first sample' and 'time of last sample' in perf script --header-only") def ValidateTimeRanges(time_ranges, time_str):	n = len(time_ranges)	for i in range(n):		start = time_ranges[i][0]		end   = time_ranges[i][1]		if i != 0 and start <= time_ranges[i - 1][1]:			BadTimeStr(time_str)		if start > end:			BadTimeStr(time_str) def TimeVal(s, dflt):	s = s.strip()	if s == "":		return dflt	a = s.split(".")	if len(a) > 2:		raise Exception(f"Bad time value'{s}'")	x = int(a[0])	if x < 0:		raise Exception("Negative time not allowed")	x *= 1000000000	if len(a) > 1:		x += int((a[1] + "000000000")[:9])	return x def BadCPUStr(cpu_str):	raise Exception(f"perf command bad cpu option: '{cpu_str}'\nCheck also 'nrcpus avail' in perf script --header-only") def ParseTimeStr(time_str, min_time, max_time):	if time_str == None or time_str == "":		return [[min_time, max_time]]	time_ranges = []	for r in time_str.split():		a = r.split(",")		if len(a) != 2:			BadTimeStr(time_str)		try:			start = TimeVal(a[0], min_time)			end   = TimeVal(a[1], max_time)		except:			BadTimeStr(time_str)		time_ranges.append([start, end])	ValidateTimeRanges(time_ranges, time_str)	return time_ranges def ParseCPUStr(cpu_str, nr_cpus):	if cpu_str == None or cpu_str == "":		return [-1]	cpus = []	for r in cpu_str.split(","):		a = r.split("-")		if len(a) < 1 or len(a) > 2:			BadCPUStr(cpu_str)		try:			start = int(a[0].strip())			if len(a) > 1:				end = int(a[1].strip())			else:				end = start		except:			BadCPUStr(cpu_str)		if start < 0 or end < 0 or end < start or end >= nr_cpus:			BadCPUStr(cpu_str)		cpus.extend(range(start, end + 1))	cpus = list(set(cpus)) # Remove duplicates	cpus.sort()	return cpus class ParallelPerf(): 	def __init__(self, a):		for arg_name in vars(a):			setattr(self, arg_name, getattr(a, arg_name))		self.orig_nr = self.nr		self.orig_cmd = list(self.cmd)		self.perf = self.cmd[0]		if os.path.exists(self.output_dir):			raise Exception(f"Output '{self.output_dir}' already exists")		if self.jobs < 0 or self.nr < 0 or self.interval < 0:			raise Exception("Bad options (negative values): try -h option for help")		if self.nr != 0 and self.interval != 0:			raise Exception("Cannot specify number of time subdivisions and time interval")		if self.jobs == 0:			self.jobs = NumberOfCPUs()		if self.nr == 0 and self.interval == 0:			if self.per_cpu:				self.nr = 1			else:				self.nr = self.jobs 	def Init(self):		if self.verbosity.debug:			print("cmd", self.cmd)		self.file_name = DetermineInputFileName(self.cmd)		self.hdr = ReadHeader(self.perf, self.file_name)		self.hdr_dict = ParseHeader(self.hdr)		self.cmd_line = HeaderField(self.hdr_dict, "cmdline") 	def ExtractTimeInfo(self):		self.min_time = TimeVal(HeaderField(self.hdr_dict, "time of first sample"), 0)		self.max_time = TimeVal(HeaderField(self.hdr_dict, "time of last sample"), 0)		self.time_str = ExtractPerfOption(self.cmd, "", "time")		self.time_ranges = ParseTimeStr(self.time_str, self.min_time, self.max_time)		if self.verbosity.debug:			print("time_ranges", self.time_ranges) 	def ExtractCPUInfo(self):		if self.per_cpu:			nr_cpus = int(HeaderField(self.hdr_dict, "nrcpus avail"))			self.cpu_str = ExtractPerfOption(self.cmd, "C", "cpu")			if self.cpu_str == None or self.cpu_str == "":				self.cpus = [ x for x in range(nr_cpus) ]			else:				self.cpus = ParseCPUStr(self.cpu_str, nr_cpus)		else:			self.cpu_str = None			self.cpus = [-1]		if self.verbosity.debug:			print("cpus", self.cpus) 	def IsIntelPT(self):		return self.cmd_line.find("intel_pt") >= 0 	def SplitTimeRanges(self):		if self.IsIntelPT() and self.interval == 0:			self.split_time_ranges_for_each_cpu = \				SplitTimeRangesByTraceDataDensity(self.time_ranges, self.cpus, self.orig_nr,								  self.orig_cmd, self.file_name, self.per_cpu,								  self.min_size, self.min_interval, self.verbosity)		elif self.nr:			self.split_time_ranges_for_each_cpu = [ SplitTimeRangesIntoN(self.time_ranges, self.nr, self.min_interval) ]		else:			self.split_time_ranges_for_each_cpu = [ SplitTimeRangesByInterval(self.time_ranges, self.interval) ] 	def CheckTimeRanges(self):		for tr in self.split_time_ranges_for_each_cpu:			# Re-combined time ranges should be the same			new_tr = RecombineTimeRanges(tr)			if new_tr != self.time_ranges:				if self.verbosity.debug:					print("tr", tr)					print("new_tr", new_tr)				raise Exception("Self test failed!") 	def OpenTimeRangeEnds(self):		for time_ranges in self.split_time_ranges_for_each_cpu:			OpenTimeRangeEnds(time_ranges, self.min_time, self.max_time) 	def CreateWorkList(self):		self.worklist = CreateWorkList(self.cmd, self.pipe_to, self.output_dir, self.cpus, self.split_time_ranges_for_each_cpu) 	def PerfDataRecordedPerCPU(self):		if "--per-thread" in self.cmd_line.split():			return False		return True 	def DefaultToPerCPU(self):		# --no-per-cpu option takes precedence		if self.no_per_cpu:			return False		if not self.PerfDataRecordedPerCPU():			return False		# Default to per-cpu for Intel PT data that was recorded per-cpu,		# because decoding can be done for each CPU separately.		if self.IsIntelPT():			return True		return False 	def Config(self):		self.Init()		self.ExtractTimeInfo()		if not self.per_cpu:			self.per_cpu = self.DefaultToPerCPU()		if self.verbosity.debug:			print("per_cpu", self.per_cpu)		self.ExtractCPUInfo()		self.SplitTimeRanges()		if self.verbosity.self_test:			self.CheckTimeRanges()		# Prefer open-ended time range to starting / ending with min_time / max_time resp.		self.OpenTimeRangeEnds()		self.CreateWorkList() 	def Run(self):		if self.dry_run:			print(len(self.worklist),"jobs:")			for w in self.worklist:				print(w.Command())			return True		result = RunWork(self.worklist, self.jobs, verbosity=self.verbosity)		if self.verbosity.verbose:			print(glb_prog_name, "done")		return result def RunParallelPerf(a):	pp = ParallelPerf(a)	pp.Config()	return pp.Run() def Main(args):	ap = argparse.ArgumentParser(		prog=glb_prog_name, formatter_class = argparse.RawDescriptionHelpFormatter,		description ="""Run a perf script command multiple times in parallel, using perf script options--cpu and --time so that each job processes a different chunk of the data.""",		epilog ="""Follow the options by '--' and then the perf script command e.g. 	$ perf record -a -- sleep 10	$ parallel-perf.py --nr=4 -- perf script --ns	All jobs finished successfully	$ tree parallel-perf-output/	parallel-perf-output/	├── time-range-0	│   ├── cmd.txt	│   └── out.txt	├── time-range-1	│   ├── cmd.txt	│   └── out.txt	├── time-range-2	│   ├── cmd.txt	│   └── out.txt	└── time-range-3	    ├── cmd.txt	    └── out.txt	$ find parallel-perf-output -name cmd.txt | sort | xargs grep -H .	parallel-perf-output/time-range-0/cmd.txt:perf script --time=,9466.504461499 --ns	parallel-perf-output/time-range-1/cmd.txt:perf script --time=9466.504461500,9469.005396999 --ns	parallel-perf-output/time-range-2/cmd.txt:perf script --time=9469.005397000,9471.506332499 --ns	parallel-perf-output/time-range-3/cmd.txt:perf script --time=9471.506332500, --ns Any perf script command can be used, including the use of perf script options--dlfilter and --script, so that the benefit of running parallel jobsnaturally extends to them also. If option --pipe-to is used, standard output is first piped through thatcommand. Beware, if the command fails (e.g. grep with no matches), it will beconsidered a fatal error. Final standard output is redirected to files named out.txt in separatesubdirectories under the output directory. Similarly, standard error iswritten to files named err.txt. In addition, files named cmd.txt contain thecorresponding perf script command. After processing, err.txt files are removedif they are empty. If any job exits with a non-zero exit code, then all jobs are killed and nomore are started. A message is printed if any job results in a non-emptyerr.txt file. There is a separate output subdirectory for each time range. If the --per-cpuoption is used, these are further grouped under cpu-n subdirectories, e.g. 	$ parallel-perf.py --per-cpu --nr=2 -- perf script --ns --cpu=0,1	All jobs finished successfully	$ tree parallel-perf-output	parallel-perf-output/	├── cpu-0	│   ├── time-range-0	│   │   ├── cmd.txt	│   │   └── out.txt	│   └── time-range-1	│       ├── cmd.txt	│       └── out.txt	└── cpu-1	    ├── time-range-0	    │   ├── cmd.txt	    │   └── out.txt	    └── time-range-1	        ├── cmd.txt	        └── out.txt	$ find parallel-perf-output -name cmd.txt | sort | xargs grep -H .	parallel-perf-output/cpu-0/time-range-0/cmd.txt:perf script --cpu=0 --time=,9469.005396999 --ns	parallel-perf-output/cpu-0/time-range-1/cmd.txt:perf script --cpu=0 --time=9469.005397000, --ns	parallel-perf-output/cpu-1/time-range-0/cmd.txt:perf script --cpu=1 --time=,9469.005396999 --ns	parallel-perf-output/cpu-1/time-range-1/cmd.txt:perf script --cpu=1 --time=9469.005397000, --ns Subdivisions of time range, and cpus if the --per-cpu option is used, areexpressed by the --time and --cpu perf script options respectively. If thesupplied perf script command has a --time option, then that time range issubdivided, otherwise the time range given by 'time of first sample' to'time of last sample' is used (refer perf script --header-only). Similarly, thesupplied perf script command may provide a --cpu option, and only those CPUswill be processed. To prevent time intervals becoming too small, the --min-interval option canbe used. Note there is special handling for processing Intel PT traces. If an interval isnot specified and the perf record command contained the intel_pt event, then thetime range will be subdivided in order to produce subdivisions that containapproximately the same amount of trace data. That is accomplished by countingdouble-quick (--itrace=qqi) samples, and choosing time ranges that encompassapproximately the same number of samples. In that case, time ranges may not bethe same for each CPU processed. For Intel PT, --per-cpu is the default, butthat can be overridden by --no-per-cpu. Note, for Intel PT, double-quickdecoding produces 1 sample for each PSB synchronization packet, which in turncome after a certain number of bytes output, determined by psb_period (referperf Intel PT documentation). The minimum number of double-quick samples thatwill define a time range can be set by the --min_size option, which defaults to64.""")	ap.add_argument("-o", "--output-dir", default="parallel-perf-output", help="output directory (default 'parallel-perf-output')")	ap.add_argument("-j", "--jobs", type=int, default=0, help="maximum number of jobs to run in parallel at one time (default is the number of CPUs)")	ap.add_argument("-n", "--nr", type=int, default=0, help="number of time subdivisions (default is the number of jobs)")	ap.add_argument("-i", "--interval", type=float, default=0, help="subdivide the time range using this time interval (in seconds e.g. 0.1 for a tenth of a second)")	ap.add_argument("-c", "--per-cpu", action="store_true", help="process data for each CPU in parallel")	ap.add_argument("-m", "--min-interval", type=float, default=glb_min_interval, help=f"minimum interval (default {glb_min_interval} seconds)")	ap.add_argument("-p", "--pipe-to", help="command to pipe output to (optional)")	ap.add_argument("-N", "--no-per-cpu", action="store_true", help="do not process data for each CPU in parallel")	ap.add_argument("-b", "--min_size", type=int, default=glb_min_samples, help="minimum data size (for Intel PT in PSBs)")	ap.add_argument("-D", "--dry-run", action="store_true", help="do not run any jobs, just show the perf script commands")	ap.add_argument("-q", "--quiet", action="store_true", help="do not print any messages except errors")	ap.add_argument("-v", "--verbose", action="store_true", help="print more messages")	ap.add_argument("-d", "--debug", action="store_true", help="print debugging messages")	cmd_line = list(args)	try:		split_pos = cmd_line.index("--")		cmd = cmd_line[split_pos + 1:]		args = cmd_line[:split_pos]	except:		cmd = None		args = cmd_line	a = ap.parse_args(args=args[1:])	a.cmd = cmd	a.verbosity = Verbosity(a.quiet, a.verbose, a.debug)	try:		if a.cmd == None:			if len(args) <= 1:				ap.print_help()				return True			raise Exception("Command line must contain '--' before perf command")		return RunParallelPerf(a)	except Exception as e:		print("Fatal error: ", str(e))		if a.debug:			raise		return False if __name__ == "__main__":	if not Main(sys.argv):		sys.exit(1)

Lines 501–989 of 989. Previous lines