Merge tag 'linux-kselftest-kunit-5.17-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest

Pull KUnit updates from Shuah Khan: "This consists of several fixes and enhancements. A few highlights: - Option --kconfig_add option allows easily tweaking kunitconfigs - make build subcommand can reconfigure if needed - doesn't error on tests without test plans - doesn't crash if no parameters are generated - defaults --jobs to # of cups - reports test parameter results as (K)TAP subtests" * tag 'linux-kselftest-kunit-5.17-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest: kunit: tool: Default --jobs to number of CPUs kunit: tool: fix newly introduced typechecker errors kunit: tool: make `build` subcommand also reconfigure if needed kunit: tool: delete kunit_parser.TestResult type kunit: tool: use dataclass instead of collections.namedtuple kunit: tool: suggest using decode_stacktrace.sh on kernel crash kunit: tool: reconfigure when the used kunitconfig changes kunit: tool: revamp message for invalid kunitconfig kunit: tool: add --kconfig_add to allow easily tweaking kunitconfigs kunit: tool: move Kconfig read_from_file/parse_from_string to package-level kunit: tool: print parsed test results fully incrementally kunit: Report test parameter results as (K)TAP subtests kunit: Don't crash if no parameters are generated kunit: tool: Report an error if any test has no subtests kunit: tool: Do not error on tests without test plans kunit: add run_checks.py script to validate kunit changes Documentation: kunit: remove claims that kunit is a mocking framework kunit: tool: fix --json output for skipped tests

Linus Torvalds committed 4 years ago

Commit bf4eebf2 parents 4369b3c ad659cc13 files · +480 −204Browse files

Unified · Split

13 files

Documentation/dev-tools/kunit/api/index.rst +1 −2
Documentation/dev-tools/kunit/api/test.rst +1 −2
Documentation/dev-tools/kunit/index.rst +1 −1
Documentation/dev-tools/kunit/start.rst +4 −4
lib/kunit/test.c +14 −11
tools/testing/kunit/kunit.py +105 −77
tools/testing/kunit/kunit_config.py +31 −30
tools/testing/kunit/kunit_json.py +5 −3
tools/testing/kunit/kunit_kernel.py +49 −27
tools/testing/kunit/kunit_parser.py +34 −23
tools/testing/kunit/kunit_tool_test.py +147 −24
tools/testing/kunit/run_checks.py +81 −0
tools/testing/kunit/test_data/test_is_test_passed-no_tests_no_plan.log +7 −0

Documentation/dev-tools/kunit/api/index.rst+1−2

@@ -12,5 +12,4 @@ following sections: · 11 unmodified lines
12	12
13	13	Documentation/dev-tools/kunit/api/test.rst
14	14
15		− - documents all of the standard testing API excluding mocking
16		− or mocking related features.
	15	+ - documents all of the standard testing API

Documentation/dev-tools/kunit/api/test.rst+1−2

@@ -4,8 +4,7 @@ · 3 unmodified lines
4	4	Test API
5	5	========
6	6
7		−This file documents all of the standard testing API excluding mocking or mocking
8		−related features.
	7	+This file documents all of the standard testing API.
9	8
10	9	.. kernel-doc:: include/kunit/test.h
11	10	:internal:

Documentation/dev-tools/kunit/index.rst+1−1

@@ -19,7 +19,7 @@ KUnit - Unit Testing for the Linux Kernel · 18 unmodified lines
19	19	What is KUnit?
20	20	==============
21	21
22		−KUnit is a lightweight unit testing and mocking framework for the Linux kernel.
	22	+KUnit is a lightweight unit testing framework for the Linux kernel.
23	23
24	24	KUnit is heavily inspired by JUnit, Python's unittest.mock, and
25	25	Googletest/Googlemock for C++. KUnit provides facilities for defining unit test

Documentation/dev-tools/kunit/start.rst+4−4

@@ -50,10 +50,10 @@ It'll warn you if you haven't included the dependencies of the options you're · 49 unmodified lines
50	50	using.
51	51
52	52	.. note::
53		− Note that removing something from the ``.kunitconfig`` will not trigger a
54		− rebuild of the ``.config`` file: the configuration is only updated if the
55		− ``.kunitconfig`` is not a subset of ``.config``. This means that you can use
56		− other tools (such as make menuconfig) to adjust other config options.
	53	+ If you change the ``.kunitconfig``, kunit.py will trigger a rebuild of the
	54	+ ``.config`` file. But you can edit the ``.config`` file directly or with
	55	+ tools like ``make menuconfig O=.kunit``. As long as its a superset of
	56	+ ``.kunitconfig``, kunit.py won't overwrite your changes.
57	57
58	58
59	59	Running the tests (KUnit Wrapper)

lib/kunit/test.c+14−11

@@ -504,25 +504,28 @@ int kunit_run_tests(struct kunit_suite *suite) · 503 unmodified lines
504	504	struct kunit_result_stats param_stats = { 0 };
505	505	test_case->status = KUNIT_SKIPPED;
506	506
507		− if (test_case->generate_params) {
	507	+ if (!test_case->generate_params) {
	508	+ /* Non-parameterised test. */
	509	+ kunit_run_case_catch_errors(suite, test_case, &test);
	510	+ kunit_update_stats(&param_stats, test.status);
	511	+ } else {
508	512	/* Get initial param. */
509	513	param_desc[0] = '\0';
510	514	test.param_value = test_case->generate_params(NULL, param_desc);
511		− }
	515	+ kunit_log(KERN_INFO, &test, KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT
	516	+ "# Subtest: %s", test_case->name);
512	517
513		− do {
514		− kunit_run_case_catch_errors(suite, test_case, &test);
	518	+ while (test.param_value) {
	519	+ kunit_run_case_catch_errors(suite, test_case, &test);
515	520
516		− if (test_case->generate_params) {
517	521	if (param_desc[0] == '\0') {
518	522	snprintf(param_desc, sizeof(param_desc),
519	523	"param-%d", test.param_index);
520	524	}
521	525
522	526	kunit_log(KERN_INFO, &test,
523		− KUNIT_SUBTEST_INDENT
524		− "# %s: %s %d - %s",
525		− test_case->name,
	527	+ KUNIT_SUBTEST_INDENT KUNIT_SUBTEST_INDENT
	528	+ "%s %d - %s",
526	529	kunit_status_to_ok_not_ok(test.status),
527	530	test.param_index + 1, param_desc);
528	531
@@ -530,11 +533,11 @@ int kunit_run_tests(struct kunit_suite *suite) · 1 unmodified line
530	533	param_desc[0] = '\0';
531	534	test.param_value = test_case->generate_params(test.param_value, param_desc);
532	535	test.param_index++;
533		− }
534	536
535		− kunit_update_stats(&param_stats, test.status);
	537	+ kunit_update_stats(&param_stats, test.status);
	538	+ }
	539	+ }
536	540
537		− } while (test.param_value);
538	541
539	542	kunit_print_test_stats(&test, param_stats);
540	543

tools/testing/kunit/kunit.py+105−77

@@ -15,38 +15,57 @@ import time · 14 unmodified lines
15	15
16	16	assert sys.version_info >= (3, 7), "Python version is too old"
17	17
18		−from collections import namedtuple
	18	+from dataclasses import dataclass
19	19	from enum import Enum, auto
20		−from typing import Iterable, Sequence, List
	20	+from typing import Any, Iterable, Sequence, List, Optional
21	21
22	22	import kunit_json
23	23	import kunit_kernel
24	24	import kunit_parser
25	25
26		−KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time'])
27		−
28		−KunitConfigRequest = namedtuple('KunitConfigRequest',
29		− ['build_dir', 'make_options'])
30		−KunitBuildRequest = namedtuple('KunitBuildRequest',
31		− ['jobs', 'build_dir', 'alltests',
32		− 'make_options'])
33		−KunitExecRequest = namedtuple('KunitExecRequest',
34		− ['timeout', 'build_dir', 'alltests',
35		− 'filter_glob', 'kernel_args', 'run_isolated'])
36		−KunitParseRequest = namedtuple('KunitParseRequest',
37		− ['raw_output', 'build_dir', 'json'])
38		−KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',
39		− 'build_dir', 'alltests', 'filter_glob',
40		− 'kernel_args', 'run_isolated', 'json', 'make_options'])
41		−
42		−KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
43		−
44	26	class KunitStatus(Enum):
45	27	SUCCESS = auto()
46	28	CONFIG_FAILURE = auto()
47	29	BUILD_FAILURE = auto()
48	30	TEST_FAILURE = auto()
49	31
	32	+@dataclass
	33	+class KunitResult:
	34	+ status: KunitStatus
	35	+ result: Any
	36	+ elapsed_time: float
	37	+
	38	+@dataclass
	39	+class KunitConfigRequest:
	40	+ build_dir: str
	41	+ make_options: Optional[List[str]]
	42	+
	43	+@dataclass
	44	+class KunitBuildRequest(KunitConfigRequest):
	45	+ jobs: int
	46	+ alltests: bool
	47	+
	48	+@dataclass
	49	+class KunitParseRequest:
	50	+ raw_output: Optional[str]
	51	+ build_dir: str
	52	+ json: Optional[str]
	53	+
	54	+@dataclass
	55	+class KunitExecRequest(KunitParseRequest):
	56	+ timeout: int
	57	+ alltests: bool
	58	+ filter_glob: str
	59	+ kernel_args: Optional[List[str]]
	60	+ run_isolated: Optional[str]
	61	+
	62	+@dataclass
	63	+class KunitRequest(KunitExecRequest, KunitBuildRequest):
	64	+ pass
	65	+
	66	+
	67	+KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
	68	+
50	69	def get_kernel_root_path() -> str:
51	70	path = sys.argv[0] if not __file__ else __file__
52	71	parts = os.path.realpath(path).split('tools/testing/kunit')
@@ -91,6 +110,14 @@ def build_tests(linux: kunit_kernel.LinuxSourceTree, · 38 unmodified lines
91	110	'built kernel successfully',
92	111	build_end - build_start)
93	112
	113	+def config_and_build_tests(linux: kunit_kernel.LinuxSourceTree,
	114	+ request: KunitBuildRequest) -> KunitResult:
	115	+ config_result = config_tests(linux, request)
	116	+ if config_result.status != KunitStatus.SUCCESS:
	117	+ return config_result
	118	+
	119	+ return build_tests(linux, request)
	120	+
94	121	def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]:
95	122	args = ['kunit.action=list']
96	123	if request.kernel_args:
@@ -121,8 +148,7 @@ def _suites_from_test_list(tests: List[str]) -> List[str]: · 24 unmodified lines
121	148
122	149
123	150
124		−def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest,
125		− parse_request: KunitParseRequest) -> KunitResult:
	151	+def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult:
126	152	filter_globs = [request.filter_glob]
127	153	if request.run_isolated:
128	154	tests = _list_tests(linux, request)
@@ -147,17 +173,23 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, · 18 unmodified lines
147	173	filter_glob=filter_glob,
148	174	build_dir=request.build_dir)
149	175
150		− result = parse_tests(parse_request, run_result)
	176	+ result = parse_tests(request, run_result)
151	177	# run_kernel() doesn't block on the kernel exiting.
152	178	# That only happens after we get the last line of output from `run_result`.
153	179	# So exec_time here actually contains parsing + execution time, which is fine.
154	180	test_end = time.time()
155	181	exec_time += test_end - test_start
156	182
157		− test_counts.add_subtest_counts(result.result.test.counts)
	183	+ test_counts.add_subtest_counts(result.result.counts)
	184	+
	185	+ if len(filter_globs) == 1 and test_counts.crashed > 0:
	186	+ bd = request.build_dir
	187	+ print('The kernel seems to have crashed; you can decode the stack traces with:')
	188	+ print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} \| tee {}/decoded.log \| {} parse'.format(
	189	+ bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0]))
158	190
159	191	kunit_status = _map_to_overall_status(test_counts.get_status())
160		− return KunitResult(status=kunit_status, result=result.result, elapsed_time=exec_time)
	192	+ return KunitResult(status=kunit_status, result=result, elapsed_time=exec_time)
161	193
162	194	def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus:
163	195	if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED):
@@ -168,14 +200,12 @@ def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: · 4 unmodified lines
168	200	def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> KunitResult:
169	201	parse_start = time.time()
170	202
171		− test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,
172		− kunit_parser.Test(),
173		− 'Tests not Parsed.')
	203	+ test_result = kunit_parser.Test()
174	204
175	205	if request.raw_output:
176	206	# Treat unparsed results as one passing test.
177		− test_result.test.status = kunit_parser.TestStatus.SUCCESS
178		− test_result.test.counts.passed = 1
	207	+ test_result.status = kunit_parser.TestStatus.SUCCESS
	208	+ test_result.counts.passed = 1
179	209
180	210	output: Iterable[str] = input_data
181	211	if request.raw_output == 'all':
@@ -193,7 +223,7 @@ def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> KunitR · 11 unmodified lines
193	223
194	224	if request.json:
195	225	json_obj = kunit_json.get_json_result(
196		− test_result=test_result,
	226	+ test=test_result,
197	227	def_config='kunit_defconfig',
198	228	build_dir=request.build_dir,
199	229	json_path=request.json)
@@ -211,27 +241,15 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, · 11 unmodified lines
211	241	request: KunitRequest) -> KunitResult:
212	242	run_start = time.time()
213	243
214		− config_request = KunitConfigRequest(request.build_dir,
215		− request.make_options)
216		− config_result = config_tests(linux, config_request)
	244	+ config_result = config_tests(linux, request)
217	245	if config_result.status != KunitStatus.SUCCESS:
218	246	return config_result
219	247
220		− build_request = KunitBuildRequest(request.jobs, request.build_dir,
221		− request.alltests,
222		− request.make_options)
223		− build_result = build_tests(linux, build_request)
	248	+ build_result = build_tests(linux, request)
224	249	if build_result.status != KunitStatus.SUCCESS:
225	250	return build_result
226	251
227		− exec_request = KunitExecRequest(request.timeout, request.build_dir,
228		− request.alltests, request.filter_glob,
229		− request.kernel_args, request.run_isolated)
230		− parse_request = KunitParseRequest(request.raw_output,
231		− request.build_dir,
232		− request.json)
233		−
234		− exec_result = exec_tests(linux, exec_request, parse_request)
	252	+ exec_result = exec_tests(linux, request)
235	253
236	254	run_end = time.time()
237	255
@@ -264,6 +282,9 @@ def massage_argv(argv: Sequence[str]) -> Sequence[str]: · 26 unmodified lines
264	282	return f'{arg}={pseudo_bool_flag_defaults[arg]}'
265	283	return list(map(massage_arg, argv))
266	284
	285	+def get_default_jobs() -> int:
	286	+ return len(os.sched_getaffinity(0))
	287	+
267	288	def add_common_opts(parser) -> None:
268	289	parser.add_argument('--build_dir',
269	290	help='As in the make command, it specifies the build '
@@ -280,6 +301,10 @@ def add_common_opts(parser) -> None: · 10 unmodified lines
280	301	' If given a directory, (e.g. lib/kunit), "/.kunitconfig" '
281	302	'will get automatically appended.',
282	303	metavar='kunitconfig')
	304	+ parser.add_argument('--kconfig_add',
	305	+ help='Additional Kconfig options to append to the '
	306	+ '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.',
	307	+ action='append')
283	308
284	309	parser.add_argument('--arch',
285	310	help=('Specifies the architecture to run tests under. '
@@ -310,7 +335,7 @@ def add_build_opts(parser) -> None: · 24 unmodified lines
310	335	parser.add_argument('--jobs',
311	336	help='As in the make command, "Specifies the number of '
312	337	'jobs (commands) to run simultaneously."',
313		− type=int, default=8, metavar='jobs')
	338	+ type=int, default=get_default_jobs(), metavar='jobs')
314	339
315	340	def add_exec_opts(parser) -> None:
316	341	parser.add_argument('--timeout',
@@ -398,20 +423,21 @@ def main(argv, linux=None): · 81 unmodified lines
398	423	if not linux:
399	424	linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
400	425	kunitconfig_path=cli_args.kunitconfig,
	426	+ kconfig_add=cli_args.kconfig_add,
401	427	arch=cli_args.arch,
402	428	cross_compile=cli_args.cross_compile,
403	429	qemu_config_path=cli_args.qemu_config)
404	430
405		− request = KunitRequest(cli_args.raw_output,
406		− cli_args.timeout,
407		− cli_args.jobs,
408		− cli_args.build_dir,
409		− cli_args.alltests,
410		− cli_args.filter_glob,
411		− cli_args.kernel_args,
412		− cli_args.run_isolated,
413		− cli_args.json,
414		− cli_args.make_options)
	431	+ request = KunitRequest(build_dir=cli_args.build_dir,
	432	+ make_options=cli_args.make_options,
	433	+ jobs=cli_args.jobs,
	434	+ alltests=cli_args.alltests,
	435	+ raw_output=cli_args.raw_output,
	436	+ json=cli_args.json,
	437	+ timeout=cli_args.timeout,
	438	+ filter_glob=cli_args.filter_glob,
	439	+ kernel_args=cli_args.kernel_args,
	440	+ run_isolated=cli_args.run_isolated)
415	441	result = run_tests(linux, request)
416	442	if result.status != KunitStatus.SUCCESS:
417	443	sys.exit(1)
@@ -423,12 +449,13 @@ def main(argv, linux=None): · 5 unmodified lines
423	449	if not linux:
424	450	linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
425	451	kunitconfig_path=cli_args.kunitconfig,
	452	+ kconfig_add=cli_args.kconfig_add,
426	453	arch=cli_args.arch,
427	454	cross_compile=cli_args.cross_compile,
428	455	qemu_config_path=cli_args.qemu_config)
429	456
430		− request = KunitConfigRequest(cli_args.build_dir,
431		− cli_args.make_options)
	457	+ request = KunitConfigRequest(build_dir=cli_args.build_dir,
	458	+ make_options=cli_args.make_options)
432	459	result = config_tests(linux, request)
433	460	kunit_parser.print_with_timestamp((
434	461	'Elapsed time: %.3fs\n') % (
@@ -439,15 +466,16 @@ def main(argv, linux=None): · 4 unmodified lines
439	466	if not linux:
440	467	linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
441	468	kunitconfig_path=cli_args.kunitconfig,
	469	+ kconfig_add=cli_args.kconfig_add,
442	470	arch=cli_args.arch,
443	471	cross_compile=cli_args.cross_compile,
444	472	qemu_config_path=cli_args.qemu_config)
445	473
446		− request = KunitBuildRequest(cli_args.jobs,
447		− cli_args.build_dir,
448		− cli_args.alltests,
449		− cli_args.make_options)
450		− result = build_tests(linux, request)
	474	+ request = KunitBuildRequest(build_dir=cli_args.build_dir,
	475	+ make_options=cli_args.make_options,
	476	+ jobs=cli_args.jobs,
	477	+ alltests=cli_args.alltests)
	478	+ result = config_and_build_tests(linux, request)
451	479	kunit_parser.print_with_timestamp((
452	480	'Elapsed time: %.3fs\n') % (
453	481	result.elapsed_time))
@@ -457,20 +485,20 @@ def main(argv, linux=None): · 3 unmodified lines
457	485	if not linux:
458	486	linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir,
459	487	kunitconfig_path=cli_args.kunitconfig,
	488	+ kconfig_add=cli_args.kconfig_add,
460	489	arch=cli_args.arch,
461	490	cross_compile=cli_args.cross_compile,
462	491	qemu_config_path=cli_args.qemu_config)
463	492
464		− exec_request = KunitExecRequest(cli_args.timeout,
465		− cli_args.build_dir,
466		− cli_args.alltests,
467		− cli_args.filter_glob,
468		− cli_args.kernel_args,
469		− cli_args.run_isolated)
470		− parse_request = KunitParseRequest(cli_args.raw_output,
471		− cli_args.build_dir,
472		− cli_args.json)
473		− result = exec_tests(linux, exec_request, parse_request)
	493	+ exec_request = KunitExecRequest(raw_output=cli_args.raw_output,
	494	+ build_dir=cli_args.build_dir,
	495	+ json=cli_args.json,
	496	+ timeout=cli_args.timeout,
	497	+ alltests=cli_args.alltests,
	498	+ filter_glob=cli_args.filter_glob,
	499	+ kernel_args=cli_args.kernel_args,
	500	+ run_isolated=cli_args.run_isolated)
	501	+ result = exec_tests(linux, exec_request)
474	502	kunit_parser.print_with_timestamp((
475	503	'Elapsed time: %.3fs\n') % (result.elapsed_time))
476	504	if result.status != KunitStatus.SUCCESS:
@@ -482,9 +510,9 @@ def main(argv, linux=None): · 5 unmodified lines
482	510	else:
483	511	with open(cli_args.file, 'r', errors='backslashreplace') as f:
484	512	kunit_output = f.read().splitlines()
485		− request = KunitParseRequest(cli_args.raw_output,
486		− None,
487		− cli_args.json)
	513	+ request = KunitParseRequest(raw_output=cli_args.raw_output,
	514	+ build_dir='',
	515	+ json=cli_args.json)
488	516	result = parse_tests(request, kunit_output)
489	517	if result.status != KunitStatus.SUCCESS:
490	518	sys.exit(1)

tools/testing/kunit/kunit_config.py+31−30

@@ -62,33 +62,34 @@ class Kconfig(object): · 61 unmodified lines
62	62	for entry in self.entries():
63	63	f.write(str(entry) + '\n')
64	64
65		− def parse_from_string(self, blob: str) -> None:
66		− """Parses a string containing KconfigEntrys and populates this Kconfig."""
67		− self._entries = []
68		− is_not_set_matcher = re.compile(CONFIG_IS_NOT_SET_PATTERN)
69		− config_matcher = re.compile(CONFIG_PATTERN)
70		− for line in blob.split('\n'):
71		− line = line.strip()
72		− if not line:
73		− continue
74		−
75		− match = config_matcher.match(line)
76		− if match:
77		− entry = KconfigEntry(match.group(1), match.group(2))
78		− self.add_entry(entry)
79		− continue
80		−
81		− empty_match = is_not_set_matcher.match(line)
82		− if empty_match:
83		− entry = KconfigEntry(empty_match.group(1), 'n')
84		− self.add_entry(entry)
85		− continue
86		−
87		− if line[0] == '#':
88		− continue
89		− else:
90		− raise KconfigParseError('Failed to parse: ' + line)
91		−
92		− def read_from_file(self, path: str) -> None:
93		− with open(path, 'r') as f:
94		− self.parse_from_string(f.read())
	65	+def parse_file(path: str) -> Kconfig:
	66	+ with open(path, 'r') as f:
	67	+ return parse_from_string(f.read())
	68	+
	69	+def parse_from_string(blob: str) -> Kconfig:
	70	+ """Parses a string containing Kconfig entries."""
	71	+ kconfig = Kconfig()
	72	+ is_not_set_matcher = re.compile(CONFIG_IS_NOT_SET_PATTERN)
	73	+ config_matcher = re.compile(CONFIG_PATTERN)
	74	+ for line in blob.split('\n'):
	75	+ line = line.strip()
	76	+ if not line:
	77	+ continue
	78	+
	79	+ match = config_matcher.match(line)
	80	+ if match:
	81	+ entry = KconfigEntry(match.group(1), match.group(2))
	82	+ kconfig.add_entry(entry)
	83	+ continue
	84	+
	85	+ empty_match = is_not_set_matcher.match(line)
	86	+ if empty_match:
	87	+ entry = KconfigEntry(empty_match.group(1), 'n')
	88	+ kconfig.add_entry(entry)
	89	+ continue
	90	+
	91	+ if line[0] == '#':
	92	+ continue
	93	+ else:
	94	+ raise KconfigParseError('Failed to parse: ' + line)
	95	+ return kconfig

tools/testing/kunit/kunit_json.py+5−3

@@ -11,7 +11,7 @@ import os · 10 unmodified lines
11	11
12	12	import kunit_parser
13	13
14		−from kunit_parser import Test, TestResult, TestStatus
	14	+from kunit_parser import Test, TestStatus
15	15	from typing import Any, Dict, Optional
16	16
17	17	JsonObj = Dict[str, Any]
@@ -30,6 +30,8 @@ def _get_group_json(test: Test, def_config: str, · 12 unmodified lines
30	30	test_case = {"name": subtest.name, "status": "FAIL"}
31	31	if subtest.status == TestStatus.SUCCESS:
32	32	test_case["status"] = "PASS"
	33	+ elif subtest.status == TestStatus.SKIPPED:
	34	+ test_case["status"] = "SKIP"
33	35	elif subtest.status == TestStatus.TEST_CRASHED:
34	36	test_case["status"] = "ERROR"
35	37	test_cases.append(test_case)
@@ -48,9 +50,9 @@ def _get_group_json(test: Test, def_config: str, · 12 unmodified lines
48	50	}
49	51	return test_group
50	52
51		−def get_json_result(test_result: TestResult, def_config: str,
	53	+def get_json_result(test: Test, def_config: str,
52	54	build_dir: Optional[str], json_path: str) -> str:
53		− test_group = _get_group_json(test_result.test, def_config, build_dir)
	55	+ test_group = _get_group_json(test, def_config, build_dir)
54	56	test_group["name"] = "KUnit Test Group"
55	57	json_obj = json.dumps(test_group, indent=4)
56	58	if json_path != 'stdout':

tools/testing/kunit/kunit_kernel.py+49−27

@@ -21,6 +21,7 @@ import qemu_config · 20 unmodified lines
21	21
22	22	KCONFIG_PATH = '.config'
23	23	KUNITCONFIG_PATH = '.kunitconfig'
	24	+OLD_KUNITCONFIG_PATH = 'last_used_kunitconfig'
24	25	DEFAULT_KUNITCONFIG_PATH = 'tools/testing/kunit/configs/default.config'
25	26	BROKEN_ALLCONFIG_PATH = 'tools/testing/kunit/configs/broken_on_uml.config'
26	27	OUTFILE_PATH = 'test.log'
@@ -116,8 +117,7 @@ class LinuxSourceTreeOperationsQemu(LinuxSourceTreeOperations): · 89 unmodified lines
116	117	self._extra_qemu_params = qemu_arch_params.extra_qemu_params
117	118
118	119	def make_arch_qemuconfig(self, base_kunitconfig: kunit_config.Kconfig) -> None:
119		− kconfig = kunit_config.Kconfig()
120		− kconfig.parse_from_string(self._kconfig)
	120	+ kconfig = kunit_config.parse_from_string(self._kconfig)
121	121	base_kunitconfig.merge_in_entries(kconfig)
122	122
123	123	def start(self, params: List[str], build_dir: str) -> subprocess.Popen:
@@ -180,6 +180,9 @@ def get_kconfig_path(build_dir) -> str: · 56 unmodified lines
180	180	def get_kunitconfig_path(build_dir) -> str:
181	181	return get_file_path(build_dir, KUNITCONFIG_PATH)
182	182
	183	+def get_old_kunitconfig_path(build_dir) -> str:
	184	+ return get_file_path(build_dir, OLD_KUNITCONFIG_PATH)
	185	+
183	186	def get_outfile_path(build_dir) -> str:
184	187	return get_file_path(build_dir, OUTFILE_PATH)
185	188
@@ -206,6 +209,7 @@ def get_source_tree_ops_from_qemu_config(config_path: str, · 20 unmodified lines
206	209	# exists as a file.
207	210	module_path = '.' + os.path.join(os.path.basename(QEMU_CONFIGS_DIR), os.path.basename(config_path))
208	211	spec = importlib.util.spec_from_file_location(module_path, config_path)
	212	+ assert spec is not None
209	213	config = importlib.util.module_from_spec(spec)
210	214	# See https://github.com/python/typeshed/pull/2626 for context.
211	215	assert isinstance(spec.loader, importlib.abc.Loader)
@@ -225,6 +229,7 @@ class LinuxSourceTree(object): · 13 unmodified lines
225	229	build_dir: str,
226	230	load_config=True,
227	231	kunitconfig_path='',
	232	+ kconfig_add: Optional[List[str]]=None,
228	233	arch=None,
229	234	cross_compile=None,
230	235	qemu_config_path=None) -> None:
@@ -249,8 +254,11 @@ class LinuxSourceTree(object): · 18 unmodified lines
249	254	if not os.path.exists(kunitconfig_path):
250	255	shutil.copyfile(DEFAULT_KUNITCONFIG_PATH, kunitconfig_path)
251	256
252		− self._kconfig = kunit_config.Kconfig()
253		− self._kconfig.read_from_file(kunitconfig_path)
	257	+ self._kconfig = kunit_config.parse_file(kunitconfig_path)
	258	+ if kconfig_add:
	259	+ kconfig = kunit_config.parse_from_string('\n'.join(kconfig_add))
	260	+ self._kconfig.merge_in_entries(kconfig)
	261	+
254	262
255	263	def clean(self) -> bool:
256	264	try:
@@ -262,17 +270,18 @@ class LinuxSourceTree(object): · 5 unmodified lines
262	270
263	271	def validate_config(self, build_dir) -> bool:
264	272	kconfig_path = get_kconfig_path(build_dir)
265		− validated_kconfig = kunit_config.Kconfig()
266		− validated_kconfig.read_from_file(kconfig_path)
267		− if not self._kconfig.is_subset_of(validated_kconfig):
268		− invalid = self._kconfig.entries() - validated_kconfig.entries()
269		− message = 'Provided Kconfig is not contained in validated .config. Following fields found in kunitconfig, ' \
270		− 'but not in .config: %s' % (
271		− ', '.join([str(e) for e in invalid])
272		− )
273		− logging.error(message)
274		− return False
275		− return True
	273	+ validated_kconfig = kunit_config.parse_file(kconfig_path)
	274	+ if self._kconfig.is_subset_of(validated_kconfig):
	275	+ return True
	276	+ invalid = self._kconfig.entries() - validated_kconfig.entries()
	277	+ message = 'Not all Kconfig options selected in kunitconfig were in the generated .config.\n' \
	278	+ 'This is probably due to unsatisfied dependencies.\n' \
	279	+ 'Missing: ' + ', '.join([str(e) for e in invalid])
	280	+ if self._arch == 'um':
	281	+ message += '\nNote: many Kconfig options aren\'t available on UML. You can try running ' \
	282	+ 'on a different architecture with something like "--arch=x86_64".'
	283	+ logging.error(message)
	284	+ return False
276	285
277	286	def build_config(self, build_dir, make_options) -> bool:
278	287	kconfig_path = get_kconfig_path(build_dir)
@@ -285,25 +294,38 @@ class LinuxSourceTree(object): · 6 unmodified lines
285	294	except ConfigError as e:
286	295	logging.error(e)
287	296	return False
288		− return self.validate_config(build_dir)
	297	+ if not self.validate_config(build_dir):
	298	+ return False
	299	+
	300	+ old_path = get_old_kunitconfig_path(build_dir)
	301	+ if os.path.exists(old_path):
	302	+ os.remove(old_path) # write_to_file appends to the file
	303	+ self._kconfig.write_to_file(old_path)
	304	+ return True
	305	+
	306	+ def _kunitconfig_changed(self, build_dir: str) -> bool:
	307	+ old_path = get_old_kunitconfig_path(build_dir)
	308	+ if not os.path.exists(old_path):
	309	+ return True
	310	+
	311	+ old_kconfig = kunit_config.parse_file(old_path)
	312	+ return old_kconfig.entries() != self._kconfig.entries()
289	313
290	314	def build_reconfig(self, build_dir, make_options) -> bool:
291	315	"""Creates a new .config if it is not a subset of the .kunitconfig."""
292	316	kconfig_path = get_kconfig_path(build_dir)
293		− if os.path.exists(kconfig_path):
294		− existing_kconfig = kunit_config.Kconfig()
295		− existing_kconfig.read_from_file(kconfig_path)
296		− self._ops.make_arch_qemuconfig(self._kconfig)
297		− if not self._kconfig.is_subset_of(existing_kconfig):
298		− print('Regenerating .config ...')
299		− os.remove(kconfig_path)
300		− return self.build_config(build_dir, make_options)
301		− else:
302		− return True
303		− else:
	317	+ if not os.path.exists(kconfig_path):
304	318	print('Generating .config ...')
305	319	return self.build_config(build_dir, make_options)
306	320
	321	+ existing_kconfig = kunit_config.parse_file(kconfig_path)
	322	+ self._ops.make_arch_qemuconfig(self._kconfig)
	323	+ if self._kconfig.is_subset_of(existing_kconfig) and not self._kunitconfig_changed(build_dir):
	324	+ return True
	325	+ print('Regenerating .config ...')
	326	+ os.remove(kconfig_path)
	327	+ return self.build_config(build_dir, make_options)
	328	+
307	329	def build_kernel(self, alltests, jobs, build_dir, make_options) -> bool:
308	330	try:
309	331	if alltests:

tools/testing/kunit/kunit_parser.py+34−23

@@ -12,14 +12,11 @@ · 11 unmodified lines
12	12	from __future__ import annotations
13	13	import re
14	14
15		−from collections import namedtuple
16		−from datetime import datetime
	15	+import datetime
17	16	from enum import Enum, auto
18	17	from functools import reduce
19	18	from typing import Iterable, Iterator, List, Optional, Tuple
20	19
21		−TestResult = namedtuple('TestResult', ['status','test','log'])
22		−
23	20	class Test(object):
24	21	"""
25	22	A class to represent a test parsed from KTAP results. All KTAP
@@ -168,42 +165,51 @@ class TestCounts: · 142 unmodified lines
168	165	class LineStream:
169	166	"""
170	167	A class to represent the lines of kernel output.
171		− Provides a peek()/pop() interface over an iterator of
	168	+ Provides a lazy peek()/pop() interface over an iterator of
172	169	(line#, text).
173	170	"""
174	171	_lines: Iterator[Tuple[int, str]]
175	172	_next: Tuple[int, str]
	173	+ _need_next: bool
176	174	_done: bool
177	175
178	176	def __init__(self, lines: Iterator[Tuple[int, str]]):
179	177	"""Creates a new LineStream that wraps the given iterator."""
180	178	self._lines = lines
181	179	self._done = False
	180	+ self._need_next = True
182	181	self._next = (0, '')
183		− self._get_next()
184	182
185	183	def _get_next(self) -> None:
186		− """Advances the LineSteam to the next line."""
	184	+ """Advances the LineSteam to the next line, if necessary."""
	185	+ if not self._need_next:
	186	+ return
187	187	try:
188	188	self._next = next(self._lines)
189	189	except StopIteration:
190	190	self._done = True
	191	+ finally:
	192	+ self._need_next = False
191	193
192	194	def peek(self) -> str:
193	195	"""Returns the current line, without advancing the LineStream.
194	196	"""
	197	+ self._get_next()
195	198	return self._next[1]
196	199
197	200	def pop(self) -> str:
198	201	"""Returns the current line and advances the LineStream to
199	202	the next line.
200	203	"""
201		− n = self._next
202		− self._get_next()
203		− return n[1]
	204	+ s = self.peek()
	205	+ if self._done:
	206	+ raise ValueError(f'LineStream: going past EOF, last line was {s}')
	207	+ self._need_next = True
	208	+ return s
204	209
205	210	def __bool__(self) -> bool:
206	211	"""Returns True if stream has more lines."""
	212	+ self._get_next()
207	213	return not self._done
208	214
209	215	# Only used by kunit_tool_test.py.
@@ -216,6 +222,7 @@ class LineStream: · 6 unmodified lines
216	222
217	223	def line_number(self) -> int:
218	224	"""Returns the line number of the current line."""
	225	+ self._get_next()
219	226	return self._next[0]
220	227
221	228	# Parsing helper methods:
@@ -340,8 +347,8 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: · 118 unmodified lines
340	347	"""
341	348	Parses test plan line and stores the expected number of subtests in
342	349	test object. Reports an error if expected count is 0.
343		− Returns False and reports missing test plan error if fails to parse
344		− test plan.
	350	+ Returns False and sets expected_count to None if there is no valid test
	351	+ plan.
345	352
346	353	Accepted format:
347	354	- '1..[number of subtests]'
@@ -356,14 +363,10 @@ def parse_test_plan(lines: LineStream, test: Test) -> bool: · 8 unmodified lines
356	363	match = TEST_PLAN.match(lines.peek())
357	364	if not match:
358	365	test.expected_count = None
359		− test.add_error('missing plan line!')
360	366	return False
361	367	test.log.append(lines.pop())
362	368	expected_count = int(match.group(1))
363	369	test.expected_count = expected_count
364		− if expected_count == 0:
365		− test.status = TestStatus.NO_TESTS
366		− test.add_error('0 tests run!')
367	370	return True
368	371
369	372	TEST_RESULT = re.compile(r'^(ok\|not ok) ([0-9]+) (- )?([^#])( # .)?$')
@@ -514,7 +517,7 @@ ANSI_LEN = len(red('')) · 144 unmodified lines
514	517
515	518	def print_with_timestamp(message: str) -> None:
516	519	"""Prints message with timestamp at beginning."""
517		− print('[%s] %s' % (datetime.now().strftime('%H:%M:%S'), message))
	520	+ print('[%s] %s' % (datetime.datetime.now().strftime('%H:%M:%S'), message))
518	521
519	522	def format_test_divider(message: str, len_message: int) -> str:
520	523	"""
@@ -590,6 +593,8 @@ def format_test_result(test: Test) -> str: · 69 unmodified lines
590	593	return (green('[PASSED] ') + test.name)
591	594	elif test.status == TestStatus.SKIPPED:
592	595	return (yellow('[SKIPPED] ') + test.name)
	596	+ elif test.status == TestStatus.NO_TESTS:
	597	+ return (yellow('[NO TESTS RUN] ') + test.name)
593	598	elif test.status == TestStatus.TEST_CRASHED:
594	599	print_log(test.log)
595	600	return (red('[CRASHED] ') + test.name)
@@ -732,6 +737,7 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: · 136 unmodified lines
732	737	# test plan
733	738	test.name = "main"
734	739	parse_test_plan(lines, test)
	740	+ parent_test = True
735	741	else:
736	742	# If KTAP/TAP header is not found, test must be subtest
737	743	# header or test result line so parse attempt to parser
@@ -745,7 +751,7 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: · 7 unmodified lines
745	751	expected_count = test.expected_count
746	752	subtests = []
747	753	test_num = 1
748		− while expected_count is None or test_num <= expected_count:
	754	+ while parent_test and (expected_count is None or test_num <= expected_count):
749	755	# Loop to parse any subtests.
750	756	# Break after parsing expected number of tests or
751	757	# if expected number of tests is unknown break when test
@@ -780,9 +786,15 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: · 28 unmodified lines
780	786	parse_test_result(lines, test, expected_num)
781	787	else:
782	788	test.add_error('missing subtest result line!')
	789	+
	790	+ # Check for there being no tests
	791	+ if parent_test and len(subtests) == 0:
	792	+ test.status = TestStatus.NO_TESTS
	793	+ test.add_error('0 tests run!')
	794	+
783	795	# Add statuses to TestCounts attribute in Test object
784	796	bubble_up_test_results(test)
785		− if parent_test:
	797	+ if parent_test and not main:
786	798	# If test has subtests and is not the main test object, print
787	799	# footer.
788	800	print_test_footer(test)
@@ -790,7 +802,7 @@ def parse_test(lines: LineStream, expected_num: int, log: List[str]) -> Test: · 1 unmodified line
790	802	print_test_result(test)
791	803	return test
792	804
793		−def parse_run_tests(kernel_output: Iterable[str]) -> TestResult:
	805	+def parse_run_tests(kernel_output: Iterable[str]) -> Test:
794	806	"""
795	807	Using kernel output, extract KTAP lines, parse the lines for test
796	808	results and print condensed test results and summary line .
@@ -799,8 +811,7 @@ def parse_run_tests(kernel_output: Iterable[str]) -> TestResult: · 2 unmodified lines
799	811	kernel_output - Iterable object contains lines of kernel output
800	812
801	813	Return:
802		− TestResult - Tuple containg status of main test object, main test
803		− object with all subtests, and log of all KTAP lines.
	814	+ Test - the main test object with all subtests.
804	815	"""
805	816	print_with_timestamp(DIVIDER)
806	817	lines = extract_tap_lines(kernel_output)
@@ -814,4 +825,4 @@ def parse_run_tests(kernel_output: Iterable[str]) -> TestResult: · 7 unmodified lines
814	825	test.status = test.counts.get_status()
815	826	print_with_timestamp(DIVIDER)
816	827	print_summary_line(test)
817		− return TestResult(test.status, test, lines)
	828	+ return test

tools/testing/kunit/kunit_tool_test.py+147−24

@@ -13,9 +13,10 @@ import tempfile, shutil # Handling test_tmpdir · 12 unmodified lines
13	13
14	14	import itertools
15	15	import json
	16	+import os
16	17	import signal
17	18	import subprocess
18		−import os
	19	+from typing import Iterable
19	20
20	21	import kunit_config
21	22	import kunit_parser
@@ -50,10 +51,9 @@ class KconfigTest(unittest.TestCase): · 28 unmodified lines
50	51	self.assertFalse(kconfig1.is_subset_of(kconfig0))
51	52
52	53	def test_read_from_file(self):
53		− kconfig = kunit_config.Kconfig()
54	54	kconfig_path = test_data_path('test_read_from_file.kconfig')
55	55
56		− kconfig.read_from_file(kconfig_path)
	56	+ kconfig = kunit_config.parse_file(kconfig_path)
57	57
58	58	expected_kconfig = kunit_config.Kconfig()
59	59	expected_kconfig.add_entry(
@@ -86,8 +86,7 @@ class KconfigTest(unittest.TestCase): · 26 unmodified lines
86	86
87	87	expected_kconfig.write_to_file(kconfig_path)
88	88
89		− actual_kconfig = kunit_config.Kconfig()
90		− actual_kconfig.read_from_file(kconfig_path)
	89	+ actual_kconfig = kunit_config.parse_file(kconfig_path)
91	90
92	91	self.assertEqual(actual_kconfig.entries(),
93	92	expected_kconfig.entries())
@@ -179,7 +178,7 @@ class KUnitParserTest(unittest.TestCase): · 85 unmodified lines
179	178	with open(empty_log) as file:
180	179	result = kunit_parser.parse_run_tests(
181	180	kunit_parser.extract_tap_lines(file.readlines()))
182		− self.assertEqual(0, len(result.test.subtests))
	181	+ self.assertEqual(0, len(result.subtests))
183	182	self.assertEqual(
184	183	kunit_parser.TestStatus.FAILURE_TO_PARSE_TESTS,
185	184	result.status)
@@ -191,7 +190,10 @@ class KUnitParserTest(unittest.TestCase): · 5 unmodified lines
191	190	result = kunit_parser.parse_run_tests(
192	191	kunit_parser.extract_tap_lines(
193	192	file.readlines()))
194		− self.assertEqual(2, result.test.counts.errors)
	193	+ # A missing test plan is not an error.
	194	+ self.assertEqual(0, result.counts.errors)
	195	+ # All tests should be accounted for.
	196	+ self.assertEqual(10, result.counts.total())
195	197	self.assertEqual(
196	198	kunit_parser.TestStatus.SUCCESS,
197	199	result.status)
@@ -201,11 +203,23 @@ class KUnitParserTest(unittest.TestCase): · 3 unmodified lines
201	203	with open(header_log) as file:
202	204	result = kunit_parser.parse_run_tests(
203	205	kunit_parser.extract_tap_lines(file.readlines()))
204		− self.assertEqual(0, len(result.test.subtests))
	206	+ self.assertEqual(0, len(result.subtests))
205	207	self.assertEqual(
206	208	kunit_parser.TestStatus.NO_TESTS,
207	209	result.status)
208	210
	211	+ def test_no_tests_no_plan(self):
	212	+ no_plan_log = test_data_path('test_is_test_passed-no_tests_no_plan.log')
	213	+ with open(no_plan_log) as file:
	214	+ result = kunit_parser.parse_run_tests(
	215	+ kunit_parser.extract_tap_lines(file.readlines()))
	216	+ self.assertEqual(0, len(result.subtests[0].subtests[0].subtests))
	217	+ self.assertEqual(
	218	+ kunit_parser.TestStatus.NO_TESTS,
	219	+ result.subtests[0].subtests[0].status)
	220	+ self.assertEqual(1, result.counts.errors)
	221	+
	222	+
209	223	def test_no_kunit_output(self):
210	224	crash_log = test_data_path('test_insufficient_memory.log')
211	225	print_mock = mock.patch('builtins.print').start()
@@ -214,7 +228,7 @@ class KUnitParserTest(unittest.TestCase): · 2 unmodified lines
214	228	kunit_parser.extract_tap_lines(file.readlines()))
215	229	print_mock.assert_any_call(StrContains('invalid KTAP input!'))
216	230	print_mock.stop()
217		− self.assertEqual(0, len(result.test.subtests))
	231	+ self.assertEqual(0, len(result.subtests))
218	232
219	233	def test_crashed_test(self):
220	234	crashed_log = test_data_path('test_is_test_passed-crash.log')
@@ -255,10 +269,10 @@ class KUnitParserTest(unittest.TestCase): · 34 unmodified lines
255	269	result.status)
256	270	self.assertEqual(
257	271	"sysctl_test",
258		− result.test.subtests[0].name)
	272	+ result.subtests[0].name)
259	273	self.assertEqual(
260	274	"example",
261		− result.test.subtests[1].name)
	275	+ result.subtests[1].name)
262	276	file.close()
263	277
264	278
@@ -269,7 +283,7 @@ class KUnitParserTest(unittest.TestCase): · 4 unmodified lines
269	283	self.assertEqual(
270	284	kunit_parser.TestStatus.SUCCESS,
271	285	result.status)
272		− self.assertEqual('kunit-resource-test', result.test.subtests[0].name)
	286	+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
273	287
274	288	def test_ignores_multiple_prefixes(self):
275	289	prefix_log = test_data_path('test_multiple_prefixes.log')
@@ -278,7 +292,7 @@ class KUnitParserTest(unittest.TestCase): · 2 unmodified lines
278	292	self.assertEqual(
279	293	kunit_parser.TestStatus.SUCCESS,
280	294	result.status)
281		− self.assertEqual('kunit-resource-test', result.test.subtests[0].name)
	295	+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
282	296
283	297	def test_prefix_mixed_kernel_output(self):
284	298	mixed_prefix_log = test_data_path('test_interrupted_tap_output.log')
@@ -287,7 +301,7 @@ class KUnitParserTest(unittest.TestCase): · 2 unmodified lines
287	301	self.assertEqual(
288	302	kunit_parser.TestStatus.SUCCESS,
289	303	result.status)
290		− self.assertEqual('kunit-resource-test', result.test.subtests[0].name)
	304	+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
291	305
292	306	def test_prefix_poundsign(self):
293	307	pound_log = test_data_path('test_pound_sign.log')
@@ -296,7 +310,7 @@ class KUnitParserTest(unittest.TestCase): · 2 unmodified lines
296	310	self.assertEqual(
297	311	kunit_parser.TestStatus.SUCCESS,
298	312	result.status)
299		− self.assertEqual('kunit-resource-test', result.test.subtests[0].name)
	313	+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
300	314
301	315	def test_kernel_panic_end(self):
302	316	panic_log = test_data_path('test_kernel_panic_interrupt.log')
@@ -305,7 +319,7 @@ class KUnitParserTest(unittest.TestCase): · 2 unmodified lines
305	319	self.assertEqual(
306	320	kunit_parser.TestStatus.TEST_CRASHED,
307	321	result.status)
308		− self.assertEqual('kunit-resource-test', result.test.subtests[0].name)
	322	+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
309	323
310	324	def test_pound_no_prefix(self):
311	325	pound_log = test_data_path('test_pound_no_prefix.log')
@@ -314,7 +328,46 @@ class KUnitParserTest(unittest.TestCase): · 2 unmodified lines
314	328	self.assertEqual(
315	329	kunit_parser.TestStatus.SUCCESS,
316	330	result.status)
317		− self.assertEqual('kunit-resource-test', result.test.subtests[0].name)
	331	+ self.assertEqual('kunit-resource-test', result.subtests[0].name)
	332	+
	333	+def line_stream_from_strs(strs: Iterable[str]) -> kunit_parser.LineStream:
	334	+ return kunit_parser.LineStream(enumerate(strs, start=1))
	335	+
	336	+class LineStreamTest(unittest.TestCase):
	337	+
	338	+ def test_basic(self):
	339	+ stream = line_stream_from_strs(['hello', 'world'])
	340	+
	341	+ self.assertTrue(stream, msg='Should be more input')
	342	+ self.assertEqual(stream.line_number(), 1)
	343	+ self.assertEqual(stream.peek(), 'hello')
	344	+ self.assertEqual(stream.pop(), 'hello')
	345	+
	346	+ self.assertTrue(stream, msg='Should be more input')
	347	+ self.assertEqual(stream.line_number(), 2)
	348	+ self.assertEqual(stream.peek(), 'world')
	349	+ self.assertEqual(stream.pop(), 'world')
	350	+
	351	+ self.assertFalse(stream, msg='Should be no more input')
	352	+ with self.assertRaisesRegex(ValueError, 'LineStream: going past EOF'):
	353	+ stream.pop()
	354	+
	355	+ def test_is_lazy(self):
	356	+ called_times = 0
	357	+ def generator():
	358	+ nonlocal called_times
	359	+ for i in range(1,5):
	360	+ called_times += 1
	361	+ yield called_times, str(called_times)
	362	+
	363	+ stream = kunit_parser.LineStream(generator())
	364	+ self.assertEqual(called_times, 0)
	365	+
	366	+ self.assertEqual(stream.pop(), '1')
	367	+ self.assertEqual(called_times, 1)
	368	+
	369	+ self.assertEqual(stream.pop(), '2')
	370	+ self.assertEqual(called_times, 2)
318	371
319	372	class LinuxSourceTreeTest(unittest.TestCase):
320	373
@@ -336,6 +389,10 @@ class LinuxSourceTreeTest(unittest.TestCase): · 15 unmodified lines
336	389	pass
337	390	kunit_kernel.LinuxSourceTree('', kunitconfig_path=dir)
338	391
	392	+ def test_kconfig_add(self):
	393	+ tree = kunit_kernel.LinuxSourceTree('', kconfig_add=['CONFIG_NOT_REAL=y'])
	394	+ self.assertIn(kunit_config.KconfigEntry('NOT_REAL', 'y'), tree._kconfig.entries())
	395	+
339	396	def test_invalid_arch(self):
340	397	with self.assertRaisesRegex(kunit_kernel.ConfigError, 'not a valid arch, options are.*x86_64'):
341	398	kunit_kernel.LinuxSourceTree('', arch='invalid')
@@ -356,6 +413,51 @@ class LinuxSourceTreeTest(unittest.TestCase): · 14 unmodified lines
356	413	with open(kunit_kernel.get_outfile_path(build_dir), 'rt') as outfile:
357	414	self.assertEqual(outfile.read(), 'hi\nbye\n', msg='Missing some output')
358	415
	416	+ def test_build_reconfig_no_config(self):
	417	+ with tempfile.TemporaryDirectory('') as build_dir:
	418	+ with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f:
	419	+ f.write('CONFIG_KUNIT=y')
	420	+
	421	+ tree = kunit_kernel.LinuxSourceTree(build_dir)
	422	+ mock_build_config = mock.patch.object(tree, 'build_config').start()
	423	+
	424	+ # Should generate the .config
	425	+ self.assertTrue(tree.build_reconfig(build_dir, make_options=[]))
	426	+ mock_build_config.assert_called_once_with(build_dir, [])
	427	+
	428	+ def test_build_reconfig_existing_config(self):
	429	+ with tempfile.TemporaryDirectory('') as build_dir:
	430	+ # Existing .config is a superset, should not touch it
	431	+ with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f:
	432	+ f.write('CONFIG_KUNIT=y')
	433	+ with open(kunit_kernel.get_old_kunitconfig_path(build_dir), 'w') as f:
	434	+ f.write('CONFIG_KUNIT=y')
	435	+ with open(kunit_kernel.get_kconfig_path(build_dir), 'w') as f:
	436	+ f.write('CONFIG_KUNIT=y\nCONFIG_KUNIT_TEST=y')
	437	+
	438	+ tree = kunit_kernel.LinuxSourceTree(build_dir)
	439	+ mock_build_config = mock.patch.object(tree, 'build_config').start()
	440	+
	441	+ self.assertTrue(tree.build_reconfig(build_dir, make_options=[]))
	442	+ self.assertEqual(mock_build_config.call_count, 0)
	443	+
	444	+ def test_build_reconfig_remove_option(self):
	445	+ with tempfile.TemporaryDirectory('') as build_dir:
	446	+ # We removed CONFIG_KUNIT_TEST=y from our .kunitconfig...
	447	+ with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f:
	448	+ f.write('CONFIG_KUNIT=y')
	449	+ with open(kunit_kernel.get_old_kunitconfig_path(build_dir), 'w') as f:
	450	+ f.write('CONFIG_KUNIT=y\nCONFIG_KUNIT_TEST=y')
	451	+ with open(kunit_kernel.get_kconfig_path(build_dir), 'w') as f:
	452	+ f.write('CONFIG_KUNIT=y\nCONFIG_KUNIT_TEST=y')
	453	+
	454	+ tree = kunit_kernel.LinuxSourceTree(build_dir)
	455	+ mock_build_config = mock.patch.object(tree, 'build_config').start()
	456	+
	457	+ # ... so we should trigger a call to build_config()
	458	+ self.assertTrue(tree.build_reconfig(build_dir, make_options=[]))
	459	+ mock_build_config.assert_called_once_with(build_dir, [])
	460	+
359	461	# TODO: add more test cases.
360	462
361	463
@@ -365,7 +467,7 @@ class KUnitJsonTest(unittest.TestCase): · 3 unmodified lines
365	467	with open(test_data_path(log_file)) as file:
366	468	test_result = kunit_parser.parse_run_tests(file)
367	469	json_obj = kunit_json.get_json_result(
368		− test_result=test_result,
	470	+ test=test_result,
369	471	def_config='kunit_defconfig',
370	472	build_dir=None,
371	473	json_path='stdout')
@@ -383,6 +485,12 @@ class KUnitJsonTest(unittest.TestCase): · 11 unmodified lines
383	485	{'name': 'example_simple_test', 'status': 'ERROR'},
384	486	result["sub_groups"][1]["test_cases"][0])
385	487
	488	+ def test_skipped_test_json(self):
	489	+ result = self._json_for('test_skip_tests.log')
	490	+ self.assertEqual(
	491	+ {'name': 'example_skip_test', 'status': 'SKIP'},
	492	+ result["sub_groups"][1]["test_cases"][1])
	493	+
386	494	def test_no_tests_json(self):
387	495	result = self._json_for('test_is_test_passed-no_tests_run_with_header.log')
388	496	self.assertEqual(0, len(result['sub_groups']))
@@ -418,8 +526,8 @@ class KUnitMainTest(unittest.TestCase): · 29 unmodified lines
418	526
419	527	def test_build_passes_args_pass(self):
420	528	kunit.main(['build'], self.linux_source_mock)
421		− self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 0)
422		− self.linux_source_mock.build_kernel.assert_called_once_with(False, 8, '.kunit', None)
	529	+ self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
	530	+ self.linux_source_mock.build_kernel.assert_called_once_with(False, kunit.get_default_jobs(), '.kunit', None)
423	531	self.assertEqual(self.linux_source_mock.run_kernel.call_count, 0)
424	532
425	533	def test_exec_passes_args_pass(self):
@@ -525,8 +633,9 @@ class KUnitMainTest(unittest.TestCase): · 99 unmodified lines
525	633
526	634	def test_build_builddir(self):
527	635	build_dir = '.kunit'
	636	+ jobs = kunit.get_default_jobs()
528	637	kunit.main(['build', '--build_dir', build_dir], self.linux_source_mock)
529		− self.linux_source_mock.build_kernel.assert_called_once_with(False, 8, build_dir, None)
	638	+ self.linux_source_mock.build_kernel.assert_called_once_with(False, jobs, build_dir, None)
530	639
531	640	def test_exec_builddir(self):
532	641	build_dir = '.kunit'
@@ -542,6 +651,7 @@ class KUnitMainTest(unittest.TestCase): · 9 unmodified lines
542	651	# Just verify that we parsed and initialized it correctly here.
543	652	mock_linux_init.assert_called_once_with('.kunit',
544	653	kunitconfig_path='mykunitconfig',
	654	+ kconfig_add=None,
545	655	arch='um',
546	656	cross_compile=None,
547	657	qemu_config_path=None)
@@ -553,6 +663,19 @@ class KUnitMainTest(unittest.TestCase): · 5 unmodified lines
553	663	# Just verify that we parsed and initialized it correctly here.
554	664	mock_linux_init.assert_called_once_with('.kunit',
555	665	kunitconfig_path='mykunitconfig',
	666	+ kconfig_add=None,
	667	+ arch='um',
	668	+ cross_compile=None,
	669	+ qemu_config_path=None)
	670	+
	671	+ @mock.patch.object(kunit_kernel, 'LinuxSourceTree')
	672	+ def test_run_kconfig_add(self, mock_linux_init):
	673	+ mock_linux_init.return_value = self.linux_source_mock
	674	+ kunit.main(['run', '--kconfig_add=CONFIG_KASAN=y', '--kconfig_add=CONFIG_KCSAN=y'])
	675	+ # Just verify that we parsed and initialized it correctly here.
	676	+ mock_linux_init.assert_called_once_with('.kunit',
	677	+ kunitconfig_path=None,
	678	+ kconfig_add=['CONFIG_KASAN=y', 'CONFIG_KCSAN=y'],
556	679	arch='um',
557	680	cross_compile=None,
558	681	qemu_config_path=None)
@@ -569,7 +692,7 @@ class KUnitMainTest(unittest.TestCase): · 10 unmodified lines
569	692	self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want
570	693
571	694	got = kunit._list_tests(self.linux_source_mock,
572		− kunit.KunitExecRequest(300, '.kunit', False, 'suite*', None, 'suite'))
	695	+ kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'suite'))
573	696
574	697	self.assertEqual(got, want)
575	698	# Should respect the user's filter glob when listing tests.
@@ -584,7 +707,7 @@ class KUnitMainTest(unittest.TestCase): · 8 unmodified lines
584	707
585	708	# Should respect the user's filter glob when listing tests.
586	709	mock_tests.assert_called_once_with(mock.ANY,
587		− kunit.KunitExecRequest(300, '.kunit', False, 'suite.test', None, 'suite'))
	710	+ kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite.test', None, 'suite'))
588	711	self.linux_source_mock.run_kernel.assert_has_calls([
589	712	mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300),
590	713	mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300),
@@ -597,7 +720,7 @@ class KUnitMainTest(unittest.TestCase): · 6 unmodified lines
597	720
598	721	# Should respect the user's filter glob when listing tests.
599	722	mock_tests.assert_called_once_with(mock.ANY,
600		− kunit.KunitExecRequest(300, '.kunit', False, 'suite*', None, 'test'))
	723	+ kunit.KunitExecRequest(None, '.kunit', None, 300, False, 'suite*', None, 'test'))
601	724	self.linux_source_mock.run_kernel.assert_has_calls([
602	725	mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300),
603	726	mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300),

tools/testing/kunit/run_checks.py+81−0

@@ -0,0 +1,81 @@
	1	+#!/usr/bin/env python3
	2	+# SPDX-License-Identifier: GPL-2.0
	3	+#
	4	+# This file runs some basic checks to verify kunit works.
	5	+# It is only of interest if you're making changes to KUnit itself.
	6	+#
	7	+# Copyright (C) 2021, Google LLC.
	8	+# Author: Daniel Latypov <dlatypov@google.com.com>
	9	+
	10	+from concurrent import futures
	11	+import datetime
	12	+import os
	13	+import shutil
	14	+import subprocess
	15	+import sys
	16	+import textwrap
	17	+from typing import Dict, List, Sequence, Tuple
	18	+
	19	+ABS_TOOL_PATH = os.path.abspath(os.path.dirname(__file__))
	20	+TIMEOUT = datetime.timedelta(minutes=5).total_seconds()
	21	+
	22	+commands: Dict[str, Sequence[str]] = {
	23	+ 'kunit_tool_test.py': ['./kunit_tool_test.py'],
	24	+ 'kunit smoke test': ['./kunit.py', 'run', '--kunitconfig=lib/kunit', '--build_dir=kunit_run_checks'],
	25	+ 'pytype': ['/bin/sh', '-c', 'pytype *.py'],
	26	+ 'mypy': ['/bin/sh', '-c', 'mypy *.py'],
	27	+}
	28	+
	29	+# The user might not have mypy or pytype installed, skip them if so.
	30	+# Note: you can install both via `$ pip install mypy pytype`
	31	+necessary_deps : Dict[str, str] = {
	32	+ 'pytype': 'pytype',
	33	+ 'mypy': 'mypy',
	34	+}
	35	+
	36	+def main(argv: Sequence[str]) -> None:
	37	+ if argv:
	38	+ raise RuntimeError('This script takes no arguments')
	39	+
	40	+ future_to_name: Dict[futures.Future, str] = {}
	41	+ executor = futures.ThreadPoolExecutor(max_workers=len(commands))
	42	+ for name, argv in commands.items():
	43	+ if name in necessary_deps and shutil.which(necessary_deps[name]) is None:
	44	+ print(f'{name}: SKIPPED, {necessary_deps[name]} not in $PATH')
	45	+ continue
	46	+ f = executor.submit(run_cmd, argv)
	47	+ future_to_name[f] = name
	48	+
	49	+ has_failures = False
	50	+ print(f'Waiting on {len(future_to_name)} checks ({", ".join(future_to_name.values())})...')
	51	+ for f in futures.as_completed(future_to_name.keys()):
	52	+ name = future_to_name[f]
	53	+ ex = f.exception()
	54	+ if not ex:
	55	+ print(f'{name}: PASSED')
	56	+ continue
	57	+
	58	+ has_failures = True
	59	+ if isinstance(ex, subprocess.TimeoutExpired):
	60	+ print(f'{name}: TIMED OUT')
	61	+ elif isinstance(ex, subprocess.CalledProcessError):
	62	+ print(f'{name}: FAILED')
	63	+ else:
	64	+ print('{name}: unexpected exception: {ex}')
	65	+ continue
	66	+
	67	+ output = ex.output
	68	+ if output:
	69	+ print(textwrap.indent(output.decode(), '> '))
	70	+ executor.shutdown()
	71	+
	72	+ if has_failures:
	73	+ sys.exit(1)
	74	+
	75	+
	76	+def run_cmd(argv: Sequence[str]):
	77	+ subprocess.check_output(argv, stderr=subprocess.STDOUT, cwd=ABS_TOOL_PATH, timeout=TIMEOUT)
	78	+
	79	+
	80	+if __name__ == '__main__':
	81	+ main(sys.argv[1:])

tools/testing/kunit/test_data/test_is_test_passed-no_tests_no_plan.log+7−0

@@ -0,0 +1,7 @@
	1	+TAP version 14
	2	+1..1
	3	+ # Subtest: suite
	4	+ 1..1
	5	+ # Subtest: case
	6	+ ok 1 - case # SKIP
	7	+ok 1 - suite