Commit 578a7155 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'linux-kselftest-kunit-fixes-5.10-rc1' of... 

Merge tag 'linux-kselftest-kunit-fixes-5.10-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest

Pull Kunit updates from Shuah Khan:
 "Several kunit tool bug fixes in flag handling, run outside kernel
  tree, make errors, and generating results"

* tag 'linux-kselftest-kunit-fixes-5.10-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest:
  kunit: tool: fix display of make errors
  kunit: tool: handle when .kunit exists but .kunitconfig does not
  kunit: tool: fix --alltests flag
  kunit: tool: allow generating test results in JSON
  kunit: tool: fix running kunit_tool from outside kernel tree
parents 0674324b 1abdd39f

tools/testing/kunit/configs/broken_on_uml.config

+1 −0
Original line number Diff line number Diff line
@@ -39,3 +39,4 @@ 
# CONFIG_QCOM_CPR is not set

# CONFIG_RESET_BRCMSTB_RESCAL is not set

# CONFIG_RESET_INTEL_GW is not set

# CONFIG_ADI_AXI_ADC is not set

tools/testing/kunit/kunit.py

+41 −17
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ from collections import namedtuple 
from enum import Enum, auto



import kunit_config

import kunit_json

import kunit_kernel

import kunit_parser
@@ -30,9 +31,9 @@ KunitBuildRequest = namedtuple('KunitBuildRequest', 
KunitExecRequest = namedtuple('KunitExecRequest',

			      ['timeout', 'build_dir', 'alltests'])

KunitParseRequest = namedtuple('KunitParseRequest',

			       ['raw_output', 'input_data'])

			       ['raw_output', 'input_data', 'build_dir', 'json'])

KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',

					   'build_dir', 'alltests',

					   'build_dir', 'alltests', 'json',

					   'make_options'])



KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
@@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult: 
	test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,

					      [],

					      'Tests not Parsed.')



	if request.raw_output:

		kunit_parser.raw_output(request.input_data)

	else:

		test_result = kunit_parser.parse_run_tests(request.input_data)

	parse_end = time.time()



	if request.json:

		json_obj = kunit_json.get_json_result(

					test_result=test_result,

					def_config='kunit_defconfig',

					build_dir=request.build_dir,

					json_path=request.json)

		if request.json == 'stdout':

			print(json_obj)



	if test_result.status != kunit_parser.TestStatus.SUCCESS:

		return KunitResult(KunitStatus.TEST_FAILURE, test_result,

				   parse_end - parse_start)
@@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, 
		return exec_result



	parse_request = KunitParseRequest(request.raw_output,

					  exec_result.result)

					  exec_result.result,

					  request.build_dir,

					  request.json)

	parse_result = parse_tests(parse_request)



	run_end = time.time()
@@ -195,7 +208,12 @@ def add_exec_opts(parser): 
def add_parse_opts(parser):

	parser.add_argument('--raw_output', help='don\'t format output from kernel',

			    action='store_true')



	parser.add_argument('--json',

			    nargs='?',

			    help='Stores test results in a JSON, and either '

			    'prints to stdout or saves to file if a '

			    'filename is specified',

			    type=str, const='stdout', default=None)



def main(argv, linux=None):

	parser = argparse.ArgumentParser(
@@ -237,10 +255,16 @@ def main(argv, linux=None): 


	cli_args = parser.parse_args(argv)



	if get_kernel_root_path():

		os.chdir(get_kernel_root_path())



	if cli_args.subcommand == 'run':

		if not os.path.exists(cli_args.build_dir):

			os.mkdir(cli_args.build_dir)



		if not os.path.exists(kunit_kernel.kunitconfig_path):

			create_default_kunitconfig()



		if not linux:

			linux = kunit_kernel.LinuxSourceTree()
@@ -249,15 +273,19 @@ def main(argv, linux=None): 
				       cli_args.jobs,

				       cli_args.build_dir,

				       cli_args.alltests,

				       cli_args.json,

				       cli_args.make_options)

		result = run_tests(linux, request)

		if result.status != KunitStatus.SUCCESS:

			sys.exit(1)

	elif cli_args.subcommand == 'config':

		if cli_args.build_dir:

			if not os.path.exists(cli_args.build_dir):

		if cli_args.build_dir and (

				not os.path.exists(cli_args.build_dir)):

			os.mkdir(cli_args.build_dir)



		if not os.path.exists(kunit_kernel.kunitconfig_path):

			create_default_kunitconfig()



		if not linux:

			linux = kunit_kernel.LinuxSourceTree()
@@ -270,10 +298,6 @@ def main(argv, linux=None): 
		if result.status != KunitStatus.SUCCESS:

			sys.exit(1)

	elif cli_args.subcommand == 'build':

		if cli_args.build_dir:

			if not os.path.exists(cli_args.build_dir):

				os.mkdir(cli_args.build_dir)



		if not linux:

			linux = kunit_kernel.LinuxSourceTree()
@@ -288,10 +312,6 @@ def main(argv, linux=None): 
		if result.status != KunitStatus.SUCCESS:

			sys.exit(1)

	elif cli_args.subcommand == 'exec':

		if cli_args.build_dir:

			if not os.path.exists(cli_args.build_dir):

				os.mkdir(cli_args.build_dir)



		if not linux:

			linux = kunit_kernel.LinuxSourceTree()
@@ -300,7 +320,9 @@ def main(argv, linux=None): 
						cli_args.alltests)

		exec_result = exec_tests(linux, exec_request)

		parse_request = KunitParseRequest(cli_args.raw_output,

						  exec_result.result)

						  exec_result.result,

						  cli_args.build_dir,

						  cli_args.json)

		result = parse_tests(parse_request)

		kunit_parser.print_with_timestamp((

			'Elapsed time: %.3fs\n') % (
@@ -314,7 +336,9 @@ def main(argv, linux=None): 
			with open(cli_args.file, 'r') as f:

				kunit_output = f.read().splitlines()

		request = KunitParseRequest(cli_args.raw_output,

					    kunit_output)

					    kunit_output,

					    cli_args.build_dir,

					    cli_args.json)

		result = parse_tests(request)

		if result.status != KunitStatus.SUCCESS:

			sys.exit(1)

tools/testing/kunit/kunit_json.py

0 → 100644
+63 −0
Original line number Diff line number Diff line 
# SPDX-License-Identifier: GPL-2.0

#

# Generates JSON from KUnit results according to

# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API

#

# Copyright (C) 2020, Google LLC.

# Author: Heidi Fahim <heidifahim@google.com>



import json

import os



import kunit_parser



from kunit_parser import TestStatus



def get_json_result(test_result, def_config, build_dir, json_path):

	sub_groups = []



	# Each test suite is mapped to a KernelCI sub_group

	for test_suite in test_result.suites:

		sub_group = {

			"name": test_suite.name,

			"arch": "UM",

			"defconfig": def_config,

			"build_environment": build_dir,

			"test_cases": [],

			"lab_name": None,

			"kernel": None,

			"job": None,

			"git_branch": "kselftest",

		}

		test_cases = []

		# TODO: Add attachments attribute in test_case with detailed

		#  failure message, see https://api.kernelci.org/schema-test-case.html#get

		for case in test_suite.cases:

			test_case = {"name": case.name, "status": "FAIL"}

			if case.status == TestStatus.SUCCESS:

				test_case["status"] = "PASS"

			elif case.status == TestStatus.TEST_CRASHED:

				test_case["status"] = "ERROR"

			test_cases.append(test_case)

		sub_group["test_cases"] = test_cases

		sub_groups.append(sub_group)

	test_group = {

		"name": "KUnit Test Group",

		"arch": "UM",

		"defconfig": def_config,

		"build_environment": build_dir,

		"sub_groups": sub_groups,

		"lab_name": None,

		"kernel": None,

		"job": None,

		"git_branch": "kselftest",

	}

	json_obj = json.dumps(test_group, indent=4)

	if json_path != 'stdout':

		with open(json_path, 'w') as result_path:

			result_path.write(json_obj)

		root = __file__.split('tools/testing/kunit/')[0]

		kunit_parser.print_with_timestamp(

			"Test results stored in %s" %

			os.path.join(root, result_path.name))

	return json_obj

tools/testing/kunit/kunit_kernel.py

+16 −11
Original line number Diff line number Diff line
@@ -36,9 +36,9 @@ class LinuxSourceTreeOperations(object): 
		try:

			subprocess.check_output(['make', 'mrproper'], stderr=subprocess.STDOUT)

		except OSError as e:

			raise ConfigError('Could not call make command: ' + e)

			raise ConfigError('Could not call make command: ' + str(e))

		except subprocess.CalledProcessError as e:

			raise ConfigError(e.output)

			raise ConfigError(e.output.decode())



	def make_olddefconfig(self, build_dir, make_options):

		command = ['make', 'ARCH=um', 'olddefconfig']
@@ -49,22 +49,27 @@ class LinuxSourceTreeOperations(object): 
		try:

			subprocess.check_output(command, stderr=subprocess.STDOUT)

		except OSError as e:

			raise ConfigError('Could not call make command: ' + e)

			raise ConfigError('Could not call make command: ' + str(e))

		except subprocess.CalledProcessError as e:

			raise ConfigError(e.output)

			raise ConfigError(e.output.decode())



	def make_allyesconfig(self):

	def make_allyesconfig(self, build_dir, make_options):

		kunit_parser.print_with_timestamp(

			'Enabling all CONFIGs for UML...')

		command = ['make', 'ARCH=um', 'allyesconfig']

		if make_options:

			command.extend(make_options)

		if build_dir:

			command += ['O=' + build_dir]

		process = subprocess.Popen(

			['make', 'ARCH=um', 'allyesconfig'],

			command,

			stdout=subprocess.DEVNULL,

			stderr=subprocess.STDOUT)

		process.wait()

		kunit_parser.print_with_timestamp(

			'Disabling broken configs to run KUnit tests...')

		with ExitStack() as es:

			config = open(KCONFIG_PATH, 'a')

			config = open(get_kconfig_path(build_dir), 'a')

			disable = open(BROKEN_ALLCONFIG_PATH, 'r').read()

			config.write(disable)

		kunit_parser.print_with_timestamp(
@@ -79,9 +84,9 @@ class LinuxSourceTreeOperations(object): 
		try:

			subprocess.check_output(command, stderr=subprocess.STDOUT)

		except OSError as e:

			raise BuildError('Could not call execute make: ' + e)

			raise BuildError('Could not call execute make: ' + str(e))

		except subprocess.CalledProcessError as e:

			raise BuildError(e.output)

			raise BuildError(e.output.decode())



	def linux_bin(self, params, timeout, build_dir, outfile):

		"""Runs the Linux UML binary. Must be named 'linux'."""
@@ -161,9 +166,9 @@ class LinuxSourceTree(object): 
			return self.build_config(build_dir, make_options)



	def build_um_kernel(self, alltests, jobs, build_dir, make_options):

		if alltests:

			self._ops.make_allyesconfig()

		try:

			if alltests:

				self._ops.make_allyesconfig(build_dir, make_options)

			self._ops.make_olddefconfig(build_dir, make_options)

			self._ops.make(jobs, build_dir, make_options)

		except (ConfigError, BuildError) as e:

tools/testing/kunit/kunit_tool_test.py

+33 −0
Original line number Diff line number Diff line
@@ -11,11 +11,13 @@ from unittest import mock 


import tempfile, shutil # Handling test_tmpdir



import json

import os



import kunit_config

import kunit_parser

import kunit_kernel

import kunit_json

import kunit



test_tmpdir = ''
@@ -230,6 +232,37 @@ class KUnitParserTest(unittest.TestCase): 
			result = kunit_parser.parse_run_tests(file.readlines())

		self.assertEqual('kunit-resource-test', result.suites[0].name)



class KUnitJsonTest(unittest.TestCase):



	def _json_for(self, log_file):

		with(open(get_absolute_path(log_file))) as file:

			test_result = kunit_parser.parse_run_tests(file)

			json_obj = kunit_json.get_json_result(

				test_result=test_result,

				def_config='kunit_defconfig',

				build_dir=None,

				json_path='stdout')

		return json.loads(json_obj)



	def test_failed_test_json(self):

		result = self._json_for(

			'test_data/test_is_test_passed-failure.log')

		self.assertEqual(

			{'name': 'example_simple_test', 'status': 'FAIL'},

			result["sub_groups"][1]["test_cases"][0])



	def test_crashed_test_json(self):

		result = self._json_for(

			'test_data/test_is_test_passed-crash.log')

		self.assertEqual(

			{'name': 'example_simple_test', 'status': 'ERROR'},

			result["sub_groups"][1]["test_cases"][0])



	def test_no_tests_json(self):

		result = self._json_for(

			'test_data/test_is_test_passed-no_tests_run.log')

		self.assertEqual(0, len(result['sub_groups']))



class StrContains(str):

	def __eq__(self, other):

		return self in other

CRA Git | Maintained and supported by SUSTech CRA and CCSE