tools/testing/kunit/kunit_tool_test.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: GPL-2.0
   3 #
   4 # A collection of tests for tools/testing/kunit/kunit.py
   5 #
   6 # Copyright (C) 2019, Google LLC.
   7 # Author: Brendan Higgins <brendanhiggins@google.com>
   8
   9 import unittest
  10 from unittest import mock
  11
  12 import tempfile, shutil # Handling test_tmpdir
  13
  14 import json
  15 import os
  16
  17 import kunit_config
  18 import kunit_parser
  19 import kunit_kernel
  20 import kunit_json
  21 import kunit
  22
  23 test_tmpdir = ''
  24
  25 def setUpModule():
  26         global test_tmpdir
  27         test_tmpdir = tempfile.mkdtemp()
  28
  29 def tearDownModule():
  30         shutil.rmtree(test_tmpdir)
  31
  32 def get_absolute_path(path):
  33         return os.path.join(os.path.dirname(__file__), path)
  34
  35 class KconfigTest(unittest.TestCase):
  36
  37         def test_is_subset_of(self):
  38                 kconfig0 = kunit_config.Kconfig()
  39                 self.assertTrue(kconfig0.is_subset_of(kconfig0))
  40
  41                 kconfig1 = kunit_config.Kconfig()
  42                 kconfig1.add_entry(kunit_config.KconfigEntry('TEST', 'y'))
  43                 self.assertTrue(kconfig1.is_subset_of(kconfig1))
  44                 self.assertTrue(kconfig0.is_subset_of(kconfig1))
  45                 self.assertFalse(kconfig1.is_subset_of(kconfig0))
  46
  47         def test_read_from_file(self):
  48                 kconfig = kunit_config.Kconfig()
  49                 kconfig_path = get_absolute_path(
  50                         'test_data/test_read_from_file.kconfig')
  51
  52                 kconfig.read_from_file(kconfig_path)
  53
  54                 expected_kconfig = kunit_config.Kconfig()
  55                 expected_kconfig.add_entry(
  56                         kunit_config.KconfigEntry('UML', 'y'))
  57                 expected_kconfig.add_entry(
  58                         kunit_config.KconfigEntry('MMU', 'y'))
  59                 expected_kconfig.add_entry(
  60                         kunit_config.KconfigEntry('TEST', 'y'))
  61                 expected_kconfig.add_entry(
  62                         kunit_config.KconfigEntry('EXAMPLE_TEST', 'y'))
  63                 expected_kconfig.add_entry(
  64                         kunit_config.KconfigEntry('MK8', 'n'))
  65
  66                 self.assertEqual(kconfig.entries(), expected_kconfig.entries())
  67
  68         def test_write_to_file(self):
  69                 kconfig_path = os.path.join(test_tmpdir, '.config')
  70
  71                 expected_kconfig = kunit_config.Kconfig()
  72                 expected_kconfig.add_entry(
  73                         kunit_config.KconfigEntry('UML', 'y'))
  74                 expected_kconfig.add_entry(
  75                         kunit_config.KconfigEntry('MMU', 'y'))
  76                 expected_kconfig.add_entry(
  77                         kunit_config.KconfigEntry('TEST', 'y'))
  78                 expected_kconfig.add_entry(
  79                         kunit_config.KconfigEntry('EXAMPLE_TEST', 'y'))
  80                 expected_kconfig.add_entry(
  81                         kunit_config.KconfigEntry('MK8', 'n'))
  82
  83                 expected_kconfig.write_to_file(kconfig_path)
  84
  85                 actual_kconfig = kunit_config.Kconfig()
  86                 actual_kconfig.read_from_file(kconfig_path)
  87
  88                 self.assertEqual(actual_kconfig.entries(),
  89                                  expected_kconfig.entries())
  90
  91 class KUnitParserTest(unittest.TestCase):
  92
  93         def assertContains(self, needle, haystack):
  94                 for line in haystack:
  95                         if needle in line:
  96                                 return
  97                 raise AssertionError('"' +
  98                         str(needle) + '" not found in "' + str(haystack) + '"!')
  99
 100         def test_output_isolated_correctly(self):
 101                 log_path = get_absolute_path(
 102                         'test_data/test_output_isolated_correctly.log')
 103                 file = open(log_path)
 104                 result = kunit_parser.isolate_kunit_output(file.readlines())
 105                 self.assertContains('TAP version 14', result)
 106                 self.assertContains('   # Subtest: example', result)
 107                 self.assertContains('   1..2', result)
 108                 self.assertContains('   ok 1 - example_simple_test', result)
 109                 self.assertContains('   ok 2 - example_mock_test', result)
 110                 self.assertContains('ok 1 - example', result)
 111                 file.close()
 112
 113         def test_output_with_prefix_isolated_correctly(self):
 114                 log_path = get_absolute_path(
 115                         'test_data/test_pound_sign.log')
 116                 with open(log_path) as file:
 117                         result = kunit_parser.isolate_kunit_output(file.readlines())
 118                 self.assertContains('TAP version 14', result)
 119                 self.assertContains('   # Subtest: kunit-resource-test', result)
 120                 self.assertContains('   1..5', result)
 121                 self.assertContains('   ok 1 - kunit_resource_test_init_resources', result)
 122                 self.assertContains('   ok 2 - kunit_resource_test_alloc_resource', result)
 123                 self.assertContains('   ok 3 - kunit_resource_test_destroy_resource', result)
 124                 self.assertContains(' foo bar   #', result)
 125                 self.assertContains('   ok 4 - kunit_resource_test_cleanup_resources', result)
 126                 self.assertContains('   ok 5 - kunit_resource_test_proper_free_ordering', result)
 127                 self.assertContains('ok 1 - kunit-resource-test', result)
 128                 self.assertContains(' foo bar   # non-kunit output', result)
 129                 self.assertContains('   # Subtest: kunit-try-catch-test', result)
 130                 self.assertContains('   1..2', result)
 131                 self.assertContains('   ok 1 - kunit_test_try_catch_successful_try_no_catch',
 132                                     result)
 133                 self.assertContains('   ok 2 - kunit_test_try_catch_unsuccessful_try_does_catch',
 134                                     result)
 135                 self.assertContains('ok 2 - kunit-try-catch-test', result)
 136                 self.assertContains('   # Subtest: string-stream-test', result)
 137                 self.assertContains('   1..3', result)
 138                 self.assertContains('   ok 1 - string_stream_test_empty_on_creation', result)
 139                 self.assertContains('   ok 2 - string_stream_test_not_empty_after_add', result)
 140                 self.assertContains('   ok 3 - string_stream_test_get_string', result)
 141                 self.assertContains('ok 3 - string-stream-test', result)
 142
 143         def test_parse_successful_test_log(self):
 144                 all_passed_log = get_absolute_path(
 145                         'test_data/test_is_test_passed-all_passed.log')
 146                 file = open(all_passed_log)
 147                 result = kunit_parser.parse_run_tests(file.readlines())
 148                 self.assertEqual(
 149                         kunit_parser.TestStatus.SUCCESS,
 150                         result.status)
 151                 file.close()
 152
 153         def test_parse_failed_test_log(self):
 154                 failed_log = get_absolute_path(
 155                         'test_data/test_is_test_passed-failure.log')
 156                 file = open(failed_log)
 157                 result = kunit_parser.parse_run_tests(file.readlines())
 158                 self.assertEqual(
 159                         kunit_parser.TestStatus.FAILURE,
 160                         result.status)
 161                 file.close()
 162
 163         def test_no_tests(self):
 164                 empty_log = get_absolute_path(
 165                         'test_data/test_is_test_passed-no_tests_run.log')
 166                 file = open(empty_log)
 167                 result = kunit_parser.parse_run_tests(
 168                         kunit_parser.isolate_kunit_output(file.readlines()))
 169                 self.assertEqual(0, len(result.suites))
 170                 self.assertEqual(
 171                         kunit_parser.TestStatus.NO_TESTS,
 172                         result.status)
 173                 file.close()
 174
 175         def test_no_kunit_output(self):
 176                 crash_log = get_absolute_path(
 177                         'test_data/test_insufficient_memory.log')
 178                 file = open(crash_log)
 179                 print_mock = mock.patch('builtins.print').start()
 180                 result = kunit_parser.parse_run_tests(
 181                         kunit_parser.isolate_kunit_output(file.readlines()))
 182                 print_mock.assert_any_call(StrContains('no tests run!'))
 183                 print_mock.stop()
 184                 file.close()
 185
 186         def test_crashed_test(self):
 187                 crashed_log = get_absolute_path(
 188                         'test_data/test_is_test_passed-crash.log')
 189                 file = open(crashed_log)
 190                 result = kunit_parser.parse_run_tests(file.readlines())
 191                 self.assertEqual(
 192                         kunit_parser.TestStatus.TEST_CRASHED,
 193                         result.status)
 194                 file.close()
 195
 196         def test_ignores_prefix_printk_time(self):
 197                 prefix_log = get_absolute_path(
 198                         'test_data/test_config_printk_time.log')
 199                 with open(prefix_log) as file:
 200                         result = kunit_parser.parse_run_tests(file.readlines())
 201                         self.assertEqual(
 202                                 kunit_parser.TestStatus.SUCCESS,
 203                                 result.status)
 204                         self.assertEqual('kunit-resource-test', result.suites[0].name)
 205
 206         def test_ignores_multiple_prefixes(self):
 207                 prefix_log = get_absolute_path(
 208                         'test_data/test_multiple_prefixes.log')
 209                 with open(prefix_log) as file:
 210                         result = kunit_parser.parse_run_tests(file.readlines())
 211                         self.assertEqual(
 212                                 kunit_parser.TestStatus.SUCCESS,
 213                                 result.status)
 214                         self.assertEqual('kunit-resource-test', result.suites[0].name)
 215
 216         def test_prefix_mixed_kernel_output(self):
 217                 mixed_prefix_log = get_absolute_path(
 218                         'test_data/test_interrupted_tap_output.log')
 219                 with open(mixed_prefix_log) as file:
 220                         result = kunit_parser.parse_run_tests(file.readlines())
 221                         self.assertEqual(
 222                                 kunit_parser.TestStatus.SUCCESS,
 223                                 result.status)
 224                         self.assertEqual('kunit-resource-test', result.suites[0].name)
 225
 226         def test_prefix_poundsign(self):
 227                 pound_log = get_absolute_path('test_data/test_pound_sign.log')
 228                 with open(pound_log) as file:
 229                         result = kunit_parser.parse_run_tests(file.readlines())
 230                         self.assertEqual(
 231                                 kunit_parser.TestStatus.SUCCESS,
 232                                 result.status)
 233                         self.assertEqual('kunit-resource-test', result.suites[0].name)
 234
 235         def test_kernel_panic_end(self):
 236                 panic_log = get_absolute_path('test_data/test_kernel_panic_interrupt.log')
 237                 with open(panic_log) as file:
 238                         result = kunit_parser.parse_run_tests(file.readlines())
 239                         self.assertEqual(
 240                                 kunit_parser.TestStatus.TEST_CRASHED,
 241                                 result.status)
 242                         self.assertEqual('kunit-resource-test', result.suites[0].name)
 243
 244         def test_pound_no_prefix(self):
 245                 pound_log = get_absolute_path('test_data/test_pound_no_prefix.log')
 246                 with open(pound_log) as file:
 247                         result = kunit_parser.parse_run_tests(file.readlines())
 248                         self.assertEqual(
 249                                 kunit_parser.TestStatus.SUCCESS,
 250                                 result.status)
 251                         self.assertEqual('kunit-resource-test', result.suites[0].name)
 252
 253 class KUnitJsonTest(unittest.TestCase):
 254
 255         def _json_for(self, log_file):
 256                 with(open(get_absolute_path(log_file))) as file:
 257                         test_result = kunit_parser.parse_run_tests(file)
 258                         json_obj = kunit_json.get_json_result(
 259                                 test_result=test_result,
 260                                 def_config='kunit_defconfig',
 261                                 build_dir=None,
 262                                 json_path='stdout')
 263                 return json.loads(json_obj)
 264
 265         def test_failed_test_json(self):
 266                 result = self._json_for(
 267                         'test_data/test_is_test_passed-failure.log')
 268                 self.assertEqual(
 269                         {'name': 'example_simple_test', 'status': 'FAIL'},
 270                         result["sub_groups"][1]["test_cases"][0])
 271
 272         def test_crashed_test_json(self):
 273                 result = self._json_for(
 274                         'test_data/test_is_test_passed-crash.log')
 275                 self.assertEqual(
 276                         {'name': 'example_simple_test', 'status': 'ERROR'},
 277                         result["sub_groups"][1]["test_cases"][0])
 278
 279         def test_no_tests_json(self):
 280                 result = self._json_for(
 281                         'test_data/test_is_test_passed-no_tests_run.log')
 282                 self.assertEqual(0, len(result['sub_groups']))
 283
 284 class StrContains(str):
 285         def __eq__(self, other):
 286                 return self in other
 287
 288 class KUnitMainTest(unittest.TestCase):
 289         def setUp(self):
 290                 path = get_absolute_path('test_data/test_is_test_passed-all_passed.log')
 291                 file = open(path)
 292                 all_passed_log = file.readlines()
 293                 self.print_patch = mock.patch('builtins.print')
 294                 self.print_mock = self.print_patch.start()
 295                 self.linux_source_mock = mock.Mock()
 296                 self.linux_source_mock.build_reconfig = mock.Mock(return_value=True)
 297                 self.linux_source_mock.build_um_kernel = mock.Mock(return_value=True)
 298                 self.linux_source_mock.run_kernel = mock.Mock(return_value=all_passed_log)
 299
 300         def tearDown(self):
 301                 self.print_patch.stop()
 302                 pass
 303
 304         def test_config_passes_args_pass(self):
 305                 kunit.main(['config', '--build_dir=.kunit'], self.linux_source_mock)
 306                 assert self.linux_source_mock.build_reconfig.call_count == 1
 307                 assert self.linux_source_mock.run_kernel.call_count == 0
 308
 309         def test_build_passes_args_pass(self):
 310                 kunit.main(['build'], self.linux_source_mock)
 311                 assert self.linux_source_mock.build_reconfig.call_count == 0
 312                 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, '.kunit', None)
 313                 assert self.linux_source_mock.run_kernel.call_count == 0
 314
 315         def test_exec_passes_args_pass(self):
 316                 kunit.main(['exec'], self.linux_source_mock)
 317                 assert self.linux_source_mock.build_reconfig.call_count == 0
 318                 assert self.linux_source_mock.run_kernel.call_count == 1
 319                 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='.kunit', timeout=300)
 320                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 321
 322         def test_run_passes_args_pass(self):
 323                 kunit.main(['run'], self.linux_source_mock)
 324                 assert self.linux_source_mock.build_reconfig.call_count == 1
 325                 assert self.linux_source_mock.run_kernel.call_count == 1
 326                 self.linux_source_mock.run_kernel.assert_called_once_with(
 327                         build_dir='.kunit', timeout=300)
 328                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 329
 330         def test_exec_passes_args_fail(self):
 331                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 332                 with self.assertRaises(SystemExit) as e:
 333                         kunit.main(['exec'], self.linux_source_mock)
 334                 assert type(e.exception) == SystemExit
 335                 assert e.exception.code == 1
 336
 337         def test_run_passes_args_fail(self):
 338                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 339                 with self.assertRaises(SystemExit) as e:
 340                         kunit.main(['run'], self.linux_source_mock)
 341                 assert type(e.exception) == SystemExit
 342                 assert e.exception.code == 1
 343                 assert self.linux_source_mock.build_reconfig.call_count == 1
 344                 assert self.linux_source_mock.run_kernel.call_count == 1
 345                 self.print_mock.assert_any_call(StrContains(' 0 tests run'))
 346
 347         def test_exec_raw_output(self):
 348                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 349                 kunit.main(['exec', '--raw_output'], self.linux_source_mock)
 350                 assert self.linux_source_mock.run_kernel.call_count == 1
 351                 for kall in self.print_mock.call_args_list:
 352                         assert kall != mock.call(StrContains('Testing complete.'))
 353                         assert kall != mock.call(StrContains(' 0 tests run'))
 354
 355         def test_run_raw_output(self):
 356                 self.linux_source_mock.run_kernel = mock.Mock(return_value=[])
 357                 kunit.main(['run', '--raw_output'], self.linux_source_mock)
 358                 assert self.linux_source_mock.build_reconfig.call_count == 1
 359                 assert self.linux_source_mock.run_kernel.call_count == 1
 360                 for kall in self.print_mock.call_args_list:
 361                         assert kall != mock.call(StrContains('Testing complete.'))
 362                         assert kall != mock.call(StrContains(' 0 tests run'))
 363
 364         def test_exec_timeout(self):
 365                 timeout = 3453
 366                 kunit.main(['exec', '--timeout', str(timeout)], self.linux_source_mock)
 367                 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir='.kunit', timeout=timeout)
 368                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 369
 370         def test_run_timeout(self):
 371                 timeout = 3453
 372                 kunit.main(['run', '--timeout', str(timeout)], self.linux_source_mock)
 373                 assert self.linux_source_mock.build_reconfig.call_count == 1
 374                 self.linux_source_mock.run_kernel.assert_called_once_with(
 375                         build_dir='.kunit', timeout=timeout)
 376                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 377
 378         def test_run_builddir(self):
 379                 build_dir = '.kunit'
 380                 kunit.main(['run', '--build_dir=.kunit'], self.linux_source_mock)
 381                 assert self.linux_source_mock.build_reconfig.call_count == 1
 382                 self.linux_source_mock.run_kernel.assert_called_once_with(
 383                         build_dir=build_dir, timeout=300)
 384                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 385
 386         def test_config_builddir(self):
 387                 build_dir = '.kunit'
 388                 kunit.main(['config', '--build_dir', build_dir], self.linux_source_mock)
 389                 assert self.linux_source_mock.build_reconfig.call_count == 1
 390
 391         def test_build_builddir(self):
 392                 build_dir = '.kunit'
 393                 kunit.main(['build', '--build_dir', build_dir], self.linux_source_mock)
 394                 self.linux_source_mock.build_um_kernel.assert_called_once_with(False, 8, build_dir, None)
 395
 396         def test_exec_builddir(self):
 397                 build_dir = '.kunit'
 398                 kunit.main(['exec', '--build_dir', build_dir], self.linux_source_mock)
 399                 self.linux_source_mock.run_kernel.assert_called_once_with(build_dir=build_dir, timeout=300)
 400                 self.print_mock.assert_any_call(StrContains('Testing complete.'))
 401
 402 if __name__ == '__main__':
 403         unittest.main()