llvm/utils/lit/tests/unit/TestRunner.py

   1 # RUN: %{python} %s
   2 #
   3 # END.
   4
   5
   6 import os.path
   7 import platform
   8 import unittest
   9
  10 import lit.discovery
  11 import lit.LitConfig
  12 import lit.Test as Test
  13 from lit.TestRunner import (
  14     ParserKind,
  15     IntegratedTestKeywordParser,
  16     parseIntegratedTestScript,
  17 )
  18
  19
  20 class TestIntegratedTestKeywordParser(unittest.TestCase):
  21     inputTestCase = None
  22
  23     @staticmethod
  24     def load_keyword_parser_lit_tests():
  25         """
  26         Create and load the LIT test suite and test objects used by
  27         TestIntegratedTestKeywordParser
  28         """
  29         # Create the global config object.
  30         lit_config = lit.LitConfig.LitConfig(
  31             progname="lit",
  32             path=[],
  33             quiet=False,
  34             useValgrind=False,
  35             valgrindLeakCheck=False,
  36             valgrindArgs=[],
  37             noExecute=False,
  38             debug=False,
  39             isWindows=(platform.system() == "Windows"),
  40             order="smart",
  41             params={},
  42         )
  43         TestIntegratedTestKeywordParser.litConfig = lit_config
  44         # Perform test discovery.
  45         test_path = os.path.dirname(os.path.dirname(__file__))
  46         inputs = [os.path.join(test_path, "Inputs/testrunner-custom-parsers/")]
  47         assert os.path.isdir(inputs[0])
  48         tests = lit.discovery.find_tests_for_inputs(lit_config, inputs)
  49         assert len(tests) == 1 and "there should only be one test"
  50         TestIntegratedTestKeywordParser.inputTestCase = tests[0]
  51
  52     @staticmethod
  53     def make_parsers():
  54         def custom_parse(line_number, line, output):
  55             if output is None:
  56                 output = []
  57             output += [part for part in line.split(" ") if part.strip()]
  58             return output
  59
  60         return [
  61             IntegratedTestKeywordParser("MY_TAG.", ParserKind.TAG),
  62             IntegratedTestKeywordParser("MY_DNE_TAG.", ParserKind.TAG),
  63             IntegratedTestKeywordParser("MY_LIST:", ParserKind.LIST),
  64             IntegratedTestKeywordParser("MY_SPACE_LIST:", ParserKind.SPACE_LIST),
  65             IntegratedTestKeywordParser("MY_BOOL:", ParserKind.BOOLEAN_EXPR),
  66             IntegratedTestKeywordParser("MY_INT:", ParserKind.INTEGER),
  67             IntegratedTestKeywordParser("MY_RUN:", ParserKind.COMMAND),
  68             IntegratedTestKeywordParser("MY_CUSTOM:", ParserKind.CUSTOM, custom_parse),
  69             IntegratedTestKeywordParser("MY_DEFINE:", ParserKind.DEFINE),
  70             IntegratedTestKeywordParser("MY_REDEFINE:", ParserKind.REDEFINE),
  71         ]
  72
  73     @staticmethod
  74     def get_parser(parser_list, keyword):
  75         for p in parser_list:
  76             if p.keyword == keyword:
  77                 return p
  78         assert False and "parser not found"
  79
  80     @staticmethod
  81     def parse_test(parser_list, allow_result=False):
  82         script = parseIntegratedTestScript(
  83             TestIntegratedTestKeywordParser.inputTestCase,
  84             additional_parsers=parser_list,
  85             require_script=False,
  86         )
  87         if isinstance(script, lit.Test.Result):
  88             assert allow_result
  89         else:
  90             assert isinstance(script, list)
  91             assert len(script) == 0
  92         return script
  93
  94     def test_tags(self):
  95         parsers = self.make_parsers()
  96         self.parse_test(parsers)
  97         tag_parser = self.get_parser(parsers, "MY_TAG.")
  98         dne_tag_parser = self.get_parser(parsers, "MY_DNE_TAG.")
  99         self.assertTrue(tag_parser.getValue())
 100         self.assertFalse(dne_tag_parser.getValue())
 101
 102     def test_lists(self):
 103         parsers = self.make_parsers()
 104         self.parse_test(parsers)
 105         list_parser = self.get_parser(parsers, "MY_LIST:")
 106         self.assertEqual(list_parser.getValue(), ["one", "two", "three", "four"])
 107
 108     def test_space_lists(self):
 109         parsers = self.make_parsers()
 110         self.parse_test(parsers)
 111         space_list_parser = self.get_parser(parsers, "MY_SPACE_LIST:")
 112         self.assertEqual(
 113             space_list_parser.getValue(),
 114             [
 115                 "orange",
 116                 "tabby",
 117                 "tortie",
 118                 "tuxedo",
 119                 "void",
 120                 "multiple",
 121                 "spaces",
 122                 "cute,",
 123                 "fluffy,",
 124                 "kittens",
 125             ],
 126         )
 127
 128     def test_commands(self):
 129         parsers = self.make_parsers()
 130         self.parse_test(parsers)
 131         cmd_parser = self.get_parser(parsers, "MY_RUN:")
 132         value = cmd_parser.getValue()
 133         self.assertEqual(len(value), 2)  # there are only two run lines
 134         self.assertEqual(value[0].command.strip(), "%dbg(MY_RUN: at line 4)  baz")
 135         self.assertEqual(value[1].command.strip(), "%dbg(MY_RUN: at line 12)  foo  bar")
 136
 137     def test_boolean(self):
 138         parsers = self.make_parsers()
 139         self.parse_test(parsers)
 140         bool_parser = self.get_parser(parsers, "MY_BOOL:")
 141         value = bool_parser.getValue()
 142         self.assertEqual(len(value), 2)  # there are only two run lines
 143         self.assertEqual(value[0].strip(), "a && (b)")
 144         self.assertEqual(value[1].strip(), "d")
 145
 146     def test_integer(self):
 147         parsers = self.make_parsers()
 148         self.parse_test(parsers)
 149         int_parser = self.get_parser(parsers, "MY_INT:")
 150         value = int_parser.getValue()
 151         self.assertEqual(len(value), 2)  # there are only two MY_INT: lines
 152         self.assertEqual(type(value[0]), int)
 153         self.assertEqual(value[0], 4)
 154         self.assertEqual(type(value[1]), int)
 155         self.assertEqual(value[1], 6)
 156
 157     def test_bad_parser_type(self):
 158         parsers = self.make_parsers() + ["BAD_PARSER_TYPE"]
 159         script = self.parse_test(parsers, allow_result=True)
 160         self.assertTrue(isinstance(script, lit.Test.Result))
 161         self.assertEqual(script.code, lit.Test.UNRESOLVED)
 162         self.assertEqual(
 163             "Additional parser must be an instance of " "IntegratedTestKeywordParser",
 164             script.output,
 165         )
 166
 167     def test_duplicate_keyword(self):
 168         parsers = self.make_parsers() + [
 169             IntegratedTestKeywordParser("KEY:", ParserKind.BOOLEAN_EXPR),
 170             IntegratedTestKeywordParser("KEY:", ParserKind.BOOLEAN_EXPR),
 171         ]
 172         script = self.parse_test(parsers, allow_result=True)
 173         self.assertTrue(isinstance(script, lit.Test.Result))
 174         self.assertEqual(script.code, lit.Test.UNRESOLVED)
 175         self.assertEqual("Parser for keyword 'KEY:' already exists", script.output)
 176
 177     def test_boolean_unterminated(self):
 178         parsers = self.make_parsers() + [
 179             IntegratedTestKeywordParser(
 180                 "MY_BOOL_UNTERMINATED:", ParserKind.BOOLEAN_EXPR
 181             )
 182         ]
 183         script = self.parse_test(parsers, allow_result=True)
 184         self.assertTrue(isinstance(script, lit.Test.Result))
 185         self.assertEqual(script.code, lit.Test.UNRESOLVED)
 186         self.assertEqual(
 187             "Test has unterminated 'MY_BOOL_UNTERMINATED:' lines " "(with '\\')",
 188             script.output,
 189         )
 190
 191     def test_custom(self):
 192         parsers = self.make_parsers()
 193         self.parse_test(parsers)
 194         custom_parser = self.get_parser(parsers, "MY_CUSTOM:")
 195         value = custom_parser.getValue()
 196         self.assertEqual(value, ["a", "b", "c"])
 197
 198     def test_defines(self):
 199         parsers = self.make_parsers()
 200         self.parse_test(parsers)
 201         cmd_parser = self.get_parser(parsers, "MY_DEFINE:")
 202         value = cmd_parser.getValue()
 203         self.assertEqual(len(value), 1)  # there's only one MY_DEFINE directive
 204         self.assertEqual(value[0].new_subst, True)
 205         self.assertEqual(value[0].name, "%{name}")
 206         self.assertEqual(value[0].value, "value one")
 207
 208     def test_redefines(self):
 209         parsers = self.make_parsers()
 210         self.parse_test(parsers)
 211         cmd_parser = self.get_parser(parsers, "MY_REDEFINE:")
 212         value = cmd_parser.getValue()
 213         self.assertEqual(len(value), 1)  # there's only one MY_REDEFINE directive
 214         self.assertEqual(value[0].new_subst, False)
 215         self.assertEqual(value[0].name, "%{name}")
 216         self.assertEqual(value[0].value, "value two")
 217
 218     def test_bad_keywords(self):
 219         def custom_parse(line_number, line, output):
 220             return output
 221
 222         try:
 223             IntegratedTestKeywordParser("TAG_NO_SUFFIX", ParserKind.TAG),
 224             self.fail("TAG_NO_SUFFIX failed to raise an exception")
 225         except ValueError as e:
 226             pass
 227         except BaseException as e:
 228             self.fail("TAG_NO_SUFFIX raised the wrong exception: %r" % e)
 229
 230         try:
 231             IntegratedTestKeywordParser("TAG_WITH_COLON:", ParserKind.TAG),
 232             self.fail("TAG_WITH_COLON: failed to raise an exception")
 233         except ValueError as e:
 234             pass
 235         except BaseException as e:
 236             self.fail("TAG_WITH_COLON: raised the wrong exception: %r" % e)
 237
 238         try:
 239             IntegratedTestKeywordParser("LIST_WITH_DOT.", ParserKind.LIST),
 240             self.fail("LIST_WITH_DOT. failed to raise an exception")
 241         except ValueError as e:
 242             pass
 243         except BaseException as e:
 244             self.fail("LIST_WITH_DOT. raised the wrong exception: %r" % e)
 245
 246         try:
 247             IntegratedTestKeywordParser("SPACE_LIST_WITH_DOT.", ParserKind.SPACE_LIST),
 248             self.fail("SPACE_LIST_WITH_DOT. failed to raise an exception")
 249         except ValueError as e:
 250             pass
 251         except BaseException as e:
 252             self.fail("SPACE_LIST_WITH_DOT. raised the wrong exception: %r" % e)
 253
 254         try:
 255             IntegratedTestKeywordParser(
 256                 "CUSTOM_NO_SUFFIX", ParserKind.CUSTOM, custom_parse
 257             ),
 258             self.fail("CUSTOM_NO_SUFFIX failed to raise an exception")
 259         except ValueError as e:
 260             pass
 261         except BaseException as e:
 262             self.fail("CUSTOM_NO_SUFFIX raised the wrong exception: %r" % e)
 263
 264         # Both '.' and ':' are allowed for CUSTOM keywords.
 265         try:
 266             IntegratedTestKeywordParser(
 267                 "CUSTOM_WITH_DOT.", ParserKind.CUSTOM, custom_parse
 268             ),
 269         except BaseException as e:
 270             self.fail("CUSTOM_WITH_DOT. raised an exception: %r" % e)
 271         try:
 272             IntegratedTestKeywordParser(
 273                 "CUSTOM_WITH_COLON:", ParserKind.CUSTOM, custom_parse
 274             ),
 275         except BaseException as e:
 276             self.fail("CUSTOM_WITH_COLON: raised an exception: %r" % e)
 277
 278         try:
 279             IntegratedTestKeywordParser("CUSTOM_NO_PARSER:", ParserKind.CUSTOM),
 280             self.fail("CUSTOM_NO_PARSER: failed to raise an exception")
 281         except ValueError as e:
 282             pass
 283         except BaseException as e:
 284             self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)
 285
 286
 287 class TestApplySubtitutions(unittest.TestCase):
 288     def test_simple(self):
 289         script = ["echo %bar"]
 290         substitutions = [("%bar", "hello")]
 291         result = lit.TestRunner.applySubstitutions(script, substitutions)
 292         self.assertEqual(result, ["echo hello"])
 293
 294     def test_multiple_substitutions(self):
 295         script = ["echo %bar %baz"]
 296         substitutions = [
 297             ("%bar", "hello"),
 298             ("%baz", "world"),
 299             ("%useless", "shouldnt expand"),
 300         ]
 301         result = lit.TestRunner.applySubstitutions(script, substitutions)
 302         self.assertEqual(result, ["echo hello world"])
 303
 304     def test_multiple_script_lines(self):
 305         script = ["%cxx %compile_flags -c -o %t.o", "%cxx %link_flags %t.o -o %t.exe"]
 306         substitutions = [
 307             ("%cxx", "clang++"),
 308             ("%compile_flags", "-std=c++11 -O3"),
 309             ("%link_flags", "-lc++"),
 310         ]
 311         result = lit.TestRunner.applySubstitutions(script, substitutions)
 312         self.assertEqual(
 313             result,
 314             ["clang++ -std=c++11 -O3 -c -o %t.o", "clang++ -lc++ %t.o -o %t.exe"],
 315         )
 316
 317     def test_recursive_substitution_real(self):
 318         script = ["%build %s"]
 319         substitutions = [
 320             ("%cxx", "clang++"),
 321             ("%compile_flags", "-std=c++11 -O3"),
 322             ("%link_flags", "-lc++"),
 323             ("%build", "%cxx %compile_flags %link_flags %s -o %t.exe"),
 324         ]
 325         result = lit.TestRunner.applySubstitutions(
 326             script, substitutions, recursion_limit=3
 327         )
 328         self.assertEqual(result, ["clang++ -std=c++11 -O3 -lc++ %s -o %t.exe %s"])
 329
 330     def test_recursive_substitution_limit(self):
 331         script = ["%rec5"]
 332         # Make sure the substitutions are not in an order where the global
 333         # substitution would appear to be recursive just because they are
 334         # processed in the right order.
 335         substitutions = [
 336             ("%rec1", "STOP"),
 337             ("%rec2", "%rec1"),
 338             ("%rec3", "%rec2"),
 339             ("%rec4", "%rec3"),
 340             ("%rec5", "%rec4"),
 341         ]
 342         for limit in [5, 6, 7]:
 343             result = lit.TestRunner.applySubstitutions(
 344                 script, substitutions, recursion_limit=limit
 345             )
 346             self.assertEqual(result, ["STOP"])
 347
 348     def test_recursive_substitution_limit_exceeded(self):
 349         script = ["%rec5"]
 350         substitutions = [
 351             ("%rec1", "STOP"),
 352             ("%rec2", "%rec1"),
 353             ("%rec3", "%rec2"),
 354             ("%rec4", "%rec3"),
 355             ("%rec5", "%rec4"),
 356         ]
 357         for limit in [0, 1, 2, 3, 4]:
 358             try:
 359                 lit.TestRunner.applySubstitutions(
 360                     script, substitutions, recursion_limit=limit
 361                 )
 362                 self.fail("applySubstitutions should have raised an exception")
 363             except ValueError:
 364                 pass
 365
 366     def test_recursive_substitution_invalid_value(self):
 367         script = ["%rec5"]
 368         substitutions = [
 369             ("%rec1", "STOP"),
 370             ("%rec2", "%rec1"),
 371             ("%rec3", "%rec2"),
 372             ("%rec4", "%rec3"),
 373             ("%rec5", "%rec4"),
 374         ]
 375         for limit in [-1, -2, -3, "foo"]:
 376             try:
 377                 lit.TestRunner.applySubstitutions(
 378                     script, substitutions, recursion_limit=limit
 379                 )
 380                 self.fail("applySubstitutions should have raised an exception")
 381             except AssertionError:
 382                 pass
 383
 384
 385 if __name__ == "__main__":
 386     TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()
 387     unittest.main(verbosity=2)