llvm/utils/lit/tests/unit/TestRunner.py

   1 # RUN: %{python} %s
   2 #
   3 # END.
   4
   5
   6 import os.path
   7 import platform
   8 import unittest
   9
  10 import lit.discovery
  11 import lit.LitConfig
  12 import lit.Test as Test
  13 from lit.TestRunner import (
  14     ParserKind,
  15     IntegratedTestKeywordParser,
  16     parseIntegratedTestScript,
  17 )
  18
  19
  20 class TestIntegratedTestKeywordParser(unittest.TestCase):
  21     inputTestCase = None
  22
  23     @staticmethod
  24     def load_keyword_parser_lit_tests():
  25         """
  26         Create and load the LIT test suite and test objects used by
  27         TestIntegratedTestKeywordParser
  28         """
  29         # Create the global config object.
  30         lit_config = lit.LitConfig.LitConfig(
  31             progname="lit",
  32             path=[],
  33             quiet=False,
  34             useValgrind=False,
  35             valgrindLeakCheck=False,
  36             valgrindArgs=[],
  37             noExecute=False,
  38             debug=False,
  39             isWindows=(platform.system() == "Windows"),
  40             order="smart",
  41             params={},
  42         )
  43         TestIntegratedTestKeywordParser.litConfig = lit_config
  44         # Perform test discovery.
  45         test_path = os.path.dirname(os.path.dirname(__file__))
  46         inputs = [os.path.join(test_path, "Inputs/testrunner-custom-parsers/")]
  47         assert os.path.isdir(inputs[0])
  48         tests = lit.discovery.find_tests_for_inputs(lit_config, inputs)
  49         assert len(tests) == 1 and "there should only be one test"
  50         TestIntegratedTestKeywordParser.inputTestCase = tests[0]
  51
  52     @staticmethod
  53     def make_parsers():
  54         def custom_parse(line_number, line, output):
  55             if output is None:
  56                 output = []
  57             output += [part for part in line.split(" ") if part.strip()]
  58             return output
  59
  60         return [
  61             IntegratedTestKeywordParser("MY_TAG.", ParserKind.TAG),
  62             IntegratedTestKeywordParser("MY_DNE_TAG.", ParserKind.TAG),
  63             IntegratedTestKeywordParser("MY_LIST:", ParserKind.LIST),
  64             IntegratedTestKeywordParser("MY_BOOL:", ParserKind.BOOLEAN_EXPR),
  65             IntegratedTestKeywordParser("MY_INT:", ParserKind.INTEGER),
  66             IntegratedTestKeywordParser("MY_RUN:", ParserKind.COMMAND),
  67             IntegratedTestKeywordParser("MY_CUSTOM:", ParserKind.CUSTOM, custom_parse),
  68             IntegratedTestKeywordParser("MY_DEFINE:", ParserKind.DEFINE),
  69             IntegratedTestKeywordParser("MY_REDEFINE:", ParserKind.REDEFINE),
  70         ]
  71
  72     @staticmethod
  73     def get_parser(parser_list, keyword):
  74         for p in parser_list:
  75             if p.keyword == keyword:
  76                 return p
  77         assert False and "parser not found"
  78
  79     @staticmethod
  80     def parse_test(parser_list, allow_result=False):
  81         script = parseIntegratedTestScript(
  82             TestIntegratedTestKeywordParser.inputTestCase,
  83             additional_parsers=parser_list,
  84             require_script=False,
  85         )
  86         if isinstance(script, lit.Test.Result):
  87             assert allow_result
  88         else:
  89             assert isinstance(script, list)
  90             assert len(script) == 0
  91         return script
  92
  93     def test_tags(self):
  94         parsers = self.make_parsers()
  95         self.parse_test(parsers)
  96         tag_parser = self.get_parser(parsers, "MY_TAG.")
  97         dne_tag_parser = self.get_parser(parsers, "MY_DNE_TAG.")
  98         self.assertTrue(tag_parser.getValue())
  99         self.assertFalse(dne_tag_parser.getValue())
 100
 101     def test_lists(self):
 102         parsers = self.make_parsers()
 103         self.parse_test(parsers)
 104         list_parser = self.get_parser(parsers, "MY_LIST:")
 105         self.assertEqual(list_parser.getValue(), ["one", "two", "three", "four"])
 106
 107     def test_commands(self):
 108         parsers = self.make_parsers()
 109         self.parse_test(parsers)
 110         cmd_parser = self.get_parser(parsers, "MY_RUN:")
 111         value = cmd_parser.getValue()
 112         self.assertEqual(len(value), 2)  # there are only two run lines
 113         self.assertEqual(value[0].command.strip(), "%dbg(MY_RUN: at line 4)  baz")
 114         self.assertEqual(value[1].command.strip(), "%dbg(MY_RUN: at line 7)  foo  bar")
 115
 116     def test_boolean(self):
 117         parsers = self.make_parsers()
 118         self.parse_test(parsers)
 119         bool_parser = self.get_parser(parsers, "MY_BOOL:")
 120         value = bool_parser.getValue()
 121         self.assertEqual(len(value), 2)  # there are only two run lines
 122         self.assertEqual(value[0].strip(), "a && (b)")
 123         self.assertEqual(value[1].strip(), "d")
 124
 125     def test_integer(self):
 126         parsers = self.make_parsers()
 127         self.parse_test(parsers)
 128         int_parser = self.get_parser(parsers, "MY_INT:")
 129         value = int_parser.getValue()
 130         self.assertEqual(len(value), 2)  # there are only two MY_INT: lines
 131         self.assertEqual(type(value[0]), int)
 132         self.assertEqual(value[0], 4)
 133         self.assertEqual(type(value[1]), int)
 134         self.assertEqual(value[1], 6)
 135
 136     def test_bad_parser_type(self):
 137         parsers = self.make_parsers() + ["BAD_PARSER_TYPE"]
 138         script = self.parse_test(parsers, allow_result=True)
 139         self.assertTrue(isinstance(script, lit.Test.Result))
 140         self.assertEqual(script.code, lit.Test.UNRESOLVED)
 141         self.assertEqual(
 142             "Additional parser must be an instance of " "IntegratedTestKeywordParser",
 143             script.output,
 144         )
 145
 146     def test_duplicate_keyword(self):
 147         parsers = self.make_parsers() + [
 148             IntegratedTestKeywordParser("KEY:", ParserKind.BOOLEAN_EXPR),
 149             IntegratedTestKeywordParser("KEY:", ParserKind.BOOLEAN_EXPR),
 150         ]
 151         script = self.parse_test(parsers, allow_result=True)
 152         self.assertTrue(isinstance(script, lit.Test.Result))
 153         self.assertEqual(script.code, lit.Test.UNRESOLVED)
 154         self.assertEqual("Parser for keyword 'KEY:' already exists", script.output)
 155
 156     def test_boolean_unterminated(self):
 157         parsers = self.make_parsers() + [
 158             IntegratedTestKeywordParser(
 159                 "MY_BOOL_UNTERMINATED:", ParserKind.BOOLEAN_EXPR
 160             )
 161         ]
 162         script = self.parse_test(parsers, allow_result=True)
 163         self.assertTrue(isinstance(script, lit.Test.Result))
 164         self.assertEqual(script.code, lit.Test.UNRESOLVED)
 165         self.assertEqual(
 166             "Test has unterminated 'MY_BOOL_UNTERMINATED:' lines " "(with '\\')",
 167             script.output,
 168         )
 169
 170     def test_custom(self):
 171         parsers = self.make_parsers()
 172         self.parse_test(parsers)
 173         custom_parser = self.get_parser(parsers, "MY_CUSTOM:")
 174         value = custom_parser.getValue()
 175         self.assertEqual(value, ["a", "b", "c"])
 176
 177     def test_defines(self):
 178         parsers = self.make_parsers()
 179         self.parse_test(parsers)
 180         cmd_parser = self.get_parser(parsers, "MY_DEFINE:")
 181         value = cmd_parser.getValue()
 182         self.assertEqual(len(value), 1)  # there's only one MY_DEFINE directive
 183         self.assertEqual(value[0].new_subst, True)
 184         self.assertEqual(value[0].name, "%{name}")
 185         self.assertEqual(value[0].value, "value one")
 186
 187     def test_redefines(self):
 188         parsers = self.make_parsers()
 189         self.parse_test(parsers)
 190         cmd_parser = self.get_parser(parsers, "MY_REDEFINE:")
 191         value = cmd_parser.getValue()
 192         self.assertEqual(len(value), 1)  # there's only one MY_REDEFINE directive
 193         self.assertEqual(value[0].new_subst, False)
 194         self.assertEqual(value[0].name, "%{name}")
 195         self.assertEqual(value[0].value, "value two")
 196
 197     def test_bad_keywords(self):
 198         def custom_parse(line_number, line, output):
 199             return output
 200
 201         try:
 202             IntegratedTestKeywordParser("TAG_NO_SUFFIX", ParserKind.TAG),
 203             self.fail("TAG_NO_SUFFIX failed to raise an exception")
 204         except ValueError as e:
 205             pass
 206         except BaseException as e:
 207             self.fail("TAG_NO_SUFFIX raised the wrong exception: %r" % e)
 208
 209         try:
 210             IntegratedTestKeywordParser("TAG_WITH_COLON:", ParserKind.TAG),
 211             self.fail("TAG_WITH_COLON: failed to raise an exception")
 212         except ValueError as e:
 213             pass
 214         except BaseException as e:
 215             self.fail("TAG_WITH_COLON: raised the wrong exception: %r" % e)
 216
 217         try:
 218             IntegratedTestKeywordParser("LIST_WITH_DOT.", ParserKind.LIST),
 219             self.fail("LIST_WITH_DOT. failed to raise an exception")
 220         except ValueError as e:
 221             pass
 222         except BaseException as e:
 223             self.fail("LIST_WITH_DOT. raised the wrong exception: %r" % e)
 224
 225         try:
 226             IntegratedTestKeywordParser(
 227                 "CUSTOM_NO_SUFFIX", ParserKind.CUSTOM, custom_parse
 228             ),
 229             self.fail("CUSTOM_NO_SUFFIX failed to raise an exception")
 230         except ValueError as e:
 231             pass
 232         except BaseException as e:
 233             self.fail("CUSTOM_NO_SUFFIX raised the wrong exception: %r" % e)
 234
 235         # Both '.' and ':' are allowed for CUSTOM keywords.
 236         try:
 237             IntegratedTestKeywordParser(
 238                 "CUSTOM_WITH_DOT.", ParserKind.CUSTOM, custom_parse
 239             ),
 240         except BaseException as e:
 241             self.fail("CUSTOM_WITH_DOT. raised an exception: %r" % e)
 242         try:
 243             IntegratedTestKeywordParser(
 244                 "CUSTOM_WITH_COLON:", ParserKind.CUSTOM, custom_parse
 245             ),
 246         except BaseException as e:
 247             self.fail("CUSTOM_WITH_COLON: raised an exception: %r" % e)
 248
 249         try:
 250             IntegratedTestKeywordParser("CUSTOM_NO_PARSER:", ParserKind.CUSTOM),
 251             self.fail("CUSTOM_NO_PARSER: failed to raise an exception")
 252         except ValueError as e:
 253             pass
 254         except BaseException as e:
 255             self.fail("CUSTOM_NO_PARSER: raised the wrong exception: %r" % e)
 256
 257
 258 class TestApplySubtitutions(unittest.TestCase):
 259     def test_simple(self):
 260         script = ["echo %bar"]
 261         substitutions = [("%bar", "hello")]
 262         result = lit.TestRunner.applySubstitutions(script, substitutions)
 263         self.assertEqual(result, ["echo hello"])
 264
 265     def test_multiple_substitutions(self):
 266         script = ["echo %bar %baz"]
 267         substitutions = [
 268             ("%bar", "hello"),
 269             ("%baz", "world"),
 270             ("%useless", "shouldnt expand"),
 271         ]
 272         result = lit.TestRunner.applySubstitutions(script, substitutions)
 273         self.assertEqual(result, ["echo hello world"])
 274
 275     def test_multiple_script_lines(self):
 276         script = ["%cxx %compile_flags -c -o %t.o", "%cxx %link_flags %t.o -o %t.exe"]
 277         substitutions = [
 278             ("%cxx", "clang++"),
 279             ("%compile_flags", "-std=c++11 -O3"),
 280             ("%link_flags", "-lc++"),
 281         ]
 282         result = lit.TestRunner.applySubstitutions(script, substitutions)
 283         self.assertEqual(
 284             result,
 285             ["clang++ -std=c++11 -O3 -c -o %t.o", "clang++ -lc++ %t.o -o %t.exe"],
 286         )
 287
 288     def test_recursive_substitution_real(self):
 289         script = ["%build %s"]
 290         substitutions = [
 291             ("%cxx", "clang++"),
 292             ("%compile_flags", "-std=c++11 -O3"),
 293             ("%link_flags", "-lc++"),
 294             ("%build", "%cxx %compile_flags %link_flags %s -o %t.exe"),
 295         ]
 296         result = lit.TestRunner.applySubstitutions(
 297             script, substitutions, recursion_limit=3
 298         )
 299         self.assertEqual(result, ["clang++ -std=c++11 -O3 -lc++ %s -o %t.exe %s"])
 300
 301     def test_recursive_substitution_limit(self):
 302         script = ["%rec5"]
 303         # Make sure the substitutions are not in an order where the global
 304         # substitution would appear to be recursive just because they are
 305         # processed in the right order.
 306         substitutions = [
 307             ("%rec1", "STOP"),
 308             ("%rec2", "%rec1"),
 309             ("%rec3", "%rec2"),
 310             ("%rec4", "%rec3"),
 311             ("%rec5", "%rec4"),
 312         ]
 313         for limit in [5, 6, 7]:
 314             result = lit.TestRunner.applySubstitutions(
 315                 script, substitutions, recursion_limit=limit
 316             )
 317             self.assertEqual(result, ["STOP"])
 318
 319     def test_recursive_substitution_limit_exceeded(self):
 320         script = ["%rec5"]
 321         substitutions = [
 322             ("%rec1", "STOP"),
 323             ("%rec2", "%rec1"),
 324             ("%rec3", "%rec2"),
 325             ("%rec4", "%rec3"),
 326             ("%rec5", "%rec4"),
 327         ]
 328         for limit in [0, 1, 2, 3, 4]:
 329             try:
 330                 lit.TestRunner.applySubstitutions(
 331                     script, substitutions, recursion_limit=limit
 332                 )
 333                 self.fail("applySubstitutions should have raised an exception")
 334             except ValueError:
 335                 pass
 336
 337     def test_recursive_substitution_invalid_value(self):
 338         script = ["%rec5"]
 339         substitutions = [
 340             ("%rec1", "STOP"),
 341             ("%rec2", "%rec1"),
 342             ("%rec3", "%rec2"),
 343             ("%rec4", "%rec3"),
 344             ("%rec5", "%rec4"),
 345         ]
 346         for limit in [-1, -2, -3, "foo"]:
 347             try:
 348                 lit.TestRunner.applySubstitutions(
 349                     script, substitutions, recursion_limit=limit
 350                 )
 351                 self.fail("applySubstitutions should have raised an exception")
 352             except AssertionError:
 353                 pass
 354
 355
 356 if __name__ == "__main__":
 357     TestIntegratedTestKeywordParser.load_keyword_parser_lit_tests()
 358     unittest.main(verbosity=2)