lldb/test/API/lldbtest.py

   1 import collections
   2 import os
   3 import re
   4 import operator
   5
   6 import lit.Test
   7 import lit.TestRunner
   8 import lit.util
   9 from lit.formats.base import TestFormat
  10
  11
  12 class LLDBTest(TestFormat):
  13     def __init__(self, dotest_cmd):
  14         self.dotest_cmd = dotest_cmd
  15
  16     def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
  17         source_path = testSuite.getSourcePath(path_in_suite)
  18         for filename in os.listdir(source_path):
  19             # Ignore dot files and excluded tests.
  20             if filename.startswith(".") or filename in localConfig.excludes:
  21                 continue
  22
  23             # Ignore files that don't start with 'Test'.
  24             if not filename.startswith("Test"):
  25                 continue
  26
  27             filepath = os.path.join(source_path, filename)
  28             if not os.path.isdir(filepath):
  29                 base, ext = os.path.splitext(filename)
  30                 if ext in localConfig.suffixes:
  31                     yield lit.Test.Test(
  32                         testSuite, path_in_suite + (filename,), localConfig
  33                     )
  34
  35     def execute(self, test, litConfig):
  36         if litConfig.noExecute:
  37             return lit.Test.PASS, ""
  38
  39         if not getattr(test.config, "lldb_enable_python", False):
  40             return (lit.Test.UNSUPPORTED, "Python module disabled")
  41
  42         if test.config.unsupported:
  43             return (lit.Test.UNSUPPORTED, "Test is unsupported")
  44
  45         testPath, testFile = os.path.split(test.getSourcePath())
  46
  47         # The Python used to run lit can be different from the Python LLDB was
  48         # build with.
  49         executable = test.config.python_executable
  50
  51         isLuaTest = testFile == test.config.lua_test_entry
  52
  53         # On Windows, the system does not always correctly interpret
  54         # shebang lines.  To make sure we can execute the tests, add
  55         # python exe as the first parameter of the command.
  56         cmd = [executable] + self.dotest_cmd + [testPath, "-p", testFile]
  57
  58         if isLuaTest:
  59             cmd.extend(["--env", "LUA_EXECUTABLE=%s" % test.config.lua_executable])
  60             cmd.extend(["--env", "LLDB_LUA_CPATH=%s" % test.config.lldb_lua_cpath])
  61
  62         timeoutInfo = None
  63         try:
  64             out, err, exitCode = lit.util.executeCommand(
  65                 cmd,
  66                 env=test.config.environment,
  67                 timeout=litConfig.maxIndividualTestTime,
  68             )
  69         except lit.util.ExecuteCommandTimeoutException as e:
  70             out = e.out
  71             err = e.err
  72             exitCode = e.exitCode
  73             timeoutInfo = "Reached timeout of {} seconds".format(
  74                 litConfig.maxIndividualTestTime
  75             )
  76
  77         output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % (" ".join(cmd), exitCode)
  78         if timeoutInfo is not None:
  79             output += """Timeout: %s\n""" % (timeoutInfo,)
  80         output += "\n"
  81
  82         if out:
  83             output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
  84         if err:
  85             output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
  86
  87         if timeoutInfo:
  88             return lit.Test.TIMEOUT, output
  89
  90         # Parse the dotest output from stderr. First get the # of total tests, in order to infer the # of passes.
  91         # Example: "Ran 5 tests in 0.042s"
  92         num_ran_regex = r"^Ran (\d+) tests? in "
  93         num_ran_results = re.search(num_ran_regex, err, re.MULTILINE)
  94
  95         # If parsing fails mark this test as unresolved.
  96         if not num_ran_results:
  97             return lit.Test.UNRESOLVED, output
  98         num_ran = int(num_ran_results.group(1))
  99
 100         # Then look for a detailed summary, which is OK or FAILED followed by optional details.
 101         # Example: "OK (skipped=1, expected failures=1)"
 102         # Example: "FAILED (failures=3)"
 103         # Example: "OK"
 104         result_regex = r"^(?:OK|FAILED)(?: \((.*)\))?\r?$"
 105         results = re.search(result_regex, err, re.MULTILINE)
 106
 107         # If parsing fails mark this test as unresolved.
 108         if not results:
 109             return lit.Test.UNRESOLVED, output
 110
 111         details = results.group(1)
 112         parsed_details = collections.defaultdict(int)
 113         if details:
 114             for detail in details.split(", "):
 115                 detail_parts = detail.split("=")
 116                 if len(detail_parts) != 2:
 117                     return lit.Test.UNRESOLVED, output
 118                 parsed_details[detail_parts[0]] = int(detail_parts[1])
 119
 120         failures = parsed_details["failures"]
 121         errors = parsed_details["errors"]
 122         skipped = parsed_details["skipped"]
 123         expected_failures = parsed_details["expected failures"]
 124         unexpected_successes = parsed_details["unexpected successes"]
 125
 126         non_pass = (
 127             failures + errors + skipped + expected_failures + unexpected_successes
 128         )
 129         passes = num_ran - non_pass
 130
 131         if exitCode:
 132             # Mark this test as FAIL if at least one test failed.
 133             if failures > 0:
 134                 return lit.Test.FAIL, output
 135             lit_results = [
 136                 (failures, lit.Test.FAIL),
 137                 (errors, lit.Test.UNRESOLVED),
 138                 (unexpected_successes, lit.Test.XPASS),
 139             ]
 140         else:
 141             # Mark this test as PASS if at least one test passed.
 142             if passes > 0:
 143                 return lit.Test.PASS, output
 144             lit_results = [
 145                 (passes, lit.Test.PASS),
 146                 (skipped, lit.Test.UNSUPPORTED),
 147                 (expected_failures, lit.Test.XFAIL),
 148             ]
 149
 150         # Return the lit result code with the maximum occurrence. Only look at
 151         # the first element and rely on the original order to break ties.
 152         return max(lit_results, key=operator.itemgetter(0))[1], output