lldb/packages/Python/lldbsuite/test/test_result.py

   1 """
   2 Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   3 See https://llvm.org/LICENSE.txt for license information.
   4 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   5
   6 Provides the LLDBTestResult class, which holds information about progress
   7 and results of a single test run.
   8 """
   9
  10 # System modules
  11 import os
  12 import traceback
  13
  14 # Third-party modules
  15 import unittest2
  16
  17 # LLDB Modules
  18 from . import configuration
  19 from lldbsuite.test_event import build_exception
  20
  21
  22 class LLDBTestResult(unittest2.TextTestResult):
  23     """
  24     Enforce a singleton pattern to allow introspection of test progress.
  25
  26     Overwrite addError(), addFailure(), and addExpectedFailure() methods
  27     to enable each test instance to track its failure/error status.  It
  28     is used in the LLDB test framework to emit detailed trace messages
  29     to a log file for easier human inspection of test failures/errors.
  30     """
  31
  32     __singleton__ = None
  33     __ignore_singleton__ = False
  34
  35     @staticmethod
  36     def getTerminalSize():
  37         import os
  38
  39         env = os.environ
  40
  41         def ioctl_GWINSZ(fd):
  42             try:
  43                 import fcntl
  44                 import termios
  45                 import struct
  46
  47                 cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
  48             except:
  49                 return
  50             return cr
  51
  52         cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
  53         if not cr:
  54             try:
  55                 fd = os.open(os.ctermid(), os.O_RDONLY)
  56                 cr = ioctl_GWINSZ(fd)
  57                 os.close(fd)
  58             except:
  59                 pass
  60         if not cr:
  61             cr = (env.get("LINES", 25), env.get("COLUMNS", 80))
  62         return int(cr[1]), int(cr[0])
  63
  64     def __init__(self, *args):
  65         if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
  66             raise Exception("LLDBTestResult instantiated more than once")
  67         super(LLDBTestResult, self).__init__(*args)
  68         LLDBTestResult.__singleton__ = self
  69         # Now put this singleton into the lldb module namespace.
  70         configuration.test_result = self
  71         # Computes the format string for displaying the counter.
  72         counterWidth = len(str(configuration.suite.countTestCases()))
  73         self.fmt = "%" + str(counterWidth) + "d: "
  74         self.indentation = " " * (counterWidth + 2)
  75         # This counts from 1 .. suite.countTestCases().
  76         self.counter = 0
  77         (width, height) = LLDBTestResult.getTerminalSize()
  78
  79     def _config_string(self, test):
  80         compiler = getattr(test, "getCompiler", None)
  81         arch = getattr(test, "getArchitecture", None)
  82         return "%s-%s" % (compiler() if compiler else "", arch() if arch else "")
  83
  84     def _exc_info_to_string(self, err, test):
  85         """Overrides superclass TestResult's method in order to append
  86         our test config info string to the exception info string."""
  87         if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
  88             return "%sConfig=%s-%s" % (
  89                 super(LLDBTestResult, self)._exc_info_to_string(err, test),
  90                 test.getArchitecture(),
  91                 test.getCompiler(),
  92             )
  93         else:
  94             return super(LLDBTestResult, self)._exc_info_to_string(err, test)
  95
  96     def getDescription(self, test):
  97         doc_first_line = test.shortDescription()
  98         if self.descriptions and doc_first_line:
  99             return "\n".join((str(test), self.indentation + doc_first_line))
 100         else:
 101             return str(test)
 102
 103     def _getTestPath(self, test):
 104         # Use test.test_filename if the test was created with
 105         # lldbinline.MakeInlineTest().
 106         if test is None:
 107             return ""
 108         elif hasattr(test, "test_filename"):
 109             return test.test_filename
 110         else:
 111             import inspect
 112
 113             return inspect.getsourcefile(test.__class__)
 114
 115     def _getFileBasedCategories(self, test):
 116         """
 117         Returns the list of categories to which this test case belongs by
 118         collecting values of "categories" files. We start at the folder the test is in
 119         and traverse the hierarchy upwards until the test-suite root directory.
 120         """
 121         start_path = self._getTestPath(test)
 122
 123         import os.path
 124
 125         folder = os.path.dirname(start_path)
 126
 127         from lldbsuite import lldb_test_root as test_root
 128
 129         if test_root != os.path.commonprefix([folder, test_root]):
 130             raise Exception(
 131                 "The test file %s is outside the test root directory" % start_path
 132             )
 133
 134         categories = set()
 135         while not os.path.samefile(folder, test_root):
 136             categories_file_name = os.path.join(folder, "categories")
 137             if os.path.exists(categories_file_name):
 138                 categories_file = open(categories_file_name, "r")
 139                 categories_str = categories_file.readline().strip()
 140                 categories_file.close()
 141                 categories.update(categories_str.split(","))
 142             folder = os.path.dirname(folder)
 143
 144         return list(categories)
 145
 146     def getCategoriesForTest(self, test):
 147         """
 148         Gets all the categories for the currently running test method in test case
 149         """
 150         test_categories = []
 151         test_method = getattr(test, test._testMethodName)
 152         if test_method is not None and hasattr(test_method, "categories"):
 153             test_categories.extend(test_method.categories)
 154
 155         test_categories.extend(self._getFileBasedCategories(test))
 156
 157         return test_categories
 158
 159     def hardMarkAsSkipped(self, test):
 160         getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
 161         getattr(
 162             test, test._testMethodName
 163         ).__func__.__unittest_skip_why__ = (
 164             "test case does not fall in any category of interest for this run"
 165         )
 166
 167     def checkExclusion(self, exclusion_list, name):
 168         if exclusion_list:
 169             import re
 170
 171             for item in exclusion_list:
 172                 if re.search(item, name):
 173                     return True
 174         return False
 175
 176     def checkCategoryExclusion(self, exclusion_list, test):
 177         return not set(exclusion_list).isdisjoint(self.getCategoriesForTest(test))
 178
 179     def startTest(self, test):
 180         if configuration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)):
 181             self.hardMarkAsSkipped(test)
 182         if self.checkExclusion(configuration.skip_tests, test.id()):
 183             self.hardMarkAsSkipped(test)
 184
 185         self.counter += 1
 186         test.test_number = self.counter
 187         if self.showAll:
 188             self.stream.write(self.fmt % self.counter)
 189         super(LLDBTestResult, self).startTest(test)
 190
 191     def addSuccess(self, test):
 192         if self.checkExclusion(
 193             configuration.xfail_tests, test.id()
 194         ) or self.checkCategoryExclusion(configuration.xfail_categories, test):
 195             self.addUnexpectedSuccess(test, None)
 196             return
 197
 198         super(LLDBTestResult, self).addSuccess(test)
 199         self.stream.write(
 200             "PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 201         )
 202
 203     def _isBuildError(self, err_tuple):
 204         exception = err_tuple[1]
 205         return isinstance(exception, build_exception.BuildError)
 206
 207     def _saveBuildErrorTuple(self, test, err):
 208         # Adjust the error description so it prints the build command and build error
 209         # rather than an uninformative Python backtrace.
 210         build_error = err[1]
 211         error_description = "{}\nTest Directory:\n{}".format(
 212             str(build_error), os.path.dirname(self._getTestPath(test))
 213         )
 214         self.errors.append((test, error_description))
 215         self._mirrorOutput = True
 216
 217     def addError(self, test, err):
 218         configuration.sdir_has_content = True
 219         if self._isBuildError(err):
 220             self._saveBuildErrorTuple(test, err)
 221         else:
 222             super(LLDBTestResult, self).addError(test, err)
 223
 224         method = getattr(test, "markError", None)
 225         if method:
 226             method()
 227         self.stream.write(
 228             "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 229         )
 230
 231     def addCleanupError(self, test, err):
 232         configuration.sdir_has_content = True
 233         super(LLDBTestResult, self).addCleanupError(test, err)
 234         method = getattr(test, "markCleanupError", None)
 235         if method:
 236             method()
 237         self.stream.write(
 238             "CLEANUP ERROR: LLDB (%s) :: %s\n%s\n"
 239             % (self._config_string(test), str(test), traceback.format_exc())
 240         )
 241
 242     def addFailure(self, test, err):
 243         if self.checkExclusion(
 244             configuration.xfail_tests, test.id()
 245         ) or self.checkCategoryExclusion(configuration.xfail_categories, test):
 246             self.addExpectedFailure(test, err, None)
 247             return
 248
 249         configuration.sdir_has_content = True
 250         super(LLDBTestResult, self).addFailure(test, err)
 251         method = getattr(test, "markFailure", None)
 252         if method:
 253             method()
 254         self.stream.write(
 255             "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 256         )
 257         if configuration.use_categories:
 258             test_categories = self.getCategoriesForTest(test)
 259             for category in test_categories:
 260                 if category in configuration.failures_per_category:
 261                     configuration.failures_per_category[category] = (
 262                         configuration.failures_per_category[category] + 1
 263                     )
 264                 else:
 265                     configuration.failures_per_category[category] = 1
 266
 267     def addExpectedFailure(self, test, err, bugnumber):
 268         configuration.sdir_has_content = True
 269         super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber)
 270         method = getattr(test, "markExpectedFailure", None)
 271         if method:
 272             method(err, bugnumber)
 273         self.stream.write(
 274             "XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 275         )
 276
 277     def addSkip(self, test, reason):
 278         configuration.sdir_has_content = True
 279         super(LLDBTestResult, self).addSkip(test, reason)
 280         method = getattr(test, "markSkippedTest", None)
 281         if method:
 282             method()
 283         self.stream.write(
 284             "UNSUPPORTED: LLDB (%s) :: %s (%s) \n"
 285             % (self._config_string(test), str(test), reason)
 286         )
 287
 288     def addUnexpectedSuccess(self, test, bugnumber):
 289         configuration.sdir_has_content = True
 290         super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber)
 291         method = getattr(test, "markUnexpectedSuccess", None)
 292         if method:
 293             method(bugnumber)
 294         self.stream.write(
 295             "XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 296         )