lldb/packages/Python/lldbsuite/test/test_result.py

   1 """
   2 Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   3 See https://llvm.org/LICENSE.txt for license information.
   4 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   5
   6 Provides the LLDBTestResult class, which holds information about progress
   7 and results of a single test run.
   8 """
   9
  10 # System modules
  11 import os
  12 import traceback
  13
  14 # Third-party modules
  15 import unittest
  16
  17 # LLDB Modules
  18 from . import configuration
  19 from lldbsuite.test_event import build_exception
  20
  21
  22 class LLDBTestResult(unittest.TextTestResult):
  23     """
  24     Enforce a singleton pattern to allow introspection of test progress.
  25
  26     Overwrite addError(), addFailure(), and addExpectedFailure() methods
  27     to enable each test instance to track its failure/error status.  It
  28     is used in the LLDB test framework to emit detailed trace messages
  29     to a log file for easier human inspection of test failures/errors.
  30     """
  31
  32     __singleton__ = None
  33     __ignore_singleton__ = False
  34
  35     @staticmethod
  36     def getTerminalSize():
  37         import os
  38
  39         env = os.environ
  40
  41         def ioctl_GWINSZ(fd):
  42             try:
  43                 import fcntl
  44                 import termios
  45                 import struct
  46
  47                 cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
  48             except:
  49                 return
  50             return cr
  51
  52         cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
  53         if not cr:
  54             try:
  55                 fd = os.open(os.ctermid(), os.O_RDONLY)
  56                 cr = ioctl_GWINSZ(fd)
  57                 os.close(fd)
  58             except:
  59                 pass
  60         if not cr:
  61             cr = (env.get("LINES", 25), env.get("COLUMNS", 80))
  62         return int(cr[1]), int(cr[0])
  63
  64     def __init__(self, *args):
  65         if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
  66             raise Exception("LLDBTestResult instantiated more than once")
  67         super(LLDBTestResult, self).__init__(*args)
  68         LLDBTestResult.__singleton__ = self
  69         # Now put this singleton into the lldb module namespace.
  70         configuration.test_result = self
  71         # Computes the format string for displaying the counter.
  72         counterWidth = len(str(configuration.suite.countTestCases()))
  73         self.fmt = "%" + str(counterWidth) + "d: "
  74         self.indentation = " " * (counterWidth + 2)
  75         # This counts from 1 .. suite.countTestCases().
  76         self.counter = 0
  77         (width, height) = LLDBTestResult.getTerminalSize()
  78
  79     def _config_string(self, test):
  80         compiler = getattr(test, "getCompiler", None)
  81         arch = getattr(test, "getArchitecture", None)
  82         return "%s-%s" % (compiler() if compiler else "", arch() if arch else "")
  83
  84     def _exc_info_to_string(self, err, test):
  85         """Overrides superclass TestResult's method in order to append
  86         our test config info string to the exception info string."""
  87         if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
  88             return "%sConfig=%s-%s" % (
  89                 super(LLDBTestResult, self)._exc_info_to_string(err, test),
  90                 test.getArchitecture(),
  91                 test.getCompiler(),
  92             )
  93         else:
  94             return super(LLDBTestResult, self)._exc_info_to_string(err, test)
  95
  96     def getDescription(self, test):
  97         doc_first_line = test.shortDescription()
  98         if self.descriptions and doc_first_line:
  99             return "\n".join((str(test), self.indentation + doc_first_line))
 100         else:
 101             return str(test)
 102
 103     def _getTestPath(self, test):
 104         # Use test.test_filename if the test was created with
 105         # lldbinline.MakeInlineTest().
 106         if test is None:
 107             return ""
 108         elif hasattr(test, "test_filename"):
 109             return test.test_filename
 110         else:
 111             import inspect
 112
 113             return inspect.getsourcefile(test.__class__)
 114
 115     def _getFileBasedCategories(self, test):
 116         """
 117         Returns the list of categories to which this test case belongs by
 118         collecting values of "categories" files. We start at the folder the test is in
 119         and traverse the hierarchy upwards until the test-suite root directory.
 120         """
 121         start_path = self._getTestPath(test)
 122
 123         import os.path
 124
 125         folder = os.path.dirname(start_path)
 126
 127         from lldbsuite import lldb_test_root as test_root
 128
 129         if test_root != os.path.commonprefix([folder, test_root]):
 130             raise Exception(
 131                 "The test file %s is outside the test root directory" % start_path
 132             )
 133
 134         categories = set()
 135         while not os.path.samefile(folder, test_root):
 136             categories_file_name = os.path.join(folder, "categories")
 137             if os.path.exists(categories_file_name):
 138                 categories_file = open(categories_file_name, "r")
 139                 categories_str = categories_file.readline().strip()
 140                 categories_file.close()
 141                 categories.update(categories_str.split(","))
 142             folder = os.path.dirname(folder)
 143
 144         return list(categories)
 145
 146     def getCategoriesForTest(self, test):
 147         """
 148         Gets all the categories for the currently running test method in test case
 149         """
 150         test_categories = []
 151         test_categories.extend(getattr(test, "categories", []))
 152
 153         test_method = getattr(test, test._testMethodName)
 154         if test_method is not None:
 155             test_categories.extend(getattr(test_method, "categories", []))
 156
 157         test_categories.extend(self._getFileBasedCategories(test))
 158
 159         return test_categories
 160
 161     def hardMarkAsSkipped(self, test):
 162         getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
 163         getattr(
 164             test, test._testMethodName
 165         ).__func__.__unittest_skip_why__ = (
 166             "test case does not fall in any category of interest for this run"
 167         )
 168
 169     def checkExclusion(self, exclusion_list, name):
 170         if exclusion_list:
 171             import re
 172
 173             for item in exclusion_list:
 174                 if re.search(item, name):
 175                     return True
 176         return False
 177
 178     def checkCategoryExclusion(self, exclusion_list, test):
 179         return not set(exclusion_list).isdisjoint(self.getCategoriesForTest(test))
 180
 181     def startTest(self, test):
 182         if configuration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)):
 183             self.hardMarkAsSkipped(test)
 184         if self.checkExclusion(configuration.skip_tests, test.id()):
 185             self.hardMarkAsSkipped(test)
 186
 187         self.counter += 1
 188         test.test_number = self.counter
 189         if self.showAll:
 190             self.stream.write(self.fmt % self.counter)
 191         super(LLDBTestResult, self).startTest(test)
 192
 193     def addSuccess(self, test):
 194         if self.checkExclusion(
 195             configuration.xfail_tests, test.id()
 196         ) or self.checkCategoryExclusion(configuration.xfail_categories, test):
 197             self.addUnexpectedSuccess(test, None)
 198             return
 199
 200         super(LLDBTestResult, self).addSuccess(test)
 201         self.stream.write(
 202             "PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 203         )
 204
 205     def _isBuildError(self, err_tuple):
 206         exception = err_tuple[1]
 207         return isinstance(exception, build_exception.BuildError)
 208
 209     def _saveBuildErrorTuple(self, test, err):
 210         # Adjust the error description so it prints the build command and build error
 211         # rather than an uninformative Python backtrace.
 212         build_error = err[1]
 213         error_description = "{}\nTest Directory:\n{}".format(
 214             str(build_error), os.path.dirname(self._getTestPath(test))
 215         )
 216         self.errors.append((test, error_description))
 217         self._mirrorOutput = True
 218
 219     def addError(self, test, err):
 220         configuration.sdir_has_content = True
 221         if self._isBuildError(err):
 222             self._saveBuildErrorTuple(test, err)
 223         else:
 224             super(LLDBTestResult, self).addError(test, err)
 225
 226         method = getattr(test, "markError", None)
 227         if method:
 228             method()
 229         self.stream.write(
 230             "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 231         )
 232
 233     def addCleanupError(self, test, err):
 234         configuration.sdir_has_content = True
 235         super(LLDBTestResult, self).addCleanupError(test, err)
 236         method = getattr(test, "markCleanupError", None)
 237         if method:
 238             method()
 239         self.stream.write(
 240             "CLEANUP ERROR: LLDB (%s) :: %s\n%s\n"
 241             % (self._config_string(test), str(test), traceback.format_exc())
 242         )
 243
 244     def addFailure(self, test, err):
 245         if self.checkExclusion(
 246             configuration.xfail_tests, test.id()
 247         ) or self.checkCategoryExclusion(configuration.xfail_categories, test):
 248             self.addExpectedFailure(test, err)
 249             return
 250
 251         configuration.sdir_has_content = True
 252         super(LLDBTestResult, self).addFailure(test, err)
 253         method = getattr(test, "markFailure", None)
 254         if method:
 255             method()
 256         self.stream.write(
 257             "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 258         )
 259         if configuration.use_categories:
 260             test_categories = self.getCategoriesForTest(test)
 261             for category in test_categories:
 262                 if category in configuration.failures_per_category:
 263                     configuration.failures_per_category[category] = (
 264                         configuration.failures_per_category[category] + 1
 265                     )
 266                 else:
 267                     configuration.failures_per_category[category] = 1
 268
 269     def addExpectedFailure(self, test, err):
 270         configuration.sdir_has_content = True
 271         super(LLDBTestResult, self).addExpectedFailure(test, err)
 272         method = getattr(test, "markExpectedFailure", None)
 273         if method:
 274             method(err)
 275         self.stream.write(
 276             "XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 277         )
 278
 279     def addSkip(self, test, reason):
 280         configuration.sdir_has_content = True
 281         super(LLDBTestResult, self).addSkip(test, reason)
 282         method = getattr(test, "markSkippedTest", None)
 283         if method:
 284             method()
 285         self.stream.write(
 286             "UNSUPPORTED: LLDB (%s) :: %s (%s) \n"
 287             % (self._config_string(test), str(test), reason)
 288         )
 289
 290     def addUnexpectedSuccess(self, test):
 291         configuration.sdir_has_content = True
 292         super(LLDBTestResult, self).addUnexpectedSuccess(test)
 293         method = getattr(test, "markUnexpectedSuccess", None)
 294         if method:
 295             method()
 296         self.stream.write(
 297             "XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
 298         )