lldb/packages/Python/lldbsuite/test/test_result.py

   1 """
   2 Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   3 See https://llvm.org/LICENSE.txt for license information.
   4 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   5
   6 Provides the LLDBTestResult class, which holds information about progress
   7 and results of a single test run.
   8 """
   9
  10 # System modules
  11 import os
  12
  13 # Third-party modules
  14 import unittest2
  15
  16 # LLDB Modules
  17 from . import configuration
  18 from lldbsuite.test_event.event_builder import EventBuilder
  19 from lldbsuite.test_event import build_exception
  20
  21
  22 class LLDBTestResult(unittest2.TextTestResult):
  23     """
  24     Enforce a singleton pattern to allow introspection of test progress.
  25
  26     Overwrite addError(), addFailure(), and addExpectedFailure() methods
  27     to enable each test instance to track its failure/error status.  It
  28     is used in the LLDB test framework to emit detailed trace messages
  29     to a log file for easier human inspection of test failures/errors.
  30     """
  31     __singleton__ = None
  32     __ignore_singleton__ = False
  33
  34     @staticmethod
  35     def getTerminalSize():
  36         import os
  37         env = os.environ
  38
  39         def ioctl_GWINSZ(fd):
  40             try:
  41                 import fcntl
  42                 import termios
  43                 import struct
  44                 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
  45                                                      '1234'))
  46             except:
  47                 return
  48             return cr
  49         cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
  50         if not cr:
  51             try:
  52                 fd = os.open(os.ctermid(), os.O_RDONLY)
  53                 cr = ioctl_GWINSZ(fd)
  54                 os.close(fd)
  55             except:
  56                 pass
  57         if not cr:
  58             cr = (env.get('LINES', 25), env.get('COLUMNS', 80))
  59         return int(cr[1]), int(cr[0])
  60
  61     def __init__(self, *args):
  62         if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
  63             raise Exception("LLDBTestResult instantiated more than once")
  64         super(LLDBTestResult, self).__init__(*args)
  65         LLDBTestResult.__singleton__ = self
  66         # Now put this singleton into the lldb module namespace.
  67         configuration.test_result = self
  68         # Computes the format string for displaying the counter.
  69         counterWidth = len(str(configuration.suite.countTestCases()))
  70         self.fmt = "%" + str(counterWidth) + "d: "
  71         self.indentation = ' ' * (counterWidth + 2)
  72         # This counts from 1 .. suite.countTestCases().
  73         self.counter = 0
  74         (width, height) = LLDBTestResult.getTerminalSize()
  75         self.results_formatter = configuration.results_formatter_object
  76
  77     def _config_string(self, test):
  78         compiler = getattr(test, "getCompiler", None)
  79         arch = getattr(test, "getArchitecture", None)
  80         return "%s-%s" % (compiler() if compiler else "",
  81                           arch() if arch else "")
  82
  83     def _exc_info_to_string(self, err, test):
  84         """Overrides superclass TestResult's method in order to append
  85         our test config info string to the exception info string."""
  86         if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
  87             return '%sConfig=%s-%s' % (super(LLDBTestResult,
  88                                              self)._exc_info_to_string(err,
  89                                                                        test),
  90                                        test.getArchitecture(),
  91                                        test.getCompiler())
  92         else:
  93             return super(LLDBTestResult, self)._exc_info_to_string(err, test)
  94
  95     def getDescription(self, test):
  96         doc_first_line = test.shortDescription()
  97         if self.descriptions and doc_first_line:
  98             return '\n'.join((str(test), self.indentation + doc_first_line))
  99         else:
 100             return str(test)
 101
 102     def _getTestPath(self, test):
 103         # Use test.test_filename if the test was created with
 104         # lldbinline.MakeInlineTest().
 105         if test is None:
 106             return ""
 107         elif hasattr(test, "test_filename"):
 108             return test.test_filename
 109         else:
 110             import inspect
 111             return inspect.getsourcefile(test.__class__)
 112
 113     def _getFileBasedCategories(self, test):
 114         """
 115         Returns the list of categories to which this test case belongs by
 116         collecting values of ".categories" files. We start at the folder the test is in
 117         and traverse the hierarchy upwards until the test-suite root directory.
 118         """
 119         start_path = self._getTestPath(test)
 120
 121         import os.path
 122         folder = os.path.dirname(start_path)
 123
 124         from lldbsuite import lldb_test_root as test_root
 125         if test_root != os.path.commonprefix([folder, test_root]):
 126             raise Exception("The test file %s is outside the test root directory" % start_path)
 127
 128         categories = set()
 129         while not os.path.samefile(folder, test_root):
 130             categories_file_name = os.path.join(folder, ".categories")
 131             if os.path.exists(categories_file_name):
 132                 categories_file = open(categories_file_name, 'r')
 133                 categories_str = categories_file.readline().strip()
 134                 categories_file.close()
 135                 categories.update(categories_str.split(','))
 136             folder = os.path.dirname(folder)
 137
 138         return list(categories)
 139
 140     def getCategoriesForTest(self, test):
 141         """
 142         Gets all the categories for the currently running test method in test case
 143         """
 144         test_categories = []
 145         test_method = getattr(test, test._testMethodName)
 146         if test_method is not None and hasattr(test_method, "categories"):
 147             test_categories.extend(test_method.categories)
 148
 149         test_categories.extend(self._getFileBasedCategories(test))
 150
 151         return test_categories
 152
 153     def hardMarkAsSkipped(self, test):
 154         getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
 155         getattr(
 156             test,
 157             test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
 158
 159     def checkExclusion(self, exclusion_list, name):
 160         if exclusion_list:
 161             import re
 162             for item in exclusion_list:
 163                 if re.search(item, name):
 164                     return True
 165         return False
 166
 167     def checkCategoryExclusion(self, exclusion_list, test):
 168         return not set(exclusion_list).isdisjoint(
 169             self.getCategoriesForTest(test))
 170
 171     def startTest(self, test):
 172         if configuration.shouldSkipBecauseOfCategories(
 173                 self.getCategoriesForTest(test)):
 174             self.hardMarkAsSkipped(test)
 175         if self.checkExclusion(
 176                 configuration.skip_tests, test.id()):
 177             self.hardMarkAsSkipped(test)
 178
 179         self.counter += 1
 180         test.test_number = self.counter
 181         if self.showAll:
 182             self.stream.write(self.fmt % self.counter)
 183         super(LLDBTestResult, self).startTest(test)
 184         if self.results_formatter:
 185             self.results_formatter.handle_event(
 186                 EventBuilder.event_for_start(test))
 187
 188     def addSuccess(self, test):
 189         if (self.checkExclusion(
 190                 configuration.xfail_tests, test.id()) or
 191             self.checkCategoryExclusion(
 192                 configuration.xfail_categories, test)):
 193             self.addUnexpectedSuccess(test, None)
 194             return
 195
 196         super(LLDBTestResult, self).addSuccess(test)
 197         self.stream.write(
 198             "PASS: LLDB (%s) :: %s\n" %
 199             (self._config_string(test), str(test)))
 200         if self.results_formatter:
 201             self.results_formatter.handle_event(
 202                 EventBuilder.event_for_success(test))
 203
 204     def _isBuildError(self, err_tuple):
 205         exception = err_tuple[1]
 206         return isinstance(exception, build_exception.BuildError)
 207
 208     def _saveBuildErrorTuple(self, test, err):
 209         # Adjust the error description so it prints the build command and build error
 210         # rather than an uninformative Python backtrace.
 211         build_error = err[1]
 212         error_description = "{}\nTest Directory:\n{}".format(
 213             str(build_error),
 214             os.path.dirname(self._getTestPath(test)))
 215         self.errors.append((test, error_description))
 216         self._mirrorOutput = True
 217
 218     def addError(self, test, err):
 219         configuration.sdir_has_content = True
 220         if self._isBuildError(err):
 221             self._saveBuildErrorTuple(test, err)
 222         else:
 223             super(LLDBTestResult, self).addError(test, err)
 224
 225         method = getattr(test, "markError", None)
 226         if method:
 227             method()
 228         self.stream.write(
 229             "FAIL: LLDB (%s) :: %s\n" %
 230             (self._config_string(test), str(test)))
 231         if self.results_formatter:
 232             # Handle build errors as a separate event type
 233             if self._isBuildError(err):
 234                 error_event = EventBuilder.event_for_build_error(test, err)
 235             else:
 236                 error_event = EventBuilder.event_for_error(test, err)
 237             self.results_formatter.handle_event(error_event)
 238
 239     def addCleanupError(self, test, err):
 240         configuration.sdir_has_content = True
 241         super(LLDBTestResult, self).addCleanupError(test, err)
 242         method = getattr(test, "markCleanupError", None)
 243         if method:
 244             method()
 245         self.stream.write(
 246             "CLEANUP ERROR: LLDB (%s) :: %s\n" %
 247             (self._config_string(test), str(test)))
 248         if self.results_formatter:
 249             self.results_formatter.handle_event(
 250                 EventBuilder.event_for_cleanup_error(
 251                     test, err))
 252
 253     def addFailure(self, test, err):
 254         if (self.checkExclusion(
 255                 configuration.xfail_tests, test.id()) or
 256             self.checkCategoryExclusion(
 257                 configuration.xfail_categories, test)):
 258             self.addExpectedFailure(test, err, None)
 259             return
 260
 261         configuration.sdir_has_content = True
 262         super(LLDBTestResult, self).addFailure(test, err)
 263         method = getattr(test, "markFailure", None)
 264         if method:
 265             method()
 266         self.stream.write(
 267             "FAIL: LLDB (%s) :: %s\n" %
 268             (self._config_string(test), str(test)))
 269         if configuration.use_categories:
 270             test_categories = self.getCategoriesForTest(test)
 271             for category in test_categories:
 272                 if category in configuration.failures_per_category:
 273                     configuration.failures_per_category[
 274                         category] = configuration.failures_per_category[category] + 1
 275                 else:
 276                     configuration.failures_per_category[category] = 1
 277         if self.results_formatter:
 278             self.results_formatter.handle_event(
 279                 EventBuilder.event_for_failure(test, err))
 280
 281     def addExpectedFailure(self, test, err, bugnumber):
 282         configuration.sdir_has_content = True
 283         super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber)
 284         method = getattr(test, "markExpectedFailure", None)
 285         if method:
 286             method(err, bugnumber)
 287         self.stream.write(
 288             "XFAIL: LLDB (%s) :: %s\n" %
 289             (self._config_string(test), str(test)))
 290         if self.results_formatter:
 291             self.results_formatter.handle_event(
 292                 EventBuilder.event_for_expected_failure(
 293                     test, err, bugnumber))
 294
 295     def addSkip(self, test, reason):
 296         configuration.sdir_has_content = True
 297         super(LLDBTestResult, self).addSkip(test, reason)
 298         method = getattr(test, "markSkippedTest", None)
 299         if method:
 300             method()
 301         self.stream.write(
 302             "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" %
 303             (self._config_string(test), str(test), reason))
 304         if self.results_formatter:
 305             self.results_formatter.handle_event(
 306                 EventBuilder.event_for_skip(test, reason))
 307
 308     def addUnexpectedSuccess(self, test, bugnumber):
 309         configuration.sdir_has_content = True
 310         super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber)
 311         method = getattr(test, "markUnexpectedSuccess", None)
 312         if method:
 313             method(bugnumber)
 314         self.stream.write(
 315             "XPASS: LLDB (%s) :: %s\n" %
 316             (self._config_string(test), str(test)))
 317         if self.results_formatter:
 318             self.results_formatter.handle_event(
 319                 EventBuilder.event_for_unexpected_success(
 320                     test, bugnumber))