lldb/packages/Python/lldbsuite/test_event/event_builder.py

   1 """
   2 Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   3 See https://llvm.org/LICENSE.txt for license information.
   4 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   5
   6 Provides a class to build Python test event data structures.
   7 """
   8
   9 from __future__ import print_function
  10 from __future__ import absolute_import
  11
  12 # System modules
  13 import inspect
  14 import time
  15 import traceback
  16
  17 # Third-party modules
  18
  19 # LLDB modules
  20 from . import build_exception
  21
  22
  23 class EventBuilder(object):
  24     """Helper class to build test result event dictionaries."""
  25
  26     BASE_DICTIONARY = None
  27
  28     # Test Event Types
  29     TYPE_JOB_RESULT = "job_result"
  30     TYPE_TEST_RESULT = "test_result"
  31     TYPE_TEST_START = "test_start"
  32     TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
  33     TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
  34     TYPE_SESSION_TERMINATE = "terminate"
  35
  36     RESULT_TYPES = {TYPE_JOB_RESULT, TYPE_TEST_RESULT}
  37
  38     # Test/Job Status Tags
  39     STATUS_EXCEPTIONAL_EXIT = "exceptional_exit"
  40     STATUS_SUCCESS = "success"
  41     STATUS_FAILURE = "failure"
  42     STATUS_EXPECTED_FAILURE = "expected_failure"
  43     STATUS_EXPECTED_TIMEOUT = "expected_timeout"
  44     STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
  45     STATUS_SKIP = "skip"
  46     STATUS_ERROR = "error"
  47     STATUS_TIMEOUT = "timeout"
  48
  49     """Test methods or jobs with a status matching any of these
  50     status values will cause a testrun failure, unless
  51     the test methods rerun and do not trigger an issue when rerun."""
  52     TESTRUN_ERROR_STATUS_VALUES = {
  53         STATUS_ERROR,
  54         STATUS_EXCEPTIONAL_EXIT,
  55         STATUS_FAILURE,
  56         STATUS_TIMEOUT}
  57
  58     @staticmethod
  59     def _get_test_name_info(test):
  60         """Returns (test-class-name, test-method-name) from a test case instance.
  61
  62         @param test a unittest.TestCase instance.
  63
  64         @return tuple containing (test class name, test method name)
  65         """
  66         test_class_components = test.id().split(".")
  67         test_class_name = ".".join(test_class_components[:-1])
  68         test_name = test_class_components[-1]
  69         return test_class_name, test_name
  70
  71     @staticmethod
  72     def bare_event(event_type):
  73         """Creates an event with default additions, event type and timestamp.
  74
  75         @param event_type the value set for the "event" key, used
  76         to distinguish events.
  77
  78         @returns an event dictionary with all default additions, the "event"
  79         key set to the passed in event_type, and the event_time value set to
  80         time.time().
  81         """
  82         if EventBuilder.BASE_DICTIONARY is not None:
  83             # Start with a copy of the "always include" entries.
  84             event = dict(EventBuilder.BASE_DICTIONARY)
  85         else:
  86             event = {}
  87
  88         event.update({
  89             "event": event_type,
  90             "event_time": time.time()
  91         })
  92         return event
  93
  94     @staticmethod
  95     def _assert_is_python_sourcefile(test_filename):
  96         if test_filename is not None:
  97             if not test_filename.endswith(".py"):
  98                 raise Exception(
  99                     "source python filename has unexpected extension: {}".format(test_filename))
 100         return test_filename
 101
 102     @staticmethod
 103     def _event_dictionary_common(test, event_type):
 104         """Returns an event dictionary setup with values for the given event type.
 105
 106         @param test the unittest.TestCase instance
 107
 108         @param event_type the name of the event type (string).
 109
 110         @return event dictionary with common event fields set.
 111         """
 112         test_class_name, test_name = EventBuilder._get_test_name_info(test)
 113
 114         # Determine the filename for the test case.  If there is an attribute
 115         # for it, use it.  Otherwise, determine from the TestCase class path.
 116         if hasattr(test, "test_filename"):
 117             test_filename = EventBuilder._assert_is_python_sourcefile(
 118                 test.test_filename)
 119         else:
 120             test_filename = EventBuilder._assert_is_python_sourcefile(
 121                 inspect.getsourcefile(test.__class__))
 122
 123         event = EventBuilder.bare_event(event_type)
 124         event.update({
 125             "test_class": test_class_name,
 126             "test_name": test_name,
 127             "test_filename": test_filename
 128         })
 129
 130         return event
 131
 132     @staticmethod
 133     def _error_tuple_class(error_tuple):
 134         """Returns the unittest error tuple's error class as a string.
 135
 136         @param error_tuple the error tuple provided by the test framework.
 137
 138         @return the error type (typically an exception) raised by the
 139         test framework.
 140         """
 141         type_var = error_tuple[0]
 142         module = inspect.getmodule(type_var)
 143         if module:
 144             return "{}.{}".format(module.__name__, type_var.__name__)
 145         else:
 146             return type_var.__name__
 147
 148     @staticmethod
 149     def _error_tuple_message(error_tuple):
 150         """Returns the unittest error tuple's error message.
 151
 152         @param error_tuple the error tuple provided by the test framework.
 153
 154         @return the error message provided by the test framework.
 155         """
 156         return str(error_tuple[1])
 157
 158     @staticmethod
 159     def _error_tuple_traceback(error_tuple):
 160         """Returns the unittest error tuple's error message.
 161
 162         @param error_tuple the error tuple provided by the test framework.
 163
 164         @return the error message provided by the test framework.
 165         """
 166         return error_tuple[2]
 167
 168     @staticmethod
 169     def _event_dictionary_test_result(test, status):
 170         """Returns an event dictionary with common test result fields set.
 171
 172         @param test a unittest.TestCase instance.
 173
 174         @param status the status/result of the test
 175         (e.g. "success", "failure", etc.)
 176
 177         @return the event dictionary
 178         """
 179         event = EventBuilder._event_dictionary_common(
 180             test, EventBuilder.TYPE_TEST_RESULT)
 181         event["status"] = status
 182         return event
 183
 184     @staticmethod
 185     def _event_dictionary_issue(test, status, error_tuple):
 186         """Returns an event dictionary with common issue-containing test result
 187         fields set.
 188
 189         @param test a unittest.TestCase instance.
 190
 191         @param status the status/result of the test
 192         (e.g. "success", "failure", etc.)
 193
 194         @param error_tuple the error tuple as reported by the test runner.
 195         This is of the form (type<error>, error).
 196
 197         @return the event dictionary
 198         """
 199         event = EventBuilder._event_dictionary_test_result(test, status)
 200         event["issue_class"] = EventBuilder._error_tuple_class(error_tuple)
 201         event["issue_message"] = EventBuilder._error_tuple_message(error_tuple)
 202         backtrace = EventBuilder._error_tuple_traceback(error_tuple)
 203         if backtrace is not None:
 204             event["issue_backtrace"] = traceback.format_tb(backtrace)
 205         return event
 206
 207     @staticmethod
 208     def event_for_start(test):
 209         """Returns an event dictionary for the test start event.
 210
 211         @param test a unittest.TestCase instance.
 212
 213         @return the event dictionary
 214         """
 215         return EventBuilder._event_dictionary_common(
 216             test, EventBuilder.TYPE_TEST_START)
 217
 218     @staticmethod
 219     def event_for_success(test):
 220         """Returns an event dictionary for a successful test.
 221
 222         @param test a unittest.TestCase instance.
 223
 224         @return the event dictionary
 225         """
 226         return EventBuilder._event_dictionary_test_result(
 227             test, EventBuilder.STATUS_SUCCESS)
 228
 229     @staticmethod
 230     def event_for_unexpected_success(test, bugnumber):
 231         """Returns an event dictionary for a test that succeeded but was
 232         expected to fail.
 233
 234         @param test a unittest.TestCase instance.
 235
 236         @param bugnumber the issue identifier for the bug tracking the
 237         fix request for the test expected to fail (but is in fact
 238         passing here).
 239
 240         @return the event dictionary
 241
 242         """
 243         event = EventBuilder._event_dictionary_test_result(
 244             test, EventBuilder.STATUS_UNEXPECTED_SUCCESS)
 245         if bugnumber:
 246             event["bugnumber"] = str(bugnumber)
 247         return event
 248
 249     @staticmethod
 250     def event_for_failure(test, error_tuple):
 251         """Returns an event dictionary for a test that failed.
 252
 253         @param test a unittest.TestCase instance.
 254
 255         @param error_tuple the error tuple as reported by the test runner.
 256         This is of the form (type<error>, error).
 257
 258         @return the event dictionary
 259         """
 260         return EventBuilder._event_dictionary_issue(
 261             test, EventBuilder.STATUS_FAILURE, error_tuple)
 262
 263     @staticmethod
 264     def event_for_expected_failure(test, error_tuple, bugnumber):
 265         """Returns an event dictionary for a test that failed as expected.
 266
 267         @param test a unittest.TestCase instance.
 268
 269         @param error_tuple the error tuple as reported by the test runner.
 270         This is of the form (type<error>, error).
 271
 272         @param bugnumber the issue identifier for the bug tracking the
 273         fix request for the test expected to fail.
 274
 275         @return the event dictionary
 276
 277         """
 278         event = EventBuilder._event_dictionary_issue(
 279             test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple)
 280         if bugnumber:
 281             event["bugnumber"] = str(bugnumber)
 282         return event
 283
 284     @staticmethod
 285     def event_for_skip(test, reason):
 286         """Returns an event dictionary for a test that was skipped.
 287
 288         @param test a unittest.TestCase instance.
 289
 290         @param reason the reason why the test is being skipped.
 291
 292         @return the event dictionary
 293         """
 294         event = EventBuilder._event_dictionary_test_result(
 295             test, EventBuilder.STATUS_SKIP)
 296         event["skip_reason"] = reason
 297         return event
 298
 299     @staticmethod
 300     def event_for_error(test, error_tuple):
 301         """Returns an event dictionary for a test that hit a test execution error.
 302
 303         @param test a unittest.TestCase instance.
 304
 305         @param error_tuple the error tuple as reported by the test runner.
 306         This is of the form (type<error>, error).
 307
 308         @return the event dictionary
 309         """
 310         event = EventBuilder._event_dictionary_issue(
 311             test, EventBuilder.STATUS_ERROR, error_tuple)
 312         event["issue_phase"] = "test"
 313         return event
 314
 315     @staticmethod
 316     def event_for_build_error(test, error_tuple):
 317         """Returns an event dictionary for a test that hit a test execution error
 318         during the test cleanup phase.
 319
 320         @param test a unittest.TestCase instance.
 321
 322         @param error_tuple the error tuple as reported by the test runner.
 323         This is of the form (type<error>, error).
 324
 325         @return the event dictionary
 326         """
 327         event = EventBuilder._event_dictionary_issue(
 328             test, EventBuilder.STATUS_ERROR, error_tuple)
 329         event["issue_phase"] = "build"
 330
 331         build_error = error_tuple[1]
 332         event["build_command"] = build_error.command
 333         event["build_error"] = build_error.build_error
 334         return event
 335
 336     @staticmethod
 337     def event_for_cleanup_error(test, error_tuple):
 338         """Returns an event dictionary for a test that hit a test execution error
 339         during the test cleanup phase.
 340
 341         @param test a unittest.TestCase instance.
 342
 343         @param error_tuple the error tuple as reported by the test runner.
 344         This is of the form (type<error>, error).
 345
 346         @return the event dictionary
 347         """
 348         event = EventBuilder._event_dictionary_issue(
 349             test, EventBuilder.STATUS_ERROR, error_tuple)
 350         event["issue_phase"] = "cleanup"
 351         return event
 352
 353     @staticmethod
 354     def event_for_job_test_add_error(test_filename, exception, backtrace):
 355         event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
 356         event["status"] = EventBuilder.STATUS_ERROR
 357         if test_filename is not None:
 358             event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
 359                 test_filename)
 360         if exception is not None and "__class__" in dir(exception):
 361             event["issue_class"] = exception.__class__
 362         event["issue_message"] = exception
 363         if backtrace is not None:
 364             event["issue_backtrace"] = backtrace
 365         return event
 366
 367     @staticmethod
 368     def event_for_job_exceptional_exit(
 369             pid, worker_index, exception_code, exception_description,
 370             test_filename, command_line):
 371         """Creates an event for a job (i.e. process) exit due to signal.
 372
 373         @param pid the process id for the job that failed
 374         @param worker_index optional id for the job queue running the process
 375         @param exception_code optional code
 376         (e.g. SIGTERM integer signal number)
 377         @param exception_description optional string containing symbolic
 378         representation of the issue (e.g. "SIGTERM")
 379         @param test_filename the path to the test filename that exited
 380         in some exceptional way.
 381         @param command_line the Popen()-style list provided as the command line
 382         for the process that timed out.
 383
 384         @return an event dictionary coding the job completion description.
 385         """
 386         event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
 387         event["status"] = EventBuilder.STATUS_EXCEPTIONAL_EXIT
 388         if pid is not None:
 389             event["pid"] = pid
 390         if worker_index is not None:
 391             event["worker_index"] = int(worker_index)
 392         if exception_code is not None:
 393             event["exception_code"] = exception_code
 394         if exception_description is not None:
 395             event["exception_description"] = exception_description
 396         if test_filename is not None:
 397             event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
 398                 test_filename)
 399         if command_line is not None:
 400             event["command_line"] = command_line
 401         return event
 402
 403     @staticmethod
 404     def event_for_job_timeout(pid, worker_index, test_filename, command_line):
 405         """Creates an event for a job (i.e. process) timeout.
 406
 407         @param pid the process id for the job that timed out
 408         @param worker_index optional id for the job queue running the process
 409         @param test_filename the path to the test filename that timed out.
 410         @param command_line the Popen-style list provided as the command line
 411         for the process that timed out.
 412
 413         @return an event dictionary coding the job completion description.
 414         """
 415         event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
 416         event["status"] = "timeout"
 417         if pid is not None:
 418             event["pid"] = pid
 419         if worker_index is not None:
 420             event["worker_index"] = int(worker_index)
 421         if test_filename is not None:
 422             event["test_filename"] = EventBuilder._assert_is_python_sourcefile(
 423                 test_filename)
 424         if command_line is not None:
 425             event["command_line"] = command_line
 426         return event
 427
 428     @staticmethod
 429     def event_for_mark_test_rerun_eligible(test):
 430         """Creates an event that indicates the specified test is explicitly
 431         eligible for rerun.
 432
 433         Note there is a mode that will enable test rerun eligibility at the
 434         global level.  These markings for explicit rerun eligibility are
 435         intended for the mode of running where only explicitly re-runnable
 436         tests are rerun upon hitting an issue.
 437
 438         @param test the TestCase instance to which this pertains.
 439
 440         @return an event that specifies the given test as being eligible to
 441         be rerun.
 442         """
 443         event = EventBuilder._event_dictionary_common(
 444             test,
 445             EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE)
 446         return event
 447
 448     @staticmethod
 449     def event_for_mark_test_expected_failure(test):
 450         """Creates an event that indicates the specified test is expected
 451         to fail.
 452
 453         @param test the TestCase instance to which this pertains.
 454
 455         @return an event that specifies the given test is expected to fail.
 456         """
 457         event = EventBuilder._event_dictionary_common(
 458             test,
 459             EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
 460         return event
 461
 462     @staticmethod
 463     def add_entries_to_all_events(entries_dict):
 464         """Specifies a dictionary of entries to add to all test events.
 465
 466         This provides a mechanism for, say, a parallel test runner to
 467         indicate to each inferior dotest.py that it should add a
 468         worker index to each.
 469
 470         Calling this method replaces all previous entries added
 471         by a prior call to this.
 472
 473         Event build methods will overwrite any entries that collide.
 474         Thus, the passed in dictionary is the base, which gets merged
 475         over by event building when keys collide.
 476
 477         @param entries_dict a dictionary containing key and value
 478         pairs that should be merged into all events created by the
 479         event generator.  May be None to clear out any extra entries.
 480         """
 481         EventBuilder.BASE_DICTIONARY = dict(entries_dict)