lldb/packages/Python/lldbsuite/test/dotest.py

   1 """
   2 A simple testing framework for lldb using python's unit testing framework.
   3
   4 Tests for lldb are written as python scripts which take advantage of the script
   5 bridging provided by LLDB.framework to interact with lldb core.
   6
   7 A specific naming pattern is followed by the .py script to be recognized as
   8 a module which implements a test scenario, namely, Test*.py.
   9
  10 To specify the directories where "Test*.py" python test scripts are located,
  11 you need to pass in a list of directory names.  By default, the current
  12 working directory is searched if nothing is specified on the command line.
  13
  14 Type:
  15
  16 ./dotest.py -h
  17
  18 for available options.
  19 """
  20
  21 from __future__ import absolute_import
  22 from __future__ import print_function
  23
  24 # System modules
  25 import atexit
  26 import datetime
  27 import errno
  28 import logging
  29 import os
  30 import platform
  31 import re
  32 import signal
  33 import subprocess
  34 import sys
  35 import tempfile
  36
  37 # Third-party modules
  38 import six
  39 import unittest2
  40
  41 # LLDB Modules
  42 import lldbsuite
  43 from . import configuration
  44 from . import dotest_args
  45 from . import lldbtest_config
  46 from . import test_categories
  47 from lldbsuite.test_event import formatter
  48 from . import test_result
  49 from lldbsuite.test_event.event_builder import EventBuilder
  50 from ..support import seven
  51
  52 def get_dotest_invocation():
  53     return ' '.join(sys.argv)
  54
  55
  56 def is_exe(fpath):
  57     """Returns true if fpath is an executable."""
  58     if fpath == None:
  59         return False
  60     return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
  61
  62
  63 def which(program):
  64     """Returns the full path to a program; None otherwise."""
  65     fpath, _ = os.path.split(program)
  66     if fpath:
  67         if is_exe(program):
  68             return program
  69     else:
  70         for path in os.environ["PATH"].split(os.pathsep):
  71             exe_file = os.path.join(path, program)
  72             if is_exe(exe_file):
  73                 return exe_file
  74     return None
  75
  76
  77 def usage(parser):
  78     parser.print_help()
  79     if configuration.verbose > 0:
  80         print("""
  81 Examples:
  82
  83 This is an example of using the -f option to pinpoint to a specific test class
  84 and test method to be run:
  85
  86 $ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command
  87 ----------------------------------------------------------------------
  88 Collected 1 test
  89
  90 test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase)
  91 Test 'frame variable this' when stopped on a class constructor. ... ok
  92
  93 ----------------------------------------------------------------------
  94 Ran 1 test in 1.396s
  95
  96 OK
  97
  98 And this is an example of using the -p option to run a single file (the filename
  99 matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'):
 100
 101 $ ./dotest.py -v -p ObjC
 102 ----------------------------------------------------------------------
 103 Collected 4 tests
 104
 105 test_break_with_dsym (TestObjCMethods.FoundationTestCase)
 106 Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
 107 test_break_with_dwarf (TestObjCMethods.FoundationTestCase)
 108 Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
 109 test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase)
 110 Lookup objective-c data types and evaluate expressions. ... ok
 111 test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase)
 112 Lookup objective-c data types and evaluate expressions. ... ok
 113
 114 ----------------------------------------------------------------------
 115 Ran 4 tests in 16.661s
 116
 117 OK
 118
 119 Running of this script also sets up the LLDB_TEST environment variable so that
 120 individual test cases can locate their supporting files correctly.  The script
 121 tries to set up Python's search paths for modules by looking at the build tree
 122 relative to this script.  See also the '-i' option in the following example.
 123
 124 Finally, this is an example of using the lldb.py module distributed/installed by
 125 Xcode4 to run against the tests under the 'forward' directory, and with the '-w'
 126 option to add some delay between two tests.  It uses ARCH=x86_64 to specify that
 127 as the architecture and CC=clang to specify the compiler used for the test run:
 128
 129 $ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward
 130
 131 Session logs for test failures/errors will go into directory '2010-11-11-13_56_16'
 132 ----------------------------------------------------------------------
 133 Collected 2 tests
 134
 135 test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
 136 Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
 137 test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
 138 Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
 139
 140 ----------------------------------------------------------------------
 141 Ran 2 tests in 5.659s
 142
 143 OK
 144
 145 The 'Session ...' verbiage is recently introduced (see also the '-s' option) to
 146 notify the directory containing the session logs for test failures or errors.
 147 In case there is any test failure/error, a similar message is appended at the
 148 end of the stderr output for your convenience.
 149
 150 ENABLING LOGS FROM TESTS
 151
 152 Option 1:
 153
 154 Writing logs into different files per test case::
 155
 156 $ ./dotest.py --channel "lldb all"
 157
 158 $ ./dotest.py --channel "lldb all" --channel "gdb-remote packets"
 159
 160 These log files are written to:
 161
 162 <session-dir>/<test-id>-host.log (logs from lldb host process)
 163 <session-dir>/<test-id>-server.log (logs from debugserver/lldb-server)
 164 <session-dir>/<test-id>-<test-result>.log (console logs)
 165
 166 By default, logs from successful runs are deleted.  Use the --log-success flag
 167 to create reference logs for debugging.
 168
 169 $ ./dotest.py --log-success
 170
 171 """)
 172     sys.exit(0)
 173
 174
 175 def parseExclusion(exclusion_file):
 176     """Parse an exclusion file, of the following format, where
 177        'skip files', 'skip methods', 'xfail files', and 'xfail methods'
 178        are the possible list heading values:
 179
 180        skip files
 181        <file name>
 182        <file name>
 183
 184        xfail methods
 185        <method name>
 186     """
 187     excl_type = None
 188
 189     with open(exclusion_file) as f:
 190         for line in f:
 191             line = line.strip()
 192             if not excl_type:
 193                 excl_type = line
 194                 continue
 195
 196             if not line:
 197                 excl_type = None
 198             elif excl_type == 'skip':
 199                 if not configuration.skip_tests:
 200                     configuration.skip_tests = []
 201                 configuration.skip_tests.append(line)
 202             elif excl_type == 'xfail':
 203                 if not configuration.xfail_tests:
 204                     configuration.xfail_tests = []
 205                 configuration.xfail_tests.append(line)
 206
 207
 208 def parseOptionsAndInitTestdirs():
 209     """Initialize the list of directories containing our unittest scripts.
 210
 211     '-h/--help as the first option prints out usage info and exit the program.
 212     """
 213
 214     do_help = False
 215
 216     platform_system = platform.system()
 217     platform_machine = platform.machine()
 218
 219     try:
 220         parser = dotest_args.create_parser()
 221         args = parser.parse_args()
 222     except:
 223         print(get_dotest_invocation())
 224         raise
 225
 226     if args.unset_env_varnames:
 227         for env_var in args.unset_env_varnames:
 228             if env_var in os.environ:
 229                 # From Python Doc: When unsetenv() is supported, deletion of items in os.environ
 230                 # is automatically translated into a corresponding call to
 231                 # unsetenv().
 232                 del os.environ[env_var]
 233                 # os.unsetenv(env_var)
 234
 235     if args.set_env_vars:
 236         for env_var in args.set_env_vars:
 237             parts = env_var.split('=', 1)
 238             if len(parts) == 1:
 239                 os.environ[parts[0]] = ""
 240             else:
 241                 os.environ[parts[0]] = parts[1]
 242
 243     if args.set_inferior_env_vars:
 244         lldbtest_config.inferior_env = ' '.join(args.set_inferior_env_vars)
 245
 246     # Only print the args if being verbose.
 247     if args.v:
 248         print(get_dotest_invocation())
 249
 250     if args.h:
 251         do_help = True
 252
 253     if args.compiler:
 254         configuration.compiler = os.path.realpath(args.compiler)
 255         if not is_exe(configuration.compiler):
 256             configuration.compiler = which(args.compiler)
 257         if not is_exe(configuration.compiler):
 258             logging.error(
 259                     '%s is not a valid compiler executable; aborting...',
 260                     args.compiler)
 261             sys.exit(-1)
 262     else:
 263         # Use a compiler appropriate appropriate for the Apple SDK if one was
 264         # specified
 265         if platform_system == 'Darwin' and args.apple_sdk:
 266             configuration.compiler = seven.get_command_output(
 267                 'xcrun -sdk "%s" -find clang 2> /dev/null' %
 268                 (args.apple_sdk))
 269         else:
 270             # 'clang' on ubuntu 14.04 is 3.4 so we try clang-3.5 first
 271             candidateCompilers = ['clang-3.5', 'clang', 'gcc']
 272             for candidate in candidateCompilers:
 273                 if which(candidate):
 274                     configuration.compiler = candidate
 275                     break
 276
 277     if args.dsymutil:
 278         os.environ['DSYMUTIL'] = args.dsymutil
 279     elif platform_system == 'Darwin':
 280         os.environ['DSYMUTIL'] = seven.get_command_output(
 281             'xcrun -find -toolchain default dsymutil')
 282
 283     if args.filecheck:
 284         # The lldb-dotest script produced by the CMake build passes in a path
 285         # to a working FileCheck binary. So does one specific Xcode project
 286         # target. However, when invoking dotest.py directly, a valid --filecheck
 287         # option needs to be given.
 288         configuration.filecheck = os.path.abspath(args.filecheck)
 289
 290     if not configuration.get_filecheck_path():
 291         logging.warning('No valid FileCheck executable; some tests may fail...')
 292         logging.warning('(Double-check the --filecheck argument to dotest.py)')
 293
 294     if args.channels:
 295         lldbtest_config.channels = args.channels
 296
 297     if args.log_success:
 298         lldbtest_config.log_success = args.log_success
 299
 300     if args.out_of_tree_debugserver:
 301         lldbtest_config.out_of_tree_debugserver = args.out_of_tree_debugserver
 302
 303     # Set SDKROOT if we are using an Apple SDK
 304     if platform_system == 'Darwin' and args.apple_sdk:
 305         os.environ['SDKROOT'] = seven.get_command_output(
 306             'xcrun --sdk "%s" --show-sdk-path 2> /dev/null' %
 307             (args.apple_sdk))
 308
 309     if args.arch:
 310         configuration.arch = args.arch
 311         if configuration.arch.startswith(
 312                 'arm') and platform_system == 'Darwin' and not args.apple_sdk:
 313             os.environ['SDKROOT'] = seven.get_command_output(
 314                 'xcrun --sdk iphoneos.internal --show-sdk-path 2> /dev/null')
 315             if not os.path.exists(os.environ['SDKROOT']):
 316                 os.environ['SDKROOT'] = seven.get_command_output(
 317                     'xcrun --sdk iphoneos --show-sdk-path 2> /dev/null')
 318     else:
 319         configuration.arch = platform_machine
 320
 321     if args.categories_list:
 322         configuration.categories_list = set(
 323             test_categories.validate(
 324                 args.categories_list, False))
 325         configuration.use_categories = True
 326     else:
 327         configuration.categories_list = []
 328
 329     if args.skip_categories:
 330         configuration.skip_categories += test_categories.validate(
 331             args.skip_categories, False)
 332
 333     if args.xfail_categories:
 334         configuration.xfail_categories += test_categories.validate(
 335             args.xfail_categories, False)
 336
 337     if args.E:
 338         os.environ['CFLAGS_EXTRAS'] = args.E
 339
 340     if args.dwarf_version:
 341         configuration.dwarf_version = args.dwarf_version
 342         # We cannot modify CFLAGS_EXTRAS because they're used in test cases
 343         # that explicitly require no debug info.
 344         os.environ['CFLAGS'] = '-gdwarf-{}'.format(configuration.dwarf_version)
 345
 346     if args.settings:
 347         for setting in args.settings:
 348             if not len(setting) == 1 or not setting[0].count('='):
 349                 logging.error('"%s" is not a setting in the form "key=value"',
 350                               setting[0])
 351                 sys.exit(-1)
 352             configuration.settings.append(setting[0].split('=', 1))
 353
 354     if args.d:
 355         sys.stdout.write(
 356             "Suspending the process %d to wait for debugger to attach...\n" %
 357             os.getpid())
 358         sys.stdout.flush()
 359         os.kill(os.getpid(), signal.SIGSTOP)
 360
 361     if args.f:
 362         if any([x.startswith('-') for x in args.f]):
 363             usage(parser)
 364         configuration.filters.extend(args.f)
 365
 366     if args.framework:
 367         configuration.lldb_framework_path = args.framework
 368
 369     if args.executable:
 370         # lldb executable is passed explicitly
 371         lldbtest_config.lldbExec = os.path.realpath(args.executable)
 372         if not is_exe(lldbtest_config.lldbExec):
 373             lldbtest_config.lldbExec = which(args.executable)
 374         if not is_exe(lldbtest_config.lldbExec):
 375             logging.error(
 376                     '%s is not a valid executable to test; aborting...',
 377                     args.executable)
 378             sys.exit(-1)
 379
 380     if args.server:
 381         os.environ['LLDB_DEBUGSERVER_PATH'] = args.server
 382
 383     if args.excluded:
 384         for excl_file in args.excluded:
 385             parseExclusion(excl_file)
 386
 387     if args.p:
 388         if args.p.startswith('-'):
 389             usage(parser)
 390         configuration.regexp = args.p
 391
 392     if args.s:
 393         configuration.sdir_name = args.s
 394     else:
 395         timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
 396         configuration.sdir_name = os.path.join(os.getcwd(), timestamp_started)
 397
 398     configuration.session_file_format = args.session_file_format
 399
 400     if args.t:
 401         os.environ['LLDB_COMMAND_TRACE'] = 'YES'
 402
 403     if args.v:
 404         configuration.verbose = 2
 405
 406     # argparse makes sure we have a number
 407     if args.sharp:
 408         configuration.count = args.sharp
 409
 410     if sys.platform.startswith('win32'):
 411         os.environ['LLDB_DISABLE_CRASH_DIALOG'] = str(
 412             args.disable_crash_dialog)
 413         os.environ['LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE'] = str(True)
 414
 415     if do_help:
 416         usage(parser)
 417
 418     if args.results_file:
 419         configuration.results_filename = args.results_file
 420
 421     if args.results_formatter:
 422         configuration.results_formatter_name = args.results_formatter
 423     if args.results_formatter_options:
 424         configuration.results_formatter_options = args.results_formatter_options
 425
 426     # Default to using the BasicResultsFormatter if no formatter is specified.
 427     if configuration.results_formatter_name is None:
 428         configuration.results_formatter_name = (
 429             "lldbsuite.test_event.formatter.results_formatter.ResultsFormatter")
 430
 431     # rerun-related arguments
 432     configuration.rerun_all_issues = args.rerun_all_issues
 433
 434     if args.lldb_platform_name:
 435         configuration.lldb_platform_name = args.lldb_platform_name
 436     if args.lldb_platform_url:
 437         configuration.lldb_platform_url = args.lldb_platform_url
 438     if args.lldb_platform_working_dir:
 439         configuration.lldb_platform_working_dir = args.lldb_platform_working_dir
 440     if args.test_build_dir:
 441         configuration.test_build_dir = args.test_build_dir
 442     if args.lldb_module_cache_dir:
 443         configuration.lldb_module_cache_dir = args.lldb_module_cache_dir
 444     else:
 445         configuration.lldb_module_cache_dir = os.path.join(
 446             configuration.test_build_dir, 'module-cache-lldb')
 447     if args.clang_module_cache_dir:
 448         configuration.clang_module_cache_dir = args.clang_module_cache_dir
 449     else:
 450         configuration.clang_module_cache_dir = os.path.join(
 451             configuration.test_build_dir, 'module-cache-clang')
 452
 453     os.environ['CLANG_MODULE_CACHE_DIR'] = configuration.clang_module_cache_dir
 454
 455     # Gather all the dirs passed on the command line.
 456     if len(args.args) > 0:
 457         configuration.testdirs = [os.path.realpath(os.path.abspath(x)) for x in args.args]
 458
 459     lldbtest_config.codesign_identity = args.codesign_identity
 460
 461
 462 def setupTestResults():
 463     """Sets up test results-related objects based on arg settings."""
 464     # Setup the results formatter configuration.
 465     formatter_config = formatter.FormatterConfig()
 466     formatter_config.filename = configuration.results_filename
 467     formatter_config.formatter_name = configuration.results_formatter_name
 468     formatter_config.formatter_options = (
 469         configuration.results_formatter_options)
 470
 471     # Create the results formatter.
 472     formatter_spec = formatter.create_results_formatter(
 473         formatter_config)
 474     if formatter_spec is not None and formatter_spec.formatter is not None:
 475         configuration.results_formatter_object = formatter_spec.formatter
 476
 477         # Send an initialize message to the formatter.
 478         initialize_event = EventBuilder.bare_event("initialize")
 479         initialize_event["worker_count"] = 1
 480
 481         formatter_spec.formatter.handle_event(initialize_event)
 482
 483         # Make sure we clean up the formatter on shutdown.
 484         if formatter_spec.cleanup_func is not None:
 485             atexit.register(formatter_spec.cleanup_func)
 486
 487
 488 def setupSysPath():
 489     """
 490     Add LLDB.framework/Resources/Python to the search paths for modules.
 491     As a side effect, we also discover the 'lldb' executable and export it here.
 492     """
 493
 494     # Get the directory containing the current script.
 495     if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ:
 496         scriptPath = os.environ["DOTEST_SCRIPT_DIR"]
 497     else:
 498         scriptPath = os.path.dirname(os.path.realpath(__file__))
 499     if not scriptPath.endswith('test'):
 500         print("This script expects to reside in lldb's test directory.")
 501         sys.exit(-1)
 502
 503     os.environ["LLDB_TEST"] = scriptPath
 504
 505     # Set up the root build directory.
 506     builddir = configuration.test_build_dir
 507     if not configuration.test_build_dir:
 508         raise Exception("test_build_dir is not set")
 509     os.environ["LLDB_BUILD"] = os.path.abspath(configuration.test_build_dir)
 510
 511     # Set up the LLDB_SRC environment variable, so that the tests can locate
 512     # the LLDB source code.
 513     os.environ["LLDB_SRC"] = lldbsuite.lldb_root
 514
 515     pluginPath = os.path.join(scriptPath, 'plugins')
 516     toolsLLDBVSCode = os.path.join(scriptPath, 'tools', 'lldb-vscode')
 517     toolsLLDBServerPath = os.path.join(scriptPath, 'tools', 'lldb-server')
 518
 519     # Insert script dir, plugin dir and lldb-server dir to the sys.path.
 520     sys.path.insert(0, pluginPath)
 521     # Adding test/tools/lldb-vscode to the path makes it easy to
 522     # "import lldb_vscode_testcase" from the VSCode tests
 523     sys.path.insert(0, toolsLLDBVSCode)
 524     # Adding test/tools/lldb-server to the path makes it easy
 525     sys.path.insert(0, toolsLLDBServerPath)
 526     # to "import lldbgdbserverutils" from the lldb-server tests
 527
 528     # This is the root of the lldb git/svn checkout
 529     # When this changes over to a package instead of a standalone script, this
 530     # will be `lldbsuite.lldb_root`
 531     lldbRootDirectory = lldbsuite.lldb_root
 532
 533     # Some of the tests can invoke the 'lldb' command directly.
 534     # We'll try to locate the appropriate executable right here.
 535
 536     # The lldb executable can be set from the command line
 537     # if it's not set, we try to find it now
 538     # first, we try the environment
 539     if not lldbtest_config.lldbExec:
 540         # First, you can define an environment variable LLDB_EXEC specifying the
 541         # full pathname of the lldb executable.
 542         if "LLDB_EXEC" in os.environ:
 543             lldbtest_config.lldbExec = os.environ["LLDB_EXEC"]
 544
 545     if not lldbtest_config.lldbExec:
 546         # Last, check the path
 547         lldbtest_config.lldbExec = which('lldb')
 548
 549     if lldbtest_config.lldbExec and not is_exe(lldbtest_config.lldbExec):
 550         print(
 551             "'{}' is not a path to a valid executable".format(
 552                 lldbtest_config.lldbExec))
 553         lldbtest_config.lldbExec = None
 554
 555     if not lldbtest_config.lldbExec:
 556         print("The 'lldb' executable cannot be located.  Some of the tests may not be run as a result.")
 557         sys.exit(-1)
 558
 559     # confusingly, this is the "bin" directory
 560     lldbLibDir = os.path.dirname(lldbtest_config.lldbExec)
 561     os.environ["LLDB_LIB_DIR"] = lldbLibDir
 562     lldbImpLibDir = os.path.join(
 563         lldbLibDir,
 564         '..',
 565         'lib') if sys.platform.startswith('win32') else lldbLibDir
 566     os.environ["LLDB_IMPLIB_DIR"] = lldbImpLibDir
 567     print("LLDB library dir:", os.environ["LLDB_LIB_DIR"])
 568     print("LLDB import library dir:", os.environ["LLDB_IMPLIB_DIR"])
 569     os.system('%s -v' % lldbtest_config.lldbExec)
 570
 571     lldbDir = os.path.dirname(lldbtest_config.lldbExec)
 572
 573     lldbVSCodeExec = os.path.join(lldbDir, "lldb-vscode")
 574     if is_exe(lldbVSCodeExec):
 575         os.environ["LLDBVSCODE_EXEC"] = lldbVSCodeExec
 576     else:
 577         if not configuration.shouldSkipBecauseOfCategories(["lldb-vscode"]):
 578             print(
 579                 "The 'lldb-vscode' executable cannot be located.  The lldb-vscode tests can not be run as a result.")
 580             configuration.skip_categories.append("lldb-vscode")
 581
 582     lldbPythonDir = None  # The directory that contains 'lldb/__init__.py'
 583     if not configuration.lldb_framework_path and os.path.exists(os.path.join(lldbLibDir, "LLDB.framework")):
 584         configuration.lldb_framework_path = os.path.join(lldbLibDir, "LLDB.framework")
 585     if configuration.lldb_framework_path:
 586         lldbtest_config.lldb_framework_path = configuration.lldb_framework_path
 587         candidatePath = os.path.join(
 588             configuration.lldb_framework_path, 'Resources', 'Python')
 589         if os.path.isfile(os.path.join(candidatePath, 'lldb/__init__.py')):
 590             lldbPythonDir = candidatePath
 591         if not lldbPythonDir:
 592             print(
 593                 'Resources/Python/lldb/__init__.py was not found in ' +
 594                 configuration.lldb_framework_path)
 595             sys.exit(-1)
 596     else:
 597         # If our lldb supports the -P option, use it to find the python path:
 598         init_in_python_dir = os.path.join('lldb', '__init__.py')
 599
 600         lldb_dash_p_result = subprocess.check_output(
 601             [lldbtest_config.lldbExec, "-P"], stderr=subprocess.STDOUT, universal_newlines=True)
 602
 603         if lldb_dash_p_result and not lldb_dash_p_result.startswith(
 604                 ("<", "lldb: invalid option:")) and not lldb_dash_p_result.startswith("Traceback"):
 605             lines = lldb_dash_p_result.splitlines()
 606
 607             # Workaround for readline vs libedit issue on FreeBSD.  If stdout
 608             # is not a terminal Python executes
 609             #     rl_variable_bind ("enable-meta-key", "off");
 610             # This produces a warning with FreeBSD's libedit because the
 611             # enable-meta-key variable is unknown.  Not an issue on Apple
 612             # because cpython commit f0ab6f9f0603 added a #ifndef __APPLE__
 613             # around the call.  See http://bugs.python.org/issue19884 for more
 614             # information.  For now we just discard the warning output.
 615             if len(lines) >= 1 and lines[0].startswith(
 616                     "bind: Invalid command"):
 617                 lines.pop(0)
 618
 619             # Taking the last line because lldb outputs
 620             # 'Cannot read termcap database;\nusing dumb terminal settings.\n'
 621             # before the path
 622             if len(lines) >= 1 and os.path.isfile(
 623                     os.path.join(lines[-1], init_in_python_dir)):
 624                 lldbPythonDir = lines[-1]
 625                 if "freebsd" in sys.platform or "linux" in sys.platform:
 626                     os.environ['LLDB_LIB_DIR'] = os.path.join(
 627                         lldbPythonDir, '..', '..')
 628
 629         if not lldbPythonDir:
 630             print(
 631                 "Unable to load lldb extension module.  Possible reasons for this include:")
 632             print("  1) LLDB was built with LLDB_ENABLE_PYTHON=0")
 633             print(
 634                 "  2) PYTHONPATH and PYTHONHOME are not set correctly.  PYTHONHOME should refer to")
 635             print(
 636                 "     the version of Python that LLDB built and linked against, and PYTHONPATH")
 637             print(
 638                 "     should contain the Lib directory for the same python distro, as well as the")
 639             print("     location of LLDB\'s site-packages folder.")
 640             print(
 641                 "  3) A different version of Python than that which was built against is exported in")
 642             print("     the system\'s PATH environment variable, causing conflicts.")
 643             print(
 644                 "  4) The executable '%s' could not be found.  Please check " %
 645                 lldbtest_config.lldbExec)
 646             print("     that it exists and is executable.")
 647
 648     if lldbPythonDir:
 649         lldbPythonDir = os.path.normpath(lldbPythonDir)
 650         # Some of the code that uses this path assumes it hasn't resolved the Versions... link.
 651         # If the path we've constructed looks like that, then we'll strip out
 652         # the Versions/A part.
 653         (before, frameWithVersion, after) = lldbPythonDir.rpartition(
 654             "LLDB.framework/Versions/A")
 655         if frameWithVersion != "":
 656             lldbPythonDir = before + "LLDB.framework" + after
 657
 658         lldbPythonDir = os.path.abspath(lldbPythonDir)
 659
 660         # If tests need to find LLDB_FRAMEWORK, now they can do it
 661         os.environ["LLDB_FRAMEWORK"] = os.path.dirname(
 662             os.path.dirname(lldbPythonDir))
 663
 664         # This is to locate the lldb.py module.  Insert it right after
 665         # sys.path[0].
 666         sys.path[1:1] = [lldbPythonDir]
 667
 668
 669 def visit_file(dir, name):
 670     # Try to match the regexp pattern, if specified.
 671     if configuration.regexp:
 672         if not re.search(configuration.regexp, name):
 673             # We didn't match the regex, we're done.
 674             return
 675
 676     if configuration.skip_tests:
 677         for file_regexp in configuration.skip_tests:
 678             if re.search(file_regexp, name):
 679                 return
 680
 681     # We found a match for our test.  Add it to the suite.
 682
 683     # Update the sys.path first.
 684     if not sys.path.count(dir):
 685         sys.path.insert(0, dir)
 686     base = os.path.splitext(name)[0]
 687
 688     # Thoroughly check the filterspec against the base module and admit
 689     # the (base, filterspec) combination only when it makes sense.
 690
 691     def check(obj, parts):
 692         for part in parts:
 693             try:
 694                 parent, obj = obj, getattr(obj, part)
 695             except AttributeError:
 696                 # The filterspec has failed.
 697                 return False
 698         return True
 699
 700     module = __import__(base)
 701
 702     def iter_filters():
 703         for filterspec in configuration.filters:
 704             parts = filterspec.split('.')
 705             if check(module, parts):
 706                 yield filterspec
 707             elif parts[0] == base and len(parts) > 1 and check(module, parts[1:]):
 708                 yield '.'.join(parts[1:])
 709             else:
 710                 for key,value in module.__dict__.items():
 711                     if check(value, parts):
 712                         yield key + '.' + filterspec
 713
 714     filtered = False
 715     for filterspec in iter_filters():
 716         filtered = True
 717         print("adding filter spec %s to module %s" % (filterspec, repr(module)))
 718         tests = unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)
 719         configuration.suite.addTests(tests)
 720
 721     # Forgo this module if the (base, filterspec) combo is invalid
 722     if configuration.filters and not filtered:
 723         return
 724
 725     if not filtered:
 726         # Add the entire file's worth of tests since we're not filtered.
 727         # Also the fail-over case when the filterspec branch
 728         # (base, filterspec) combo doesn't make sense.
 729         configuration.suite.addTests(
 730             unittest2.defaultTestLoader.loadTestsFromName(base))
 731
 732
 733 def visit(prefix, dir, names):
 734     """Visitor function for os.path.walk(path, visit, arg)."""
 735
 736     dir_components = set(dir.split(os.sep))
 737     excluded_components = set(['.svn', '.git'])
 738     if dir_components.intersection(excluded_components):
 739         return
 740
 741     # Gather all the Python test file names that follow the Test*.py pattern.
 742     python_test_files = [
 743         name
 744         for name in names
 745         if name.endswith('.py') and name.startswith(prefix)]
 746
 747     # Visit all the python test files.
 748     for name in python_test_files:
 749         try:
 750             # Ensure we error out if we have multiple tests with the same
 751             # base name.
 752             # Future improvement: find all the places where we work with base
 753             # names and convert to full paths.  We have directory structure
 754             # to disambiguate these, so we shouldn't need this constraint.
 755             if name in configuration.all_tests:
 756                 raise Exception("Found multiple tests with the name %s" % name)
 757             configuration.all_tests.add(name)
 758
 759             # Run the relevant tests in the python file.
 760             visit_file(dir, name)
 761         except Exception as ex:
 762             # Convert this exception to a test event error for the file.
 763             test_filename = os.path.abspath(os.path.join(dir, name))
 764             if configuration.results_formatter_object is not None:
 765                 # Grab the backtrace for the exception.
 766                 import traceback
 767                 backtrace = traceback.format_exc()
 768
 769                 # Generate the test event.
 770                 configuration.results_formatter_object.handle_event(
 771                     EventBuilder.event_for_job_test_add_error(
 772                         test_filename, ex, backtrace))
 773             raise
 774
 775
 776 def setSetting(setting, value):
 777     import lldb
 778     ci = lldb.DBG.GetCommandInterpreter()
 779     res = lldb.SBCommandReturnObject()
 780     cmd = 'setting set %s %s'%(setting, value)
 781     print(cmd)
 782     ci.HandleCommand(cmd, res, False)
 783     if not res.Succeeded():
 784         raise Exception('failed to run "%s"'%cmd)
 785
 786 # ======================================== #
 787 #                                          #
 788 # Execution of the test driver starts here #
 789 #                                          #
 790 # ======================================== #
 791
 792
 793 def checkDsymForUUIDIsNotOn():
 794     cmd = ["defaults", "read", "com.apple.DebugSymbols"]
 795     process = subprocess.Popen(
 796         cmd,
 797         stdout=subprocess.PIPE,
 798         stderr=subprocess.STDOUT)
 799     cmd_output = process.stdout.read()
 800     output_str = cmd_output.decode("utf-8")
 801     if "DBGFileMappedPaths = " in output_str:
 802         print("%s =>" % ' '.join(cmd))
 803         print(output_str)
 804         print(
 805             "Disable automatic lookup and caching of dSYMs before running the test suite!")
 806         print("Exiting...")
 807         sys.exit(0)
 808
 809
 810 def exitTestSuite(exitCode=None):
 811     import lldb
 812     lldb.SBDebugger.Terminate()
 813     if exitCode:
 814         sys.exit(exitCode)
 815
 816
 817 def getVersionForSDK(sdk):
 818     sdk = str.lower(sdk)
 819     full_path = seven.get_command_output('xcrun -sdk %s --show-sdk-path' % sdk)
 820     basename = os.path.basename(full_path)
 821     basename = os.path.splitext(basename)[0]
 822     basename = str.lower(basename)
 823     ver = basename.replace(sdk, '')
 824     return ver
 825
 826
 827 def setDefaultTripleForPlatform():
 828     if configuration.lldb_platform_name == 'ios-simulator':
 829         triple_str = 'x86_64-apple-ios%s' % (
 830             getVersionForSDK('iphonesimulator'))
 831         os.environ['TRIPLE'] = triple_str
 832         return {'TRIPLE': triple_str}
 833     return {}
 834
 835
 836 def checkCompiler():
 837     # Add some intervention here to sanity check that the compiler requested is sane.
 838     # If found not to be an executable program, we abort.
 839     c = configuration.compiler
 840     if which(c):
 841         return
 842
 843     if not sys.platform.startswith("darwin"):
 844         raise Exception(c + " is not a valid compiler")
 845
 846     pipe = subprocess.Popen(
 847         ['xcrun', '-find', c], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 848     cmd_output = pipe.stdout.read()
 849     if not cmd_output or "not found" in cmd_output:
 850         raise Exception(c + " is not a valid compiler")
 851
 852     configuration.compiler = cmd_output.split('\n')[0]
 853     print("'xcrun -find %s' returning %s" % (c, configuration.compiler))
 854
 855 def canRunLibcxxTests():
 856     from lldbsuite.test import lldbplatformutil
 857
 858     platform = lldbplatformutil.getPlatform()
 859
 860     if lldbplatformutil.target_is_android() or lldbplatformutil.platformIsDarwin():
 861         return True, "libc++ always present"
 862
 863     if platform == "linux":
 864         if os.path.isdir("/usr/include/c++/v1"):
 865             return True, "Headers found, let's hope they work"
 866         with tempfile.NamedTemporaryFile() as f:
 867             cmd = [configuration.compiler, "-xc++", "-stdlib=libc++", "-o", f.name, "-"]
 868             p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
 869             _, stderr = p.communicate("#include <algorithm>\nint main() {}")
 870             if not p.returncode:
 871                 return True, "Compiling with -stdlib=libc++ works"
 872             return False, "Compiling with -stdlib=libc++ fails with the error: %s" % stderr
 873
 874     return False, "Don't know how to build with libc++ on %s" % platform
 875
 876 def checkLibcxxSupport():
 877     result, reason = canRunLibcxxTests()
 878     if result:
 879         return # libc++ supported
 880     if "libc++" in configuration.categories_list:
 881         return # libc++ category explicitly requested, let it run.
 882     print("Libc++ tests will not be run because: " + reason)
 883     configuration.skip_categories.append("libc++")
 884
 885 def canRunLibstdcxxTests():
 886     from lldbsuite.test import lldbplatformutil
 887
 888     platform = lldbplatformutil.getPlatform()
 889     if lldbplatformutil.target_is_android():
 890         platform = "android"
 891     if platform == "linux":
 892         return True, "libstdcxx always present"
 893     return False, "Don't know how to build with libstdcxx on %s" % platform
 894
 895 def checkLibstdcxxSupport():
 896     result, reason = canRunLibstdcxxTests()
 897     if result:
 898         return # libstdcxx supported
 899     if "libstdcxx" in configuration.categories_list:
 900         return # libstdcxx category explicitly requested, let it run.
 901     print("libstdcxx tests will not be run because: " + reason)
 902     configuration.skip_categories.append("libstdcxx")
 903
 904 def canRunWatchpointTests():
 905     from lldbsuite.test import lldbplatformutil
 906
 907     platform = lldbplatformutil.getPlatform()
 908     if platform == "netbsd":
 909         if os.geteuid() == 0:
 910             return True, "root can always write dbregs"
 911         try:
 912             output = subprocess.check_output(["/sbin/sysctl", "-n",
 913               "security.models.extensions.user_set_dbregs"]).decode().strip()
 914             if output == "1":
 915                 return True, "security.models.extensions.user_set_dbregs enabled"
 916         except subprocess.CalledProcessError:
 917             pass
 918         return False, "security.models.extensions.user_set_dbregs disabled"
 919     return True, "watchpoint support available"
 920
 921 def checkWatchpointSupport():
 922     result, reason = canRunWatchpointTests()
 923     if result:
 924         return # watchpoints supported
 925     if "watchpoint" in configuration.categories_list:
 926         return # watchpoint category explicitly requested, let it run.
 927     print("watchpoint tests will not be run because: " + reason)
 928     configuration.skip_categories.append("watchpoint")
 929
 930 def checkDebugInfoSupport():
 931     import lldb
 932
 933     platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2]
 934     compiler = configuration.compiler
 935     skipped = []
 936     for cat in test_categories.debug_info_categories:
 937         if cat in configuration.categories_list:
 938             continue # Category explicitly requested, let it run.
 939         if test_categories.is_supported_on_platform(cat, platform, compiler):
 940             continue
 941         configuration.skip_categories.append(cat)
 942         skipped.append(cat)
 943     if skipped:
 944         print("Skipping following debug info categories:", skipped)
 945
 946 def run_suite():
 947     # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
 948     # does not exist before proceeding to running the test suite.
 949     if sys.platform.startswith("darwin"):
 950         checkDsymForUUIDIsNotOn()
 951
 952     # Start the actions by first parsing the options while setting up the test
 953     # directories, followed by setting up the search paths for lldb utilities;
 954     # then, we walk the directory trees and collect the tests into our test suite.
 955     #
 956     parseOptionsAndInitTestdirs()
 957
 958     # Setup test results (test results formatter and output handling).
 959     setupTestResults()
 960
 961     setupSysPath()
 962
 963
 964     # For the time being, let's bracket the test runner within the
 965     # lldb.SBDebugger.Initialize()/Terminate() pair.
 966     import lldb
 967
 968     # Now we can also import lldbutil
 969     from lldbsuite.test import lldbutil
 970
 971     # Create a singleton SBDebugger in the lldb namespace.
 972     lldb.DBG = lldb.SBDebugger.Create()
 973
 974     if configuration.lldb_platform_name:
 975         print("Setting up remote platform '%s'" %
 976               (configuration.lldb_platform_name))
 977         lldb.remote_platform = lldb.SBPlatform(
 978             configuration.lldb_platform_name)
 979         if not lldb.remote_platform.IsValid():
 980             print(
 981                 "error: unable to create the LLDB platform named '%s'." %
 982                 (configuration.lldb_platform_name))
 983             exitTestSuite(1)
 984         if configuration.lldb_platform_url:
 985             # We must connect to a remote platform if a LLDB platform URL was
 986             # specified
 987             print(
 988                 "Connecting to remote platform '%s' at '%s'..." %
 989                 (configuration.lldb_platform_name, configuration.lldb_platform_url))
 990             platform_connect_options = lldb.SBPlatformConnectOptions(
 991                 configuration.lldb_platform_url)
 992             err = lldb.remote_platform.ConnectRemote(platform_connect_options)
 993             if err.Success():
 994                 print("Connected.")
 995             else:
 996                 print("error: failed to connect to remote platform using URL '%s': %s" % (
 997                     configuration.lldb_platform_url, err))
 998                 exitTestSuite(1)
 999         else:
1000             configuration.lldb_platform_url = None
1001
1002     platform_changes = setDefaultTripleForPlatform()
1003     first = True
1004     for key in platform_changes:
1005         if first:
1006             print("Environment variables setup for platform support:")
1007             first = False
1008         print("%s = %s" % (key, platform_changes[key]))
1009
1010     if configuration.lldb_platform_working_dir:
1011         print("Setting remote platform working directory to '%s'..." %
1012               (configuration.lldb_platform_working_dir))
1013         error = lldb.remote_platform.MakeDirectory(
1014             configuration.lldb_platform_working_dir, 448)  # 448 = 0o700
1015         if error.Fail():
1016             raise Exception("making remote directory '%s': %s" % (
1017                 configuration.lldb_platform_working_dir, error))
1018
1019         if not lldb.remote_platform.SetWorkingDirectory(
1020                 configuration.lldb_platform_working_dir):
1021             raise Exception("failed to set working directory '%s'" % configuration.lldb_platform_working_dir)
1022         lldb.DBG.SetSelectedPlatform(lldb.remote_platform)
1023     else:
1024         lldb.remote_platform = None
1025         configuration.lldb_platform_working_dir = None
1026         configuration.lldb_platform_url = None
1027
1028     # Set up the working directory.
1029     # Note that it's not dotest's job to clean this directory.
1030     build_dir = configuration.test_build_dir
1031     lldbutil.mkdir_p(build_dir)
1032
1033     target_platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2]
1034
1035     checkLibcxxSupport()
1036     checkLibstdcxxSupport()
1037     checkWatchpointSupport()
1038     checkDebugInfoSupport()
1039
1040     # Don't do debugserver tests on anything except OS X.
1041     configuration.dont_do_debugserver_test = (
1042             "linux" in target_platform or
1043             "freebsd" in target_platform or
1044             "netbsd" in target_platform or
1045             "windows" in target_platform)
1046
1047     # Don't do lldb-server (llgs) tests on anything except Linux and Windows.
1048     configuration.dont_do_llgs_test = not (
1049             "linux" in target_platform or
1050             "netbsd" in target_platform or
1051             "windows" in target_platform)
1052
1053     # Collect tests from the specified testing directories. If a test
1054     # subdirectory filter is explicitly specified, limit the search to that
1055     # subdirectory.
1056     exclusive_test_subdir = configuration.get_absolute_path_to_exclusive_test_subdir()
1057     if exclusive_test_subdir:
1058         dirs_to_search = [exclusive_test_subdir]
1059     else:
1060         dirs_to_search = configuration.testdirs
1061     for testdir in dirs_to_search:
1062         for (dirpath, dirnames, filenames) in os.walk(testdir):
1063             visit('Test', dirpath, filenames)
1064
1065     #
1066     # Now that we have loaded all the test cases, run the whole test suite.
1067     #
1068
1069     # Set any user-overridden settings.
1070     for key, value in configuration.settings:
1071         setSetting(key, value)
1072
1073     # Install the control-c handler.
1074     unittest2.signals.installHandler()
1075
1076     lldbutil.mkdir_p(configuration.sdir_name)
1077     os.environ["LLDB_SESSION_DIRNAME"] = configuration.sdir_name
1078
1079     sys.stderr.write(
1080         "\nSession logs for test failures/errors/unexpected successes"
1081         " will go into directory '%s'\n" %
1082         configuration.sdir_name)
1083     sys.stderr.write("Command invoked: %s\n" % get_dotest_invocation())
1084
1085     #
1086     # Invoke the default TextTestRunner to run the test suite
1087     #
1088     checkCompiler()
1089
1090     if configuration.verbose:
1091         print("compiler=%s" % configuration.compiler)
1092
1093     # Iterating over all possible architecture and compiler combinations.
1094     os.environ["ARCH"] = configuration.arch
1095     os.environ["CC"] = configuration.compiler
1096     configString = "arch=%s compiler=%s" % (configuration.arch,
1097                                             configuration.compiler)
1098
1099     # Output the configuration.
1100     if configuration.verbose:
1101         sys.stderr.write("\nConfiguration: " + configString + "\n")
1102
1103     # First, write out the number of collected test cases.
1104     if configuration.verbose:
1105         sys.stderr.write(configuration.separator + "\n")
1106         sys.stderr.write(
1107             "Collected %d test%s\n\n" %
1108             (configuration.suite.countTestCases(),
1109              configuration.suite.countTestCases() != 1 and "s" or ""))
1110
1111     # Invoke the test runner.
1112     if configuration.count == 1:
1113         result = unittest2.TextTestRunner(
1114             stream=sys.stderr,
1115             verbosity=configuration.verbose,
1116             resultclass=test_result.LLDBTestResult).run(
1117             configuration.suite)
1118     else:
1119         # We are invoking the same test suite more than once.  In this case,
1120         # mark __ignore_singleton__ flag as True so the signleton pattern is
1121         # not enforced.
1122         test_result.LLDBTestResult.__ignore_singleton__ = True
1123         for i in range(configuration.count):
1124
1125             result = unittest2.TextTestRunner(
1126                 stream=sys.stderr,
1127                 verbosity=configuration.verbose,
1128                 resultclass=test_result.LLDBTestResult).run(
1129                 configuration.suite)
1130
1131     configuration.failed = not result.wasSuccessful()
1132
1133     if configuration.sdir_has_content and configuration.verbose:
1134         sys.stderr.write(
1135             "Session logs for test failures/errors/unexpected successes"
1136             " can be found in directory '%s'\n" %
1137             configuration.sdir_name)
1138
1139     if configuration.use_categories and len(
1140             configuration.failures_per_category) > 0:
1141         sys.stderr.write("Failures per category:\n")
1142         for category in configuration.failures_per_category:
1143             sys.stderr.write(
1144                 "%s - %d\n" %
1145                 (category, configuration.failures_per_category[category]))
1146
1147     # Exiting.
1148     exitTestSuite(configuration.failed)
1149
1150 if __name__ == "__main__":
1151     print(
1152         __file__ +
1153         " is for use as a module only.  It should not be run as a standalone script.")
1154     sys.exit(-1)