lldb/packages/Python/lldbsuite/test/dotest.py

   1 """
   2 A simple testing framework for lldb using python's unit testing framework.
   3
   4 Tests for lldb are written as python scripts which take advantage of the script
   5 bridging provided by LLDB.framework to interact with lldb core.
   6
   7 A specific naming pattern is followed by the .py script to be recognized as
   8 a module which implements a test scenario, namely, Test*.py.
   9
  10 To specify the directories where "Test*.py" python test scripts are located,
  11 you need to pass in a list of directory names.  By default, the current
  12 working directory is searched if nothing is specified on the command line.
  13
  14 Type:
  15
  16 ./dotest.py -h
  17
  18 for available options.
  19 """
  20
  21 # System modules
  22 import atexit
  23 import datetime
  24 import errno
  25 import logging
  26 import os
  27 import platform
  28 import re
  29 import shutil
  30 import signal
  31 import subprocess
  32 import sys
  33 import tempfile
  34
  35 # Third-party modules
  36 import unittest2
  37
  38 # LLDB Modules
  39 import lldbsuite
  40 from . import configuration
  41 from . import dotest_args
  42 from . import lldbtest_config
  43 from . import test_categories
  44 from . import test_result
  45 from ..support import seven
  46
  47
  48 def is_exe(fpath):
  49     """Returns true if fpath is an executable."""
  50     if fpath == None:
  51         return False
  52     if sys.platform == "win32":
  53         if not fpath.endswith(".exe"):
  54             fpath += ".exe"
  55     return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
  56
  57
  58 def which(program):
  59     """Returns the full path to a program; None otherwise."""
  60     fpath, _ = os.path.split(program)
  61     if fpath:
  62         if is_exe(program):
  63             return program
  64     else:
  65         for path in os.environ["PATH"].split(os.pathsep):
  66             exe_file = os.path.join(path, program)
  67             if is_exe(exe_file):
  68                 return exe_file
  69     return None
  70
  71
  72 def usage(parser):
  73     parser.print_help()
  74     if configuration.verbose > 0:
  75         print(
  76             """
  77 Examples:
  78
  79 This is an example of using the -f option to pinpoint to a specific test class
  80 and test method to be run:
  81
  82 $ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command
  83 ----------------------------------------------------------------------
  84 Collected 1 test
  85
  86 test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase)
  87 Test 'frame variable this' when stopped on a class constructor. ... ok
  88
  89 ----------------------------------------------------------------------
  90 Ran 1 test in 1.396s
  91
  92 OK
  93
  94 And this is an example of using the -p option to run a single file (the filename
  95 matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'):
  96
  97 $ ./dotest.py -v -p ObjC
  98 ----------------------------------------------------------------------
  99 Collected 4 tests
 100
 101 test_break_with_dsym (TestObjCMethods.FoundationTestCase)
 102 Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
 103 test_break_with_dwarf (TestObjCMethods.FoundationTestCase)
 104 Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
 105 test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase)
 106 Lookup objective-c data types and evaluate expressions. ... ok
 107 test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase)
 108 Lookup objective-c data types and evaluate expressions. ... ok
 109
 110 ----------------------------------------------------------------------
 111 Ran 4 tests in 16.661s
 112
 113 OK
 114
 115 Running of this script also sets up the LLDB_TEST environment variable so that
 116 individual test cases can locate their supporting files correctly.  The script
 117 tries to set up Python's search paths for modules by looking at the build tree
 118 relative to this script.  See also the '-i' option in the following example.
 119
 120 Finally, this is an example of using the lldb.py module distributed/installed by
 121 Xcode4 to run against the tests under the 'forward' directory, and with the '-w'
 122 option to add some delay between two tests.  It uses ARCH=x86_64 to specify that
 123 as the architecture and CC=clang to specify the compiler used for the test run:
 124
 125 $ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward
 126
 127 Session logs for test failures/errors will go into directory '2010-11-11-13_56_16'
 128 ----------------------------------------------------------------------
 129 Collected 2 tests
 130
 131 test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
 132 Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
 133 test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
 134 Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
 135
 136 ----------------------------------------------------------------------
 137 Ran 2 tests in 5.659s
 138
 139 OK
 140
 141 The 'Session ...' verbiage is recently introduced (see also the '-s' option) to
 142 notify the directory containing the session logs for test failures or errors.
 143 In case there is any test failure/error, a similar message is appended at the
 144 end of the stderr output for your convenience.
 145
 146 ENABLING LOGS FROM TESTS
 147
 148 Option 1:
 149
 150 Writing logs into different files per test case::
 151
 152 $ ./dotest.py --channel "lldb all"
 153
 154 $ ./dotest.py --channel "lldb all" --channel "gdb-remote packets"
 155
 156 These log files are written to:
 157
 158 <session-dir>/<test-id>-host.log (logs from lldb host process)
 159 <session-dir>/<test-id>-server.log (logs from debugserver/lldb-server)
 160 <session-dir>/<test-id>-<test-result>.log (console logs)
 161
 162 By default, logs from successful runs are deleted.  Use the --log-success flag
 163 to create reference logs for debugging.
 164
 165 $ ./dotest.py --log-success
 166
 167 """
 168         )
 169     sys.exit(0)
 170
 171
 172 def parseExclusion(exclusion_file):
 173     """Parse an exclusion file, of the following format, where
 174     'skip files', 'skip methods', 'xfail files', and 'xfail methods'
 175     are the possible list heading values:
 176
 177     skip files
 178     <file name>
 179     <file name>
 180
 181     xfail methods
 182     <method name>
 183     """
 184     excl_type = None
 185
 186     with open(exclusion_file) as f:
 187         for line in f:
 188             line = line.strip()
 189             if not excl_type:
 190                 excl_type = line
 191                 continue
 192
 193             if not line:
 194                 excl_type = None
 195             elif excl_type == "skip":
 196                 if not configuration.skip_tests:
 197                     configuration.skip_tests = []
 198                 configuration.skip_tests.append(line)
 199             elif excl_type == "xfail":
 200                 if not configuration.xfail_tests:
 201                     configuration.xfail_tests = []
 202                 configuration.xfail_tests.append(line)
 203
 204
 205 def parseOptionsAndInitTestdirs():
 206     """Initialize the list of directories containing our unittest scripts.
 207
 208     '-h/--help as the first option prints out usage info and exit the program.
 209     """
 210
 211     do_help = False
 212
 213     platform_system = platform.system()
 214     platform_machine = platform.machine()
 215
 216     try:
 217         parser = dotest_args.create_parser()
 218         args = parser.parse_args()
 219     except:
 220         raise
 221
 222     if args.unset_env_varnames:
 223         for env_var in args.unset_env_varnames:
 224             if env_var in os.environ:
 225                 # From Python Doc: When unsetenv() is supported, deletion of items in os.environ
 226                 # is automatically translated into a corresponding call to
 227                 # unsetenv().
 228                 del os.environ[env_var]
 229                 # os.unsetenv(env_var)
 230
 231     if args.set_env_vars:
 232         for env_var in args.set_env_vars:
 233             parts = env_var.split("=", 1)
 234             if len(parts) == 1:
 235                 os.environ[parts[0]] = ""
 236             else:
 237                 os.environ[parts[0]] = parts[1]
 238
 239     if args.set_inferior_env_vars:
 240         lldbtest_config.inferior_env = " ".join(args.set_inferior_env_vars)
 241
 242     if args.h:
 243         do_help = True
 244
 245     if args.compiler:
 246         configuration.compiler = os.path.abspath(args.compiler)
 247         if not is_exe(configuration.compiler):
 248             configuration.compiler = which(args.compiler)
 249         if not is_exe(configuration.compiler):
 250             logging.error(
 251                 "%s is not a valid compiler executable; aborting...", args.compiler
 252             )
 253             sys.exit(-1)
 254     else:
 255         # Use a compiler appropriate appropriate for the Apple SDK if one was
 256         # specified
 257         if platform_system == "Darwin" and args.apple_sdk:
 258             configuration.compiler = seven.get_command_output(
 259                 'xcrun -sdk "%s" -find clang 2> /dev/null' % (args.apple_sdk)
 260             )
 261         else:
 262             # 'clang' on ubuntu 14.04 is 3.4 so we try clang-3.5 first
 263             candidateCompilers = ["clang-3.5", "clang", "gcc"]
 264             for candidate in candidateCompilers:
 265                 if which(candidate):
 266                     configuration.compiler = candidate
 267                     break
 268
 269     if args.dsymutil:
 270         configuration.dsymutil = args.dsymutil
 271     elif platform_system == "Darwin":
 272         configuration.dsymutil = seven.get_command_output(
 273             "xcrun -find -toolchain default dsymutil"
 274         )
 275     if args.llvm_tools_dir:
 276         configuration.filecheck = shutil.which("FileCheck", path=args.llvm_tools_dir)
 277         configuration.yaml2obj = shutil.which("yaml2obj", path=args.llvm_tools_dir)
 278
 279     if not configuration.get_filecheck_path():
 280         logging.warning("No valid FileCheck executable; some tests may fail...")
 281         logging.warning("(Double-check the --llvm-tools-dir argument to dotest.py)")
 282
 283     if args.libcxx_include_dir or args.libcxx_library_dir:
 284         if args.lldb_platform_name:
 285             logging.warning(
 286                 "Custom libc++ is not supported for remote runs: ignoring --libcxx arguments"
 287             )
 288         elif not (args.libcxx_include_dir and args.libcxx_library_dir):
 289             logging.error(
 290                 "Custom libc++ requires both --libcxx-include-dir and --libcxx-library-dir"
 291             )
 292             sys.exit(-1)
 293     configuration.libcxx_include_dir = args.libcxx_include_dir
 294     configuration.libcxx_include_target_dir = args.libcxx_include_target_dir
 295     configuration.libcxx_library_dir = args.libcxx_library_dir
 296
 297     if args.channels:
 298         lldbtest_config.channels = args.channels
 299
 300     if args.log_success:
 301         lldbtest_config.log_success = args.log_success
 302
 303     if args.out_of_tree_debugserver:
 304         lldbtest_config.out_of_tree_debugserver = args.out_of_tree_debugserver
 305
 306     # Set SDKROOT if we are using an Apple SDK
 307     if platform_system == "Darwin" and args.apple_sdk:
 308         configuration.sdkroot = seven.get_command_output(
 309             'xcrun --sdk "%s" --show-sdk-path 2> /dev/null' % (args.apple_sdk)
 310         )
 311         if not configuration.sdkroot:
 312             logging.error("No SDK found with the name %s; aborting...", args.apple_sdk)
 313             sys.exit(-1)
 314
 315     if args.arch:
 316         configuration.arch = args.arch
 317     else:
 318         configuration.arch = platform_machine
 319
 320     if args.categories_list:
 321         configuration.categories_list = set(
 322             test_categories.validate(args.categories_list, False)
 323         )
 324         configuration.use_categories = True
 325     else:
 326         configuration.categories_list = []
 327
 328     if args.skip_categories:
 329         configuration.skip_categories += test_categories.validate(
 330             args.skip_categories, False
 331         )
 332
 333     if args.xfail_categories:
 334         configuration.xfail_categories += test_categories.validate(
 335             args.xfail_categories, False
 336         )
 337
 338     if args.E:
 339         os.environ["CFLAGS_EXTRAS"] = args.E
 340
 341     if args.dwarf_version:
 342         configuration.dwarf_version = args.dwarf_version
 343         # We cannot modify CFLAGS_EXTRAS because they're used in test cases
 344         # that explicitly require no debug info.
 345         os.environ["CFLAGS"] = "-gdwarf-{}".format(configuration.dwarf_version)
 346
 347     if args.settings:
 348         for setting in args.settings:
 349             if not len(setting) == 1 or not setting[0].count("="):
 350                 logging.error(
 351                     '"%s" is not a setting in the form "key=value"', setting[0]
 352                 )
 353                 sys.exit(-1)
 354             setting_list = setting[0].split("=", 1)
 355             configuration.settings.append((setting_list[0], setting_list[1]))
 356
 357     if args.d:
 358         sys.stdout.write(
 359             "Suspending the process %d to wait for debugger to attach...\n"
 360             % os.getpid()
 361         )
 362         sys.stdout.flush()
 363         os.kill(os.getpid(), signal.SIGSTOP)
 364
 365     if args.f:
 366         if any([x.startswith("-") for x in args.f]):
 367             usage(parser)
 368         configuration.filters.extend(args.f)
 369
 370     if args.framework:
 371         configuration.lldb_framework_path = args.framework
 372
 373     if args.executable:
 374         # lldb executable is passed explicitly
 375         lldbtest_config.lldbExec = os.path.abspath(args.executable)
 376         if not is_exe(lldbtest_config.lldbExec):
 377             lldbtest_config.lldbExec = which(args.executable)
 378         if not is_exe(lldbtest_config.lldbExec):
 379             logging.error(
 380                 "%s is not a valid executable to test; aborting...", args.executable
 381             )
 382             sys.exit(-1)
 383
 384     if args.excluded:
 385         for excl_file in args.excluded:
 386             parseExclusion(excl_file)
 387
 388     if args.p:
 389         if args.p.startswith("-"):
 390             usage(parser)
 391         configuration.regexp = args.p
 392
 393     if args.t:
 394         os.environ["LLDB_COMMAND_TRACE"] = "YES"
 395
 396     if args.v:
 397         configuration.verbose = 2
 398
 399     # argparse makes sure we have a number
 400     if args.sharp:
 401         configuration.count = args.sharp
 402
 403     if sys.platform.startswith("win32"):
 404         os.environ["LLDB_DISABLE_CRASH_DIALOG"] = str(args.disable_crash_dialog)
 405         os.environ["LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE"] = str(True)
 406
 407     if do_help:
 408         usage(parser)
 409
 410     if args.lldb_platform_name:
 411         configuration.lldb_platform_name = args.lldb_platform_name
 412     if args.lldb_platform_url:
 413         configuration.lldb_platform_url = args.lldb_platform_url
 414     if args.lldb_platform_working_dir:
 415         configuration.lldb_platform_working_dir = args.lldb_platform_working_dir
 416     if platform_system == "Darwin" and args.apple_sdk:
 417         configuration.apple_sdk = args.apple_sdk
 418     if args.test_build_dir:
 419         configuration.test_build_dir = args.test_build_dir
 420     if args.lldb_module_cache_dir:
 421         configuration.lldb_module_cache_dir = args.lldb_module_cache_dir
 422     else:
 423         configuration.lldb_module_cache_dir = os.path.join(
 424             configuration.test_build_dir, "module-cache-lldb"
 425         )
 426     if args.clang_module_cache_dir:
 427         configuration.clang_module_cache_dir = args.clang_module_cache_dir
 428     else:
 429         configuration.clang_module_cache_dir = os.path.join(
 430             configuration.test_build_dir, "module-cache-clang"
 431         )
 432
 433     if args.lldb_libs_dir:
 434         configuration.lldb_libs_dir = args.lldb_libs_dir
 435
 436     if args.enabled_plugins:
 437         configuration.enabled_plugins = args.enabled_plugins
 438
 439     # Gather all the dirs passed on the command line.
 440     if len(args.args) > 0:
 441         configuration.testdirs = [
 442             os.path.realpath(os.path.abspath(x)) for x in args.args
 443         ]
 444
 445     lldbtest_config.codesign_identity = args.codesign_identity
 446
 447
 448 def registerFaulthandler():
 449     try:
 450         import faulthandler
 451     except ImportError:
 452         # faulthandler is not available until python3
 453         return
 454
 455     faulthandler.enable()
 456     # faulthandler.register is not available on Windows.
 457     if getattr(faulthandler, "register", None):
 458         faulthandler.register(signal.SIGTERM, chain=True)
 459
 460
 461 def setupSysPath():
 462     """
 463     Add LLDB.framework/Resources/Python to the search paths for modules.
 464     As a side effect, we also discover the 'lldb' executable and export it here.
 465     """
 466
 467     # Get the directory containing the current script.
 468     if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ:
 469         scriptPath = os.environ["DOTEST_SCRIPT_DIR"]
 470     else:
 471         scriptPath = os.path.dirname(os.path.abspath(__file__))
 472     if not scriptPath.endswith("test"):
 473         print("This script expects to reside in lldb's test directory.")
 474         sys.exit(-1)
 475
 476     os.environ["LLDB_TEST"] = scriptPath
 477
 478     # Set up the root build directory.
 479     if not configuration.test_build_dir:
 480         raise Exception("test_build_dir is not set")
 481     configuration.test_build_dir = os.path.abspath(configuration.test_build_dir)
 482
 483     # Set up the LLDB_SRC environment variable, so that the tests can locate
 484     # the LLDB source code.
 485     os.environ["LLDB_SRC"] = lldbsuite.lldb_root
 486
 487     pluginPath = os.path.join(scriptPath, "plugins")
 488     toolsLLDBDAP = os.path.join(scriptPath, "tools", "lldb-dap")
 489     toolsLLDBServerPath = os.path.join(scriptPath, "tools", "lldb-server")
 490     intelpt = os.path.join(scriptPath, "tools", "intelpt")
 491
 492     # Insert script dir, plugin dir and lldb-server dir to the sys.path.
 493     sys.path.insert(0, pluginPath)
 494     # Adding test/tools/lldb-dap to the path makes it easy to
 495     # "import lldb_dap_testcase" from the DAP tests
 496     sys.path.insert(0, toolsLLDBDAP)
 497     # Adding test/tools/lldb-server to the path makes it easy
 498     # to "import lldbgdbserverutils" from the lldb-server tests
 499     sys.path.insert(0, toolsLLDBServerPath)
 500     # Adding test/tools/intelpt to the path makes it easy
 501     # to "import intelpt_testcase" from the lldb-server tests
 502     sys.path.insert(0, intelpt)
 503
 504     # This is the root of the lldb git/svn checkout
 505     # When this changes over to a package instead of a standalone script, this
 506     # will be `lldbsuite.lldb_root`
 507     lldbRootDirectory = lldbsuite.lldb_root
 508
 509     # Some of the tests can invoke the 'lldb' command directly.
 510     # We'll try to locate the appropriate executable right here.
 511
 512     # The lldb executable can be set from the command line
 513     # if it's not set, we try to find it now
 514     # first, we try the environment
 515     if not lldbtest_config.lldbExec:
 516         # First, you can define an environment variable LLDB_EXEC specifying the
 517         # full pathname of the lldb executable.
 518         if "LLDB_EXEC" in os.environ:
 519             lldbtest_config.lldbExec = os.environ["LLDB_EXEC"]
 520
 521     if not lldbtest_config.lldbExec:
 522         # Last, check the path
 523         lldbtest_config.lldbExec = which("lldb")
 524
 525     if lldbtest_config.lldbExec and not is_exe(lldbtest_config.lldbExec):
 526         print(
 527             "'{}' is not a path to a valid executable".format(lldbtest_config.lldbExec)
 528         )
 529         lldbtest_config.lldbExec = None
 530
 531     if not lldbtest_config.lldbExec:
 532         print(
 533             "The 'lldb' executable cannot be located.  Some of the tests may not be run as a result."
 534         )
 535         sys.exit(-1)
 536
 537     os.system("%s -v" % lldbtest_config.lldbExec)
 538
 539     lldbDir = os.path.dirname(lldbtest_config.lldbExec)
 540
 541     lldbDAPExec = os.path.join(lldbDir, "lldb-dap")
 542     if is_exe(lldbDAPExec):
 543         os.environ["LLDBDAP_EXEC"] = lldbDAPExec
 544     else:
 545         if not configuration.shouldSkipBecauseOfCategories(["lldb-dap"]):
 546             print(
 547                 "The 'lldb-dap' executable cannot be located.  The lldb-dap tests can not be run as a result."
 548             )
 549             configuration.skip_categories.append("lldb-dap")
 550
 551     lldbPythonDir = None  # The directory that contains 'lldb/__init__.py'
 552
 553     # If our lldb supports the -P option, use it to find the python path:
 554     lldb_dash_p_result = subprocess.check_output(
 555         [lldbtest_config.lldbExec, "-P"], universal_newlines=True
 556     )
 557     if lldb_dash_p_result:
 558         for line in lldb_dash_p_result.splitlines():
 559             if os.path.isdir(line) and os.path.exists(
 560                 os.path.join(line, "lldb", "__init__.py")
 561             ):
 562                 lldbPythonDir = line
 563                 break
 564
 565     if not lldbPythonDir:
 566         print(
 567             "Unable to load lldb extension module.  Possible reasons for this include:"
 568         )
 569         print("  1) LLDB was built with LLDB_ENABLE_PYTHON=0")
 570         print(
 571             "  2) PYTHONPATH and PYTHONHOME are not set correctly.  PYTHONHOME should refer to"
 572         )
 573         print(
 574             "     the version of Python that LLDB built and linked against, and PYTHONPATH"
 575         )
 576         print(
 577             "     should contain the Lib directory for the same python distro, as well as the"
 578         )
 579         print("     location of LLDB's site-packages folder.")
 580         print(
 581             "  3) A different version of Python than that which was built against is exported in"
 582         )
 583         print("     the system's PATH environment variable, causing conflicts.")
 584         print(
 585             "  4) The executable '%s' could not be found.  Please check "
 586             % lldbtest_config.lldbExec
 587         )
 588         print("     that it exists and is executable.")
 589
 590     if lldbPythonDir:
 591         lldbPythonDir = os.path.normpath(lldbPythonDir)
 592         # Some of the code that uses this path assumes it hasn't resolved the Versions... link.
 593         # If the path we've constructed looks like that, then we'll strip out
 594         # the Versions/A part.
 595         (before, frameWithVersion, after) = lldbPythonDir.rpartition(
 596             "LLDB.framework/Versions/A"
 597         )
 598         if frameWithVersion != "":
 599             lldbPythonDir = before + "LLDB.framework" + after
 600
 601         lldbPythonDir = os.path.abspath(lldbPythonDir)
 602
 603         if "freebsd" in sys.platform or "linux" in sys.platform:
 604             os.environ["LLDB_LIB_DIR"] = os.path.join(lldbPythonDir, "..", "..")
 605
 606         # If tests need to find LLDB_FRAMEWORK, now they can do it
 607         os.environ["LLDB_FRAMEWORK"] = os.path.dirname(os.path.dirname(lldbPythonDir))
 608
 609         # This is to locate the lldb.py module.  Insert it right after
 610         # sys.path[0].
 611         sys.path[1:1] = [lldbPythonDir]
 612
 613
 614 def visit_file(dir, name):
 615     # Try to match the regexp pattern, if specified.
 616     if configuration.regexp:
 617         if not re.search(configuration.regexp, name):
 618             # We didn't match the regex, we're done.
 619             return
 620
 621     if configuration.skip_tests:
 622         for file_regexp in configuration.skip_tests:
 623             if re.search(file_regexp, name):
 624                 return
 625
 626     # We found a match for our test.  Add it to the suite.
 627
 628     # Update the sys.path first.
 629     if not sys.path.count(dir):
 630         sys.path.insert(0, dir)
 631     base = os.path.splitext(name)[0]
 632
 633     # Thoroughly check the filterspec against the base module and admit
 634     # the (base, filterspec) combination only when it makes sense.
 635
 636     def check(obj, parts):
 637         for part in parts:
 638             try:
 639                 parent, obj = obj, getattr(obj, part)
 640             except AttributeError:
 641                 # The filterspec has failed.
 642                 return False
 643         return True
 644
 645     module = __import__(base)
 646
 647     def iter_filters():
 648         for filterspec in configuration.filters:
 649             parts = filterspec.split(".")
 650             if check(module, parts):
 651                 yield filterspec
 652             elif parts[0] == base and len(parts) > 1 and check(module, parts[1:]):
 653                 yield ".".join(parts[1:])
 654             else:
 655                 for key, value in module.__dict__.items():
 656                     if check(value, parts):
 657                         yield key + "." + filterspec
 658
 659     filtered = False
 660     for filterspec in iter_filters():
 661         filtered = True
 662         print("adding filter spec %s to module %s" % (filterspec, repr(module)))
 663         tests = unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)
 664         configuration.suite.addTests(tests)
 665
 666     # Forgo this module if the (base, filterspec) combo is invalid
 667     if configuration.filters and not filtered:
 668         return
 669
 670     if not filtered:
 671         # Add the entire file's worth of tests since we're not filtered.
 672         # Also the fail-over case when the filterspec branch
 673         # (base, filterspec) combo doesn't make sense.
 674         configuration.suite.addTests(
 675             unittest2.defaultTestLoader.loadTestsFromName(base)
 676         )
 677
 678
 679 def visit(prefix, dir, names):
 680     """Visitor function for os.path.walk(path, visit, arg)."""
 681
 682     dir_components = set(dir.split(os.sep))
 683     excluded_components = set([".svn", ".git"])
 684     if dir_components.intersection(excluded_components):
 685         return
 686
 687     # Gather all the Python test file names that follow the Test*.py pattern.
 688     python_test_files = [
 689         name for name in names if name.endswith(".py") and name.startswith(prefix)
 690     ]
 691
 692     # Visit all the python test files.
 693     for name in python_test_files:
 694         # Ensure we error out if we have multiple tests with the same
 695         # base name.
 696         # Future improvement: find all the places where we work with base
 697         # names and convert to full paths.  We have directory structure
 698         # to disambiguate these, so we shouldn't need this constraint.
 699         if name in configuration.all_tests:
 700             raise Exception("Found multiple tests with the name %s" % name)
 701         configuration.all_tests.add(name)
 702
 703         # Run the relevant tests in the python file.
 704         visit_file(dir, name)
 705
 706
 707 # ======================================== #
 708 #                                          #
 709 # Execution of the test driver starts here #
 710 #                                          #
 711 # ======================================== #
 712
 713
 714 def checkDsymForUUIDIsNotOn():
 715     cmd = ["defaults", "read", "com.apple.DebugSymbols"]
 716     process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 717     cmd_output = process.stdout.read()
 718     output_str = cmd_output.decode("utf-8")
 719     if "DBGFileMappedPaths = " in output_str:
 720         print("%s =>" % " ".join(cmd))
 721         print(output_str)
 722         print(
 723             "Disable automatic lookup and caching of dSYMs before running the test suite!"
 724         )
 725         print("Exiting...")
 726         sys.exit(0)
 727
 728
 729 def exitTestSuite(exitCode=None):
 730     # lldb.py does SBDebugger.Initialize().
 731     # Call SBDebugger.Terminate() on exit.
 732     import lldb
 733
 734     lldb.SBDebugger.Terminate()
 735     if exitCode:
 736         sys.exit(exitCode)
 737
 738
 739 def getVersionForSDK(sdk):
 740     sdk = str.lower(sdk)
 741     full_path = seven.get_command_output("xcrun -sdk %s --show-sdk-path" % sdk)
 742     basename = os.path.basename(full_path)
 743     basename = os.path.splitext(basename)[0]
 744     basename = str.lower(basename)
 745     ver = basename.replace(sdk, "")
 746     return ver
 747
 748
 749 def checkCompiler():
 750     # Add some intervention here to sanity check that the compiler requested is sane.
 751     # If found not to be an executable program, we abort.
 752     c = configuration.compiler
 753     if which(c):
 754         return
 755
 756     if not sys.platform.startswith("darwin"):
 757         raise Exception(c + " is not a valid compiler")
 758
 759     pipe = subprocess.Popen(
 760         ["xcrun", "-find", c], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
 761     )
 762     cmd_output = pipe.stdout.read()
 763     if not cmd_output or "not found" in cmd_output:
 764         raise Exception(c + " is not a valid compiler")
 765
 766     configuration.compiler = cmd_output.split("\n")[0]
 767     print("'xcrun -find %s' returning %s" % (c, configuration.compiler))
 768
 769
 770 def canRunLibcxxTests():
 771     from lldbsuite.test import lldbplatformutil
 772
 773     platform = lldbplatformutil.getPlatform()
 774
 775     if lldbplatformutil.target_is_android() or lldbplatformutil.platformIsDarwin():
 776         return True, "libc++ always present"
 777
 778     if platform == "linux":
 779         with tempfile.NamedTemporaryFile() as f:
 780             cmd = [configuration.compiler, "-xc++", "-stdlib=libc++", "-o", f.name, "-"]
 781             p = subprocess.Popen(
 782                 cmd,
 783                 stdin=subprocess.PIPE,
 784                 stdout=subprocess.PIPE,
 785                 stderr=subprocess.PIPE,
 786                 universal_newlines=True,
 787             )
 788             _, stderr = p.communicate("#include <cassert>\nint main() {}")
 789             if not p.returncode:
 790                 return True, "Compiling with -stdlib=libc++ works"
 791             return (
 792                 False,
 793                 "Compiling with -stdlib=libc++ fails with the error: %s" % stderr,
 794             )
 795
 796     return False, "Don't know how to build with libc++ on %s" % platform
 797
 798
 799 def checkLibcxxSupport():
 800     result, reason = canRunLibcxxTests()
 801     if result:
 802         return  # libc++ supported
 803     if "libc++" in configuration.categories_list:
 804         return  # libc++ category explicitly requested, let it run.
 805     if configuration.verbose:
 806         print("libc++ tests will not be run because: " + reason)
 807     configuration.skip_categories.append("libc++")
 808
 809
 810 def canRunLibstdcxxTests():
 811     from lldbsuite.test import lldbplatformutil
 812
 813     platform = lldbplatformutil.getPlatform()
 814     if lldbplatformutil.target_is_android():
 815         platform = "android"
 816     if platform == "linux":
 817         return True, "libstdcxx always present"
 818     return False, "Don't know how to build with libstdcxx on %s" % platform
 819
 820
 821 def checkLibstdcxxSupport():
 822     result, reason = canRunLibstdcxxTests()
 823     if result:
 824         return  # libstdcxx supported
 825     if "libstdcxx" in configuration.categories_list:
 826         return  # libstdcxx category explicitly requested, let it run.
 827     if configuration.verbose:
 828         print("libstdcxx tests will not be run because: " + reason)
 829     configuration.skip_categories.append("libstdcxx")
 830
 831
 832 def canRunWatchpointTests():
 833     from lldbsuite.test import lldbplatformutil
 834
 835     platform = lldbplatformutil.getPlatform()
 836     if platform == "netbsd":
 837         if os.geteuid() == 0:
 838             return True, "root can always write dbregs"
 839         try:
 840             output = (
 841                 subprocess.check_output(
 842                     ["/sbin/sysctl", "-n", "security.models.extensions.user_set_dbregs"]
 843                 )
 844                 .decode()
 845                 .strip()
 846             )
 847             if output == "1":
 848                 return True, "security.models.extensions.user_set_dbregs enabled"
 849         except subprocess.CalledProcessError:
 850             pass
 851         return False, "security.models.extensions.user_set_dbregs disabled"
 852     elif platform == "freebsd" and configuration.arch == "aarch64":
 853         import lldb
 854
 855         if lldb.SBPlatform.GetHostPlatform().GetOSMajorVersion() < 13:
 856             return False, "Watchpoint support on arm64 requires FreeBSD 13.0"
 857     return True, "watchpoint support available"
 858
 859
 860 def checkWatchpointSupport():
 861     result, reason = canRunWatchpointTests()
 862     if result:
 863         return  # watchpoints supported
 864     if "watchpoint" in configuration.categories_list:
 865         return  # watchpoint category explicitly requested, let it run.
 866     if configuration.verbose:
 867         print("watchpoint tests will not be run because: " + reason)
 868     configuration.skip_categories.append("watchpoint")
 869
 870
 871 def checkObjcSupport():
 872     from lldbsuite.test import lldbplatformutil
 873
 874     if not lldbplatformutil.platformIsDarwin():
 875         if configuration.verbose:
 876             print("objc tests will be skipped because of unsupported platform")
 877         configuration.skip_categories.append("objc")
 878
 879
 880 def checkDebugInfoSupport():
 881     from lldbsuite.test import lldbplatformutil
 882
 883     platform = lldbplatformutil.getPlatform()
 884     compiler = configuration.compiler
 885     for cat in test_categories.debug_info_categories:
 886         if cat in configuration.categories_list:
 887             continue  # Category explicitly requested, let it run.
 888         if test_categories.is_supported_on_platform(cat, platform, compiler):
 889             continue
 890         configuration.skip_categories.append(cat)
 891
 892
 893 def checkDebugServerSupport():
 894     from lldbsuite.test import lldbplatformutil
 895     import lldb
 896
 897     skip_msg = "Skipping %s tests, as they are not compatible with remote testing on this platform"
 898     if lldbplatformutil.platformIsDarwin():
 899         configuration.skip_categories.append("llgs")
 900         if lldb.remote_platform:
 901             # <rdar://problem/34539270>
 902             configuration.skip_categories.append("debugserver")
 903             if configuration.verbose:
 904                 print(skip_msg % "debugserver")
 905     else:
 906         configuration.skip_categories.append("debugserver")
 907         if lldb.remote_platform and lldbplatformutil.getPlatform() == "windows":
 908             configuration.skip_categories.append("llgs")
 909             if configuration.verbose:
 910                 print(skip_msg % "lldb-server")
 911
 912
 913 def checkForkVForkSupport():
 914     from lldbsuite.test import lldbplatformutil
 915
 916     platform = lldbplatformutil.getPlatform()
 917     if platform not in ["freebsd", "linux", "netbsd"]:
 918         configuration.skip_categories.append("fork")
 919
 920
 921 def run_suite():
 922     # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
 923     # does not exist before proceeding to running the test suite.
 924     if sys.platform.startswith("darwin"):
 925         checkDsymForUUIDIsNotOn()
 926
 927     # Start the actions by first parsing the options while setting up the test
 928     # directories, followed by setting up the search paths for lldb utilities;
 929     # then, we walk the directory trees and collect the tests into our test suite.
 930     #
 931     parseOptionsAndInitTestdirs()
 932
 933     # Print a stack trace if the test hangs or is passed SIGTERM.
 934     registerFaulthandler()
 935
 936     setupSysPath()
 937
 938     import lldb
 939
 940     lldb.SBDebugger.Initialize()
 941     lldb.SBDebugger.PrintStackTraceOnError()
 942
 943     # Use host platform by default.
 944     lldb.remote_platform = None
 945     lldb.selected_platform = lldb.SBPlatform.GetHostPlatform()
 946
 947     # Now we can also import lldbutil
 948     from lldbsuite.test import lldbutil
 949
 950     if configuration.lldb_platform_name:
 951         print("Setting up remote platform '%s'" % (configuration.lldb_platform_name))
 952         lldb.remote_platform = lldb.SBPlatform(configuration.lldb_platform_name)
 953         lldb.selected_platform = lldb.remote_platform
 954         if not lldb.remote_platform.IsValid():
 955             print(
 956                 "error: unable to create the LLDB platform named '%s'."
 957                 % (configuration.lldb_platform_name)
 958             )
 959             exitTestSuite(1)
 960         if configuration.lldb_platform_url:
 961             # We must connect to a remote platform if a LLDB platform URL was
 962             # specified
 963             print(
 964                 "Connecting to remote platform '%s' at '%s'..."
 965                 % (configuration.lldb_platform_name, configuration.lldb_platform_url)
 966             )
 967             platform_connect_options = lldb.SBPlatformConnectOptions(
 968                 configuration.lldb_platform_url
 969             )
 970             err = lldb.remote_platform.ConnectRemote(platform_connect_options)
 971             if err.Success():
 972                 print("Connected.")
 973             else:
 974                 print(
 975                     "error: failed to connect to remote platform using URL '%s': %s"
 976                     % (configuration.lldb_platform_url, err)
 977                 )
 978                 exitTestSuite(1)
 979         else:
 980             configuration.lldb_platform_url = None
 981
 982     if configuration.lldb_platform_working_dir:
 983         print(
 984             "Setting remote platform working directory to '%s'..."
 985             % (configuration.lldb_platform_working_dir)
 986         )
 987         error = lldb.remote_platform.MakeDirectory(
 988             configuration.lldb_platform_working_dir, 448
 989         )  # 448 = 0o700
 990         if error.Fail():
 991             raise Exception(
 992                 "making remote directory '%s': %s"
 993                 % (configuration.lldb_platform_working_dir, error)
 994             )
 995
 996         if not lldb.remote_platform.SetWorkingDirectory(
 997             configuration.lldb_platform_working_dir
 998         ):
 999             raise Exception(
1000                 "failed to set working directory '%s'"
1001                 % configuration.lldb_platform_working_dir
1002             )
1003         lldb.selected_platform = lldb.remote_platform
1004     else:
1005         lldb.remote_platform = None
1006         configuration.lldb_platform_working_dir = None
1007         configuration.lldb_platform_url = None
1008
1009     # Set up the working directory.
1010     # Note that it's not dotest's job to clean this directory.
1011     lldbutil.mkdir_p(configuration.test_build_dir)
1012
1013     checkLibcxxSupport()
1014     checkLibstdcxxSupport()
1015     checkWatchpointSupport()
1016     checkDebugInfoSupport()
1017     checkDebugServerSupport()
1018     checkObjcSupport()
1019     checkForkVForkSupport()
1020
1021     skipped_categories_list = ", ".join(configuration.skip_categories)
1022     print(
1023         "Skipping the following test categories: {}".format(
1024             configuration.skip_categories
1025         )
1026     )
1027
1028     for testdir in configuration.testdirs:
1029         for dirpath, dirnames, filenames in os.walk(testdir):
1030             visit("Test", dirpath, filenames)
1031
1032     #
1033     # Now that we have loaded all the test cases, run the whole test suite.
1034     #
1035
1036     # Install the control-c handler.
1037     unittest2.signals.installHandler()
1038
1039     #
1040     # Invoke the default TextTestRunner to run the test suite
1041     #
1042     checkCompiler()
1043
1044     if configuration.verbose:
1045         print("compiler=%s" % configuration.compiler)
1046
1047     # Iterating over all possible architecture and compiler combinations.
1048     configString = "arch=%s compiler=%s" % (configuration.arch, configuration.compiler)
1049
1050     # Output the configuration.
1051     if configuration.verbose:
1052         sys.stderr.write("\nConfiguration: " + configString + "\n")
1053
1054     # First, write out the number of collected test cases.
1055     if configuration.verbose:
1056         sys.stderr.write(configuration.separator + "\n")
1057         sys.stderr.write(
1058             "Collected %d test%s\n\n"
1059             % (
1060                 configuration.suite.countTestCases(),
1061                 configuration.suite.countTestCases() != 1 and "s" or "",
1062             )
1063         )
1064
1065     if configuration.suite.countTestCases() == 0:
1066         logging.error("did not discover any matching tests")
1067         exitTestSuite(1)
1068
1069     # Invoke the test runner.
1070     if configuration.count == 1:
1071         result = unittest2.TextTestRunner(
1072             stream=sys.stderr,
1073             verbosity=configuration.verbose,
1074             resultclass=test_result.LLDBTestResult,
1075         ).run(configuration.suite)
1076     else:
1077         # We are invoking the same test suite more than once.  In this case,
1078         # mark __ignore_singleton__ flag as True so the signleton pattern is
1079         # not enforced.
1080         test_result.LLDBTestResult.__ignore_singleton__ = True
1081         for i in range(configuration.count):
1082             result = unittest2.TextTestRunner(
1083                 stream=sys.stderr,
1084                 verbosity=configuration.verbose,
1085                 resultclass=test_result.LLDBTestResult,
1086             ).run(configuration.suite)
1087
1088     configuration.failed = not result.wasSuccessful()
1089
1090     if configuration.sdir_has_content and configuration.verbose:
1091         sys.stderr.write(
1092             "Session logs for test failures/errors/unexpected successes"
1093             " can be found in the test build directory\n"
1094         )
1095
1096     if configuration.use_categories and len(configuration.failures_per_category) > 0:
1097         sys.stderr.write("Failures per category:\n")
1098         for category in configuration.failures_per_category:
1099             sys.stderr.write(
1100                 "%s - %d\n" % (category, configuration.failures_per_category[category])
1101             )
1102
1103     # Exiting.
1104     exitTestSuite(configuration.failed)
1105
1106
1107 if __name__ == "__main__":
1108     print(
1109         __file__
1110         + " is for use as a module only.  It should not be run as a standalone script."
1111     )
1112     sys.exit(-1)