compiler-rt/test/lit.common.cfg.py

   1 # -*- Python -*-
   2
   3 # Configuration file for 'lit' test runner.
   4 # This file contains common rules for various compiler-rt testsuites.
   5 # It is mostly copied from lit.cfg.py used by Clang.
   6 import os
   7 import platform
   8 import re
   9 import subprocess
  10 import json
  11
  12 import lit.formats
  13 import lit.util
  14
  15 # Get shlex.quote if available (added in 3.3), and fall back to pipes.quote if
  16 # it's not available.
  17 try:
  18   import shlex
  19   sh_quote = shlex.quote
  20 except:
  21   import pipes
  22   sh_quote = pipes.quote
  23
  24 def find_compiler_libdir():
  25   """
  26     Returns the path to library resource directory used
  27     by the compiler.
  28   """
  29   if config.compiler_id != 'Clang':
  30     lit_config.warning(f'Determining compiler\'s runtime directory is not supported for {config.compiler_id}')
  31     # TODO: Support other compilers.
  32     return None
  33   def get_path_from_clang(args, allow_failure):
  34     clang_cmd = [
  35       config.clang.strip(),
  36       f'--target={config.target_triple}',
  37     ]
  38     clang_cmd.extend(args)
  39     path = None
  40     try:
  41       result = subprocess.run(
  42         clang_cmd,
  43         stdout=subprocess.PIPE,
  44         stderr=subprocess.PIPE,
  45         check=True
  46       )
  47       path = result.stdout.decode().strip()
  48     except subprocess.CalledProcessError as e:
  49       msg = f'Failed to run {clang_cmd}\nrc:{e.returncode}\nstdout:{e.stdout}\ne.stderr{e.stderr}'
  50       if allow_failure:
  51         lit_config.warning(msg)
  52       else:
  53         lit_config.fatal(msg)
  54     return path, clang_cmd
  55
  56   # Try using `-print-runtime-dir`. This is only supported by very new versions of Clang.
  57   # so allow failure here.
  58   runtime_dir, clang_cmd = get_path_from_clang(shlex.split(config.target_cflags)
  59                                                + ['-print-runtime-dir'],
  60                                                allow_failure=True)
  61   if runtime_dir:
  62     if os.path.exists(runtime_dir):
  63       return os.path.realpath(runtime_dir)
  64     # TODO(dliew): This should be a fatal error but it seems to trip the `llvm-clang-win-x-aarch64`
  65     # bot which is likely misconfigured
  66     lit_config.warning(
  67       f'Path reported by clang does not exist: \"{runtime_dir}\". '
  68       f'This path was found by running {clang_cmd}.'
  69     )
  70     return None
  71
  72   # Fall back for older AppleClang that doesn't support `-print-runtime-dir`
  73   # Note `-print-file-name=<path to compiler-rt lib>` was broken for Apple
  74   # platforms so we can't use that approach here (see https://reviews.llvm.org/D101682).
  75   if config.host_os == 'Darwin':
  76     lib_dir, _ = get_path_from_clang(['-print-file-name=lib'], allow_failure=False)
  77     runtime_dir = os.path.join(lib_dir, 'darwin')
  78     if not os.path.exists(runtime_dir):
  79       lit_config.fatal(f'Path reported by clang does not exist: {runtime_dir}')
  80     return os.path.realpath(runtime_dir)
  81
  82   lit_config.warning('Failed to determine compiler\'s runtime directory')
  83   return None
  84
  85
  86 # Choose between lit's internal shell pipeline runner and a real shell.  If
  87 # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
  88 use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
  89 if use_lit_shell:
  90     # 0 is external, "" is default, and everything else is internal.
  91     execute_external = (use_lit_shell == "0")
  92 else:
  93     # Otherwise we default to internal on Windows and external elsewhere, as
  94     # bash on Windows is usually very slow.
  95     execute_external = (not sys.platform in ['win32'])
  96
  97 # Allow expanding substitutions that are based on other substitutions
  98 config.recursiveExpansionLimit = 10
  99
 100 # Setup test format.
 101 config.test_format = lit.formats.ShTest(execute_external)
 102 if execute_external:
 103   config.available_features.add('shell')
 104
 105 compiler_id = getattr(config, 'compiler_id', None)
 106 if compiler_id == "Clang":
 107   if platform.system() != 'Windows':
 108     config.cxx_mode_flags = ["--driver-mode=g++"]
 109   else:
 110     config.cxx_mode_flags = []
 111   # We assume that sanitizers should provide good enough error
 112   # reports and stack traces even with minimal debug info.
 113   config.debug_info_flags = ["-gline-tables-only"]
 114   if platform.system() == 'Windows':
 115     # On Windows, use CodeView with column info instead of DWARF. Both VS and
 116     # windbg do not behave well when column info is enabled, but users have
 117     # requested it because it makes ASan reports more precise.
 118     config.debug_info_flags.append("-gcodeview")
 119     config.debug_info_flags.append("-gcolumn-info")
 120 elif compiler_id == 'GNU':
 121   config.cxx_mode_flags = ["-x c++"]
 122   config.debug_info_flags = ["-g"]
 123 else:
 124   lit_config.fatal("Unsupported compiler id: %r" % compiler_id)
 125 # Add compiler ID to the list of available features.
 126 config.available_features.add(compiler_id)
 127
 128 # When LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=on, the initial value of
 129 # config.compiler_rt_libdir (COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR) has the
 130 # triple as the trailing path component. The value is incorrect for -m32/-m64.
 131 # Adjust config.compiler_rt accordingly.
 132 if config.enable_per_target_runtime_dir:
 133     if '-m32' in shlex.split(config.target_cflags):
 134         config.compiler_rt_libdir = re.sub(r'/x86_64(?=-[^/]+$)', '/i386', config.compiler_rt_libdir)
 135     elif '-m64' in shlex.split(config.target_cflags):
 136         config.compiler_rt_libdir = re.sub(r'/i386(?=-[^/]+$)', '/x86_64', config.compiler_rt_libdir)
 137
 138 # Ask the compiler for the path to libraries it is going to use. If this
 139 # doesn't match config.compiler_rt_libdir then it means we might be testing the
 140 # compiler's own runtime libraries rather than the ones we just built.
 141 # Warn about about this and handle appropriately.
 142 compiler_libdir = find_compiler_libdir()
 143 if compiler_libdir:
 144   compiler_rt_libdir_real = os.path.realpath(config.compiler_rt_libdir)
 145   if compiler_libdir != compiler_rt_libdir_real:
 146     lit_config.warning(
 147       'Compiler lib dir != compiler-rt lib dir\n'
 148       f'Compiler libdir:     "{compiler_libdir}"\n'
 149       f'compiler-rt libdir:  "{compiler_rt_libdir_real}"')
 150     if config.test_standalone_build_libs:
 151       # Use just built runtime libraries, i.e. the the libraries this built just built.
 152       if not config.test_suite_supports_overriding_runtime_lib_path:
 153         # Test suite doesn't support this configuration.
 154         # TODO(dliew): This should be an error but it seems several bots are
 155         # testing incorrectly and having this as an error breaks them.
 156         lit_config.warning(
 157             'COMPILER_RT_TEST_STANDALONE_BUILD_LIBS=ON, but this test suite '
 158             'does not support testing the just-built runtime libraries '
 159             'when the test compiler is configured to use different runtime '
 160             'libraries. Either modify this test suite to support this test '
 161             'configuration, or set COMPILER_RT_TEST_STANDALONE_BUILD_LIBS=OFF '
 162             'to test the runtime libraries included in the compiler instead.'
 163         )
 164     else:
 165       # Use Compiler's resource library directory instead.
 166       config.compiler_rt_libdir = compiler_libdir
 167     lit_config.note(f'Testing using libraries in "{config.compiler_rt_libdir}"')
 168
 169 # If needed, add cflag for shadow scale.
 170 if config.asan_shadow_scale != '':
 171   config.target_cflags += " -mllvm -asan-mapping-scale=" + config.asan_shadow_scale
 172 if config.memprof_shadow_scale != '':
 173   config.target_cflags += " -mllvm -memprof-mapping-scale=" + config.memprof_shadow_scale
 174
 175 # Clear some environment variables that might affect Clang.
 176 possibly_dangerous_env_vars = ['ASAN_OPTIONS', 'DFSAN_OPTIONS', 'HWASAN_OPTIONS',
 177                                'LSAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS',
 178                                'COMPILER_PATH', 'RC_DEBUG_OPTIONS',
 179                                'CINDEXTEST_PREAMBLE_FILE', 'LIBRARY_PATH',
 180                                'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH',
 181                                'OBJC_INCLUDE_PATH', 'OBJCPLUS_INCLUDE_PATH',
 182                                'LIBCLANG_TIMING', 'LIBCLANG_OBJTRACKING',
 183                                'LIBCLANG_LOGGING', 'LIBCLANG_BGPRIO_INDEX',
 184                                'LIBCLANG_BGPRIO_EDIT', 'LIBCLANG_NOTHREADS',
 185                                'LIBCLANG_RESOURCE_USAGE',
 186                                'LIBCLANG_CODE_COMPLETION_LOGGING',
 187                                'XRAY_OPTIONS']
 188 # Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it.
 189 if platform.system() != 'Windows':
 190     possibly_dangerous_env_vars.append('INCLUDE')
 191 for name in possibly_dangerous_env_vars:
 192   if name in config.environment:
 193     del config.environment[name]
 194
 195 # Tweak PATH to include llvm tools dir.
 196 if (not config.llvm_tools_dir) or (not os.path.exists(config.llvm_tools_dir)):
 197   lit_config.fatal("Invalid llvm_tools_dir config attribute: %r" % config.llvm_tools_dir)
 198 path = os.path.pathsep.join((config.llvm_tools_dir, config.environment['PATH']))
 199 config.environment['PATH'] = path
 200
 201 # Help MSVS link.exe find the standard libraries.
 202 # Make sure we only try to use it when targetting Windows.
 203 if platform.system() == 'Windows' and '-win' in config.target_triple:
 204   config.environment['LIB'] = os.environ['LIB']
 205
 206 config.available_features.add(config.host_os.lower())
 207
 208 if config.target_triple.startswith("ppc") or config.target_triple.startswith("powerpc"):
 209   config.available_features.add("ppc")
 210
 211 if re.match(r'^x86_64.*-linux', config.target_triple):
 212   config.available_features.add("x86_64-linux")
 213
 214 config.available_features.add("host-byteorder-" + sys.byteorder + "-endian")
 215
 216 if config.have_zlib == "1":
 217   config.available_features.add("zlib")
 218
 219 # Use ugly construction to explicitly prohibit "clang", "clang++" etc.
 220 # in RUN lines.
 221 config.substitutions.append(
 222     (' clang', """\n\n*** Do not use 'clangXXX' in tests,
 223      instead define '%clangXXX' substitution in lit config. ***\n\n""") )
 224
 225 if config.host_os == 'NetBSD':
 226   nb_commands_dir = os.path.join(config.compiler_rt_src_root,
 227                                  "test", "sanitizer_common", "netbsd_commands")
 228   config.netbsd_noaslr_prefix = ('sh ' +
 229                                  os.path.join(nb_commands_dir, 'run_noaslr.sh'))
 230   config.netbsd_nomprotect_prefix = ('sh ' +
 231                                      os.path.join(nb_commands_dir,
 232                                                   'run_nomprotect.sh'))
 233   config.substitutions.append( ('%run_nomprotect',
 234                                 config.netbsd_nomprotect_prefix) )
 235 else:
 236   config.substitutions.append( ('%run_nomprotect', '%run') )
 237
 238 # Copied from libcxx's config.py
 239 def get_lit_conf(name, default=None):
 240     # Allow overriding on the command line using --param=<name>=<val>
 241     val = lit_config.params.get(name, None)
 242     if val is None:
 243         val = getattr(config, name, None)
 244         if val is None:
 245             val = default
 246     return val
 247
 248 emulator = get_lit_conf('emulator', None)
 249
 250 def get_ios_commands_dir():
 251   return os.path.join(config.compiler_rt_src_root, "test", "sanitizer_common", "ios_commands")
 252
 253 # Allow tests to be executed on a simulator or remotely.
 254 if emulator:
 255   config.substitutions.append( ('%run', emulator) )
 256   config.substitutions.append( ('%env ', "env ") )
 257   # TODO: Implement `%device_rm` to perform removal of files in the emulator.
 258   # For now just make it a no-op.
 259   lit_config.warning('%device_rm is not implemented')
 260   config.substitutions.append( ('%device_rm', 'echo ') )
 261   config.compile_wrapper = ""
 262 elif config.host_os == 'Darwin' and config.apple_platform != "osx":
 263   # Darwin tests can be targetting macOS, a device or a simulator. All devices
 264   # are declared as "ios", even for iOS derivatives (tvOS, watchOS). Similarly,
 265   # all simulators are "iossim". See the table below.
 266   #
 267   # =========================================================================
 268   # Target             | Feature set
 269   # =========================================================================
 270   # macOS              | darwin
 271   # iOS device         | darwin, ios
 272   # iOS simulator      | darwin, ios, iossim
 273   # tvOS device        | darwin, ios, tvos
 274   # tvOS simulator     | darwin, ios, iossim, tvos, tvossim
 275   # watchOS device     | darwin, ios, watchos
 276   # watchOS simulator  | darwin, ios, iossim, watchos, watchossim
 277   # =========================================================================
 278
 279   ios_or_iossim = "iossim" if config.apple_platform.endswith("sim") else "ios"
 280
 281   config.available_features.add('ios')
 282   device_id_env = "SANITIZER_" + ios_or_iossim.upper() + "_TEST_DEVICE_IDENTIFIER"
 283   if ios_or_iossim == "iossim":
 284     config.available_features.add('iossim')
 285     if device_id_env not in os.environ:
 286       lit_config.fatal(
 287         '{} must be set in the environment when running iossim tests'.format(
 288           device_id_env))
 289   if config.apple_platform != "ios" and config.apple_platform != "iossim":
 290     config.available_features.add(config.apple_platform)
 291
 292   ios_commands_dir = get_ios_commands_dir()
 293
 294   run_wrapper = os.path.join(ios_commands_dir, ios_or_iossim + "_run.py")
 295   env_wrapper = os.path.join(ios_commands_dir, ios_or_iossim + "_env.py")
 296   compile_wrapper = os.path.join(ios_commands_dir, ios_or_iossim + "_compile.py")
 297   prepare_script = os.path.join(ios_commands_dir, ios_or_iossim + "_prepare.py")
 298
 299   if device_id_env in os.environ:
 300     config.environment[device_id_env] = os.environ[device_id_env]
 301   config.substitutions.append(('%run', run_wrapper))
 302   config.substitutions.append(('%env ', env_wrapper + " "))
 303   # Current implementation of %device_rm uses the run_wrapper to do
 304   # the work.
 305   config.substitutions.append(('%device_rm', '{} rm '.format(run_wrapper)))
 306   config.compile_wrapper = compile_wrapper
 307
 308   try:
 309     prepare_output = subprocess.check_output([prepare_script, config.apple_platform, config.clang]).decode().strip()
 310   except subprocess.CalledProcessError as e:
 311     print("Command failed:")
 312     print(e.output)
 313     raise e
 314   if len(prepare_output) > 0: print(prepare_output)
 315   prepare_output_json = prepare_output.split("\n")[-1]
 316   prepare_output = json.loads(prepare_output_json)
 317   config.environment.update(prepare_output["env"])
 318 elif config.android:
 319   config.available_features.add('android')
 320   compile_wrapper = os.path.join(config.compiler_rt_src_root, "test", "sanitizer_common", "android_commands", "android_compile.py") + " "
 321   config.compile_wrapper = compile_wrapper
 322   config.substitutions.append( ('%run', "") )
 323   config.substitutions.append( ('%env ', "env ") )
 324 else:
 325   config.substitutions.append( ('%run', "") )
 326   config.substitutions.append( ('%env ', "env ") )
 327   # When running locally %device_rm is a no-op.
 328   config.substitutions.append( ('%device_rm', 'echo ') )
 329   config.compile_wrapper = ""
 330
 331 # Define CHECK-%os to check for OS-dependent output.
 332 config.substitutions.append( ('CHECK-%os', ("CHECK-" + config.host_os)))
 333
 334 # Define %arch to check for architecture-dependent output.
 335 config.substitutions.append( ('%arch', (config.host_arch)))
 336
 337 if config.host_os == 'Windows':
 338   # FIXME: This isn't quite right. Specifically, it will succeed if the program
 339   # does not crash but exits with a non-zero exit code. We ought to merge
 340   # KillTheDoctor and not --crash to make the latter more useful and remove the
 341   # need for this substitution.
 342   config.expect_crash = "not KillTheDoctor "
 343 else:
 344   config.expect_crash = "not --crash "
 345
 346 config.substitutions.append( ("%expect_crash ", config.expect_crash) )
 347
 348 target_arch = getattr(config, 'target_arch', None)
 349 if target_arch:
 350   config.available_features.add(target_arch + '-target-arch')
 351   if target_arch in ['x86_64', 'i386']:
 352     config.available_features.add('x86-target-arch')
 353   config.available_features.add(target_arch + '-' + config.host_os.lower())
 354
 355 compiler_rt_debug = getattr(config, 'compiler_rt_debug', False)
 356 if not compiler_rt_debug:
 357   config.available_features.add('compiler-rt-optimized')
 358
 359 libdispatch = getattr(config, 'compiler_rt_intercept_libdispatch', False)
 360 if libdispatch:
 361   config.available_features.add('libdispatch')
 362
 363 sanitizer_can_use_cxxabi = getattr(config, 'sanitizer_can_use_cxxabi', True)
 364 if sanitizer_can_use_cxxabi:
 365   config.available_features.add('cxxabi')
 366
 367 if not getattr(config, 'sanitizer_uses_static_cxxabi', False):
 368   config.available_features.add('shared_cxxabi')
 369
 370 if not getattr(config, 'sanitizer_uses_static_unwind', False):
 371   config.available_features.add('shared_unwind')
 372
 373 if config.has_lld:
 374   config.available_features.add('lld-available')
 375
 376 if config.use_lld:
 377   config.available_features.add('lld')
 378
 379 if config.can_symbolize:
 380   config.available_features.add('can-symbolize')
 381
 382 if config.gwp_asan:
 383   config.available_features.add('gwp_asan')
 384
 385 lit.util.usePlatformSdkOnDarwin(config, lit_config)
 386
 387 min_macos_deployment_target_substitutions = [
 388   (10, 11),
 389   (10, 12),
 390 ]
 391 # TLS requires watchOS 3+
 392 config.substitutions.append( ('%darwin_min_target_with_tls_support', '%min_macos_deployment_target=10.12') )
 393
 394 if config.host_os == 'Darwin':
 395   osx_version = (10, 0, 0)
 396   try:
 397     osx_version = subprocess.check_output(["sw_vers", "-productVersion"],
 398                                           universal_newlines=True)
 399     osx_version = tuple(int(x) for x in osx_version.split('.'))
 400     if len(osx_version) == 2: osx_version = (osx_version[0], osx_version[1], 0)
 401     if osx_version >= (10, 11):
 402       config.available_features.add('osx-autointerception')
 403       config.available_features.add('osx-ld64-live_support')
 404     if osx_version >= (10, 15):
 405       config.available_features.add('osx-swift-runtime')
 406   except subprocess.CalledProcessError:
 407     pass
 408
 409   config.darwin_osx_version = osx_version
 410
 411   # Detect x86_64h
 412   try:
 413     output = subprocess.check_output(["sysctl", "hw.cpusubtype"])
 414     output_re = re.match("^hw.cpusubtype: ([0-9]+)$", output)
 415     if output_re:
 416       cpu_subtype = int(output_re.group(1))
 417       if cpu_subtype == 8: # x86_64h
 418         config.available_features.add('x86_64h')
 419   except:
 420     pass
 421
 422   # 32-bit iOS simulator is deprecated and removed in latest Xcode.
 423   if config.apple_platform == "iossim":
 424     if config.target_arch == "i386":
 425       config.unsupported = True
 426
 427   def get_macos_aligned_version(macos_vers):
 428     platform = config.apple_platform
 429     if platform == 'osx':
 430       return macos_vers
 431
 432     macos_major, macos_minor = macos_vers
 433     assert macos_major >= 10
 434
 435     if macos_major == 10:  # macOS 10.x
 436       major = macos_minor
 437       minor = 0
 438     else:                  # macOS 11+
 439       major = macos_major + 5
 440       minor = macos_minor
 441
 442     assert major >= 11
 443
 444     if platform.startswith('ios') or platform.startswith('tvos'):
 445       major -= 2
 446     elif platform.startswith('watch'):
 447       major -= 9
 448     else:
 449       lit_config.fatal("Unsupported apple platform '{}'".format(platform))
 450
 451     return (major, minor)
 452
 453   for vers in min_macos_deployment_target_substitutions:
 454     flag = config.apple_platform_min_deployment_target_flag
 455     major, minor = get_macos_aligned_version(vers)
 456     if 'mtargetos' in flag:
 457       sim = '-simulator' if 'sim' in config.apple_platform else ''
 458       config.substitutions.append( ('%%min_macos_deployment_target=%s.%s' % vers, '{}{}.{}{}'.format(flag, major, minor, sim)) )
 459     else:
 460       config.substitutions.append( ('%%min_macos_deployment_target=%s.%s' % vers, '{}={}.{}'.format(flag, major, minor)) )
 461 else:
 462   for vers in min_macos_deployment_target_substitutions:
 463     config.substitutions.append( ('%%min_macos_deployment_target=%s.%s' % vers, '') )
 464
 465 if config.android:
 466   env = os.environ.copy()
 467   if config.android_serial:
 468     env['ANDROID_SERIAL'] = config.android_serial
 469     config.environment['ANDROID_SERIAL'] = config.android_serial
 470
 471   adb = os.environ.get('ADB', 'adb')
 472
 473   # These are needed for tests to upload/download temp files, such as
 474   # suppression-files, to device.
 475   config.substitutions.append( ('%device_rundir/', "/data/local/tmp/Output/") )
 476   config.substitutions.append( ('%push_to_device', "%s -s '%s' push " % (adb, env['ANDROID_SERIAL']) ) )
 477   config.substitutions.append( ('%adb_shell ', "%s -s '%s' shell " % (adb, env['ANDROID_SERIAL']) ) )
 478   config.substitutions.append( ('%device_rm', "%s -s '%s' shell 'rm ' " % (adb, env['ANDROID_SERIAL']) ) )
 479
 480   try:
 481     android_api_level_str = subprocess.check_output([adb, "shell", "getprop", "ro.build.version.sdk"], env=env).rstrip()
 482     android_api_codename = subprocess.check_output([adb, "shell", "getprop", "ro.build.version.codename"], env=env).rstrip().decode("utf-8")
 483   except (subprocess.CalledProcessError, OSError):
 484     lit_config.fatal("Failed to read ro.build.version.sdk (using '%s' as adb)" % adb)
 485   try:
 486     android_api_level = int(android_api_level_str)
 487   except ValueError:
 488     lit_config.fatal("Failed to read ro.build.version.sdk (using '%s' as adb): got '%s'" % (adb, android_api_level_str))
 489   android_api_level = min(android_api_level, int(config.android_api_level))
 490   for required in [26, 28, 29, 30]:
 491     if android_api_level >= required:
 492       config.available_features.add('android-%s' % required)
 493   # FIXME: Replace with appropriate version when availible.
 494   if android_api_level > 30 or (android_api_level == 30 and android_api_codename == 'S'):
 495     config.available_features.add('android-thread-properties-api')
 496
 497   # Prepare the device.
 498   android_tmpdir = '/data/local/tmp/Output'
 499   subprocess.check_call([adb, "shell", "mkdir", "-p", android_tmpdir], env=env)
 500   for file in config.android_files_to_push:
 501     subprocess.check_call([adb, "push", file, android_tmpdir], env=env)
 502 else:
 503   config.substitutions.append( ('%device_rundir/', "") )
 504   config.substitutions.append( ('%push_to_device', "echo ") )
 505   config.substitutions.append( ('%adb_shell', "echo ") )
 506
 507 if config.host_os == 'Linux':
 508   # detect whether we are using glibc, and which version
 509   # NB: 'ldd' is just one of the tools commonly installed as part of glibc/musl
 510   ldd_ver_cmd = subprocess.Popen(['ldd', '--version'],
 511                                  stdout=subprocess.PIPE,
 512                                  stderr=subprocess.DEVNULL,
 513                                  env={'LANG': 'C'})
 514   sout, _ = ldd_ver_cmd.communicate()
 515   ver_lines = sout.splitlines()
 516   if not config.android and len(ver_lines) and ver_lines[0].startswith(b"ldd "):
 517     from distutils.version import LooseVersion
 518     ver = LooseVersion(ver_lines[0].split()[-1].decode())
 519     for required in ["2.27", "2.30", "2.34"]:
 520       if ver >= LooseVersion(required):
 521         config.available_features.add("glibc-" + required)
 522
 523 sancovcc_path = os.path.join(config.llvm_tools_dir, "sancov")
 524 if os.path.exists(sancovcc_path):
 525   config.available_features.add("has_sancovcc")
 526   config.substitutions.append( ("%sancovcc ", sancovcc_path) )
 527
 528 def liblto_path():
 529   return os.path.join(config.llvm_shlib_dir, 'libLTO.dylib')
 530
 531 def is_darwin_lto_supported():
 532   return os.path.exists(liblto_path())
 533
 534 def is_binutils_lto_supported():
 535   if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'LLVMgold.so')):
 536     return False
 537
 538   # We require both ld.bfd and ld.gold exist and support plugins. They are in
 539   # the same repository 'binutils-gdb' and usually built together.
 540   for exe in (config.gnu_ld_executable, config.gold_executable):
 541     try:
 542       ld_cmd = subprocess.Popen([exe, '--help'], stdout=subprocess.PIPE, env={'LANG': 'C'})
 543       ld_out = ld_cmd.stdout.read().decode()
 544       ld_cmd.wait()
 545     except OSError:
 546       return False
 547     if not '-plugin' in ld_out:
 548       return False
 549
 550   return True
 551
 552 def is_windows_lto_supported():
 553   return os.path.exists(os.path.join(config.llvm_tools_dir, 'lld-link.exe'))
 554
 555 if config.host_os == 'Darwin' and is_darwin_lto_supported():
 556   config.lto_supported = True
 557   config.lto_flags = [ '-Wl,-lto_library,' + liblto_path() ]
 558 elif config.host_os in ['Linux', 'FreeBSD', 'NetBSD']:
 559   config.lto_supported = False
 560   if config.use_lld:
 561     config.lto_supported = True
 562   if is_binutils_lto_supported():
 563     config.available_features.add('binutils_lto')
 564     config.lto_supported = True
 565
 566   if config.lto_supported:
 567     if config.use_lld:
 568       config.lto_flags = ["-fuse-ld=lld"]
 569     else:
 570       config.lto_flags = ["-fuse-ld=gold"]
 571 elif config.host_os == 'Windows' and is_windows_lto_supported():
 572   config.lto_supported = True
 573   config.lto_flags = ["-fuse-ld=lld"]
 574 else:
 575   config.lto_supported = False
 576
 577 if config.lto_supported:
 578   config.available_features.add('lto')
 579   if config.use_thinlto:
 580     config.available_features.add('thinlto')
 581     config.lto_flags += ["-flto=thin"]
 582   else:
 583     config.lto_flags += ["-flto"]
 584
 585 if config.have_rpc_xdr_h:
 586   config.available_features.add('sunrpc')
 587
 588 # Ask llvm-config about assertion mode.
 589 try:
 590   llvm_config_cmd = subprocess.Popen(
 591       [os.path.join(config.llvm_tools_dir, 'llvm-config'), '--assertion-mode'],
 592       stdout = subprocess.PIPE,
 593       env=config.environment)
 594 except OSError as e:
 595   print("Could not launch llvm-config in " + config.llvm_tools_dir)
 596   print("    Failed with error #{0}: {1}".format(e.errno, e.strerror))
 597   exit(42)
 598
 599 if re.search(r'ON', llvm_config_cmd.stdout.read().decode('ascii')):
 600   config.available_features.add('asserts')
 601 llvm_config_cmd.wait()
 602
 603 # Sanitizer tests tend to be flaky on Windows due to PR24554, so add some
 604 # retries. We don't do this on otther platforms because it's slower.
 605 if platform.system() == 'Windows':
 606   config.test_retry_attempts = 2
 607
 608 # No throttling on non-Darwin platforms.
 609 lit_config.parallelism_groups['shadow-memory'] = None
 610
 611 if platform.system() == 'Darwin':
 612   ios_device = config.apple_platform != 'osx' and not config.apple_platform.endswith('sim')
 613   # Force sequential execution when running tests on iOS devices.
 614   if ios_device:
 615     lit_config.warning('Forcing sequential execution for iOS device tests')
 616     lit_config.parallelism_groups['ios-device'] = 1
 617     config.parallelism_group = 'ios-device'
 618
 619   # Only run up to 3 processes that require shadow memory simultaneously on
 620   # 64-bit Darwin. Using more scales badly and hogs the system due to
 621   # inefficient handling of large mmap'd regions (terabytes) by the kernel.
 622   else:
 623     lit_config.warning('Throttling sanitizer tests that require shadow memory on Darwin')
 624     lit_config.parallelism_groups['shadow-memory'] = 3
 625
 626 # Multiple substitutions are necessary to support multiple shared objects used
 627 # at once.
 628 # Note that substitutions with numbers have to be defined first to avoid
 629 # being subsumed by substitutions with smaller postfix.
 630 for postfix in ["2", "1", ""]:
 631   if config.host_os == 'Darwin':
 632     config.substitutions.append( ("%ld_flags_rpath_exe" + postfix, '-Wl,-rpath,@executable_path/ %dynamiclib' + postfix) )
 633     config.substitutions.append( ("%ld_flags_rpath_so" + postfix, '-install_name @rpath/`basename %dynamiclib{}`'.format(postfix)) )
 634   elif config.host_os in ('FreeBSD', 'NetBSD', 'OpenBSD'):
 635     config.substitutions.append( ("%ld_flags_rpath_exe" + postfix, "-Wl,-z,origin -Wl,-rpath,\$ORIGIN -L%T -l%xdynamiclib_namespec" + postfix) )
 636     config.substitutions.append( ("%ld_flags_rpath_so" + postfix, '') )
 637   elif config.host_os == 'Linux':
 638     config.substitutions.append( ("%ld_flags_rpath_exe" + postfix, "-Wl,-rpath,\$ORIGIN -L%T -l%xdynamiclib_namespec" + postfix) )
 639     config.substitutions.append( ("%ld_flags_rpath_so" + postfix, '') )
 640   elif config.host_os == 'SunOS':
 641     config.substitutions.append( ("%ld_flags_rpath_exe" + postfix, "-Wl,-R\$ORIGIN -L%T -l%xdynamiclib_namespec" + postfix) )
 642     config.substitutions.append( ("%ld_flags_rpath_so" + postfix, '') )
 643
 644   # Must be defined after the substitutions that use %dynamiclib.
 645   config.substitutions.append( ("%dynamiclib" + postfix, '%T/%xdynamiclib_filename' + postfix) )
 646   config.substitutions.append( ("%xdynamiclib_filename" + postfix, 'lib%xdynamiclib_namespec{}.so'.format(postfix)) )
 647   config.substitutions.append( ("%xdynamiclib_namespec", '%basename_t.dynamic') )
 648
 649 # Provide a substitution that can be used to tell Clang to use a static libstdc++.
 650 # The substitution expands to nothing on non Linux platforms.
 651 # FIXME: This should check the target OS, not the host OS.
 652 if config.host_os == 'Linux':
 653   config.substitutions.append( ("%linux_static_libstdcplusplus", "-stdlib=libstdc++ -static-libstdc++") )
 654 else:
 655   config.substitutions.append( ("%linux_static_libstdcplusplus", "") )
 656
 657 config.default_sanitizer_opts = []
 658 if config.host_os == 'Darwin':
 659   # On Darwin, we default to `abort_on_error=1`, which would make tests run
 660   # much slower. Let's override this and run lit tests with 'abort_on_error=0'.
 661   config.default_sanitizer_opts += ['abort_on_error=0']
 662   config.default_sanitizer_opts += ['log_to_syslog=0']
 663   if lit.util.which('log'):
 664     # Querying the log can only done by a privileged user so
 665     # so check if we can query the log.
 666     exit_code = -1
 667     with open('/dev/null', 'r') as f:
 668       # Run a `log show` command the should finish fairly quickly and produce very little output.
 669       exit_code = subprocess.call(['log', 'show', '--last', '1m', '--predicate', '1 == 0'], stdout=f, stderr=f)
 670     if exit_code == 0:
 671       config.available_features.add('darwin_log_cmd')
 672     else:
 673       lit_config.warning('log command found but cannot queried')
 674   else:
 675     lit_config.warning('log command not found. Some tests will be skipped.')
 676 elif config.android:
 677   config.default_sanitizer_opts += ['abort_on_error=0']
 678
 679 # Allow tests to use REQUIRES=stable-runtime.  For use when you cannot use XFAIL
 680 # because the test hangs or fails on one configuration and not the other.
 681 if config.android or (config.target_arch not in ['arm', 'armhf', 'aarch64']):
 682   config.available_features.add('stable-runtime')
 683
 684 if config.asan_shadow_scale:
 685   config.available_features.add("shadow-scale-%s" % config.asan_shadow_scale)
 686 else:
 687   config.available_features.add("shadow-scale-3")
 688
 689 if config.memprof_shadow_scale:
 690   config.available_features.add("memprof-shadow-scale-%s" % config.memprof_shadow_scale)
 691 else:
 692   config.available_features.add("memprof-shadow-scale-3")
 693
 694 if config.expensive_checks:
 695   config.available_features.add("expensive_checks")
 696
 697 # Propagate the LLD/LTO into the clang config option, so nothing else is needed.
 698 run_wrapper = []
 699 target_cflags = [getattr(config, 'target_cflags', None)]
 700 extra_cflags = []
 701
 702 if config.use_lto and config.lto_supported:
 703   extra_cflags += config.lto_flags
 704 elif config.use_lto and (not config.lto_supported):
 705   config.unsupported = True
 706
 707 if config.use_lld and config.has_lld and not config.use_lto:
 708   extra_cflags += ["-fuse-ld=lld"]
 709 elif config.use_lld and (not config.has_lld):
 710   config.unsupported = True
 711
 712 # Append any extra flags passed in lit_config
 713 append_target_cflags = lit_config.params.get('append_target_cflags', None)
 714 if append_target_cflags:
 715   lit_config.note('Appending to extra_cflags: "{}"'.format(append_target_cflags))
 716   extra_cflags += [append_target_cflags]
 717
 718 config.clang = " " + " ".join(run_wrapper + [config.compile_wrapper, config.clang]) + " "
 719 config.target_cflags = " " + " ".join(target_cflags + extra_cflags) + " "
 720
 721 if config.host_os == 'Darwin':
 722   config.substitutions.append((
 723     "%get_pid_from_output",
 724     "{} {}/get_pid_from_output.py".format(
 725       sh_quote(config.python_executable),
 726       sh_quote(get_ios_commands_dir())
 727     ))
 728   )
 729   config.substitutions.append(
 730     ("%print_crashreport_for_pid",
 731     "{} {}/print_crashreport_for_pid.py".format(
 732       sh_quote(config.python_executable),
 733       sh_quote(get_ios_commands_dir())
 734     ))
 735   )