tools/clang/scripts/update.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Windows can't run .sh files, so this is a Python implementation of
   7 update.sh. This script should replace update.sh on all platforms eventually."""
   8
   9 import argparse
  10 import contextlib
  11 import cStringIO
  12 import glob
  13 import os
  14 import pipes
  15 import re
  16 import shutil
  17 import subprocess
  18 import stat
  19 import sys
  20 import tarfile
  21 import time
  22 import urllib2
  23 import zipfile
  24
  25 # Do NOT CHANGE this if you don't know what you're doing -- see
  26 # https://code.google.com/p/chromium/wiki/UpdatingClang
  27 # Reverting problematic clang rolls is safe, though.
  28 # Note: this revision is only used for Windows. Other platforms use update.sh.
  29 # TODO(thakis): Use the same revision on Windows and non-Windows.
  30 # TODO(thakis): Remove update.sh, use update.py everywhere.
  31 LLVM_WIN_REVISION = '245965'
  32
  33 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
  34 if use_head_revision:
  35   LLVM_WIN_REVISION = 'HEAD'
  36
  37 # This is incremented when pushing a new build of Clang at the same revision.
  38 CLANG_SUB_REVISION=1
  39
  40 PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION)
  41
  42 # Path constants. (All of these should be absolute paths.)
  43 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  44 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
  45 THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
  46 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
  47 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
  48 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
  49                                           'llvm-bootstrap-install')
  50 CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
  51 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
  52                               'Release+Asserts')
  53 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
  54 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
  55 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
  56 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
  57 LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
  58 LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
  59 LLVM_BUILD_TOOLS_DIR = os.path.abspath(
  60     os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
  61 STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')
  62 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
  63 VERSION = '3.8.0'
  64
  65 # URL for pre-built binaries.
  66 CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
  67
  68 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
  69 if 'LLVM_REPO_URL' in os.environ:
  70   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
  71
  72
  73 def DownloadUrl(url, output_file):
  74   """Download url into output_file."""
  75   CHUNK_SIZE = 4096
  76   TOTAL_DOTS = 10
  77   sys.stdout.write('Downloading %s ' % url)
  78   sys.stdout.flush()
  79   response = urllib2.urlopen(url)
  80   total_size = int(response.info().getheader('Content-Length').strip())
  81   bytes_done = 0
  82   dots_printed = 0
  83   while True:
  84     chunk = response.read(CHUNK_SIZE)
  85     if not chunk:
  86       break
  87     output_file.write(chunk)
  88     bytes_done += len(chunk)
  89     num_dots = TOTAL_DOTS * bytes_done / total_size
  90     sys.stdout.write('.' * (num_dots - dots_printed))
  91     sys.stdout.flush()
  92     dots_printed = num_dots
  93   print ' Done.'
  94
  95
  96 def ReadStampFile():
  97   """Return the contents of the stamp file, or '' if it doesn't exist."""
  98   try:
  99     with open(STAMP_FILE, 'r') as f:
 100       return f.read()
 101   except IOError:
 102     return ''
 103
 104
 105 def WriteStampFile(s):
 106   """Write s to the stamp file."""
 107   if not os.path.exists(os.path.dirname(STAMP_FILE)):
 108     os.makedirs(os.path.dirname(STAMP_FILE))
 109   with open(STAMP_FILE, 'w') as f:
 110     f.write(s)
 111
 112
 113 def GetSvnRevision(svn_repo):
 114   """Returns current revision of the svn repo at svn_repo."""
 115   svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
 116   m = re.search(r'Revision: (\d+)', svn_info)
 117   return m.group(1)
 118
 119
 120 def RmTree(dir):
 121   """Delete dir."""
 122   def ChmodAndRetry(func, path, _):
 123     # Subversion can leave read-only files around.
 124     if not os.access(path, os.W_OK):
 125       os.chmod(path, stat.S_IWUSR)
 126       return func(path)
 127     raise
 128
 129   shutil.rmtree(dir, onerror=ChmodAndRetry)
 130
 131
 132 def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
 133   """Run command and return success (True) or failure; or if fail_hard is
 134      True, exit on failure.  If msvc_arch is set, runs the command in a
 135      shell with the msvc tools for that architecture."""
 136
 137   if msvc_arch and sys.platform == 'win32':
 138     command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
 139
 140   # https://docs.python.org/2/library/subprocess.html:
 141   # "On Unix with shell=True [...] if args is a sequence, the first item
 142   # specifies the command string, and any additional items will be treated as
 143   # additional arguments to the shell itself.  That is to say, Popen does the
 144   # equivalent of:
 145   #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
 146   #
 147   # We want to pass additional arguments to command[0], not to the shell,
 148   # so manually join everything into a single string.
 149   # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
 150   # quote c:\path but svn can't handle quoted paths on Windows.  Since on
 151   # Windows follow-on args are passed to args[0] instead of the shell, don't
 152   # do the single-string transformation there.
 153   if sys.platform != 'win32':
 154     command = ' '.join([pipes.quote(c) for c in command])
 155   print 'Running', command
 156   if subprocess.call(command, env=env, shell=True) == 0:
 157     return True
 158   print 'Failed.'
 159   if fail_hard:
 160     sys.exit(1)
 161   return False
 162
 163
 164 def CopyFile(src, dst):
 165   """Copy a file from src to dst."""
 166   shutil.copy(src, dst)
 167   print "Copying %s to %s" % (src, dst)
 168
 169
 170 def CopyDirectoryContents(src, dst, filename_filter=None):
 171   """Copy the files from directory src to dst
 172   with an optional filename filter."""
 173   if not os.path.exists(dst):
 174     os.makedirs(dst)
 175   for root, _, files in os.walk(src):
 176     for f in files:
 177       if filename_filter and not re.match(filename_filter, f):
 178         continue
 179       CopyFile(os.path.join(root, f), dst)
 180
 181
 182 def Checkout(name, url, dir):
 183   """Checkout the SVN module at url into dir. Use name for the log message."""
 184   print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir)
 185
 186   command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir]
 187   if RunCommand(command, fail_hard=False):
 188     return
 189
 190   if os.path.isdir(dir):
 191     print "Removing %s." % (dir)
 192     RmTree(dir)
 193
 194   print "Retrying."
 195   RunCommand(command)
 196
 197
 198 def RevertPreviouslyPatchedFiles():
 199   print 'Reverting previously patched files'
 200   files = [
 201     '%(clang)s/test/Index/crash-recovery-modules.m',
 202     '%(clang)s/unittests/libclang/LibclangTest.cpp',
 203     '%(compiler_rt)s/lib/asan/asan_rtl.cc',
 204     '%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc',
 205     '%(llvm)s/test/DebugInfo/gmlt.ll',
 206     '%(llvm)s/lib/CodeGen/SpillPlacement.cpp',
 207     '%(llvm)s/lib/CodeGen/SpillPlacement.h',
 208     '%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp',
 209     '%(clang)s/test/Driver/env.c',
 210     '%(clang)s/lib/Frontend/InitPreprocessor.cpp',
 211     '%(clang)s/test/Frontend/exceptions.c',
 212     '%(clang)s/test/Preprocessor/predefined-exceptions.m',
 213     '%(llvm)s/test/Bindings/Go/go.test',
 214     '%(clang)s/lib/Parse/ParseExpr.cpp',
 215     '%(clang)s/lib/Parse/ParseTemplate.cpp',
 216     '%(clang)s/lib/Sema/SemaDeclCXX.cpp',
 217     '%(clang)s/lib/Sema/SemaExprCXX.cpp',
 218     '%(clang)s/test/SemaCXX/default2.cpp',
 219     '%(clang)s/test/SemaCXX/typo-correction-delayed.cpp',
 220     '%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc',
 221     '%(compiler_rt)s/test/tsan/signal_segv_handler.cc',
 222     '%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc',
 223     '%(compiler_rt)s/cmake/config-ix.cmake',
 224     '%(compiler_rt)s/CMakeLists.txt',
 225     '%(compiler_rt)s/lib/ubsan/ubsan_platform.h',
 226     ]
 227   for f in files:
 228     f = f % {
 229         'clang': CLANG_DIR,
 230         'compiler_rt': COMPILER_RT_DIR,
 231         'llvm': LLVM_DIR,
 232         }
 233     if os.path.exists(f):
 234       os.remove(f)  # For unversioned files.
 235       RunCommand(['svn', 'revert', f])
 236
 237
 238 def ApplyLocalPatches():
 239   # There's no patch program on Windows by default.  We don't need patches on
 240   # Windows yet, and maybe this not working on Windows will motivate us to
 241   # remove patches over time.
 242   assert sys.platform != 'win32'
 243
 244   # No patches.
 245
 246
 247 def DeleteChromeToolsShim():
 248   OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
 249   shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
 250   shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
 251
 252
 253 def CreateChromeToolsShim():
 254   """Hooks the Chrome tools into the LLVM build.
 255
 256   Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
 257   detects implicit tools in the tools subdirectory, so this helper install a
 258   shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
 259
 260   Note that the shim directory name intentionally has no - or _. The implicit
 261   tool detection logic munges them in a weird way."""
 262   assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
 263   os.mkdir(CHROME_TOOLS_SHIM_DIR)
 264   with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
 265     f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
 266             'Do not edit.\n')
 267     f.write('# Since tools/clang is located in another directory, use the \n')
 268     f.write('# two arg version to specify where build artifacts go. CMake\n')
 269     f.write('# disallows reuse of the same binary dir for multiple source\n')
 270     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
 271     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
 272     f.write('if (CHROMIUM_TOOLS_SRC)\n')
 273     f.write('  add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
 274               '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
 275     f.write('endif (CHROMIUM_TOOLS_SRC)\n')
 276
 277
 278 def AddCMakeToPath():
 279   """Download CMake and add it to PATH."""
 280   if sys.platform == 'win32':
 281     zip_name = 'cmake-3.2.2-win32-x86.zip'
 282     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
 283                              'cmake-3.2.2-win32-x86', 'bin')
 284   else:
 285     suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
 286     zip_name = 'cmake310_%s.tgz' % suffix
 287     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin')
 288   if not os.path.exists(cmake_dir):
 289     if not os.path.exists(LLVM_BUILD_TOOLS_DIR):
 290       os.makedirs(LLVM_BUILD_TOOLS_DIR)
 291     # The cmake archive is smaller than 20 MB, small enough to keep in memory:
 292     with contextlib.closing(cStringIO.StringIO()) as f:
 293       DownloadUrl(CDS_URL + '/tools/' + zip_name, f)
 294       f.seek(0)
 295       if zip_name.endswith('.zip'):
 296         zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR)
 297       else:
 298         tarfile.open(mode='r:gz', fileobj=f).extractall(path=
 299             LLVM_BUILD_TOOLS_DIR)
 300   os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
 301
 302 vs_version = None
 303 def GetVSVersion():
 304   global vs_version
 305   if vs_version:
 306     return vs_version
 307
 308   # Try using the toolchain in depot_tools.
 309   # This sets environment variables used by SelectVisualStudioVersion below.
 310   sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
 311   import vs_toolchain
 312   vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
 313
 314   # Use gyp to find the MSVS installation, either in depot_tools as per above,
 315   # or a system-wide installation otherwise.
 316   sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
 317   import gyp.MSVSVersion
 318   vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013')
 319   return vs_version
 320
 321
 322 def UpdateClang(args):
 323   print 'Updating Clang to %s...' % PACKAGE_VERSION
 324   if ReadStampFile() == PACKAGE_VERSION:
 325     print 'Already up to date.'
 326     return 0
 327
 328   # Reset the stamp file in case the build is unsuccessful.
 329   WriteStampFile('')
 330
 331   if not args.force_local_build:
 332     cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
 333     cds_full_url = CDS_URL + '/Win/' + cds_file
 334
 335     # Check if there's a prebuilt binary and if so just fetch that. That's
 336     # faster, and goma relies on having matching binary hashes on client and
 337     # server too.
 338     print 'Trying to download prebuilt clang'
 339
 340     # clang packages are smaller than 50 MB, small enough to keep in memory.
 341     with contextlib.closing(cStringIO.StringIO()) as f:
 342       try:
 343         DownloadUrl(cds_full_url, f)
 344         f.seek(0)
 345         # TODO(thakis): Delete LLVM_BUILD_DIR before extracting.
 346         tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR)
 347         print 'clang %s unpacked' % PACKAGE_VERSION
 348         # Download the gold plugin if requested to by an environment variable.
 349         # This is used by the CFI ClusterFuzz bot.
 350         if 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ:
 351           RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
 352         WriteStampFile(PACKAGE_VERSION)
 353         return 0
 354       except urllib2.HTTPError:
 355         print 'Did not find prebuilt clang %s, building locally' % cds_file
 356
 357   AddCMakeToPath()
 358
 359   RevertPreviouslyPatchedFiles()
 360   DeleteChromeToolsShim()
 361
 362   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
 363   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
 364   if sys.platform == 'win32':
 365     Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
 366   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
 367   if sys.platform == 'darwin':
 368     # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
 369     # (i.e. this is needed for bootstrap builds).
 370     Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
 371     # While we're bundling our own libc++ on OS X, we need to compile libc++abi
 372     # into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
 373     Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR)
 374
 375   if args.with_patches and sys.platform != 'win32':
 376     ApplyLocalPatches()
 377
 378   cc, cxx = None, None
 379   libstdcpp = None
 380   if args.gcc_toolchain:  # This option is only used on Linux.
 381     # Use the specified gcc installation for building.
 382     cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
 383     cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
 384
 385     if not os.access(cc, os.X_OK):
 386       print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
 387       print '"%s" does not appear to be valid.' % cc
 388       return 1
 389
 390     # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
 391     # compiler, etc.) find the .so.
 392     libstdcpp = subprocess.check_output(
 393         [cxx, '-print-file-name=libstdc++.so.6']).rstrip()
 394     os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
 395
 396   cflags = cxxflags = ldflags = []
 397
 398   # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
 399   # needed, on OS X it requires libc++. clang only automatically links to libc++
 400   # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
 401   # on OS X versions as old as 10.7.
 402   # TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle
 403   # libc++.dylib.  Remove this once all bots are on 10.7+, then use
 404   # -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7.
 405   deployment_target = ''
 406
 407   if sys.platform == 'darwin':
 408     # When building on 10.9, /usr/include usually doesn't exist, and while
 409     # Xcode's clang automatically sets a sysroot, self-built clangs don't.
 410     cflags = ['-isysroot', subprocess.check_output(
 411         ['xcrun', '--show-sdk-path']).rstrip()]
 412     cxxflags = ['-stdlib=libc++', '-nostdinc++',
 413                 '-I' + os.path.join(LIBCXX_DIR, 'include')] + cflags
 414     if args.bootstrap:
 415       deployment_target = '10.6'
 416
 417   base_cmake_args = ['-GNinja',
 418                      '-DCMAKE_BUILD_TYPE=Release',
 419                      '-DLLVM_ENABLE_ASSERTIONS=ON',
 420                      '-DLLVM_ENABLE_THREADS=OFF',
 421                      ]
 422
 423   if args.bootstrap:
 424     print 'Building bootstrap compiler'
 425     if not os.path.exists(LLVM_BOOTSTRAP_DIR):
 426       os.makedirs(LLVM_BOOTSTRAP_DIR)
 427     os.chdir(LLVM_BOOTSTRAP_DIR)
 428     bootstrap_args = base_cmake_args + [
 429         '-DLLVM_TARGETS_TO_BUILD=host',
 430         '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
 431         '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 432         '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
 433         ]
 434     if cc is not None:  bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
 435     if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
 436     RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
 437     RunCommand(['ninja'], msvc_arch='x64')
 438     if args.run_tests:
 439       RunCommand(['ninja', 'check-all'], msvc_arch='x64')
 440     RunCommand(['ninja', 'install'], msvc_arch='x64')
 441     if args.gcc_toolchain:
 442       # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
 443       # compiler can start.
 444       CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
 445
 446     if sys.platform == 'win32':
 447       cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
 448       cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
 449       # CMake has a hard time with backslashes in compiler paths:
 450       # https://stackoverflow.com/questions/13050827
 451       cc = cc.replace('\\', '/')
 452       cxx = cxx.replace('\\', '/')
 453     else:
 454       cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
 455       cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
 456
 457     if args.gcc_toolchain:
 458       # Tell the bootstrap compiler to use a specific gcc prefix to search
 459       # for standard library headers and shared object files.
 460       cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
 461       cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
 462     print 'Building final compiler'
 463
 464   if sys.platform == 'darwin':
 465     # Build libc++.dylib while some bots are still on OS X 10.6.
 466     libcxxbuild = os.path.join(LLVM_BUILD_DIR, 'libcxxbuild')
 467     if os.path.isdir(libcxxbuild):
 468       RmTree(libcxxbuild)
 469     libcxxflags = ['-O3', '-std=c++11', '-fstrict-aliasing']
 470
 471     # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
 472     # into different subdirectories.
 473     os.makedirs(os.path.join(libcxxbuild, 'libcxx'))
 474     os.chdir(os.path.join(libcxxbuild, 'libcxx'))
 475     RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
 476                 glob.glob(os.path.join(LIBCXX_DIR, 'src', '*.cpp')))
 477
 478     os.makedirs(os.path.join(libcxxbuild, 'libcxxabi'))
 479     os.chdir(os.path.join(libcxxbuild, 'libcxxabi'))
 480     RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
 481                glob.glob(os.path.join(LIBCXXABI_DIR, 'src', '*.cpp')) +
 482                ['-I' + os.path.join(LIBCXXABI_DIR, 'include')])
 483
 484     os.chdir(libcxxbuild)
 485     libdir = os.path.join(LIBCXX_DIR, 'lib')
 486     RunCommand(['cc'] + glob.glob('libcxx/*.o') + glob.glob('libcxxabi/*.o') +
 487         ['-o', 'libc++.1.dylib', '-dynamiclib', '-nodefaultlibs',
 488          '-current_version', '1', '-compatibility_version', '1', '-lSystem',
 489          '-install_name', '@executable_path/libc++.dylib',
 490          '-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp',
 491          '-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp',
 492          '-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp'])
 493     if os.path.exists('libc++.dylib'):
 494       os.remove('libc++.dylib')
 495     os.symlink('libc++.1.dylib', 'libc++.dylib')
 496     ldflags += ['-stdlib=libc++', '-L' + libcxxbuild]
 497
 498     if args.bootstrap:
 499       # Now that the libc++ headers have been installed and libc++.dylib is
 500       # built, delete the libc++ checkout again so that it's not part of the
 501       # main build below -- the libc++(abi) tests don't pass on OS X in
 502       # bootstrap builds (http://llvm.org/PR24068)
 503       RmTree(LIBCXX_DIR)
 504       RmTree(LIBCXXABI_DIR)
 505       cxxflags = ['-stdlib=libc++', '-nostdinc++',
 506                   '-I' + os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR,
 507                                       'include/c++/v1')
 508                   ] + cflags
 509
 510   # Build clang.
 511   binutils_incdir = ''
 512   if sys.platform.startswith('linux'):
 513     binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
 514
 515   # If building at head, define a macro that plugins can use for #ifdefing
 516   # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa.
 517   if use_head_revision:
 518     cflags += ['-DLLVM_FORCE_HEAD_REVISION']
 519     cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
 520
 521   CreateChromeToolsShim()
 522
 523   deployment_env = None
 524   if deployment_target:
 525     deployment_env = os.environ.copy()
 526     deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
 527
 528   cmake_args = base_cmake_args + [
 529       '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
 530       '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
 531       '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 532       '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
 533       '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
 534       '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
 535       '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
 536       '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
 537       '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
 538       '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
 539   # TODO(thakis): Unconditionally append this to base_cmake_args instead once
 540   # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
 541   cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
 542   if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
 543   if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
 544
 545   if not os.path.exists(LLVM_BUILD_DIR):
 546     os.makedirs(LLVM_BUILD_DIR)
 547   os.chdir(LLVM_BUILD_DIR)
 548   RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
 549              msvc_arch='x64', env=deployment_env)
 550
 551   if args.gcc_toolchain:
 552     # Copy in the right stdlibc++.so.6 so clang can start.
 553     if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
 554       os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
 555     libstdcpp = subprocess.check_output(
 556         [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
 557     CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
 558
 559   RunCommand(['ninja'], msvc_arch='x64')
 560
 561   if args.tools:
 562     # If any Chromium tools were built, install those now.
 563     RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
 564
 565   if sys.platform == 'darwin':
 566     CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'),
 567              os.path.join(LLVM_BUILD_DIR, 'bin'))
 568     # See http://crbug.com/256342
 569     RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
 570   elif sys.platform.startswith('linux'):
 571     RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
 572
 573   # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
 574   # TODO(hans): Remove once the regular build above produces this.
 575   if not os.path.exists(COMPILER_RT_BUILD_DIR):
 576     os.makedirs(COMPILER_RT_BUILD_DIR)
 577   os.chdir(COMPILER_RT_BUILD_DIR)
 578   # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
 579   # above).
 580   #if args.bootstrap and sys.platform == 'win32':
 581     # The bootstrap compiler produces 64-bit binaries by default.
 582     #cflags += ['-m32']
 583     #cxxflags += ['-m32']
 584   compiler_rt_args = base_cmake_args + [
 585       '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 586       '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
 587   if sys.platform != 'win32':
 588     compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
 589                          os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
 590                         '-DSANITIZER_MIN_OSX_VERSION="10.7"']
 591   RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR],
 592               msvc_arch='x86', env=deployment_env)
 593   RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
 594
 595   # TODO(hans): Make this (and the .gypi and .isolate files) version number
 596   # independent.
 597   if sys.platform == 'win32':
 598     platform = 'windows'
 599   elif sys.platform == 'darwin':
 600     platform = 'darwin'
 601   else:
 602     assert sys.platform.startswith('linux')
 603     platform = 'linux'
 604   asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
 605                                      VERSION, 'lib', platform)
 606   asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 607                                      VERSION, 'lib', platform)
 608   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 609                         r'^.*-i386\.lib$')
 610   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 611                         r'^.*-i386\.dll$')
 612
 613   CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
 614            os.path.join(asan_rt_lib_dst_dir, '..', '..'))
 615
 616   if sys.platform == 'win32':
 617     # Make an extra copy of the sanitizer headers, to be put on the include path
 618     # of the fallback compiler.
 619     sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 620                                          VERSION, 'include', 'sanitizer')
 621     aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 622                                              VERSION, 'include_sanitizer',
 623                                              'sanitizer')
 624     if not os.path.exists(aux_sanitizer_include_dir):
 625       os.makedirs(aux_sanitizer_include_dir)
 626     for _, _, files in os.walk(sanitizer_include_dir):
 627       for f in files:
 628         CopyFile(os.path.join(sanitizer_include_dir, f),
 629                  aux_sanitizer_include_dir)
 630
 631   # Run tests.
 632   if args.run_tests or use_head_revision:
 633     os.chdir(LLVM_BUILD_DIR)
 634     RunCommand(GetVSVersion().SetupScript('x64') +
 635                ['&&', 'ninja', 'cr-check-all'])
 636   if args.run_tests:
 637     os.chdir(LLVM_BUILD_DIR)
 638     RunCommand(GetVSVersion().SetupScript('x64') +
 639                ['&&', 'ninja', 'check-all'])
 640
 641   WriteStampFile(PACKAGE_VERSION)
 642   print 'Clang update was successful.'
 643   return 0
 644
 645
 646 def main():
 647   if not sys.platform in ['win32', 'cygwin']:
 648     # For non-Windows, fall back to update.sh.
 649     # TODO(hans): Make update.py replace update.sh completely.
 650
 651     # This script is called by gclient. gclient opens its hooks subprocesses
 652     # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does
 653     # custom output processing that breaks printing '\r' characters for
 654     # single-line updating status messages as printed by curl and wget.
 655     # Work around this by setting stderr of the update.sh process to stdin (!):
 656     # gclient doesn't redirect stdin, and while stdin itself is read-only, a
 657     # dup()ed sys.stdin is writable, try
 658     #   fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
 659     # TODO: Fix gclient instead, http://crbug.com/95350
 660     if '--no-stdin-hack' in sys.argv:
 661       sys.argv.remove('--no-stdin-hack')
 662       stderr = None
 663     else:
 664       try:
 665         stderr = os.fdopen(os.dup(sys.stdin.fileno()))
 666       except:
 667         stderr = sys.stderr
 668     return subprocess.call(
 669         [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:],
 670         stderr=stderr)
 671
 672   parser = argparse.ArgumentParser(description='Build Clang.')
 673   parser.add_argument('--bootstrap', action='store_true',
 674                       help='first build clang with CC, then with itself.')
 675   parser.add_argument('--if-needed', action='store_true',
 676                       help="run only if the script thinks clang is needed")
 677   parser.add_argument('--force-local-build', action='store_true',
 678                       help="don't try to download prebuild binaries")
 679   parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
 680                       'version be used for building; --gcc-toolchain=/opt/foo '
 681                       'picks /opt/foo/bin/gcc')
 682   parser.add_argument('--print-revision', action='store_true',
 683                       help='print current clang revision and exit.')
 684   parser.add_argument('--print-clang-version', action='store_true',
 685                       help='print current clang version (e.g. x.y.z) and exit.')
 686   parser.add_argument('--run-tests', action='store_true',
 687                       help='run tests after building; only for local builds')
 688   parser.add_argument('--tools', nargs='*',
 689                       help='select which chrome tools to build',
 690                       default=['plugins', 'blink_gc_plugin'])
 691   parser.add_argument('--without-patches', action='store_false',
 692                       help="don't apply patches (default)", dest='with_patches',
 693                       default=True)
 694
 695   # For now, these flags are only used for the non-Windows flow, but argparser
 696   # gets mad if it sees a flag it doesn't recognize.
 697   parser.add_argument('--no-stdin-hack', action='store_true')
 698
 699   args = parser.parse_args()
 700
 701   if args.if_needed:
 702     is_clang_required = False
 703     # clang is always used on Mac and Linux.
 704     if sys.platform == 'darwin' or sys.platform.startswith('linux'):
 705       is_clang_required = True
 706     # clang requested via $GYP_DEFINES.
 707     if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
 708                  os.environ.get('GYP_DEFINES', '')):
 709       is_clang_required = True
 710     # clang previously downloaded, keep it up-to-date.
 711     # If you don't want this, delete third_party/llvm-build on your machine.
 712     if os.path.isdir(LLVM_BUILD_DIR):
 713       is_clang_required = True
 714     if not is_clang_required:
 715       return 0
 716     if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
 717       print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
 718       return 0
 719
 720   global LLVM_WIN_REVISION, PACKAGE_VERSION
 721   if args.print_revision:
 722     if use_head_revision:
 723       print GetSvnRevision(LLVM_DIR)
 724     else:
 725       print PACKAGE_VERSION
 726     return 0
 727
 728   if args.print_clang_version:
 729     sys.stdout.write(VERSION)
 730     return 0
 731
 732   # Don't buffer stdout, so that print statements are immediately flushed.
 733   # Do this only after --print-revision has been handled, else we'll get
 734   # an error message when this script is run from gn for some reason.
 735   sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
 736
 737   if use_head_revision:
 738     # Use a real revision number rather than HEAD to make sure that the stamp
 739     # file logic works.
 740     LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL)
 741     PACKAGE_VERSION = LLVM_WIN_REVISION + '-0'
 742
 743     args.force_local_build = True
 744     # Skip local patches when using HEAD: they probably don't apply anymore.
 745     args.with_patches = False
 746
 747   return UpdateClang(args)
 748
 749
 750 if __name__ == '__main__':
 751   sys.exit(main())