tools/clang/scripts/update.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Windows can't run .sh files, so this is a Python implementation of
   7 update.sh. This script should replace update.sh on all platforms eventually."""
   8
   9 import argparse
  10 import contextlib
  11 import cStringIO
  12 import glob
  13 import os
  14 import pipes
  15 import re
  16 import shutil
  17 import subprocess
  18 import stat
  19 import sys
  20 import tarfile
  21 import time
  22 import urllib2
  23 import zipfile
  24
  25 # Do NOT CHANGE this if you don't know what you're doing -- see
  26 # https://code.google.com/p/chromium/wiki/UpdatingClang
  27 # Reverting problematic clang rolls is safe, though.
  28 # Note: this revision is only used for Windows. Other platforms use update.sh.
  29 # TODO(thakis): Use the same revision on Windows and non-Windows.
  30 # TODO(thakis): Remove update.sh, use update.py everywhere.
  31 LLVM_WIN_REVISION = '245402'
  32
  33 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
  34 if use_head_revision:
  35   LLVM_WIN_REVISION = 'HEAD'
  36
  37 # This is incremented when pushing a new build of Clang at the same revision.
  38 CLANG_SUB_REVISION=1
  39
  40 PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION)
  41
  42 # Path constants. (All of these should be absolute paths.)
  43 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  44 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
  45 THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
  46 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
  47 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
  48 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
  49                                           'llvm-bootstrap-install')
  50 CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
  51 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
  52                               'Release+Asserts')
  53 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
  54 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
  55 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
  56 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
  57 LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
  58 LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
  59 LLVM_BUILD_TOOLS_DIR = os.path.abspath(
  60     os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
  61 STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')
  62 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
  63 VERSION = '3.8.0'
  64
  65 # URL for pre-built binaries.
  66 CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
  67
  68 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
  69 if 'LLVM_REPO_URL' in os.environ:
  70   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
  71
  72
  73 def DownloadUrl(url, output_file):
  74   """Download url into output_file."""
  75   CHUNK_SIZE = 4096
  76   TOTAL_DOTS = 10
  77   sys.stdout.write('Downloading %s ' % url)
  78   sys.stdout.flush()
  79   response = urllib2.urlopen(url)
  80   total_size = int(response.info().getheader('Content-Length').strip())
  81   bytes_done = 0
  82   dots_printed = 0
  83   while True:
  84     chunk = response.read(CHUNK_SIZE)
  85     if not chunk:
  86       break
  87     output_file.write(chunk)
  88     bytes_done += len(chunk)
  89     num_dots = TOTAL_DOTS * bytes_done / total_size
  90     sys.stdout.write('.' * (num_dots - dots_printed))
  91     sys.stdout.flush()
  92     dots_printed = num_dots
  93   print ' Done.'
  94
  95
  96 def ReadStampFile():
  97   """Return the contents of the stamp file, or '' if it doesn't exist."""
  98   try:
  99     with open(STAMP_FILE, 'r') as f:
 100       return f.read()
 101   except IOError:
 102     return ''
 103
 104
 105 def WriteStampFile(s):
 106   """Write s to the stamp file."""
 107   if not os.path.exists(os.path.dirname(STAMP_FILE)):
 108     os.makedirs(os.path.dirname(STAMP_FILE))
 109   with open(STAMP_FILE, 'w') as f:
 110     f.write(s)
 111
 112
 113 def GetSvnRevision(svn_repo):
 114   """Returns current revision of the svn repo at svn_repo."""
 115   svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
 116   m = re.search(r'Revision: (\d+)', svn_info)
 117   return m.group(1)
 118
 119
 120 def RmTree(dir):
 121   """Delete dir."""
 122   def ChmodAndRetry(func, path, _):
 123     # Subversion can leave read-only files around.
 124     if not os.access(path, os.W_OK):
 125       os.chmod(path, stat.S_IWUSR)
 126       return func(path)
 127     raise
 128
 129   shutil.rmtree(dir, onerror=ChmodAndRetry)
 130
 131
 132 def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
 133   """Run command and return success (True) or failure; or if fail_hard is
 134      True, exit on failure.  If msvc_arch is set, runs the command in a
 135      shell with the msvc tools for that architecture."""
 136
 137   if msvc_arch and sys.platform == 'win32':
 138     command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
 139
 140   # https://docs.python.org/2/library/subprocess.html:
 141   # "On Unix with shell=True [...] if args is a sequence, the first item
 142   # specifies the command string, and any additional items will be treated as
 143   # additional arguments to the shell itself.  That is to say, Popen does the
 144   # equivalent of:
 145   #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
 146   #
 147   # We want to pass additional arguments to command[0], not to the shell,
 148   # so manually join everything into a single string.
 149   # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
 150   # quote c:\path but svn can't handle quoted paths on Windows.  Since on
 151   # Windows follow-on args are passed to args[0] instead of the shell, don't
 152   # do the single-string transformation there.
 153   if sys.platform != 'win32':
 154     command = ' '.join([pipes.quote(c) for c in command])
 155   print 'Running', command
 156   if subprocess.call(command, env=env, shell=True) == 0:
 157     return True
 158   print 'Failed.'
 159   if fail_hard:
 160     sys.exit(1)
 161   return False
 162
 163
 164 def CopyFile(src, dst):
 165   """Copy a file from src to dst."""
 166   shutil.copy(src, dst)
 167   print "Copying %s to %s" % (src, dst)
 168
 169
 170 def CopyDirectoryContents(src, dst, filename_filter=None):
 171   """Copy the files from directory src to dst
 172   with an optional filename filter."""
 173   if not os.path.exists(dst):
 174     os.makedirs(dst)
 175   for root, _, files in os.walk(src):
 176     for f in files:
 177       if filename_filter and not re.match(filename_filter, f):
 178         continue
 179       CopyFile(os.path.join(root, f), dst)
 180
 181
 182 def Checkout(name, url, dir):
 183   """Checkout the SVN module at url into dir. Use name for the log message."""
 184   print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir)
 185
 186   command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir]
 187   if RunCommand(command, fail_hard=False):
 188     return
 189
 190   if os.path.isdir(dir):
 191     print "Removing %s." % (dir)
 192     RmTree(dir)
 193
 194   print "Retrying."
 195   RunCommand(command)
 196
 197
 198 def RevertPreviouslyPatchedFiles():
 199   print 'Reverting previously patched files'
 200   files = [
 201     '%(clang)s/test/Index/crash-recovery-modules.m',
 202     '%(clang)s/unittests/libclang/LibclangTest.cpp',
 203     '%(compiler_rt)s/lib/asan/asan_rtl.cc',
 204     '%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc',
 205     '%(llvm)s/test/DebugInfo/gmlt.ll',
 206     '%(llvm)s/lib/CodeGen/SpillPlacement.cpp',
 207     '%(llvm)s/lib/CodeGen/SpillPlacement.h',
 208     '%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp',
 209     '%(clang)s/test/Driver/env.c',
 210     '%(clang)s/lib/Frontend/InitPreprocessor.cpp',
 211     '%(clang)s/test/Frontend/exceptions.c',
 212     '%(clang)s/test/Preprocessor/predefined-exceptions.m',
 213     '%(llvm)s/test/Bindings/Go/go.test',
 214     '%(clang)s/lib/Parse/ParseExpr.cpp',
 215     '%(clang)s/lib/Parse/ParseTemplate.cpp',
 216     '%(clang)s/lib/Sema/SemaDeclCXX.cpp',
 217     '%(clang)s/lib/Sema/SemaExprCXX.cpp',
 218     '%(clang)s/test/SemaCXX/default2.cpp',
 219     '%(clang)s/test/SemaCXX/typo-correction-delayed.cpp',
 220     '%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc',
 221     '%(compiler_rt)s/test/tsan/signal_segv_handler.cc',
 222     '%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc',
 223     '%(compiler_rt)s/cmake/config-ix.cmake',
 224     '%(compiler_rt)s/CMakeLists.txt',
 225     '%(compiler_rt)s/lib/ubsan/ubsan_platform.h',
 226     ]
 227   for f in files:
 228     f = f % {
 229         'clang': CLANG_DIR,
 230         'compiler_rt': COMPILER_RT_DIR,
 231         'llvm': LLVM_DIR,
 232         }
 233     if os.path.exists(f):
 234       os.remove(f)  # For unversioned files.
 235       RunCommand(['svn', 'revert', f])
 236
 237
 238 def ApplyLocalPatches():
 239   # There's no patch program on Windows by default.  We don't need patches on
 240   # Windows yet, and maybe this not working on Windows will motivate us to
 241   # remove patches over time.
 242   assert sys.platform != 'win32'
 243
 244   # No patches.
 245
 246
 247 def DeleteChromeToolsShim():
 248   OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
 249   shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
 250   shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
 251
 252
 253 def CreateChromeToolsShim():
 254   """Hooks the Chrome tools into the LLVM build.
 255
 256   Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
 257   detects implicit tools in the tools subdirectory, so this helper install a
 258   shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
 259
 260   Note that the shim directory name intentionally has no - or _. The implicit
 261   tool detection logic munges them in a weird way."""
 262   assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
 263   os.mkdir(CHROME_TOOLS_SHIM_DIR)
 264   with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
 265     f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
 266             'Do not edit.\n')
 267     f.write('# Since tools/clang is located in another directory, use the \n')
 268     f.write('# two arg version to specify where build artifacts go. CMake\n')
 269     f.write('# disallows reuse of the same binary dir for multiple source\n')
 270     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
 271     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
 272     f.write('if (CHROMIUM_TOOLS_SRC)\n')
 273     f.write('  add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
 274               '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
 275     f.write('endif (CHROMIUM_TOOLS_SRC)\n')
 276
 277
 278 def AddCMakeToPath():
 279   """Download CMake and add it to PATH."""
 280   if sys.platform == 'win32':
 281     zip_name = 'cmake-3.2.2-win32-x86.zip'
 282     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
 283                              'cmake-3.2.2-win32-x86', 'bin')
 284   else:
 285     suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
 286     zip_name = 'cmake310_%s.tgz' % suffix
 287     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin')
 288   if not os.path.exists(cmake_dir):
 289     if not os.path.exists(LLVM_BUILD_TOOLS_DIR):
 290       os.makedirs(LLVM_BUILD_TOOLS_DIR)
 291     # The cmake archive is smaller than 20 MB, small enough to keep in memory:
 292     with contextlib.closing(cStringIO.StringIO()) as f:
 293       DownloadUrl(CDS_URL + '/tools/' + zip_name, f)
 294       f.seek(0)
 295       if zip_name.endswith('.zip'):
 296         zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR)
 297       else:
 298         tarfile.open(mode='r:gz', fileobj=f).extractall(path=
 299             LLVM_BUILD_TOOLS_DIR)
 300   os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
 301
 302 vs_version = None
 303 def GetVSVersion():
 304   global vs_version
 305   if vs_version:
 306     return vs_version
 307
 308   # Try using the toolchain in depot_tools.
 309   # This sets environment variables used by SelectVisualStudioVersion below.
 310   sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
 311   import vs_toolchain
 312   vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
 313
 314   # Use gyp to find the MSVS installation, either in depot_tools as per above,
 315   # or a system-wide installation otherwise.
 316   sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
 317   import gyp.MSVSVersion
 318   vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013')
 319   return vs_version
 320
 321
 322 def UpdateClang(args):
 323   print 'Updating Clang to %s...' % PACKAGE_VERSION
 324   if ReadStampFile() == PACKAGE_VERSION:
 325     print 'Already up to date.'
 326     return 0
 327
 328   # Reset the stamp file in case the build is unsuccessful.
 329   WriteStampFile('')
 330
 331   if not args.force_local_build:
 332     cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
 333     cds_full_url = CDS_URL + '/Win/' + cds_file
 334
 335     # Check if there's a prebuilt binary and if so just fetch that. That's
 336     # faster, and goma relies on having matching binary hashes on client and
 337     # server too.
 338     print 'Trying to download prebuilt clang'
 339
 340     # clang packages are smaller than 50 MB, small enough to keep in memory.
 341     with contextlib.closing(cStringIO.StringIO()) as f:
 342       try:
 343         DownloadUrl(cds_full_url, f)
 344         f.seek(0)
 345         # TODO(thakis): Delete LLVM_BUILD_DIR before extracting.
 346         tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR)
 347         print 'clang %s unpacked' % PACKAGE_VERSION
 348         # Download the gold plugin if requested to by an environment variable.
 349         # This is used by the CFI ClusterFuzz bot.
 350         if 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ:
 351           RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
 352         WriteStampFile(PACKAGE_VERSION)
 353         return 0
 354       except urllib2.HTTPError:
 355         print 'Did not find prebuilt clang %s, building locally' % cds_file
 356
 357   AddCMakeToPath()
 358
 359   RevertPreviouslyPatchedFiles()
 360   DeleteChromeToolsShim()
 361
 362   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
 363   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
 364   if sys.platform == 'win32':
 365     Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
 366   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
 367   if sys.platform == 'darwin':
 368     # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
 369     # (i.e. this is needed for bootstrap builds).
 370     Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
 371     # While we're bundling our own libc++ on OS X, we need to compile libc++abi
 372     # into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
 373     Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR)
 374
 375   if args.with_patches and sys.platform != 'win32':
 376     ApplyLocalPatches()
 377
 378   cc, cxx = None, None
 379   libstdcpp = None
 380   if args.gcc_toolchain:  # This option is only used on Linux.
 381     # Use the specified gcc installation for building.
 382     cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
 383     cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
 384
 385     if not os.access(cc, os.X_OK):
 386       print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
 387       print '"%s" does not appear to be valid.' % cc
 388       return 1
 389
 390     # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
 391     # compiler, etc.) find the .so.
 392     libstdcpp = subprocess.check_output(
 393         [cxx, '-print-file-name=libstdc++.so.6']).rstrip()
 394     os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
 395
 396   cflags = cxxflags = ldflags = []
 397
 398   # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
 399   # needed, on OS X it requires libc++. clang only automatically links to libc++
 400   # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
 401   # on OS X versions as old as 10.7.
 402   # TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle
 403   # libc++.dylib.  Remove this once all bots are on 10.7+, then use
 404   # -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7.
 405   deployment_target = ''
 406
 407   if sys.platform == 'darwin':
 408     # When building on 10.9, /usr/include usually doesn't exist, and while
 409     # Xcode's clang automatically sets a sysroot, self-built clangs don't.
 410     cflags = ['-isysroot', subprocess.check_output(
 411         ['xcrun', '--show-sdk-path']).rstrip()]
 412     cxxflags = ['-stdlib=libc++', '-nostdinc++',
 413                 '-I' + os.path.join(LIBCXX_DIR, 'include')] + cflags
 414     if args.bootstrap:
 415       deployment_target = '10.6'
 416
 417   base_cmake_args = ['-GNinja',
 418                      '-DCMAKE_BUILD_TYPE=Release',
 419                      '-DLLVM_ENABLE_ASSERTIONS=ON',
 420                      '-DLLVM_ENABLE_THREADS=OFF',
 421                      ]
 422
 423   if args.bootstrap:
 424     print 'Building bootstrap compiler'
 425     if not os.path.exists(LLVM_BOOTSTRAP_DIR):
 426       os.makedirs(LLVM_BOOTSTRAP_DIR)
 427     os.chdir(LLVM_BOOTSTRAP_DIR)
 428     bootstrap_args = base_cmake_args + [
 429         '-DLLVM_TARGETS_TO_BUILD=host',
 430         '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
 431         '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 432         '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
 433         ]
 434     if cc is not None:  bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
 435     if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
 436     RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
 437     RunCommand(['ninja'], msvc_arch='x64')
 438     if args.run_tests:
 439       RunCommand(['ninja', 'check-all'], msvc_arch='x64')
 440     RunCommand(['ninja', 'install'], msvc_arch='x64')
 441     if args.gcc_toolchain:
 442       # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
 443       # compiler can start.
 444       CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
 445
 446     if sys.platform == 'win32':
 447       cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
 448       cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
 449       # CMake has a hard time with backslashes in compiler paths:
 450       # https://stackoverflow.com/questions/13050827
 451       cc = cc.replace('\\', '/')
 452       cxx = cxx.replace('\\', '/')
 453     else:
 454       cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
 455       cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
 456
 457     if args.gcc_toolchain:
 458       # Tell the bootstrap compiler to use a specific gcc prefix to search
 459       # for standard library headers and shared object files.
 460       cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
 461       cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
 462     print 'Building final compiler'
 463
 464   if sys.platform == 'darwin':
 465     # Build libc++.dylib while some bots are still on OS X 10.6.
 466     libcxxbuild = os.path.join(LLVM_BUILD_DIR, 'libcxxbuild')
 467     if os.path.isdir(libcxxbuild):
 468       RmTree(libcxxbuild)
 469     libcxxflags = ['-O3', '-std=c++11', '-fstrict-aliasing']
 470
 471     # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
 472     # into different subdirectories.
 473     os.makedirs(os.path.join(libcxxbuild, 'libcxx'))
 474     os.chdir(os.path.join(libcxxbuild, 'libcxx'))
 475     RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
 476                 glob.glob(os.path.join(LIBCXX_DIR, 'src', '*.cpp')))
 477
 478     os.makedirs(os.path.join(libcxxbuild, 'libcxxabi'))
 479     os.chdir(os.path.join(libcxxbuild, 'libcxxabi'))
 480     RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
 481                glob.glob(os.path.join(LIBCXXABI_DIR, 'src', '*.cpp')) +
 482                ['-I' + os.path.join(LIBCXXABI_DIR, 'include')])
 483
 484     os.chdir(libcxxbuild)
 485     libdir = os.path.join(LIBCXX_DIR, 'lib')
 486     RunCommand(['cc'] + glob.glob('libcxx/*.o') + glob.glob('libcxxabi/*.o') +
 487         ['-o', 'libc++.1.dylib', '-dynamiclib', '-nodefaultlibs',
 488          '-current_version', '1', '-compatibility_version', '1', '-lSystem',
 489          '-install_name', '@executable_path/libc++.dylib',
 490          '-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp',
 491          '-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp',
 492          '-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp'])
 493     if os.path.exists('libc++.dylib'):
 494       os.remove('libc++.dylib')
 495     os.symlink('libc++.1.dylib', 'libc++.dylib')
 496     ldflags += ['-stdlib=libc++', '-L' + libcxxbuild]
 497
 498     if args.bootstrap:
 499       # Now that the libc++ headers have been installed and libc++.dylib is
 500       # built, delete the libc++ checkout again so that it's not part of the
 501       # main build below -- the libc++(abi) tests don't pass on OS X in
 502       # bootstrap builds (http://llvm.org/PR24068)
 503       RmTree(LIBCXX_DIR)
 504       RmTree(LIBCXXABI_DIR)
 505       cxxflags = ['-stdlib=libc++', '-nostdinc++',
 506                   '-I' + os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR,
 507                                       'include/c++/v1')
 508                   ] + cflags
 509
 510   # Build clang.
 511   binutils_incdir = ''
 512   if sys.platform.startswith('linux'):
 513     binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
 514
 515   # If building at head, define a macro that plugins can use for #ifdefing
 516   # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa.
 517   if use_head_revision:
 518     cflags += ['-DLLVM_FORCE_HEAD_REVISION']
 519     cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
 520
 521   CreateChromeToolsShim()
 522
 523   deployment_env = None
 524   if deployment_target:
 525     deployment_env = os.environ.copy()
 526     deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
 527
 528   cmake_args = base_cmake_args + [
 529       '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
 530       '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 531       '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
 532       '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
 533       '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
 534       '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
 535       '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
 536       '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
 537       '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
 538   # TODO(thakis): Unconditionally append this to base_cmake_args instead once
 539   # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
 540   cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
 541   if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
 542   if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
 543
 544   if not os.path.exists(LLVM_BUILD_DIR):
 545     os.makedirs(LLVM_BUILD_DIR)
 546   os.chdir(LLVM_BUILD_DIR)
 547   RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
 548              msvc_arch='x64', env=deployment_env)
 549
 550   if args.gcc_toolchain:
 551     # Copy in the right stdlibc++.so.6 so clang can start.
 552     if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
 553       os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
 554     libstdcpp = subprocess.check_output(
 555         [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
 556     CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
 557
 558   RunCommand(['ninja'], msvc_arch='x64')
 559
 560   if args.tools:
 561     # If any Chromium tools were built, install those now.
 562     RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
 563
 564   if sys.platform == 'darwin':
 565     CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'),
 566              os.path.join(LLVM_BUILD_DIR, 'bin'))
 567     # See http://crbug.com/256342
 568     RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
 569   elif sys.platform.startswith('linux'):
 570     RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
 571
 572   # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
 573   # TODO(hans): Remove once the regular build above produces this.
 574   if not os.path.exists(COMPILER_RT_BUILD_DIR):
 575     os.makedirs(COMPILER_RT_BUILD_DIR)
 576   os.chdir(COMPILER_RT_BUILD_DIR)
 577   # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
 578   # above).
 579   #if args.bootstrap and sys.platform == 'win32':
 580     # The bootstrap compiler produces 64-bit binaries by default.
 581     #cflags += ['-m32']
 582     #cxxflags += ['-m32']
 583   compiler_rt_args = base_cmake_args + [
 584       '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 585       '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
 586   if sys.platform != 'win32':
 587     compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
 588                          os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
 589                         '-DSANITIZER_MIN_OSX_VERSION="10.7"']
 590   RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR],
 591               msvc_arch='x86', env=deployment_env)
 592   RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
 593
 594   # TODO(hans): Make this (and the .gypi and .isolate files) version number
 595   # independent.
 596   if sys.platform == 'win32':
 597     platform = 'windows'
 598   elif sys.platform == 'darwin':
 599     platform = 'darwin'
 600   else:
 601     assert sys.platform.startswith('linux')
 602     platform = 'linux'
 603   asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
 604                                      VERSION, 'lib', platform)
 605   asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 606                                      VERSION, 'lib', platform)
 607   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 608                         r'^.*-i386\.lib$')
 609   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 610                         r'^.*-i386\.dll$')
 611
 612   CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
 613            os.path.join(asan_rt_lib_dst_dir, '..', '..'))
 614
 615   if sys.platform == 'win32':
 616     # Make an extra copy of the sanitizer headers, to be put on the include path
 617     # of the fallback compiler.
 618     sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 619                                          VERSION, 'include', 'sanitizer')
 620     aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 621                                              VERSION, 'include_sanitizer',
 622                                              'sanitizer')
 623     if not os.path.exists(aux_sanitizer_include_dir):
 624       os.makedirs(aux_sanitizer_include_dir)
 625     for _, _, files in os.walk(sanitizer_include_dir):
 626       for f in files:
 627         CopyFile(os.path.join(sanitizer_include_dir, f),
 628                  aux_sanitizer_include_dir)
 629
 630   # Run tests.
 631   if args.run_tests or use_head_revision:
 632     os.chdir(LLVM_BUILD_DIR)
 633     RunCommand(GetVSVersion().SetupScript('x64') +
 634                ['&&', 'ninja', 'cr-check-all'])
 635   if args.run_tests:
 636     os.chdir(LLVM_BUILD_DIR)
 637     RunCommand(GetVSVersion().SetupScript('x64') +
 638                ['&&', 'ninja', 'check-all'])
 639
 640   WriteStampFile(PACKAGE_VERSION)
 641   print 'Clang update was successful.'
 642   return 0
 643
 644
 645 def main():
 646   if not sys.platform in ['win32', 'cygwin']:
 647     # For non-Windows, fall back to update.sh.
 648     # TODO(hans): Make update.py replace update.sh completely.
 649
 650     # This script is called by gclient. gclient opens its hooks subprocesses
 651     # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does
 652     # custom output processing that breaks printing '\r' characters for
 653     # single-line updating status messages as printed by curl and wget.
 654     # Work around this by setting stderr of the update.sh process to stdin (!):
 655     # gclient doesn't redirect stdin, and while stdin itself is read-only, a
 656     # dup()ed sys.stdin is writable, try
 657     #   fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
 658     # TODO: Fix gclient instead, http://crbug.com/95350
 659     if '--no-stdin-hack' in sys.argv:
 660       sys.argv.remove('--no-stdin-hack')
 661       stderr = None
 662     else:
 663       try:
 664         stderr = os.fdopen(os.dup(sys.stdin.fileno()))
 665       except:
 666         stderr = sys.stderr
 667     return subprocess.call(
 668         [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:],
 669         stderr=stderr)
 670
 671   parser = argparse.ArgumentParser(description='Build Clang.')
 672   parser.add_argument('--bootstrap', action='store_true',
 673                       help='first build clang with CC, then with itself.')
 674   parser.add_argument('--if-needed', action='store_true',
 675                       help="run only if the script thinks clang is needed")
 676   parser.add_argument('--force-local-build', action='store_true',
 677                       help="don't try to download prebuild binaries")
 678   parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
 679                       'version be used for building; --gcc-toolchain=/opt/foo '
 680                       'picks /opt/foo/bin/gcc')
 681   parser.add_argument('--print-revision', action='store_true',
 682                       help='print current clang revision and exit.')
 683   parser.add_argument('--print-clang-version', action='store_true',
 684                       help='print current clang version (e.g. x.y.z) and exit.')
 685   parser.add_argument('--run-tests', action='store_true',
 686                       help='run tests after building; only for local builds')
 687   parser.add_argument('--tools', nargs='*',
 688                       help='select which chrome tools to build',
 689                       default=['plugins', 'blink_gc_plugin'])
 690   parser.add_argument('--without-patches', action='store_false',
 691                       help="don't apply patches (default)", dest='with_patches',
 692                       default=True)
 693
 694   # For now, these flags are only used for the non-Windows flow, but argparser
 695   # gets mad if it sees a flag it doesn't recognize.
 696   parser.add_argument('--no-stdin-hack', action='store_true')
 697
 698   args = parser.parse_args()
 699
 700   if args.if_needed:
 701     is_clang_required = False
 702     # clang is always used on Mac and Linux.
 703     if sys.platform == 'darwin' or sys.platform.startswith('linux'):
 704       is_clang_required = True
 705     # clang requested via $GYP_DEFINES.
 706     if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
 707                  os.environ.get('GYP_DEFINES', '')):
 708       is_clang_required = True
 709     # clang previously downloaded, keep it up-to-date.
 710     # If you don't want this, delete third_party/llvm-build on your machine.
 711     if os.path.isdir(LLVM_BUILD_DIR):
 712       is_clang_required = True
 713     if not is_clang_required:
 714       return 0
 715     if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
 716       print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
 717       return 0
 718
 719   global LLVM_WIN_REVISION, PACKAGE_VERSION
 720   if args.print_revision:
 721     if use_head_revision:
 722       print GetSvnRevision(LLVM_DIR)
 723     else:
 724       print PACKAGE_VERSION
 725     return 0
 726
 727   if args.print_clang_version:
 728     sys.stdout.write(VERSION)
 729     return 0
 730
 731   # Don't buffer stdout, so that print statements are immediately flushed.
 732   # Do this only after --print-revision has been handled, else we'll get
 733   # an error message when this script is run from gn for some reason.
 734   sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
 735
 736   if use_head_revision:
 737     # Use a real revision number rather than HEAD to make sure that the stamp
 738     # file logic works.
 739     LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL)
 740     PACKAGE_VERSION = LLVM_WIN_REVISION + '-0'
 741
 742     args.force_local_build = True
 743     # Skip local patches when using HEAD: they probably don't apply anymore.
 744     args.with_patches = False
 745
 746   return UpdateClang(args)
 747
 748
 749 if __name__ == '__main__':
 750   sys.exit(main())