tools/clang/scripts/update.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Windows can't run .sh files, so this is a Python implementation of
   7 update.sh. This script should replace update.sh on all platforms eventually."""
   8
   9 import argparse
  10 import contextlib
  11 import cStringIO
  12 import glob
  13 import os
  14 import pipes
  15 import re
  16 import shutil
  17 import subprocess
  18 import stat
  19 import sys
  20 import tarfile
  21 import time
  22 import urllib2
  23 import zipfile
  24
  25 # Do NOT CHANGE this if you don't know what you're doing -- see
  26 # https://code.google.com/p/chromium/wiki/UpdatingClang
  27 # Reverting problematic clang rolls is safe, though.
  28 # Note: this revision is only used for Windows. Other platforms use update.sh.
  29 # TODO(thakis): Use the same revision on Windows and non-Windows.
  30 # TODO(thakis): Remove update.sh, use update.py everywhere.
  31 LLVM_WIN_REVISION = '243039'
  32
  33 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
  34 if use_head_revision:
  35   LLVM_WIN_REVISION = 'HEAD'
  36
  37 # This is incremented when pushing a new build of Clang at the same revision.
  38 CLANG_SUB_REVISION=1
  39
  40 PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION)
  41
  42 # Path constants. (All of these should be absolute paths.)
  43 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  44 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
  45 THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
  46 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
  47 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
  48 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
  49                                           'llvm-bootstrap-install')
  50 CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
  51 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
  52                               'Release+Asserts')
  53 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
  54 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
  55 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
  56 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
  57 LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
  58 LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
  59 LLVM_BUILD_TOOLS_DIR = os.path.abspath(
  60     os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
  61 STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')
  62 BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
  63 VERSION = '3.8.0'
  64
  65 # URL for pre-built binaries.
  66 CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
  67
  68 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
  69 if 'LLVM_REPO_URL' in os.environ:
  70   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
  71
  72
  73 def DownloadUrl(url, output_file):
  74   """Download url into output_file."""
  75   CHUNK_SIZE = 4096
  76   TOTAL_DOTS = 10
  77   sys.stdout.write('Downloading %s ' % url)
  78   sys.stdout.flush()
  79   response = urllib2.urlopen(url)
  80   total_size = int(response.info().getheader('Content-Length').strip())
  81   bytes_done = 0
  82   dots_printed = 0
  83   while True:
  84     chunk = response.read(CHUNK_SIZE)
  85     if not chunk:
  86       break
  87     output_file.write(chunk)
  88     bytes_done += len(chunk)
  89     num_dots = TOTAL_DOTS * bytes_done / total_size
  90     sys.stdout.write('.' * (num_dots - dots_printed))
  91     sys.stdout.flush()
  92     dots_printed = num_dots
  93   print ' Done.'
  94
  95
  96 def ReadStampFile():
  97   """Return the contents of the stamp file, or '' if it doesn't exist."""
  98   try:
  99     with open(STAMP_FILE, 'r') as f:
 100       return f.read()
 101   except IOError:
 102     return ''
 103
 104
 105 def WriteStampFile(s):
 106   """Write s to the stamp file."""
 107   if not os.path.exists(os.path.dirname(STAMP_FILE)):
 108     os.makedirs(os.path.dirname(STAMP_FILE))
 109   with open(STAMP_FILE, 'w') as f:
 110     f.write(s)
 111
 112
 113 def GetSvnRevision(svn_repo):
 114   """Returns current revision of the svn repo at svn_repo."""
 115   svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
 116   m = re.search(r'Revision: (\d+)', svn_info)
 117   return m.group(1)
 118
 119
 120 def RmTree(dir):
 121   """Delete dir."""
 122   def ChmodAndRetry(func, path, _):
 123     # Subversion can leave read-only files around.
 124     if not os.access(path, os.W_OK):
 125       os.chmod(path, stat.S_IWUSR)
 126       return func(path)
 127     raise
 128
 129   shutil.rmtree(dir, onerror=ChmodAndRetry)
 130
 131
 132 def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
 133   """Run command and return success (True) or failure; or if fail_hard is
 134      True, exit on failure.  If msvc_arch is set, runs the command in a
 135      shell with the msvc tools for that architecture."""
 136
 137   if msvc_arch and sys.platform == 'win32':
 138     command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
 139
 140   # https://docs.python.org/2/library/subprocess.html:
 141   # "On Unix with shell=True [...] if args is a sequence, the first item
 142   # specifies the command string, and any additional items will be treated as
 143   # additional arguments to the shell itself.  That is to say, Popen does the
 144   # equivalent of:
 145   #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
 146   #
 147   # We want to pass additional arguments to command[0], not to the shell,
 148   # so manually join everything into a single string.
 149   # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
 150   # quote c:\path but svn can't handle quoted paths on Windows.  Since on
 151   # Windows follow-on args are passed to args[0] instead of the shell, don't
 152   # do the single-string transformation there.
 153   if sys.platform != 'win32':
 154     command = ' '.join([pipes.quote(c) for c in command])
 155   print 'Running', command
 156   if subprocess.call(command, env=env, shell=True) == 0:
 157     return True
 158   print 'Failed.'
 159   if fail_hard:
 160     sys.exit(1)
 161   return False
 162
 163
 164 def CopyFile(src, dst):
 165   """Copy a file from src to dst."""
 166   shutil.copy(src, dst)
 167   print "Copying %s to %s" % (src, dst)
 168
 169
 170 def CopyDirectoryContents(src, dst, filename_filter=None):
 171   """Copy the files from directory src to dst
 172   with an optional filename filter."""
 173   if not os.path.exists(dst):
 174     os.makedirs(dst)
 175   for root, _, files in os.walk(src):
 176     for f in files:
 177       if filename_filter and not re.match(filename_filter, f):
 178         continue
 179       CopyFile(os.path.join(root, f), dst)
 180
 181
 182 def Checkout(name, url, dir):
 183   """Checkout the SVN module at url into dir. Use name for the log message."""
 184   print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir)
 185
 186   command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir]
 187   if RunCommand(command, fail_hard=False):
 188     return
 189
 190   if os.path.isdir(dir):
 191     print "Removing %s." % (dir)
 192     RmTree(dir)
 193
 194   print "Retrying."
 195   RunCommand(command)
 196
 197
 198 def RevertPreviouslyPatchedFiles():
 199   print 'Reverting previously patched files'
 200   files = [
 201     '%(clang)s/test/Index/crash-recovery-modules.m',
 202     '%(clang)s/unittests/libclang/LibclangTest.cpp',
 203     '%(compiler_rt)s/lib/asan/asan_rtl.cc',
 204     '%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc',
 205     '%(llvm)s/test/DebugInfo/gmlt.ll',
 206     '%(llvm)s/lib/CodeGen/SpillPlacement.cpp',
 207     '%(llvm)s/lib/CodeGen/SpillPlacement.h',
 208     '%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp',
 209     '%(clang)s/test/Driver/env.c',
 210     '%(clang)s/lib/Frontend/InitPreprocessor.cpp',
 211     '%(clang)s/test/Frontend/exceptions.c',
 212     '%(clang)s/test/Preprocessor/predefined-exceptions.m',
 213     '%(llvm)s/test/Bindings/Go/go.test',
 214     '%(clang)s/lib/Parse/ParseExpr.cpp',
 215     '%(clang)s/lib/Parse/ParseTemplate.cpp',
 216     '%(clang)s/lib/Sema/SemaDeclCXX.cpp',
 217     '%(clang)s/lib/Sema/SemaExprCXX.cpp',
 218     '%(clang)s/test/SemaCXX/default2.cpp',
 219     '%(clang)s/test/SemaCXX/typo-correction-delayed.cpp',
 220     '%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc',
 221     '%(compiler_rt)s/test/tsan/signal_segv_handler.cc',
 222     '%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc',
 223     '%(compiler_rt)s/cmake/config-ix.cmake',
 224     '%(compiler_rt)s/CMakeLists.txt',
 225     '%(compiler_rt)s/lib/ubsan/ubsan_platform.h',
 226     ]
 227   for f in files:
 228     f = f % {
 229         'clang': CLANG_DIR,
 230         'compiler_rt': COMPILER_RT_DIR,
 231         'llvm': LLVM_DIR,
 232         }
 233     if os.path.exists(f):
 234       os.remove(f)  # For unversioned files.
 235       RunCommand(['svn', 'revert', f])
 236
 237
 238 def ApplyLocalPatches():
 239   # There's no patch program on Windows by default.  We don't need patches on
 240   # Windows yet, and maybe this not working on Windows will motivate us to
 241   # remove patches over time.
 242   assert sys.platform != 'win32'
 243
 244   # Apply patch for tests failing with --disable-pthreads (llvm.org/PR11974)
 245   clang_patches = [ r"""\
 246 --- test/Index/crash-recovery-modules.m (revision 202554)
 247 +++ test/Index/crash-recovery-modules.m (working copy)
 248 @@ -12,6 +12,8 @@
 249
 250  // REQUIRES: crash-recovery
 251  // REQUIRES: shell
 252 +// XFAIL: *
 253 +//    (PR11974)
 254
 255  @import Crash;
 256 """, r"""\
 257 --- unittests/libclang/LibclangTest.cpp (revision 215949)
 258 +++ unittests/libclang/LibclangTest.cpp (working copy)
 259 @@ -431,7 +431,7 @@
 260    EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU));
 261  }
 262
 263 -TEST_F(LibclangReparseTest, ReparseWithModule) {
 264 +TEST_F(LibclangReparseTest, DISABLED_ReparseWithModule) {
 265    const char *HeaderTop = "#ifndef H\n#define H\nstruct Foo { int bar;";
 266    const char *HeaderBottom = "\n};\n#endif\n";
 267    const char *MFile = "#include \"HeaderFile.h\"\nint main() {"
 268 """
 269       ]
 270
 271   # This Go bindings test doesn't work after bootstrap on Linux, PR21552.
 272   llvm_patches = [ r"""\
 273 --- test/Bindings/Go/go.test    (revision 223109)
 274 +++ test/Bindings/Go/go.test    (working copy)
 275 @@ -1,3 +1,3 @@
 276 -; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm
 277 +; RUN: true
 278
 279  ; REQUIRES: shell
 280 """
 281       ]
 282
 283   # The UBSan run-time, which is now bundled with the ASan run-time, doesn't
 284   # work on Mac OS X 10.8 (PR23539).
 285   compiler_rt_patches = [ r"""\
 286 --- CMakeLists.txt      (revision 241602)
 287 +++ CMakeLists.txt      (working copy)
 288 @@ -305,6 +305,7 @@
 289        list(APPEND SANITIZER_COMMON_SUPPORTED_OS iossim)
 290      endif()
 291    endif()
 292 +  set(SANITIZER_MIN_OSX_VERSION "10.7")
 293    if(SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.7")
 294      message(FATAL_ERROR "Too old OS X version: ${SANITIZER_MIN_OSX_VERSION}")
 295    endif()
 296 """
 297       ]
 298
 299   for path, patches in [(LLVM_DIR, llvm_patches),
 300                         (CLANG_DIR, clang_patches),
 301                         (COMPILER_RT_DIR, compiler_rt_patches)]:
 302     print 'Applying patches in', path
 303     for patch in patches:
 304       print patch
 305       p = subprocess.Popen( ['patch', '-p0', '-d', path], stdin=subprocess.PIPE)
 306       (stdout, stderr) = p.communicate(input=patch)
 307       if p.returncode != 0:
 308         raise RuntimeError('stdout %s, stderr %s' % (stdout, stderr))
 309
 310
 311 def DeleteChromeToolsShim():
 312   OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
 313   shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
 314   shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
 315
 316
 317 def CreateChromeToolsShim():
 318   """Hooks the Chrome tools into the LLVM build.
 319
 320   Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
 321   detects implicit tools in the tools subdirectory, so this helper install a
 322   shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
 323
 324   Note that the shim directory name intentionally has no - or _. The implicit
 325   tool detection logic munges them in a weird way."""
 326   assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
 327   os.mkdir(CHROME_TOOLS_SHIM_DIR)
 328   with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
 329     f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
 330             'Do not edit.\n')
 331     f.write('# Since tools/clang is located in another directory, use the \n')
 332     f.write('# two arg version to specify where build artifacts go. CMake\n')
 333     f.write('# disallows reuse of the same binary dir for multiple source\n')
 334     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
 335     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
 336     f.write('if (CHROMIUM_TOOLS_SRC)\n')
 337     f.write('  add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
 338               '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
 339     f.write('endif (CHROMIUM_TOOLS_SRC)\n')
 340
 341
 342 def AddCMakeToPath():
 343   """Download CMake and add it to PATH."""
 344   if sys.platform == 'win32':
 345     zip_name = 'cmake-3.2.2-win32-x86.zip'
 346     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
 347                              'cmake-3.2.2-win32-x86', 'bin')
 348   else:
 349     suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
 350     zip_name = 'cmake310_%s.tgz' % suffix
 351     cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin')
 352   if not os.path.exists(cmake_dir):
 353     if not os.path.exists(LLVM_BUILD_TOOLS_DIR):
 354       os.makedirs(LLVM_BUILD_TOOLS_DIR)
 355     # The cmake archive is smaller than 20 MB, small enough to keep in memory:
 356     with contextlib.closing(cStringIO.StringIO()) as f:
 357       DownloadUrl(CDS_URL + '/tools/' + zip_name, f)
 358       f.seek(0)
 359       if zip_name.endswith('.zip'):
 360         zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR)
 361       else:
 362         tarfile.open(mode='r:gz', fileobj=f).extractall(path=
 363             LLVM_BUILD_TOOLS_DIR)
 364   os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
 365
 366 vs_version = None
 367 def GetVSVersion():
 368   global vs_version
 369   if vs_version:
 370     return vs_version
 371
 372   # Try using the toolchain in depot_tools.
 373   # This sets environment variables used by SelectVisualStudioVersion below.
 374   sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
 375   import vs_toolchain
 376   vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
 377
 378   # Use gyp to find the MSVS installation, either in depot_tools as per above,
 379   # or a system-wide installation otherwise.
 380   sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
 381   import gyp.MSVSVersion
 382   vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013')
 383   return vs_version
 384
 385
 386 def UpdateClang(args):
 387   print 'Updating Clang to %s...' % PACKAGE_VERSION
 388   if ReadStampFile() == PACKAGE_VERSION:
 389     print 'Already up to date.'
 390     return 0
 391
 392   # Reset the stamp file in case the build is unsuccessful.
 393   WriteStampFile('')
 394
 395   if not args.force_local_build:
 396     cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
 397     cds_full_url = CDS_URL + '/Win/' + cds_file
 398
 399     # Check if there's a prebuilt binary and if so just fetch that. That's
 400     # faster, and goma relies on having matching binary hashes on client and
 401     # server too.
 402     print 'Trying to download prebuilt clang'
 403
 404     # clang packages are smaller than 50 MB, small enough to keep in memory.
 405     with contextlib.closing(cStringIO.StringIO()) as f:
 406       try:
 407         DownloadUrl(cds_full_url, f)
 408         f.seek(0)
 409         # TODO(thakis): Delete LLVM_BUILD_DIR before extracting.
 410         tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR)
 411         print 'clang %s unpacked' % PACKAGE_VERSION
 412         # Download the gold plugin if requested to by an environment variable.
 413         # This is used by the CFI ClusterFuzz bot.
 414         if 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ:
 415           RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
 416         WriteStampFile(PACKAGE_VERSION)
 417         return 0
 418       except urllib2.HTTPError:
 419         print 'Did not find prebuilt clang %s, building locally' % cds_file
 420
 421   AddCMakeToPath()
 422
 423   RevertPreviouslyPatchedFiles()
 424   DeleteChromeToolsShim()
 425
 426   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
 427   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
 428   if sys.platform == 'win32':
 429     Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
 430   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
 431   if sys.platform == 'darwin':
 432     # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
 433     # (i.e. this is needed for bootstrap builds).
 434     Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
 435     # While we're bundling our own libc++ on OS X, we need to compile libc++abi
 436     # into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
 437     Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR)
 438
 439   if args.with_patches and sys.platform != 'win32':
 440     ApplyLocalPatches()
 441
 442   cc, cxx = None, None
 443   libstdcpp = None
 444   if args.gcc_toolchain:  # This option is only used on Linux.
 445     # Use the specified gcc installation for building.
 446     cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
 447     cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
 448
 449     if not os.access(cc, os.X_OK):
 450       print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
 451       print '"%s" does not appear to be valid.' % cc
 452       return 1
 453
 454     # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
 455     # compiler, etc.) find the .so.
 456     libstdcpp = subprocess.check_output(
 457         [cxx, '-print-file-name=libstdc++.so.6']).rstrip()
 458     os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
 459
 460   cflags = cxxflags = ldflags = []
 461
 462   # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
 463   # needed, on OS X it requires libc++. clang only automatically links to libc++
 464   # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
 465   # on OS X versions as old as 10.7.
 466   # TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle
 467   # libc++.dylib.  Remove this once all bots are on 10.7+, then use
 468   # -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7.
 469   deployment_target = ''
 470
 471   if sys.platform == 'darwin':
 472     # When building on 10.9, /usr/include usually doesn't exist, and while
 473     # Xcode's clang automatically sets a sysroot, self-built clangs don't.
 474     cflags = ['-isysroot', subprocess.check_output(
 475         ['xcrun', '--show-sdk-path']).rstrip()]
 476     cxxflags = ['-stdlib=libc++', '-nostdinc++',
 477                 '-I' + os.path.join(LIBCXX_DIR, 'include')] + cflags
 478     if args.bootstrap:
 479       deployment_target = '10.6'
 480
 481   base_cmake_args = ['-GNinja',
 482                      '-DCMAKE_BUILD_TYPE=Release',
 483                      '-DLLVM_ENABLE_ASSERTIONS=ON',
 484                      '-DLLVM_ENABLE_THREADS=OFF',
 485                      ]
 486
 487   if args.bootstrap:
 488     print 'Building bootstrap compiler'
 489     if not os.path.exists(LLVM_BOOTSTRAP_DIR):
 490       os.makedirs(LLVM_BOOTSTRAP_DIR)
 491     os.chdir(LLVM_BOOTSTRAP_DIR)
 492     bootstrap_args = base_cmake_args + [
 493         '-DLLVM_TARGETS_TO_BUILD=host',
 494         '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
 495         '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 496         '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
 497         ]
 498     if cc is not None:  bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
 499     if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
 500     RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
 501     RunCommand(['ninja'], msvc_arch='x64')
 502     if args.run_tests:
 503       RunCommand(['ninja', 'check-all'], msvc_arch='x64')
 504     RunCommand(['ninja', 'install'], msvc_arch='x64')
 505     if args.gcc_toolchain:
 506       # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
 507       # compiler can start.
 508       CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
 509
 510     if sys.platform == 'win32':
 511       cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
 512       cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
 513       # CMake has a hard time with backslashes in compiler paths:
 514       # https://stackoverflow.com/questions/13050827
 515       cc = cc.replace('\\', '/')
 516       cxx = cxx.replace('\\', '/')
 517     else:
 518       cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
 519       cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
 520
 521     if args.gcc_toolchain:
 522       # Tell the bootstrap compiler to use a specific gcc prefix to search
 523       # for standard library headers and shared object files.
 524       cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
 525       cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
 526     print 'Building final compiler'
 527
 528   if sys.platform == 'darwin':
 529     # Build libc++.dylib while some bots are still on OS X 10.6.
 530     libcxxbuild = os.path.join(LLVM_BUILD_DIR, 'libcxxbuild')
 531     if os.path.isdir(libcxxbuild):
 532       RmTree(libcxxbuild)
 533     libcxxflags = ['-O3', '-std=c++11', '-fstrict-aliasing']
 534
 535     # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
 536     # into different subdirectories.
 537     os.makedirs(os.path.join(libcxxbuild, 'libcxx'))
 538     os.chdir(os.path.join(libcxxbuild, 'libcxx'))
 539     RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
 540                 glob.glob(os.path.join(LIBCXX_DIR, 'src', '*.cpp')))
 541
 542     os.makedirs(os.path.join(libcxxbuild, 'libcxxabi'))
 543     os.chdir(os.path.join(libcxxbuild, 'libcxxabi'))
 544     RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
 545                glob.glob(os.path.join(LIBCXXABI_DIR, 'src', '*.cpp')) +
 546                ['-I' + os.path.join(LIBCXXABI_DIR, 'include')])
 547
 548     os.chdir(libcxxbuild)
 549     libdir = os.path.join(LIBCXX_DIR, 'lib')
 550     RunCommand(['cc'] + glob.glob('libcxx/*.o') + glob.glob('libcxxabi/*.o') +
 551         ['-o', 'libc++.1.dylib', '-dynamiclib', '-nodefaultlibs',
 552          '-current_version', '1', '-compatibility_version', '1', '-lSystem',
 553          '-install_name', '@executable_path/libc++.dylib',
 554          '-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp',
 555          '-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp',
 556          '-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp'])
 557     if os.path.exists('libc++.dylib'):
 558       os.remove('libc++.dylib')
 559     os.symlink('libc++.1.dylib', 'libc++.dylib')
 560     ldflags += ['-stdlib=libc++', '-L' + libcxxbuild]
 561
 562     if args.bootstrap:
 563       # Now that the libc++ headers have been installed and libc++.dylib is
 564       # built, delete the libc++ checkout again so that it's not part of the
 565       # main build below -- the libc++(abi) tests don't pass on OS X in
 566       # bootstrap builds (http://llvm.org/PR24068)
 567       RmTree(LIBCXX_DIR)
 568       RmTree(LIBCXXABI_DIR)
 569       cxxflags = ['-stdlib=libc++', '-nostdinc++',
 570                   '-I' + os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR,
 571                                       'include/c++/v1')
 572                   ] + cflags
 573
 574   # Build clang.
 575   binutils_incdir = ''
 576   if sys.platform.startswith('linux'):
 577     binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
 578
 579   # If building at head, define a macro that plugins can use for #ifdefing
 580   # out code that builds at head, but not at LLVM_WIN_REVISION or vice versa.
 581   if use_head_revision:
 582     cflags += ['-DLLVM_FORCE_HEAD_REVISION']
 583     cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
 584
 585   CreateChromeToolsShim()
 586
 587   deployment_env = None
 588   if deployment_target:
 589     deployment_env = os.environ.copy()
 590     deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
 591
 592   cmake_args = base_cmake_args + [
 593       '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
 594       '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 595       '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
 596       '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
 597       '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
 598       '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
 599       '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
 600       '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
 601       '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
 602   # TODO(thakis): Unconditionally append this to base_cmake_args instead once
 603   # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
 604   cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
 605   if cc is not None:  cc_args.append('-DCMAKE_C_COMPILER=' + cc)
 606   if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
 607
 608   if not os.path.exists(LLVM_BUILD_DIR):
 609     os.makedirs(LLVM_BUILD_DIR)
 610   os.chdir(LLVM_BUILD_DIR)
 611   RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
 612              msvc_arch='x64', env=deployment_env)
 613
 614   if args.gcc_toolchain:
 615     # Copy in the right stdlibc++.so.6 so clang can start.
 616     if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
 617       os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
 618     libstdcpp = subprocess.check_output(
 619         [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
 620     CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
 621
 622   RunCommand(['ninja'], msvc_arch='x64')
 623
 624   if args.tools:
 625     # If any Chromium tools were built, install those now.
 626     RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
 627
 628   if sys.platform == 'darwin':
 629     CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'),
 630              os.path.join(LLVM_BUILD_DIR, 'bin'))
 631     # See http://crbug.com/256342
 632     RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
 633   elif sys.platform.startswith('linux'):
 634     RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
 635
 636   # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
 637   # TODO(hans): Remove once the regular build above produces this.
 638   if not os.path.exists(COMPILER_RT_BUILD_DIR):
 639     os.makedirs(COMPILER_RT_BUILD_DIR)
 640   os.chdir(COMPILER_RT_BUILD_DIR)
 641   # TODO(thakis): Add this once compiler-rt can build with clang-cl (see
 642   # above).
 643   #if args.bootstrap and sys.platform == 'win32':
 644     # The bootstrap compiler produces 64-bit binaries by default.
 645     #cflags += ['-m32']
 646     #cxxflags += ['-m32']
 647   compiler_rt_args = base_cmake_args + [
 648       '-DCMAKE_C_FLAGS=' + ' '.join(cflags),
 649       '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
 650   if sys.platform != 'win32':
 651     compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
 652                          os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config')]
 653   RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR],
 654               msvc_arch='x86', env=deployment_env)
 655   RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
 656
 657   # TODO(hans): Make this (and the .gypi and .isolate files) version number
 658   # independent.
 659   if sys.platform == 'win32':
 660     platform = 'windows'
 661   elif sys.platform == 'darwin':
 662     platform = 'darwin'
 663   else:
 664     assert sys.platform.startswith('linux')
 665     platform = 'linux'
 666   asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
 667                                      VERSION, 'lib', platform)
 668   asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 669                                      VERSION, 'lib', platform)
 670   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 671                         r'^.*-i386\.lib$')
 672   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 673                         r'^.*-i386\.dll$')
 674
 675   CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
 676            os.path.join(asan_rt_lib_dst_dir, '..', '..'))
 677
 678   if sys.platform == 'win32':
 679     # Make an extra copy of the sanitizer headers, to be put on the include path
 680     # of the fallback compiler.
 681     sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 682                                          VERSION, 'include', 'sanitizer')
 683     aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 684                                              VERSION, 'include_sanitizer',
 685                                              'sanitizer')
 686     if not os.path.exists(aux_sanitizer_include_dir):
 687       os.makedirs(aux_sanitizer_include_dir)
 688     for _, _, files in os.walk(sanitizer_include_dir):
 689       for f in files:
 690         CopyFile(os.path.join(sanitizer_include_dir, f),
 691                  aux_sanitizer_include_dir)
 692
 693   # Run tests.
 694   if args.run_tests or use_head_revision:
 695     os.chdir(LLVM_BUILD_DIR)
 696     RunCommand(GetVSVersion().SetupScript('x64') +
 697                ['&&', 'ninja', 'cr-check-all'])
 698   if args.run_tests:
 699     os.chdir(LLVM_BUILD_DIR)
 700     RunCommand(GetVSVersion().SetupScript('x64') +
 701                ['&&', 'ninja', 'check-all'])
 702
 703   WriteStampFile(PACKAGE_VERSION)
 704   print 'Clang update was successful.'
 705   return 0
 706
 707
 708 def main():
 709   if not sys.platform in ['win32', 'cygwin']:
 710     # For non-Windows, fall back to update.sh.
 711     # TODO(hans): Make update.py replace update.sh completely.
 712
 713     # This script is called by gclient. gclient opens its hooks subprocesses
 714     # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does
 715     # custom output processing that breaks printing '\r' characters for
 716     # single-line updating status messages as printed by curl and wget.
 717     # Work around this by setting stderr of the update.sh process to stdin (!):
 718     # gclient doesn't redirect stdin, and while stdin itself is read-only, a
 719     # dup()ed sys.stdin is writable, try
 720     #   fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
 721     # TODO: Fix gclient instead, http://crbug.com/95350
 722     if '--no-stdin-hack' in sys.argv:
 723       sys.argv.remove('--no-stdin-hack')
 724       stderr = None
 725     else:
 726       try:
 727         stderr = os.fdopen(os.dup(sys.stdin.fileno()))
 728       except:
 729         stderr = sys.stderr
 730     return subprocess.call(
 731         [os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:],
 732         stderr=stderr)
 733
 734   parser = argparse.ArgumentParser(description='Build Clang.')
 735   parser.add_argument('--bootstrap', action='store_true',
 736                       help='first build clang with CC, then with itself.')
 737   parser.add_argument('--if-needed', action='store_true',
 738                       help="run only if the script thinks clang is needed")
 739   parser.add_argument('--force-local-build', action='store_true',
 740                       help="don't try to download prebuild binaries")
 741   parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
 742                       'version be used for building; --gcc-toolchain=/opt/foo '
 743                       'picks /opt/foo/bin/gcc')
 744   parser.add_argument('--print-revision', action='store_true',
 745                       help='print current clang revision and exit.')
 746   parser.add_argument('--print-clang-version', action='store_true',
 747                       help='print current clang version (e.g. x.y.z) and exit.')
 748   parser.add_argument('--run-tests', action='store_true',
 749                       help='run tests after building; only for local builds')
 750   parser.add_argument('--tools', nargs='*',
 751                       help='select which chrome tools to build',
 752                       default=['plugins', 'blink_gc_plugin'])
 753   parser.add_argument('--without-patches', action='store_false',
 754                       help="don't apply patches (default)", dest='with_patches',
 755                       default=True)
 756
 757   # For now, these flags are only used for the non-Windows flow, but argparser
 758   # gets mad if it sees a flag it doesn't recognize.
 759   parser.add_argument('--no-stdin-hack', action='store_true')
 760
 761   args = parser.parse_args()
 762
 763   if args.if_needed:
 764     is_clang_required = False
 765     # clang is always used on Mac and Linux.
 766     if sys.platform == 'darwin' or sys.platform.startswith('linux'):
 767       is_clang_required = True
 768     # clang requested via $GYP_DEFINES.
 769     if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
 770                  os.environ.get('GYP_DEFINES', '')):
 771       is_clang_required = True
 772     # clang previously downloaded, keep it up-to-date.
 773     # If you don't want this, delete third_party/llvm-build on your machine.
 774     if os.path.isdir(LLVM_BUILD_DIR):
 775       is_clang_required = True
 776     if not is_clang_required:
 777       return 0
 778     if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
 779       print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
 780       return 0
 781
 782   global LLVM_WIN_REVISION, PACKAGE_VERSION
 783   if args.print_revision:
 784     if use_head_revision:
 785       print GetSvnRevision(LLVM_DIR)
 786     else:
 787       print PACKAGE_VERSION
 788     return 0
 789
 790   if args.print_clang_version:
 791     sys.stdout.write(VERSION)
 792     return 0
 793
 794   # Don't buffer stdout, so that print statements are immediately flushed.
 795   # Do this only after --print-revision has been handled, else we'll get
 796   # an error message when this script is run from gn for some reason.
 797   sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
 798
 799   if use_head_revision:
 800     # Use a real revision number rather than HEAD to make sure that the stamp
 801     # file logic works.
 802     LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL)
 803     PACKAGE_VERSION = LLVM_WIN_REVISION + '-0'
 804
 805     args.force_local_build = True
 806     # Skip local patches when using HEAD: they probably don't apply anymore.
 807     args.with_patches = False
 808
 809   return UpdateClang(args)
 810
 811
 812 if __name__ == '__main__':
 813   sys.exit(main())