tools/clang/scripts/update.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Windows can't run .sh files, so this is a Python implementation of
   7 update.sh. This script should replace update.sh on all platforms eventually."""
   8
   9 import argparse
  10 import os
  11 import re
  12 import shutil
  13 import subprocess
  14 import stat
  15 import sys
  16 import time
  17
  18 # Do NOT CHANGE this if you don't know what you're doing -- see
  19 # https://code.google.com/p/chromium/wiki/UpdatingClang
  20 # Reverting problematic clang rolls is safe, though.
  21 # Note: this revision is only used for Windows. Other platforms use update.sh.
  22 LLVM_WIN_REVISION = 'HEAD'
  23
  24 # ASan on Windows is useful enough to use it even while the clang/win is still
  25 # in bringup. Use a pinned revision to make it slightly more stable.
  26 if (re.search(r'\b(asan)=1', os.environ.get('GYP_DEFINES', '')) and
  27     not 'LLVM_FORCE_HEAD_REVISION' in os.environ):
  28   LLVM_WIN_REVISION = '233530'
  29
  30 # Path constants. (All of these should be absolute paths.)
  31 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  32 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
  33 LLVM_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm')
  34 CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
  35 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
  36                               'Release+Asserts')
  37 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
  38 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
  39 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
  40 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
  41 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
  42 VERSION = '3.7.0'
  43
  44 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
  45 if 'LLVM_REPO_URL' in os.environ:
  46   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
  47
  48
  49 def ReadStampFile():
  50   """Return the contents of the stamp file, or '' if it doesn't exist."""
  51   try:
  52     with open(STAMP_FILE, 'r') as f:
  53       return f.read();
  54   except IOError:
  55     return ''
  56
  57
  58 def WriteStampFile(s):
  59   """Write s to the stamp file."""
  60   if not os.path.exists(LLVM_BUILD_DIR):
  61     os.makedirs(LLVM_BUILD_DIR)
  62   with open(STAMP_FILE, 'w') as f:
  63     f.write(s)
  64
  65
  66 def RmTree(dir):
  67   """Delete dir."""
  68   def ChmodAndRetry(func, path, _):
  69     # Subversion can leave read-only files around.
  70     if not os.access(path, os.W_OK):
  71       os.chmod(path, stat.S_IWUSR)
  72       return func(path)
  73     raise
  74
  75   shutil.rmtree(dir, onerror=ChmodAndRetry)
  76
  77
  78 def ClobberChromiumBuildFiles():
  79   """Clobber Chomium build files."""
  80   print 'Clobbering Chromium build files...'
  81   out_dir = os.path.join(CHROMIUM_DIR, 'out')
  82   if os.path.isdir(out_dir):
  83     RmTree(out_dir)
  84     print 'Removed Chromium out dir: %s.' % (out_dir)
  85
  86
  87 def RunCommand(command, fail_hard=True):
  88   """Run command and return success (True) or failure; or if fail_hard is
  89      True, exit on failure."""
  90
  91   print 'Running %s' % (str(command))
  92   if subprocess.call(command, shell=True) == 0:
  93     return True
  94   print 'Failed.'
  95   if fail_hard:
  96     sys.exit(1)
  97   return False
  98
  99
 100 def CopyFile(src, dst):
 101   """Copy a file from src to dst."""
 102   shutil.copy(src, dst)
 103   print "Copying %s to %s" % (src, dst)
 104
 105
 106 def CopyDirectoryContents(src, dst, filename_filter=None):
 107   """Copy the files from directory src to dst
 108   with an optional filename filter."""
 109   if not os.path.exists(dst):
 110     os.makedirs(dst)
 111   for root, _, files in os.walk(src):
 112     for f in files:
 113       if filename_filter and not re.match(filename_filter, f):
 114         continue
 115       CopyFile(os.path.join(root, f), dst)
 116
 117
 118 def Checkout(name, url, dir):
 119   """Checkout the SVN module at url into dir. Use name for the log message."""
 120   print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir)
 121
 122   command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir]
 123   if RunCommand(command, fail_hard=False):
 124     return
 125
 126   if os.path.isdir(dir):
 127     print "Removing %s." % (dir)
 128     RmTree(dir)
 129
 130   print "Retrying."
 131   RunCommand(command)
 132
 133
 134 def DeleteChromeToolsShim():
 135   shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
 136
 137
 138 def CreateChromeToolsShim():
 139   """Hooks the Chrome tools into the LLVM build.
 140
 141   Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
 142   detects implicit tools in the tools subdirectory, so this helper install a
 143   shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
 144
 145   Note that the shim directory name intentionally has no - or _. The implicit
 146   tool detection logic munges them in a weird way."""
 147   assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
 148   os.mkdir(CHROME_TOOLS_SHIM_DIR)
 149   with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
 150     f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
 151             'Do not edit.\n')
 152     f.write('# Since tools/clang is located in another directory, use the \n')
 153     f.write('# two arg version to specify where build artifacts go. CMake\n')
 154     f.write('# disallows reuse of the same binary dir for multiple source\n')
 155     f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
 156     f.write('add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
 157             '${CMAKE_CURRENT_BINARY_DIR}/a)\n')
 158
 159
 160 def AddCMakeToPath():
 161   """Look for CMake and add it to PATH if it's not there already."""
 162   try:
 163     # First check if cmake is already on PATH.
 164     subprocess.call(['cmake', '--version'])
 165     return
 166   except OSError as e:
 167     if e.errno != os.errno.ENOENT:
 168       raise
 169
 170   cmake_locations = ['C:\\Program Files (x86)\\CMake\\bin',
 171                      'C:\\Program Files (x86)\\CMake 2.8\\bin']
 172   for d in cmake_locations:
 173     if os.path.isdir(d):
 174       os.environ['PATH'] = os.environ.get('PATH', '') + os.pathsep + d
 175       return
 176   print 'Failed to find CMake!'
 177   sys.exit(1)
 178
 179
 180 vs_version = None
 181 def GetVSVersion():
 182   global vs_version
 183   if vs_version:
 184     return vs_version
 185
 186   # Try using the toolchain in depot_tools.
 187   # This sets environment variables used by SelectVisualStudioVersion below.
 188   sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
 189   import vs_toolchain
 190   vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
 191
 192   # Use gyp to find the MSVS installation, either in depot_tools as per above,
 193   # or a system-wide installation otherwise.
 194   sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
 195   import gyp.MSVSVersion
 196   vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013')
 197   return vs_version
 198
 199
 200 def SubversionCmakeArg():
 201   # Since cmake's find_program can only find .exe and .com,
 202   # svn.bat in depot_tools will be ignored.
 203   default_pathext = ('.com', '.exe', '.bat', '.cmd')
 204   for path in os.environ.get('PATH', '').split(os.pathsep):
 205     for ext in default_pathext:
 206       candidate = os.path.join(path, 'svn' + ext)
 207       if os.path.isfile(candidate):
 208         return '-DSubversion_SVN_EXECUTABLE=%s' % candidate
 209   return ''
 210
 211
 212 def UpdateClang(args):
 213   print 'Updating Clang to %s...' % (LLVM_WIN_REVISION)
 214   if LLVM_WIN_REVISION != 'HEAD' and ReadStampFile() == LLVM_WIN_REVISION:
 215     print 'Already up to date.'
 216     return 0
 217
 218   AddCMakeToPath()
 219   if args.clobber:
 220     ClobberChromiumBuildFiles()
 221
 222   # Reset the stamp file in case the build is unsuccessful.
 223   WriteStampFile('')
 224
 225   DeleteChromeToolsShim();
 226   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
 227   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
 228   Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
 229   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
 230   CreateChromeToolsShim();
 231
 232   if not os.path.exists(LLVM_BUILD_DIR):
 233     os.makedirs(LLVM_BUILD_DIR)
 234   os.chdir(LLVM_BUILD_DIR)
 235
 236   cmake_args = ['-GNinja', '-DCMAKE_BUILD_TYPE=Release',
 237                 '-DLLVM_ENABLE_ASSERTIONS=ON', SubversionCmakeArg(),
 238                 '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(
 239                     CHROMIUM_DIR, 'tools', 'clang')]
 240   if args.tools:
 241     cmake_args.append('-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools))
 242
 243   RunCommand(GetVSVersion().SetupScript('x64') +
 244              ['&&', 'cmake'] + cmake_args + [LLVM_DIR])
 245   RunCommand(GetVSVersion().SetupScript('x64') + ['&&', 'ninja', 'all'])
 246
 247   # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
 248   # TODO(hans): Remove once the regular build above produces this.
 249   if not os.path.exists(COMPILER_RT_BUILD_DIR):
 250     os.makedirs(COMPILER_RT_BUILD_DIR)
 251   os.chdir(COMPILER_RT_BUILD_DIR)
 252   RunCommand(GetVSVersion().SetupScript('x86') +
 253              ['&&', 'cmake'] + cmake_args + [LLVM_DIR])
 254   RunCommand(GetVSVersion().SetupScript('x86') + ['&&', 'ninja', 'compiler-rt'])
 255
 256   # TODO(hans): Make this (and the .gypi and .isolate files) version number
 257   # independent.
 258   asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
 259                                      VERSION, 'lib', 'windows')
 260   asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 261                                      VERSION, 'lib', 'windows')
 262   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 263                         r'^.*-i386\.lib$')
 264   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 265                         r'^.*-i386\.dll$')
 266
 267   CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
 268            os.path.join(asan_rt_lib_dst_dir, '..', '..'))
 269
 270   # Make an extra copy of the sanitizer headers, to be put on the include path
 271   # of the fallback compiler.
 272   sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', VERSION,
 273                                        'include', 'sanitizer')
 274   aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 275                                            VERSION, 'include_sanitizer',
 276                                            'sanitizer')
 277   if not os.path.exists(aux_sanitizer_include_dir):
 278     os.makedirs(aux_sanitizer_include_dir)
 279   for _, _, files in os.walk(sanitizer_include_dir):
 280     for f in files:
 281       CopyFile(os.path.join(sanitizer_include_dir, f),
 282                aux_sanitizer_include_dir)
 283
 284   WriteStampFile(LLVM_WIN_REVISION)
 285   print 'Clang update was successful.'
 286   return 0
 287
 288
 289 def main():
 290   if not sys.platform in ['win32', 'cygwin']:
 291     # For non-Windows, fall back to update.sh.
 292     # TODO(hans): Make update.py replace update.sh completely.
 293
 294     # This script is called by gclient. gclient opens its hooks subprocesses
 295     # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does
 296     # custom output processing that breaks printing '\r' characters for
 297     # single-line updating status messages as printed by curl and wget.
 298     # Work around this by setting stderr of the update.sh process to stdin (!):
 299     # gclient doesn't redirect stdin, and while stdin itself is read-only, a
 300     # dup()ed sys.stdin is writable, try
 301     #   fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
 302     # TODO: Fix gclient instead, http://crbug.com/95350
 303     return subprocess.call(
 304         [os.path.join(os.path.dirname(__file__), 'update.sh')] +  sys.argv[1:],
 305         stderr=os.fdopen(os.dup(sys.stdin.fileno())))
 306
 307   if not re.search(r'\b(clang|asan)=1', os.environ.get('GYP_DEFINES', '')):
 308     print 'Skipping Clang update (clang=1 was not set in GYP_DEFINES).'
 309     return 0
 310
 311   if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
 312     print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
 313     return 0
 314
 315   parser = argparse.ArgumentParser(description='Build Clang.')
 316   parser.add_argument('--no-clobber', dest='clobber', action='store_false')
 317   parser.add_argument('--tools', nargs='*')
 318   # For now, this flag is only used for the non-Windows flow, but argparser gets
 319   # mad if it sees a flag it doesn't recognize.
 320   parser.add_argument('--if-needed', action='store_true')
 321   return UpdateClang(parser.parse_args())
 322
 323
 324 if __name__ == '__main__':
 325   sys.exit(main())