tools/clang/scripts/update.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Windows can't run .sh files, so this is a Python implementation of
   7 update.sh. This script should replace update.sh on all platforms eventually."""
   8
   9 import os
  10 import re
  11 import shutil
  12 import subprocess
  13 import stat
  14 import sys
  15 import time
  16
  17 # Do NOT CHANGE this if you don't know what you're doing -- see
  18 # https://code.google.com/p/chromium/wiki/UpdatingClang
  19 # Reverting problematic clang rolls is safe, though.
  20 # Note: this revision is only used for Windows. Other platforms use update.sh.
  21 LLVM_WIN_REVISION = 'HEAD'
  22
  23 # ASan on Windows is useful enough to use it even while the clang/win is still
  24 # in bringup. Use a pinned revision to make it slightly more stable.
  25 if (re.search(r'\b(asan)=1', os.environ.get('GYP_DEFINES', '')) and
  26     not 'LLVM_FORCE_HEAD_REVISION' in os.environ):
  27   LLVM_WIN_REVISION = '229860'
  28
  29 # Path constants. (All of these should be absolute paths.)
  30 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
  31 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
  32 LLVM_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm')
  33 LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
  34                               'Release+Asserts')
  35 COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
  36 CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
  37 LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
  38 COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
  39 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
  40 VERSION = '3.7.0'
  41
  42 LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
  43 if 'LLVM_REPO_URL' in os.environ:
  44   LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
  45
  46
  47 def ReadStampFile():
  48   """Return the contents of the stamp file, or '' if it doesn't exist."""
  49   try:
  50     with open(STAMP_FILE, 'r') as f:
  51       return f.read();
  52   except IOError:
  53     return ''
  54
  55
  56 def WriteStampFile(s):
  57   """Write s to the stamp file."""
  58   if not os.path.exists(LLVM_BUILD_DIR):
  59     os.makedirs(LLVM_BUILD_DIR)
  60   with open(STAMP_FILE, 'w') as f:
  61     f.write(s)
  62
  63
  64 def RmTree(dir):
  65   """Delete dir."""
  66   def ChmodAndRetry(func, path, _):
  67     # Subversion can leave read-only files around.
  68     if not os.access(path, os.W_OK):
  69       os.chmod(path, stat.S_IWUSR)
  70       return func(path)
  71     raise
  72
  73   shutil.rmtree(dir, onerror=ChmodAndRetry)
  74
  75
  76 def ClobberChromiumBuildFiles():
  77   """Clobber Chomium build files."""
  78   print 'Clobbering Chromium build files...'
  79   out_dir = os.path.join(CHROMIUM_DIR, 'out')
  80   if os.path.isdir(out_dir):
  81     RmTree(out_dir)
  82     print 'Removed Chromium out dir: %s.' % (out_dir)
  83
  84
  85 def RunCommand(command, fail_hard=True):
  86   """Run command and return success (True) or failure; or if fail_hard is
  87      True, exit on failure."""
  88
  89   print 'Running %s' % (str(command))
  90   if subprocess.call(command, shell=True) == 0:
  91     return True
  92   print 'Failed.'
  93   if fail_hard:
  94     sys.exit(1)
  95   return False
  96
  97
  98 def CopyFile(src, dst):
  99   """Copy a file from src to dst."""
 100   shutil.copy(src, dst)
 101   print "Copying %s to %s" % (src, dst)
 102
 103
 104 def CopyDirectoryContents(src, dst, filename_filter=None):
 105   """Copy the files from directory src to dst
 106   with an optional filename filter."""
 107   if not os.path.exists(dst):
 108     os.makedirs(dst)
 109   for root, _, files in os.walk(src):
 110     for f in files:
 111       if filename_filter and not re.match(filename_filter, f):
 112         continue
 113       CopyFile(os.path.join(root, f), dst)
 114
 115
 116 def Checkout(name, url, dir):
 117   """Checkout the SVN module at url into dir. Use name for the log message."""
 118   print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir)
 119
 120   command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir]
 121   if RunCommand(command, fail_hard=False):
 122     return
 123
 124   if os.path.isdir(dir):
 125     print "Removing %s." % (dir)
 126     RmTree(dir)
 127
 128   print "Retrying."
 129   RunCommand(command)
 130
 131
 132 def AddCMakeToPath():
 133   """Look for CMake and add it to PATH if it's not there already."""
 134   try:
 135     # First check if cmake is already on PATH.
 136     subprocess.call(['cmake', '--version'])
 137     return
 138   except OSError as e:
 139     if e.errno != os.errno.ENOENT:
 140       raise
 141
 142   cmake_locations = ['C:\\Program Files (x86)\\CMake\\bin',
 143                      'C:\\Program Files (x86)\\CMake 2.8\\bin']
 144   for d in cmake_locations:
 145     if os.path.isdir(d):
 146       os.environ['PATH'] = os.environ.get('PATH', '') + os.pathsep + d
 147       return
 148   print 'Failed to find CMake!'
 149   sys.exit(1)
 150
 151
 152 vs_version = None
 153 def GetVSVersion():
 154   global vs_version
 155   if vs_version:
 156     return vs_version
 157
 158   # Try using the toolchain in depot_tools.
 159   # This sets environment variables used by SelectVisualStudioVersion below.
 160   sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
 161   import vs_toolchain
 162   vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
 163
 164   # Use gyp to find the MSVS installation, either in depot_tools as per above,
 165   # or a system-wide installation otherwise.
 166   sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
 167   import gyp.MSVSVersion
 168   vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013')
 169   return vs_version
 170
 171
 172 def SubversionCmakeArg():
 173   # Since cmake's find_program can only find .exe and .com,
 174   # svn.bat in depot_tools will be ignored.
 175   default_pathext = ('.com', '.exe', '.bat', '.cmd')
 176   for path in os.environ.get('PATH', '').split(os.pathsep):
 177     for ext in default_pathext:
 178       candidate = os.path.join(path, 'svn' + ext)
 179       if os.path.isfile(candidate):
 180         return '-DSubversion_SVN_EXECUTABLE=%s' % candidate
 181   return ''
 182
 183
 184 def UpdateClang():
 185   print 'Updating Clang to %s...' % (LLVM_WIN_REVISION)
 186   if LLVM_WIN_REVISION != 'HEAD' and ReadStampFile() == LLVM_WIN_REVISION:
 187     print 'Already up to date.'
 188     return 0
 189
 190   AddCMakeToPath()
 191   ClobberChromiumBuildFiles()
 192
 193   # Reset the stamp file in case the build is unsuccessful.
 194   WriteStampFile('')
 195
 196   Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
 197   Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
 198   Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
 199   Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
 200
 201   if not os.path.exists(LLVM_BUILD_DIR):
 202     os.makedirs(LLVM_BUILD_DIR)
 203   os.chdir(LLVM_BUILD_DIR)
 204
 205   RunCommand(GetVSVersion().SetupScript('x64') +
 206              ['&&', 'cmake', '-GNinja', '-DCMAKE_BUILD_TYPE=Release',
 207               '-DLLVM_ENABLE_ASSERTIONS=ON', SubversionCmakeArg(), LLVM_DIR])
 208   RunCommand(GetVSVersion().SetupScript('x64') + ['&&', 'ninja', 'all'])
 209
 210   # Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
 211   # TODO(hans): Remove once the regular build above produces this.
 212   if not os.path.exists(COMPILER_RT_BUILD_DIR):
 213     os.makedirs(COMPILER_RT_BUILD_DIR)
 214   os.chdir(COMPILER_RT_BUILD_DIR)
 215   RunCommand(GetVSVersion().SetupScript('x86') +
 216              ['&&', 'cmake', '-GNinja', '-DCMAKE_BUILD_TYPE=Release',
 217               '-DLLVM_ENABLE_ASSERTIONS=ON', LLVM_DIR])
 218   RunCommand(GetVSVersion().SetupScript('x86') + ['&&', 'ninja', 'compiler-rt'])
 219
 220   # TODO(hans): Make this (and the .gypi and .isolate files) version number
 221   # independent.
 222   asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
 223                                      VERSION, 'lib', 'windows')
 224   asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 225                                      VERSION, 'lib', 'windows')
 226   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 227                         r'^.*-i386\.lib$')
 228   CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
 229                         r'^.*-i386\.dll$')
 230
 231   CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
 232            os.path.join(asan_rt_lib_dst_dir, '..', '..'))
 233
 234   # Make an extra copy of the sanitizer headers, to be put on the include path
 235   # of the fallback compiler.
 236   sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', VERSION,
 237                                        'include', 'sanitizer')
 238   aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
 239                                            VERSION, 'include_sanitizer',
 240                                            'sanitizer')
 241   if not os.path.exists(aux_sanitizer_include_dir):
 242     os.makedirs(aux_sanitizer_include_dir)
 243   for _, _, files in os.walk(sanitizer_include_dir):
 244     for f in files:
 245       CopyFile(os.path.join(sanitizer_include_dir, f),
 246                aux_sanitizer_include_dir)
 247
 248   WriteStampFile(LLVM_WIN_REVISION)
 249   print 'Clang update was successful.'
 250   return 0
 251
 252
 253 def main():
 254   if not sys.platform in ['win32', 'cygwin']:
 255     # For non-Windows, fall back to update.sh.
 256     # TODO(hans): Make update.py replace update.sh completely.
 257
 258     # This script is called by gclient. gclient opens its hooks subprocesses
 259     # with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does
 260     # custom output processing that breaks printing '\r' characters for
 261     # single-line updating status messages as printed by curl and wget.
 262     # Work around this by setting stderr of the update.sh process to stdin (!):
 263     # gclient doesn't redirect stdin, and while stdin itself is read-only, a
 264     # dup()ed sys.stdin is writable, try
 265     #   fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
 266     # TODO: Fix gclient instead, http://crbug.com/95350
 267     return subprocess.call(
 268         [os.path.join(os.path.dirname(__file__), 'update.sh')] +  sys.argv[1:],
 269         stderr=os.fdopen(os.dup(sys.stdin.fileno())))
 270
 271   if not re.search(r'\b(clang|asan)=1', os.environ.get('GYP_DEFINES', '')):
 272     print 'Skipping Clang update (clang=1 was not set in GYP_DEFINES).'
 273     return 0
 274
 275   if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
 276     print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
 277     return 0
 278
 279   return UpdateClang()
 280
 281
 282 if __name__ == '__main__':
 283   sys.exit(main())