tools/clang/scripts/package.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """This script will check out llvm and clang, and then package the results up
   7 to a tgz file."""
   8
   9 import argparse
  10 import fnmatch
  11 import itertools
  12 import os
  13 import shutil
  14 import subprocess
  15 import sys
  16 import tarfile
  17
  18 # Path constants.
  19 THIS_DIR = os.path.dirname(__file__)
  20 THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party')
  21 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
  22 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
  23 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
  24                                           'llvm-bootstrap-install')
  25 LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build')
  26 LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts')
  27 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
  28
  29
  30 def Tee(output, logfile):
  31   logfile.write(output)
  32   print output,
  33
  34
  35 def TeeCmd(cmd, logfile, fail_hard=True):
  36   """Runs cmd and writes the output to both stdout and logfile."""
  37   # Reading from PIPE can deadlock if one buffer is full but we wait on a
  38   # different one.  To work around this, pipe the subprocess's stderr to
  39   # its stdout buffer and don't give it a stdin.
  40   # shell=True is required in cmd.exe since depot_tools has an svn.bat, and
  41   # bat files only work with shell=True set.
  42   proc = subprocess.Popen(cmd, bufsize=1, shell=sys.platform == 'win32',
  43                           stdin=open(os.devnull), stdout=subprocess.PIPE,
  44                           stderr=subprocess.STDOUT)
  45   for line in iter(proc.stdout.readline,''):
  46     Tee(line, logfile)
  47     if proc.poll() is not None:
  48       break
  49   exit_code = proc.wait()
  50   if exit_code != 0 and fail_hard:
  51     print 'Failed:', cmd
  52     sys.exit(1)
  53
  54
  55 def PrintTarProgress(tarinfo):
  56   print 'Adding', tarinfo.name
  57   return tarinfo
  58
  59
  60 def main():
  61   if sys.platform == 'win32':
  62     try:
  63       subprocess.check_output(['grep', '--help'], shell=True)
  64     except subprocess.CalledProcessError:
  65       print 'Add gnuwin32 to your PATH, then try again.'
  66       return 1
  67
  68   parser = argparse.ArgumentParser(description='build and package clang')
  69   parser.add_argument('--gcc-toolchain',
  70                       help="the prefix for the GCC version used for building. "
  71                            "For /opt/foo/bin/gcc, pass "
  72                            "'--gcc-toolchain '/opt/foo'")
  73
  74   args = parser.parse_args()
  75
  76   with open('buildlog.txt', 'w') as log:
  77     Tee('Diff in llvm:\n', log)
  78     TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False)
  79     TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False)
  80     Tee('Diff in llvm/tools/clang:\n', log)
  81     TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')],
  82            log, fail_hard=False)
  83     TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')],
  84            log, fail_hard=False)
  85     # TODO(thakis): compiler-rt is in projects/compiler-rt on Windows but
  86     # llvm/compiler-rt elsewhere. So this diff call is currently only right on
  87     # Windows.
  88     Tee('Diff in llvm/compiler-rt:\n', log)
  89     TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')],
  90            log, fail_hard=False)
  91     TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')],
  92            log, fail_hard=False)
  93     Tee('Diff in llvm/projects/libcxx:\n', log)
  94     TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxx')],
  95            log, fail_hard=False)
  96     TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxx')],
  97            log, fail_hard=False)
  98     Tee('Diff in llvm/projects/libcxxabi:\n', log)
  99     TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxxabi')],
 100            log, fail_hard=False)
 101     TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxxabi')],
 102            log, fail_hard=False)
 103
 104     Tee('Starting build\n', log)
 105
 106     # Do a clobber build.
 107     shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True)
 108     shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True)
 109     shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True)
 110
 111     build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'),
 112                  '--bootstrap', '--force-local-build', '--run-tests',
 113                  '--no-stdin-hack']
 114     if args.gcc_toolchain is not None:
 115       build_cmd.extend(['--gcc-toolchain', args.gcc_toolchain])
 116     TeeCmd(build_cmd, log)
 117
 118   stamp = open(STAMP_FILE).read().rstrip()
 119   pdir = 'clang-' + stamp
 120   print pdir
 121   shutil.rmtree(pdir, ignore_errors=True)
 122
 123   # Copy a whitelist of files to the directory we're going to tar up.
 124   # This supports the same patterns that the fnmatch module understands.
 125   exe_ext = '.exe' if sys.platform == 'win32' else ''
 126   want = ['bin/llvm-symbolizer' + exe_ext,
 127           'lib/clang/*/asan_blacklist.txt',
 128           'lib/clang/*/cfi_blacklist.txt',
 129           # Copy built-in headers (lib/clang/3.x.y/include).
 130           'lib/clang/*/include/*',
 131           ]
 132   if sys.platform == 'win32':
 133     want.append('bin/clang-cl.exe')
 134     want.append('bin/lld-link.exe')
 135   else:
 136     so_ext = 'dylib' if sys.platform == 'darwin' else 'so'
 137     want.extend(['bin/clang',
 138                  'lib/libFindBadConstructs.' + so_ext,
 139                  'lib/libBlinkGCPlugin.' + so_ext,
 140                  ])
 141   if sys.platform == 'darwin':
 142     want.extend(['bin/libc++.1.dylib',
 143                  # Copy only the OSX (ASan and profile) and iossim (ASan)
 144                  # runtime libraries:
 145                  'lib/clang/*/lib/darwin/*asan_osx*',
 146                  'lib/clang/*/lib/darwin/*asan_iossim*',
 147                  'lib/clang/*/lib/darwin/*profile_osx*',
 148                  ])
 149   elif sys.platform.startswith('linux'):
 150     # Copy only
 151     # lib/clang/*/lib/linux/libclang_rt.{[atm]san,san,ubsan,profile}-*.a ,
 152     # but not dfsan.
 153     want.extend(['lib/clang/*/lib/linux/*[atm]san*',
 154                  'lib/clang/*/lib/linux/*ubsan*',
 155                  'lib/clang/*/lib/linux/*libclang_rt.san*',
 156                  'lib/clang/*/lib/linux/*profile*',
 157                  'lib/clang/*/msan_blacklist.txt',
 158                  ])
 159   elif sys.platform == 'win32':
 160     want.extend(['lib/clang/*/lib/windows/clang_rt.asan*.dll',
 161                  'lib/clang/*/lib/windows/clang_rt.asan*.lib',
 162                  'lib/clang/*/include_sanitizer/*',
 163                  ])
 164   if args.gcc_toolchain is not None:
 165     # Copy the stdlibc++.so.6 we linked Clang against so it can run.
 166     want.append('lib/libstdc++.so.6')
 167
 168   for root, dirs, files in os.walk(LLVM_RELEASE_DIR):
 169     # root: third_party/llvm-build/Release+Asserts/lib/..., rel_root: lib/...
 170     rel_root = root[len(LLVM_RELEASE_DIR)+1:]
 171     rel_files = [os.path.join(rel_root, f) for f in files]
 172     wanted_files = list(set(itertools.chain.from_iterable(
 173         fnmatch.filter(rel_files, p) for p in want)))
 174     if wanted_files:
 175       # Guaranteed to not yet exist at this point:
 176       os.makedirs(os.path.join(pdir, rel_root))
 177     for f in wanted_files:
 178       src = os.path.join(LLVM_RELEASE_DIR, f)
 179       dest = os.path.join(pdir, f)
 180       shutil.copy(src, dest)
 181       # Strip libraries.
 182       if sys.platform == 'darwin' and f.endswith('.dylib'):
 183         # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
 184         # @executable_path.
 185         # TODO(glider): this is transitional. We'll need to fix the dylib
 186         # name either in our build system, or in Clang. See also
 187         # http://crbug.com/344836.
 188         subprocess.call(['install_name_tool', '-id',
 189                          '@executable_path/' + os.path.basename(dest), dest])
 190         subprocess.call(['strip', '-x', dest])
 191       elif (sys.platform.startswith('linux') and
 192             os.path.splitext(f)[1] in ['.so', '.a']):
 193         subprocess.call(['strip', '-g', dest])
 194
 195   # Set up symlinks.
 196   if sys.platform != 'win32':
 197     os.symlink('clang', os.path.join(pdir, 'bin', 'clang++'))
 198     os.symlink('clang', os.path.join(pdir, 'bin', 'clang-cl'))
 199   if sys.platform == 'darwin':
 200     os.symlink('libc++.1.dylib', os.path.join(pdir, 'bin', 'libc++.dylib'))
 201     # Also copy libc++ headers.
 202     shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'),
 203                     os.path.join(pdir, 'include', 'c++'))
 204
 205   # Copy buildlog over.
 206   shutil.copy('buildlog.txt', pdir)
 207
 208   # Create archive.
 209   tar_entries = ['bin', 'lib', 'buildlog.txt']
 210   if sys.platform == 'darwin':
 211     tar_entries += ['include']
 212   with tarfile.open(pdir + '.tgz', 'w:gz') as tar:
 213     for entry in tar_entries:
 214       tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress)
 215
 216   if sys.platform == 'darwin':
 217     platform = 'Mac'
 218   elif sys.platform == 'win32':
 219     platform = 'Win'
 220   else:
 221     platform = 'Linux_x64'
 222
 223   print 'To upload, run:'
 224   print ('gsutil cp -a public-read %s.tgz '
 225          'gs://chromium-browser-clang/%s/%s.tgz') % (pdir, platform, pdir)
 226
 227   # Zip up gold plugin on Linux.
 228   if sys.platform.startswith('linux'):
 229     golddir = 'llvmgold-' + stamp
 230     shutil.rmtree(golddir, ignore_errors=True)
 231     os.makedirs(os.path.join(golddir, 'lib'))
 232     shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'lib', 'LLVMgold.so'),
 233                 os.path.join(golddir, 'lib'))
 234     with tarfile.open(golddir + '.tgz', 'w:gz') as tar:
 235       tar.add(os.path.join(golddir, 'lib'), arcname='lib',
 236               filter=PrintTarProgress)
 237     print ('gsutil cp -a public-read %s.tgz '
 238            'gs://chromium-browser-clang/%s/%s.tgz') % (golddir, platform,
 239                                                        golddir)
 240
 241   # FIXME: Warn if the file already exists on the server.
 242
 243
 244 if __name__ == '__main__':
 245   sys.exit(main())