third_party/instrumented_libraries/download_build_install.py

   1 #!/usr/bin/python
   2 # Copyright 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Downloads, builds (with instrumentation) and installs shared libraries."""
   7
   8 import argparse
   9 import os
  10 import platform
  11 import re
  12 import shlex
  13 import shutil
  14 import subprocess
  15 import sys
  16
  17 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__))
  18
  19 class ScopedChangeDirectory(object):
  20   """Changes current working directory and restores it back automatically."""
  21
  22   def __init__(self, path):
  23     self.path = path
  24     self.old_path = ''
  25
  26   def __enter__(self):
  27     self.old_path = os.getcwd()
  28     os.chdir(self.path)
  29     return self
  30
  31   def __exit__(self, exc_type, exc_value, traceback):
  32     os.chdir(self.old_path)
  33
  34 def get_package_build_dependencies(package):
  35   command = 'apt-get -s build-dep %s | grep Inst | cut -d " " -f 2' % package
  36   command_result = subprocess.Popen(command, stdout=subprocess.PIPE,
  37                                     shell=True)
  38   if command_result.wait():
  39     raise Exception('Failed to determine build dependencies for %s' % package)
  40   build_dependencies = [l.strip() for l in command_result.stdout]
  41   return build_dependencies
  42
  43
  44 def check_package_build_dependencies(package):
  45   build_dependencies = get_package_build_dependencies(package)
  46   if len(build_dependencies):
  47     print >> sys.stderr, 'Please, install build-dependencies for %s' % package
  48     print >> sys.stderr, 'One-liner for APT:'
  49     print >> sys.stderr, 'sudo apt-get -y --no-remove build-dep %s' % package
  50     sys.exit(1)
  51
  52
  53 def shell_call(command, verbose=False, environment=None):
  54   """ Wrapper on subprocess.Popen
  55
  56   Calls command with specific environment and verbosity using
  57   subprocess.Popen
  58
  59   Args:
  60     command: Command to run in shell.
  61     verbose: If False, hides all stdout and stderr in case of successful build.
  62         Otherwise, always prints stdout and stderr.
  63     environment: Parameter 'env' for subprocess.Popen.
  64
  65   Returns:
  66     None
  67
  68   Raises:
  69     Exception: if return code after call is not zero.
  70   """
  71   child = subprocess.Popen(
  72       command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
  73       env=environment, shell=True)
  74   stdout, stderr = child.communicate()
  75   if verbose or child.returncode:
  76     print stdout
  77   if child.returncode:
  78     raise Exception('Failed to run: %s' % command)
  79
  80
  81 def run_shell_commands(commands, verbose=False, environment=None):
  82   for command in commands:
  83     shell_call(command, verbose, environment)
  84
  85
  86 def fix_rpaths(destdir):
  87   # TODO(earthdok): reimplement fix_rpaths.sh in Python.
  88   shell_call("%s/fix_rpaths.sh %s/lib" % (SCRIPT_ABSOLUTE_PATH, destdir))
  89
  90
  91 def destdir_configure_make_install(parsed_arguments, environment,
  92                                    install_prefix):
  93   configure_command = './configure %s' % parsed_arguments.extra_configure_flags
  94   configure_command += ' --libdir=/lib/'
  95   # Installing to a temporary directory allows us to safely clean up the .la
  96   # files below.
  97   destdir = '%s/debian/instrumented_build' % os.getcwd()
  98   # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR.
  99   make_command = 'make DESTDIR=%s BUILDROOT=%s INSTALL_ROOT=%s' % (destdir,
 100                                                                    destdir,
 101                                                                    destdir)
 102   build_and_install_in_destdir = [
 103     configure_command,
 104     '%s -j%s' % (make_command, parsed_arguments.jobs),
 105     # Parallel install is flaky for some packages.
 106     '%s install -j1' % make_command,
 107     # Kill the .la files. They contain absolute paths, and will cause build
 108     # errors in dependent libraries.
 109     'rm %s/lib/*.la -f' % destdir
 110   ]
 111   run_shell_commands(build_and_install_in_destdir,
 112                      parsed_arguments.verbose, environment)
 113   fix_rpaths(destdir)
 114   run_shell_commands([
 115       # Now move the contents of the temporary destdir to their final place.
 116       # We only care for the contents of lib/.
 117       'mkdir -p %s/lib' % install_prefix,
 118       'cp %s/lib/* %s/lib/ -rdf' % (destdir, install_prefix)],
 119                      parsed_arguments.verbose, environment)
 120
 121
 122 def nss_make_and_copy(parsed_arguments, environment, install_prefix):
 123   # NSS uses a build system that's different from configure/make/install. All
 124   # flags must be passed as arguments to make.
 125   make_args = []
 126   # Do an optimized build.
 127   make_args.append('BUILD_OPT=1')
 128   # Set USE_64=1 on x86_64 systems.
 129   if platform.architecture()[0] == '64bit':
 130     make_args.append('USE_64=1')
 131   # Passing C(XX)FLAGS overrides the defaults, and EXTRA_C(XX)FLAGS is not
 132   # supported. Append our extra flags to CC/CXX.
 133   make_args.append('CC="%s %s"' % (environment['CC'], environment['CFLAGS']))
 134   make_args.append('CXX="%s %s"' %
 135                    (environment['CXX'], environment['CXXFLAGS']))
 136   # We need to override ZDEFS_FLAG at least to prevent -Wl,-z,defs.
 137   # Might as well use this to pass the linker flags, since ZDEF_FLAG is always
 138   # added during linking on Linux.
 139   make_args.append('ZDEFS_FLAG="-Wl,-z,nodefs %s"' % environment['LDFLAGS'])
 140   make_args.append('NSPR_INCLUDE_DIR=/usr/include/nspr')
 141   make_args.append('NSPR_LIB_DIR=%s/lib' % install_prefix)
 142   make_args.append('NSS_ENABLE_ECC=1')
 143   # Make sure we don't override the default flags.
 144   for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']:
 145     del environment[variable]
 146   with ScopedChangeDirectory('nss') as cd_nss:
 147     # -j is not supported
 148     shell_call('make %s' % ' '.join(make_args), parsed_arguments.verbose,
 149                environment)
 150     fix_rpaths(os.getcwd())
 151     # 'make install' is not supported. Copy the DSOs manually.
 152     install_dir = '%s/lib/' % install_prefix
 153     for (dirpath, dirnames, filenames) in os.walk('./lib/'):
 154       for filename in filenames:
 155         if filename.endswith('.so'):
 156           full_path = os.path.join(dirpath, filename)
 157           if parsed_arguments.verbose:
 158             print 'download_build_install.py: installing %s' % full_path
 159           shutil.copy(full_path, install_dir)
 160
 161
 162 def libcap2_make_install(parsed_arguments, environment, install_prefix):
 163   # libcap2 doesn't come with a configure script
 164   make_args = [
 165       '%s="%s"' % (name, environment[name])
 166       for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']]
 167   shell_call('make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args)),
 168              parsed_arguments.verbose, environment)
 169   destdir = '%s/debian/instrumented_build' % os.getcwd()
 170   install_args = [
 171       'DESTDIR=%s' % destdir,
 172       # Do not install in lib64/.
 173       'lib=lib',
 174       # Skip a step that requires sudo.
 175       'RAISE_SETFCAP=no'
 176   ]
 177   shell_call('make -j%s install %s' %
 178              (parsed_arguments.jobs, ' '.join(install_args)),
 179              parsed_arguments.verbose, environment)
 180   fix_rpaths(destdir)
 181   run_shell_commands([
 182       # Now move the contents of the temporary destdir to their final place.
 183       # We only care for the contents of lib/.
 184       'mkdir -p %s/lib' % install_prefix,
 185       'cp %s/lib/* %s/lib/ -rdf' % (destdir, install_prefix)],
 186                       parsed_arguments.verbose, environment)
 187
 188
 189 def libpci3_make_install(parsed_arguments, environment, install_prefix):
 190   # pciutils doesn't have a configure script
 191   # This build script follows debian/rules.
 192
 193   # Find out the package version. We'll use this when creating symlinks.
 194   dir_name = os.path.split(os.getcwd())[-1]
 195   match = re.match('pciutils-(\d+\.\d+\.\d+)', dir_name)
 196   if match is None:
 197     raise Exception(
 198         'Unable to guess libpci3 version from directory name: %s' %  dir_name)
 199   version = match.group(1)
 200
 201   # `make install' will create a "$(DESTDIR)-udeb" directory alongside destdir.
 202   # We don't want that in our product dir, so we use an intermediate directory.
 203   destdir = '%s/debian/pciutils' % os.getcwd()
 204   make_args = [
 205       '%s="%s"' % (name, environment[name])
 206       for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']]
 207   make_args.append('SHARED=yes')
 208   # pciutils-3.2.1 (Trusty) fails to build due to unresolved libkmod symbols.
 209   # The binary package has no dependencies on libkmod, so it looks like it was
 210   # actually built without libkmod support.
 211   make_args.append('LIBKMOD=no')
 212   paths = [
 213       'LIBDIR=/lib/',
 214       'PREFIX=/usr',
 215       'SBINDIR=/usr/bin',
 216       'IDSDIR=/usr/share/misc',
 217   ]
 218   install_args = ['DESTDIR=%s' % destdir]
 219   run_shell_commands([
 220       'mkdir -p %s-udeb/usr/bin' % destdir,
 221       'make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args + paths)),
 222       'make -j%s %s install' % (
 223           parsed_arguments.jobs,
 224           ' '.join(install_args + paths))],
 225                      parsed_arguments.verbose, environment)
 226   fix_rpaths(destdir)
 227   # Now install the DSOs to their final place.
 228   run_shell_commands([
 229       'mkdir -p %s/lib' % install_prefix,
 230       'install -m 644 lib/libpci.so* %s/lib/' % install_prefix,
 231       'ln -sf libpci.so.%s %s/lib/libpci.so.3' % (version, install_prefix)],
 232                      parsed_arguments.verbose, environment)
 233
 234
 235 def build_and_install(parsed_arguments, environment, install_prefix):
 236   if parsed_arguments.build_method == 'destdir':
 237     destdir_configure_make_install(
 238         parsed_arguments, environment, install_prefix)
 239   elif parsed_arguments.build_method == 'custom_nss':
 240     nss_make_and_copy(parsed_arguments, environment, install_prefix)
 241   elif parsed_arguments.build_method == 'custom_libcap':
 242     libcap2_make_install(parsed_arguments, environment, install_prefix)
 243   elif parsed_arguments.build_method == 'custom_libpci3':
 244     libpci3_make_install(parsed_arguments, environment, install_prefix)
 245   else:
 246     raise Exception('Unrecognized build method: %s' %
 247                     parsed_arguments.build_method)
 248
 249
 250 def unescape_flags(s):
 251   # GYP escapes the build flags as if they are going to be inserted directly
 252   # into the command line. Since we pass them via CFLAGS/LDFLAGS, we must drop
 253   # the double quotes accordingly.
 254   return ' '.join(shlex.split(s))
 255
 256
 257 def build_environment(parsed_arguments, product_directory, install_prefix):
 258   environment = os.environ.copy()
 259   # The CC/CXX environment variables take precedence over the command line
 260   # flags.
 261   if 'CC' not in environment and parsed_arguments.cc:
 262     environment['CC'] = parsed_arguments.cc
 263   if 'CXX' not in environment and parsed_arguments.cxx:
 264     environment['CXX'] = parsed_arguments.cxx
 265
 266   cflags = unescape_flags(parsed_arguments.cflags)
 267   if parsed_arguments.sanitizer_blacklist:
 268     cflags += ' -fsanitize-blacklist=%s/%s' % (
 269         SCRIPT_ABSOLUTE_PATH,
 270         parsed_arguments.sanitizer_blacklist)
 271   environment['CFLAGS'] = cflags
 272   environment['CXXFLAGS'] = cflags
 273
 274   ldflags = unescape_flags(parsed_arguments.ldflags)
 275   # Make sure the linker searches the instrumented libraries dir for
 276   # library dependencies.
 277   environment['LDFLAGS'] = '%s -L%s/lib' % (ldflags, install_prefix)
 278
 279   if parsed_arguments.sanitizer_type == 'asan':
 280     # Do not report leaks during the build process.
 281     environment['ASAN_OPTIONS'] = '%s:detect_leaks=0' % \
 282         environment.get('ASAN_OPTIONS', '')
 283
 284   # libappindicator1 needs this.
 285   environment['CSC'] = '/usr/bin/mono-csc'
 286   return environment
 287
 288
 289
 290 def download_build_install(parsed_arguments):
 291   product_directory = os.path.normpath('%s/%s' % (
 292       SCRIPT_ABSOLUTE_PATH,
 293       parsed_arguments.product_directory))
 294
 295   install_prefix = '%s/instrumented_libraries/%s' % (
 296       product_directory,
 297       parsed_arguments.sanitizer_type)
 298
 299   environment = build_environment(parsed_arguments, product_directory,
 300                                   install_prefix)
 301
 302   package_directory = '%s/%s' % (parsed_arguments.intermediate_directory,
 303                                  parsed_arguments.package)
 304
 305   # Clobber by default, unless the developer wants to hack on the package's
 306   # source code.
 307   clobber = (environment.get('INSTRUMENTED_LIBRARIES_NO_CLOBBER', '') != '1')
 308
 309   download_source = True
 310   if os.path.exists(package_directory):
 311     if clobber:
 312       shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose)
 313     else:
 314       download_source = False
 315   if download_source:
 316     os.makedirs(package_directory)
 317
 318   with ScopedChangeDirectory(package_directory) as cd_package:
 319     if download_source:
 320       shell_call('apt-get source %s' % parsed_arguments.package,
 321                  parsed_arguments.verbose)
 322     # There should be exactly one subdirectory after downloading a package.
 323     subdirectories = [d for d in os.listdir('.') if os.path.isdir(d)]
 324     if len(subdirectories) != 1:
 325       raise Exception('apt-get source %s must create exactly one subdirectory.'
 326          % parsed_arguments.package)
 327     with ScopedChangeDirectory(subdirectories[0]):
 328       # Here we are in the package directory.
 329       if download_source:
 330         # Patch/run_before_build steps are only done once.
 331         if parsed_arguments.patch:
 332           shell_call(
 333               'patch -p1 -i %s/%s' %
 334               (os.path.relpath(cd_package.old_path),
 335                parsed_arguments.patch),
 336               parsed_arguments.verbose)
 337         if parsed_arguments.run_before_build:
 338           shell_call(
 339               '%s/%s' %
 340               (os.path.relpath(cd_package.old_path),
 341                parsed_arguments.run_before_build),
 342               parsed_arguments.verbose)
 343       try:
 344         build_and_install(parsed_arguments, environment, install_prefix)
 345       except Exception as exception:
 346         print exception
 347         print 'Failed to build package %s.' % parsed_arguments.package
 348         print ('Probably, some of its dependencies are not installed: %s' %
 349                ' '.join(get_package_build_dependencies(parsed_arguments.package)))
 350         sys.exit(1)
 351
 352   # Touch a txt file to indicate package is installed.
 353   open('%s/%s.txt' % (install_prefix, parsed_arguments.package), 'w').close()
 354
 355   # Remove downloaded package and generated temporary build files.
 356   # Failed builds intentionally skip this step, in order to aid in tracking down
 357   # build failures.
 358   if clobber:
 359     shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose)
 360
 361 def main():
 362   argument_parser = argparse.ArgumentParser(
 363       description='Download, build and install instrumented package')
 364
 365   argument_parser.add_argument('-j', '--jobs', type=int, default=1)
 366   argument_parser.add_argument('-p', '--package', required=True)
 367   argument_parser.add_argument(
 368       '-i', '--product-directory', default='.',
 369       help='Relative path to the directory with chrome binaries')
 370   argument_parser.add_argument(
 371       '-m', '--intermediate-directory', default='.',
 372       help='Relative path to the directory for temporary build files')
 373   argument_parser.add_argument('--extra-configure-flags', default='')
 374   argument_parser.add_argument('--cflags', default='')
 375   argument_parser.add_argument('--ldflags', default='')
 376   argument_parser.add_argument('-s', '--sanitizer-type', required=True,
 377                                choices=['asan', 'msan', 'tsan'])
 378   argument_parser.add_argument('-v', '--verbose', action='store_true')
 379   argument_parser.add_argument('--check-build-deps', action='store_true')
 380   argument_parser.add_argument('--cc')
 381   argument_parser.add_argument('--cxx')
 382   argument_parser.add_argument('--patch', default='')
 383   # This should be a shell script to run before building specific libraries.
 384   # This will be run after applying the patch above.
 385   argument_parser.add_argument('--run-before-build', default='')
 386   argument_parser.add_argument('--build-method', default='destdir')
 387   argument_parser.add_argument('--sanitizer-blacklist', default='')
 388
 389   # Ignore all empty arguments because in several cases gyp passes them to the
 390   # script, but ArgumentParser treats them as positional arguments instead of
 391   # ignoring (and doesn't have such options).
 392   parsed_arguments = argument_parser.parse_args(
 393       [arg for arg in sys.argv[1:] if len(arg) != 0])
 394   # Ensure current working directory is this script directory.
 395   os.chdir(SCRIPT_ABSOLUTE_PATH)
 396   # Ensure all build dependencies are installed.
 397   if parsed_arguments.check_build_deps:
 398     check_package_build_dependencies(parsed_arguments.package)
 399
 400   download_build_install(parsed_arguments)
 401
 402
 403 if __name__ == '__main__':
 404   main()