third_party/instrumented_libraries/download_build_install.py

   1 #!/usr/bin/python
   2 # Copyright 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Downloads, builds (with instrumentation) and installs shared libraries."""
   7
   8 import argparse
   9 import os
  10 import platform
  11 import shutil
  12 import subprocess
  13 import sys
  14
  15 # Build parameters for different sanitizers.
  16 # We use XORIGIN as RPATH and after building library replace it to $ORIGIN
  17 # The reason: this flag goes through configure script and makefiles
  18 # differently for different libraries. So the dollar sign '$' should be
  19 # differently escaped. Instead of having problems with that it just
  20 # uses XORIGIN to build library and after that replaces it to $ORIGIN
  21 # directly in .so file.
  22 SUPPORTED_SANITIZERS = {
  23     'asan': {
  24         'compiler_flags': '-O2 -fsanitize=address -gline-tables-only -fPIC -w '
  25                           '-U_FORITFY_SOURCE',
  26         'linker_flags': '-fsanitize=address -Wl,-z,origin -Wl,-R,XORIGIN/.'
  27     },
  28     'msan': {
  29         'compiler_flags': '-O2 -fsanitize=memory '
  30                           '-fsanitize-memory-track-origins '
  31                           '-gline-tables-only -fPIC -w -U_FORTIFY_SOURCE',
  32         'linker_flags': '-fsanitize=memory -Wl,-z,origin -Wl,-R,XORIGIN/.'
  33     },
  34     'tsan': {
  35         'compiler_flags': '-O2 -fsanitize=thread -gline-tables-only -fPIC -w '
  36                           '-U_FORTIFY_SOURCE',
  37         'linker_flags': '-fsanitize=thread -Wl,-z,origin -Wl,-R,XORIGIN/.'
  38     },
  39 }
  40
  41
  42 class ScopedChangeDirectory(object):
  43   """Changes current working directory and restores it back automatically."""
  44
  45   def __init__(self, path):
  46     self.path = path
  47     self.old_path = ''
  48
  49   def __enter__(self):
  50     self.old_path = os.getcwd()
  51     os.chdir(self.path)
  52     return self
  53
  54   def __exit__(self, exc_type, exc_value, traceback):
  55     os.chdir(self.old_path)
  56
  57
  58 def get_script_absolute_path():
  59   return os.path.dirname(os.path.abspath(__file__))
  60
  61
  62 def get_library_build_dependencies(library):
  63   command = 'apt-get -s build-dep %s | grep Inst | cut -d " " -f 2' % library
  64   command_result = subprocess.Popen(command, stdout=subprocess.PIPE,
  65                                     shell=True)
  66   if command_result.wait():
  67     raise Exception('Failed to determine build dependencies for %s' % library)
  68   build_dependencies = [l.strip() for l in command_result.stdout]
  69   return build_dependencies
  70
  71
  72 def check_library_build_dependencies(library):
  73   build_dependencies = get_library_build_dependencies(library)
  74   if len(build_dependencies):
  75     print >> sys.stderr, 'Please, install build-dependencies for %s' % library
  76     print >> sys.stderr, 'One-liner for APT:'
  77     print >> sys.stderr, 'sudo apt-get -y --no-remove build-dep %s' % library
  78     sys.exit(1)
  79
  80
  81 def shell_call(command, verbose=False, environment=None):
  82   """ Wrapper on subprocess.Popen
  83
  84   Calls command with specific environment and verbosity using
  85   subprocess.Popen
  86
  87   Args:
  88     command: Command to run in shell.
  89     verbose: If False, hides all stdout and stderr in case of successful build.
  90         Otherwise, always prints stdout and stderr.
  91     environment: Parameter 'env' for subprocess.Popen.
  92
  93   Returns:
  94     None
  95
  96   Raises:
  97     Exception: if return code after call is not zero.
  98   """
  99   child = subprocess.Popen(
 100       command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 101       env=environment, shell=True)
 102   stdout, stderr = child.communicate()
 103   if verbose or child.returncode:
 104     print stdout
 105   if child.returncode:
 106     raise Exception('Failed to run: %s' % command)
 107
 108
 109 def run_shell_commands(commands, verbose=False, environment=None):
 110   for command in commands:
 111     shell_call(command, verbose, environment)
 112
 113
 114 def destdir_configure_make_install(parsed_arguments, environment,
 115                                    install_prefix):
 116   configure_command = './configure %s' % parsed_arguments.extra_configure_flags
 117   configure_command += ' --libdir=/lib/'
 118   # Installing to a temporary directory allows us to safely clean up the .la
 119   # files below.
 120   destdir = '%s/debian/instrumented_build' % os.getcwd()
 121   # Some makefiles use BUILDROOT instead of DESTDIR.
 122   make_command = 'make -j%s DESTDIR=%s BUILDROOT=%s' % (
 123       parsed_arguments.jobs, destdir, destdir),
 124   run_shell_commands([
 125       configure_command,
 126       make_command,
 127       '%s install' % make_command,
 128       # Kill the .la files. They contain absolute paths, and will cause build
 129       # errors in dependent libraries.
 130       'rm %s/lib/*.la -f' % destdir,
 131       # Now move the contents of the temporary destdir to their final place.
 132       'cp %s/* %s/ -rdf' % (destdir, install_prefix)],
 133                      parsed_arguments.verbose, environment)
 134
 135
 136 def prefix_configure_make_install(parsed_arguments, environment,
 137                                   install_prefix):
 138   configure_command = './configure %s --prefix=%s' % (
 139       parsed_arguments.extra_configure_flags, install_prefix)
 140   shell_call(configure_command, parsed_arguments.verbose, environment)
 141   shell_call('make -j%s' % parsed_arguments.jobs,
 142              parsed_arguments.verbose, environment)
 143   shell_call('make -j%s install' % parsed_arguments.jobs,
 144              parsed_arguments.verbose, environment)
 145
 146
 147 def nss_make_and_copy(parsed_arguments, environment, install_prefix):
 148   # NSS uses a build system that's different from configure/make/install. All
 149   # flags must be passed as arguments to make.
 150   make_args = []
 151   # Do an optimized build.
 152   make_args.append('BUILD_OPT=1')
 153   # Set USE_64=1 on x86_64 systems.
 154   if platform.architecture()[0] == '64bit':
 155     make_args.append('USE_64=1')
 156   # Passing C(XX)FLAGS overrides the defaults, and EXTRA_C(XX)FLAGS is not
 157   # supported. Append our extra flags to CC/CXX.
 158   make_args.append('CC="%s %s"' % (environment['CC'], environment['CFLAGS']))
 159   make_args.append('CXX="%s %s"' %
 160                    (environment['CXX'], environment['CXXFLAGS']))
 161   # We need to override ZDEFS_FLAGS at least to prevent -Wl,-z,defs.
 162   # Might as well use this to pass the linker flags, since ZDEF_FLAGS is always
 163   # added during linking on Linux.
 164   make_args.append('ZDEFS_FLAG="-Wl,-z,nodefs %s"' % environment['LDFLAGS'])
 165   make_args.append('NSPR_INCLUDE_DIR=/usr/include/nspr')
 166   make_args.append('NSPR_LIB_DIR=%s/lib' % install_prefix)
 167   make_args.append('NSS_ENABLE_ECC=1')
 168   with ScopedChangeDirectory('nss') as cd_nss:
 169     # -j is not supported
 170     shell_call('make %s' % ' '.join(make_args), parsed_arguments.verbose,
 171                environment)
 172     # 'make install' is not supported. Copy the DSOs manually.
 173     install_dir = '%s/lib/' % install_prefix
 174     for (dirpath, dirnames, filenames) in os.walk('./lib/'):
 175       for filename in filenames:
 176         if filename.endswith('.so'):
 177           full_path = os.path.join(dirpath, filename)
 178           if parsed_arguments.verbose:
 179             print 'download_build_install.py: installing %s' % full_path
 180           shutil.copy(full_path, install_dir)
 181
 182
 183 def libcap2_make_install(parsed_arguments, environment, install_prefix):
 184   # libcap2 doesn't come with a configure script
 185   make_args = [
 186       '%s="%s"' % (name, environment[name])
 187       for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']]
 188   shell_call('make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args)),
 189              parsed_arguments.verbose, environment)
 190   install_args = [
 191       'DESTDIR=%s' % install_prefix,
 192       # Do not install in lib64/.
 193       'lib=lib',
 194       # Skip a step that requires sudo.
 195       'RAISE_SETFCAP=no'
 196   ]
 197   shell_call('make -j%s install %s' %
 198              (parsed_arguments.jobs, ' '.join(install_args)),
 199              parsed_arguments.verbose, environment)
 200
 201
 202 def libpci3_make_install(parsed_arguments, environment, install_prefix):
 203   # pciutils doesn't have a configure script
 204   # This build script follows debian/rules.
 205
 206   # `make install' will create a "$(DESTDIR)-udeb" directory alongside destdir.
 207   # We don't want that in our product dir, so we use an intermediate directory.
 208   destdir = '%s/debian/pciutils' % os.getcwd()
 209   make_args = [
 210       '%s="%s"' % (name, environment[name])
 211       for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']]
 212   make_args.append('SHARED=yes')
 213   paths = [
 214       'LIBDIR=/lib/',
 215       'PREFIX=/usr',
 216       'SBINDIR=/usr/bin',
 217       'IDSDIR=/usr/share/misc',
 218   ]
 219   install_args = ['DESTDIR=%s' % destdir]
 220   run_shell_commands([
 221       'mkdir -p %s-udeb/usr/bin' % destdir,
 222       'make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args + paths)),
 223       'make -j%s %s install' % (
 224           parsed_arguments.jobs,
 225           ' '.join(install_args + paths))],
 226                      parsed_arguments.verbose, environment)
 227   # Now move the contents of the temporary destdir to their final place.
 228   run_shell_commands([
 229       'cp %s/* %s/ -rd' % (destdir, install_prefix),
 230       'install -m 644 lib/libpci.so* %s/lib/' % install_prefix,
 231       'ln -sf libpci.so.3.1.8 %s/lib/libpci.so.3' % install_prefix],
 232                      parsed_arguments.verbose, environment)
 233
 234
 235 def build_and_install(parsed_arguments, environment, install_prefix):
 236   if parsed_arguments.build_method == 'destdir':
 237     destdir_configure_make_install(
 238         parsed_arguments, environment, install_prefix)
 239   elif parsed_arguments.build_method == 'prefix':
 240     prefix_configure_make_install(parsed_arguments, environment, install_prefix)
 241   elif parsed_arguments.build_method == 'custom_nss':
 242     nss_make_and_copy(parsed_arguments, environment, install_prefix)
 243   elif parsed_arguments.build_method == 'custom_libcap':
 244     libcap2_make_install(parsed_arguments, environment, install_prefix)
 245   elif parsed_arguments.build_method == 'custom_pango':
 246     parsed_arguments.extra_configure_flags += \
 247       ' --x-libraries=%s/lib' % install_prefix
 248     parsed_arguments.extra_configure_flags += \
 249       ' --x-includes=%s/include' % install_prefix
 250     prefix_configure_make_install(parsed_arguments, environment, install_prefix)
 251   elif parsed_arguments.build_method == 'custom_libpci3':
 252     libpci3_make_install(parsed_arguments, environment, install_prefix)
 253   else:
 254     raise Exception('Unrecognized build method: %s' %
 255                     parsed_arguments.build_method)
 256
 257
 258 def download_build_install(parsed_arguments):
 259   sanitizer_params = SUPPORTED_SANITIZERS[parsed_arguments.sanitizer_type]
 260
 261   environment = os.environ.copy()
 262   # Usage of environment variables CC and CXX prefers usage flags --c-compiler
 263   # and --cxx-compiler
 264   if 'CC' not in environment and parsed_arguments.cc:
 265     environment['CC'] = parsed_arguments.cc
 266   if 'CXX' not in environment and parsed_arguments.cxx:
 267     environment['CXX'] = parsed_arguments.cxx
 268
 269   product_directory = os.path.normpath('%s/%s' % (
 270       get_script_absolute_path(),
 271       parsed_arguments.product_directory))
 272
 273   compiler_flags = sanitizer_params['compiler_flags']
 274   if parsed_arguments.sanitizer_blacklist:
 275     compiler_flags += ' -fsanitize-blacklist=%s/%s' % (
 276         product_directory,
 277         parsed_arguments.sanitizer_blacklist)
 278   environment['CFLAGS'] = '%s %s' % (compiler_flags,
 279                                      parsed_arguments.extra_cflags)
 280   environment['CXXFLAGS'] = '%s %s' % (
 281       compiler_flags,
 282       parsed_arguments.extra_cxxflags)
 283
 284   install_prefix = '%s/instrumented_libraries/%s' % (
 285       product_directory,
 286       parsed_arguments.sanitizer_type)
 287
 288   # Make sure the linker searches the instrumented libraries dir for
 289   # library dependencies.
 290   environment['LDFLAGS'] = '%s -L%s/lib %s' % (
 291       sanitizer_params['linker_flags'],
 292       install_prefix, parsed_arguments.extra_ldflags)
 293
 294   library_directory = '%s/%s' % (parsed_arguments.intermediate_directory,
 295                                  parsed_arguments.library)
 296
 297   # A failed build might have left a dirty source tree behind.
 298   if os.path.exists(library_directory):
 299     shell_call('rm -rf %s' % library_directory, parsed_arguments.verbose)
 300   os.makedirs(library_directory)
 301
 302   with ScopedChangeDirectory(library_directory) as cd_library:
 303     shell_call('apt-get source %s' % parsed_arguments.library,
 304                parsed_arguments.verbose)
 305     # There should be exactly one subdirectory after downloading a package.
 306     subdirectories = [d for d in os.listdir('.') if os.path.isdir(d)]
 307     if len(subdirectories) != 1:
 308       raise (Exception('There was not one directory after downloading '
 309                        'a package %s' % parsed_arguments.library))
 310     with ScopedChangeDirectory(subdirectories[0]):
 311       # Here we are in the package directory.
 312       if parsed_arguments.run_before_build:
 313         shell_call(
 314             '%s/%s' %
 315             (os.path.relpath(cd_library.old_path),
 316              parsed_arguments.run_before_build),
 317             parsed_arguments.verbose)
 318       try:
 319         build_and_install(parsed_arguments, environment, install_prefix)
 320       except Exception as exception:
 321         print exception
 322         print 'Failed to build library %s.' % parsed_arguments.library
 323         print ('Probably, some of its dependencies are not installed: %s' %
 324                ' '.join(get_library_build_dependencies(parsed_arguments.library)))
 325         sys.exit(1)
 326
 327   # Touch a txt file to indicate library is installed.
 328   open('%s/%s.txt' % (install_prefix, parsed_arguments.library), 'w').close()
 329
 330   # Remove downloaded package and generated temporary build files.
 331   # Failed builds intentionally skip this step, in order to aid in tracking down
 332   # build failures.
 333   shell_call('rm -rf %s' % library_directory, parsed_arguments.verbose)
 334
 335
 336 def main():
 337   argument_parser = argparse.ArgumentParser(
 338       description='Download, build and install instrumented library')
 339
 340   argument_parser.add_argument('-j', '--jobs', type=int, default=1)
 341   argument_parser.add_argument('-l', '--library', required=True)
 342   argument_parser.add_argument(
 343       '-i', '--product-directory', default='.',
 344       help='Relative path to the directory with chrome binaries')
 345   argument_parser.add_argument(
 346       '-m', '--intermediate-directory', default='.',
 347       help='Relative path to the directory for temporary build files')
 348   argument_parser.add_argument('--extra-configure-flags', default='')
 349   argument_parser.add_argument('--extra-cflags', default='')
 350   argument_parser.add_argument('--extra-cxxflags', default='')
 351   argument_parser.add_argument('--extra-ldflags', default='')
 352   argument_parser.add_argument('-s', '--sanitizer-type', required=True,
 353                                choices=SUPPORTED_SANITIZERS.keys())
 354   argument_parser.add_argument('-v', '--verbose', action='store_true')
 355   argument_parser.add_argument('--check-build-deps', action='store_true')
 356   argument_parser.add_argument('--cc')
 357   argument_parser.add_argument('--cxx')
 358   # This should be a shell script to run before building specific libraries
 359   # e.g. extracting archives with sources, patching makefiles, etc.
 360   argument_parser.add_argument('--run-before-build', default='')
 361   argument_parser.add_argument('--build-method', default='destdir')
 362   argument_parser.add_argument('--sanitizer-blacklist', default='')
 363
 364   # Ignore all empty arguments because in several cases gyp passes them to the
 365   # script, but ArgumentParser treats them as positional arguments instead of
 366   # ignoring (and doesn't have such options).
 367   parsed_arguments = argument_parser.parse_args(
 368       [arg for arg in sys.argv[1:] if len(arg) != 0])
 369   # Ensure current working directory is this script directory.
 370   os.chdir(get_script_absolute_path())
 371   # Ensure all build dependencies are installed.
 372   if parsed_arguments.check_build_deps:
 373     check_library_build_dependencies(parsed_arguments.library)
 374
 375   download_build_install(parsed_arguments)
 376
 377
 378 if __name__ == '__main__':
 379   main()