third_party/instrumented_libraries/scripts/download_build_install.py

   1 #!/usr/bin/python
   2 # Copyright 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Downloads, builds (with instrumentation) and installs shared libraries."""
   7
   8 import argparse
   9 import os
  10 import platform
  11 import re
  12 import shlex
  13 import shutil
  14 import subprocess
  15 import sys
  16
  17 SCRIPT_ABSOLUTE_PATH = os.path.dirname(os.path.abspath(__file__))
  18
  19 def unescape_flags(s):
  20   """Un-escapes build flags received from GYP.
  21
  22   GYP escapes build flags as if they are to be inserted directly into a command
  23   line, wrapping each flag in double quotes. When flags are passed via
  24   CFLAGS/LDFLAGS instead, double quotes must be dropped.
  25   """
  26   return ' '.join(shlex.split(s))
  27
  28
  29 def real_path(path_relative_to_gyp):
  30   """Returns the absolute path to a file.
  31
  32   GYP generates paths relative to the location of the .gyp file, which is one
  33   level above the location of this script. This function converts them to
  34   absolute paths.
  35   """
  36   return os.path.realpath(os.path.join(SCRIPT_ABSOLUTE_PATH, '..',
  37                                        path_relative_to_gyp))
  38
  39
  40 class InstrumentedPackageBuilder(object):
  41   """Checks out and builds a single instrumented package."""
  42   def __init__(self, args, clobber):
  43     self._cc = args.cc
  44     self._cxx = args.cxx
  45     self._extra_configure_flags = args.extra_configure_flags
  46     self._jobs = args.jobs
  47     self._libdir = args.libdir
  48     self._package = args.package
  49     self._patch = real_path(args.patch) if args.patch else None
  50     self._pre_build = \
  51         real_path(args.pre_build) if args.pre_build else None
  52     self._sanitizer = args.sanitizer
  53     self._verbose = args.verbose
  54     self._clobber = clobber
  55     self._working_dir = os.path.join(
  56         real_path(args.intermediate_dir), self._package, '')
  57
  58     product_dir = real_path(args.product_dir)
  59     self._destdir = os.path.join(
  60         product_dir, 'instrumented_libraries', self._sanitizer)
  61     self._source_archives_dir = os.path.join(
  62         product_dir, 'instrumented_libraries', 'sources', self._package)
  63
  64     self._cflags = unescape_flags(args.cflags)
  65     if args.sanitizer_blacklist:
  66       blacklist_file = real_path(args.sanitizer_blacklist)
  67       self._cflags += ' -fsanitize-blacklist=%s' % blacklist_file
  68
  69     self._ldflags = unescape_flags(args.ldflags)
  70
  71     self.init_build_env()
  72
  73     # Initialized later.
  74     self._source_dir = None
  75     self._source_archives = None
  76
  77   def init_build_env(self):
  78     self._build_env = os.environ.copy()
  79
  80     self._build_env['CC'] = self._cc
  81     self._build_env['CXX'] = self._cxx
  82
  83     self._build_env['CFLAGS'] = self._cflags
  84     self._build_env['CXXFLAGS'] = self._cflags
  85     self._build_env['LDFLAGS'] = self._ldflags
  86
  87     if self._sanitizer == 'asan':
  88       # Do not report leaks during the build process.
  89       self._build_env['ASAN_OPTIONS'] = \
  90           '%s:detect_leaks=0' % self._build_env.get('ASAN_OPTIONS', '')
  91
  92     # libappindicator1 needs this.
  93     self._build_env['CSC'] = '/usr/bin/mono-csc'
  94
  95   def shell_call(self, command, env=None, cwd=None):
  96     """Wrapper around subprocess.Popen().
  97
  98     Calls command with specific environment and verbosity using
  99     subprocess.Popen().
 100     """
 101     child = subprocess.Popen(
 102         command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 103         env=env, shell=True, cwd=cwd)
 104     stdout, stderr = child.communicate()
 105     if self._verbose or child.returncode:
 106       print stdout
 107     if child.returncode:
 108       raise Exception('Failed to run: %s' % command)
 109
 110   def maybe_download_source(self):
 111     """Checks out the source code (if needed).
 112
 113     Checks out the source code for the package, if required (i.e. unless running
 114     in no-clobber mode). Initializes self._source_dir and self._source_archives.
 115     """
 116     get_fresh_source = self._clobber or not os.path.exists(self._working_dir)
 117     if get_fresh_source:
 118       self.shell_call('rm -rf %s' % self._working_dir)
 119       os.makedirs(self._working_dir)
 120       self.shell_call('apt-get source %s' % self._package,
 121                       cwd=self._working_dir)
 122
 123     (dirpath, dirnames, filenames) = os.walk(self._working_dir).next()
 124
 125     if len(dirnames) != 1:
 126       raise Exception(
 127           '`apt-get source %s\' must create exactly one subdirectory.'
 128               % self._package)
 129     self._source_dir = os.path.join(dirpath, dirnames[0], '')
 130
 131     if len(filenames) == 0:
 132       raise Exception('Can\'t find source archives after `apt-get source %s\'.'
 133          % self._package)
 134     self._source_archives = \
 135         [os.path.join(dirpath, filename) for filename in filenames]
 136
 137     return get_fresh_source
 138
 139   def patch_source(self):
 140     if self._patch:
 141       self.shell_call('patch -p1 -i %s' % self._patch, cwd=self._source_dir)
 142     if self._pre_build:
 143       self.shell_call(self._pre_build, cwd=self._source_dir)
 144
 145   def copy_source_archives(self):
 146     """Copies the downloaded source archives to the output dir.
 147
 148     For license compliance purposes, every Chromium build that includes
 149     instrumented libraries must include their full source code.
 150     """
 151     self.shell_call('rm -rf %s' % self._source_archives_dir)
 152     os.makedirs(self._source_archives_dir)
 153     for filename in self._source_archives:
 154       shutil.copy(filename, self._source_archives_dir)
 155     if self._patch:
 156       shutil.copy(self._patch, self._source_archives_dir)
 157
 158   def download_build_install(self):
 159     got_fresh_source = self.maybe_download_source()
 160     if got_fresh_source:
 161       self.patch_source()
 162       self.copy_source_archives()
 163
 164     self.shell_call('mkdir -p %s' % self.dest_libdir())
 165
 166     try:
 167       self.build_and_install()
 168     except Exception as exception:
 169       print 'ERROR: Failed to build package %s. Have you run ' \
 170             'src/third_party/instrumented_libraries/scripts/' \
 171             'install-build-deps.sh?' % \
 172             self._package
 173       print
 174       raise
 175
 176     # Touch a text file to indicate package is installed.
 177     stamp_file = os.path.join(self._destdir, '%s.txt' % self._package)
 178     open(stamp_file, 'w').close()
 179
 180     # Remove downloaded package and generated temporary build files. Failed
 181     # builds intentionally skip this step to help debug build failures.
 182     if self._clobber:
 183       self.shell_call('rm -rf %s' % self._working_dir)
 184
 185   def fix_rpaths(self, directory):
 186     # TODO(earthdok): reimplement fix_rpaths.sh in Python.
 187     script = real_path('scripts/fix_rpaths.sh')
 188     self.shell_call("%s %s" % (script, directory))
 189
 190   def temp_dir(self):
 191     """Returns the directory which will be passed to `make install'."""
 192     return os.path.join(self._source_dir, 'debian', 'instrumented_build')
 193
 194   def temp_libdir(self):
 195     """Returns the directory under temp_dir() containing the DSOs."""
 196     return os.path.join(self.temp_dir(), self._libdir)
 197
 198   def dest_libdir(self):
 199     """Returns the final location of the DSOs."""
 200     return os.path.join(self._destdir, self._libdir)
 201
 202   def cleanup_after_install(self):
 203     """Removes unneeded files in self.temp_libdir()."""
 204     # .la files are not needed, nuke them.
 205     # In case --no-static is not supported, nuke any static libraries we built.
 206     self.shell_call(
 207         'find %s -name *.la -or -name *.a | xargs rm -f' % self.temp_libdir())
 208     # .pc files are not needed.
 209     self.shell_call('rm %s/pkgconfig -rf' % self.temp_libdir())
 210
 211   def make(self, args, jobs=None, env=None, cwd=None):
 212     """Invokes `make'.
 213
 214     Invokes `make' with the specified args, using self._build_env and
 215     self._source_dir by default.
 216     """
 217     if jobs is None:
 218       jobs = self._jobs
 219     if cwd is None:
 220       cwd = self._source_dir
 221     if env is None:
 222       env = self._build_env
 223     cmd = ['make', '-j%s' % jobs] + args
 224     self.shell_call(' '.join(cmd), env=env, cwd=cwd)
 225
 226   def make_install(self, args, **kwargs):
 227     """Invokes `make install'."""
 228     self.make(['install'] + args, **kwargs)
 229
 230   def build_and_install(self):
 231     """Builds and installs the DSOs.
 232
 233     Builds the package with ./configure + make, installs it to a temporary
 234     location, then moves the relevant files to their permanent location.
 235     """
 236     configure_cmd = './configure --libdir=/%s/ %s' % (
 237         self._libdir, self._extra_configure_flags)
 238     self.shell_call(configure_cmd, env=self._build_env, cwd=self._source_dir)
 239
 240     # Some makefiles use BUILDROOT or INSTALL_ROOT instead of DESTDIR.
 241     args = ['DESTDIR', 'BUILDROOT', 'INSTALL_ROOT']
 242     make_args = ['%s=%s' % (name, self.temp_dir()) for name in args]
 243     self.make(make_args)
 244
 245     # Some packages don't support parallel install. Use -j1 always.
 246     self.make_install(make_args, jobs=1)
 247
 248     self.cleanup_after_install()
 249
 250     self.fix_rpaths(self.temp_libdir())
 251
 252     # Now move the contents of the temporary destdir to their final place.
 253     # We only care for the contents of LIBDIR.
 254     self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(),
 255                                           self.dest_libdir()))
 256
 257
 258 class LibcapBuilder(InstrumentedPackageBuilder):
 259   def build_and_install(self):
 260     # libcap2 doesn't have a configure script
 261     build_args = ['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']
 262     make_args = [
 263         '%s="%s"' % (name, self._build_env[name]) for name in build_args
 264     ]
 265     self.make(make_args)
 266
 267     install_args = [
 268         'DESTDIR=%s' % self.temp_dir(),
 269         'lib=%s' % self._libdir,
 270         # Skip a step that requires sudo.
 271         'RAISE_SETFCAP=no'
 272     ]
 273     self.make_install(install_args)
 274
 275     self.cleanup_after_install()
 276
 277     self.fix_rpaths(self.temp_libdir())
 278
 279     # Now move the contents of the temporary destdir to their final place.
 280     # We only care for the contents of LIBDIR.
 281     self.shell_call('cp %s/* %s/ -rdf' % (self.temp_libdir(),
 282                                           self.dest_libdir()))
 283
 284
 285 class Libpci3Builder(InstrumentedPackageBuilder):
 286   def package_version(self):
 287     """Guesses libpci3 version from source directory name."""
 288     dir_name = os.path.split(os.path.normpath(self._source_dir))[-1]
 289     match = re.match('pciutils-(\d+\.\d+\.\d+)', dir_name)
 290     if match is None:
 291       raise Exception(
 292           'Unable to guess libpci3 version from directory name: %s' %  dir_name)
 293     return match.group(1)
 294
 295   def temp_libdir(self):
 296     # DSOs have to be picked up from <source_dir>/lib, since `make install'
 297     # doesn't actualy install them anywhere.
 298     return os.path.join(self._source_dir, 'lib')
 299
 300   def build_and_install(self):
 301     # pciutils doesn't have a configure script
 302     # This build process follows debian/rules.
 303     self.shell_call('mkdir -p %s-udeb/usr/bin' % self.temp_dir())
 304
 305     build_args = ['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']
 306     make_args = [
 307         '%s="%s"' % (name, self._build_env[name]) for name in build_args
 308     ]
 309     make_args += [
 310         'LIBDIR=/%s/' % self._libdir,
 311         'PREFIX=/usr',
 312         'SBINDIR=/usr/bin',
 313         'IDSDIR=/usr/share/misc',
 314         'SHARED=yes',
 315         # pciutils-3.2.1 (Trusty) fails to build due to unresolved libkmod
 316         # symbols. The binary package has no dependencies on libkmod, so it
 317         # looks like it was actually built without libkmod support.
 318        'LIBKMOD=no',
 319     ]
 320     self.make(make_args)
 321
 322     # `make install' is not needed.
 323     self.fix_rpaths(self.temp_libdir())
 324
 325     # Now install the DSOs to their final place.
 326     self.shell_call(
 327         'install -m 644 %s/libpci.so* %s' % (self.temp_libdir(),
 328                                              self.dest_libdir()))
 329     self.shell_call(
 330         'ln -sf libpci.so.%s %s/libpci.so.3' % (self.package_version(),
 331                                                 self.dest_libdir()))
 332
 333
 334 class NSSBuilder(InstrumentedPackageBuilder):
 335   def build_and_install(self):
 336     # NSS uses a build system that's different from configure/make/install. All
 337     # flags must be passed as arguments to make.
 338     make_args = [
 339         # Do an optimized build.
 340         'BUILD_OPT=1',
 341         # CFLAGS/CXXFLAGS should not be used, as doing so overrides the flags in
 342         # the makefile completely. The only way to append our flags is to tack
 343         # them onto CC/CXX.
 344         'CC="%s %s"' % (self._build_env['CC'], self._build_env['CFLAGS']),
 345         'CXX="%s %s"' % (self._build_env['CXX'], self._build_env['CXXFLAGS']),
 346         # We need to override ZDEFS_FLAG at least to avoid -Wl,-z,defs, which
 347         # is not compatible with sanitizers. We also need some way to pass
 348         # LDFLAGS without overriding the defaults. Conveniently, ZDEF_FLAG is
 349         # always appended to link flags when building NSS on Linux, so we can
 350         # just add our LDFLAGS here.
 351         'ZDEFS_FLAG="-Wl,-z,nodefs %s"' % self._build_env['LDFLAGS'],
 352         'NSPR_INCLUDE_DIR=/usr/include/nspr',
 353         'NSPR_LIB_DIR=%s' % self.dest_libdir(),
 354         'NSS_ENABLE_ECC=1'
 355     ]
 356     if platform.architecture()[0] == '64bit':
 357       make_args.append('USE_64=1')
 358
 359     # Make sure we don't override the default flags in the makefile.
 360     for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']:
 361       del self._build_env[variable]
 362
 363     # Hardcoded paths.
 364     temp_dir = os.path.join(self._source_dir, 'nss')
 365     temp_libdir = os.path.join(temp_dir, 'lib')
 366
 367     # Parallel build is not supported. Also, the build happens in
 368     # <source_dir>/nss.
 369     self.make(make_args, jobs=1, cwd=temp_dir)
 370
 371     self.fix_rpaths(temp_libdir)
 372
 373     # 'make install' is not supported. Copy the DSOs manually.
 374     for (dirpath, dirnames, filenames) in os.walk(temp_libdir):
 375       for filename in filenames:
 376         if filename.endswith('.so'):
 377           full_path = os.path.join(dirpath, filename)
 378           if self._verbose:
 379             print 'download_build_install.py: installing %s' % full_path
 380           shutil.copy(full_path, self.dest_libdir())
 381
 382
 383 def main():
 384   parser = argparse.ArgumentParser(
 385       description='Download, build and install an instrumented package.')
 386
 387   parser.add_argument('-j', '--jobs', type=int, default=1)
 388   parser.add_argument('-p', '--package', required=True)
 389   parser.add_argument(
 390       '-i', '--product-dir', default='.',
 391       help='Relative path to the directory with chrome binaries')
 392   parser.add_argument(
 393       '-m', '--intermediate-dir', default='.',
 394       help='Relative path to the directory for temporary build files')
 395   parser.add_argument('--extra-configure-flags', default='')
 396   parser.add_argument('--cflags', default='')
 397   parser.add_argument('--ldflags', default='')
 398   parser.add_argument('-s', '--sanitizer', required=True,
 399                                choices=['asan', 'msan', 'tsan'])
 400   parser.add_argument('-v', '--verbose', action='store_true')
 401   parser.add_argument('--cc')
 402   parser.add_argument('--cxx')
 403   parser.add_argument('--patch', default='')
 404   # This should be a shell script to run before building specific libraries.
 405   # This will be run after applying the patch above.
 406   parser.add_argument('--pre-build', default='')
 407   parser.add_argument('--build-method', default='destdir')
 408   parser.add_argument('--sanitizer-blacklist', default='')
 409   # The LIBDIR argument to configure/make.
 410   parser.add_argument('--libdir', default='lib')
 411
 412   # Ignore all empty arguments because in several cases gyp passes them to the
 413   # script, but ArgumentParser treats them as positional arguments instead of
 414   # ignoring (and doesn't have such options).
 415   args = parser.parse_args([arg for arg in sys.argv[1:] if len(arg) != 0])
 416
 417   # Clobber by default, unless the developer wants to hack on the package's
 418   # source code.
 419   clobber = \
 420         (os.environ.get('INSTRUMENTED_LIBRARIES_NO_CLOBBER', '') != '1')
 421
 422   if args.build_method == 'destdir':
 423     builder = InstrumentedPackageBuilder(args, clobber)
 424   elif args.build_method == 'custom_nss':
 425     builder = NSSBuilder(args, clobber)
 426   elif args.build_method == 'custom_libcap':
 427     builder = LibcapBuilder(args, clobber)
 428   elif args.build_method == 'custom_libpci3':
 429     builder = Libpci3Builder(args, clobber)
 430   else:
 431     raise Exception('Unrecognized build method: %s' % args.build_method)
 432
 433   builder.download_build_install()
 434
 435 if __name__ == '__main__':
 436   main()