tools/auto_bisect/fetch_build.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This module contains functions for fetching and extracting archived builds.
   6
   7 The builds may be stored in different places by different types of builders;
   8 for example, builders on tryserver.chromium.perf stores builds in one place,
   9 while builders on chromium.linux store builds in another.
  10
  11 This module can be either imported or run as a stand-alone script to download
  12 and extract a build.
  13
  14 Usage: fetch_build.py <type> <revision> <output_dir> [options]
  15 """
  16
  17 import argparse
  18 import errno
  19 import logging
  20 import os
  21 import shutil
  22 import sys
  23 import zipfile
  24
  25 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH.
  26 from telemetry.util import cloud_storage
  27
  28 import bisect_utils
  29
  30 # Possible builder types.
  31 PERF_BUILDER = 'perf'
  32 FULL_BUILDER = 'full'
  33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf'
  34
  35
  36 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER,
  37                            target_arch='ia32', target_platform='chromium',
  38                            deps_patch_sha=None, extra_src=None):
  39   """Returns the location where a build archive is expected to be.
  40
  41   Args:
  42     revision: Revision string, e.g. a git commit hash or SVN revision.
  43     builder_type: Type of build archive.
  44     target_arch: Architecture, e.g. "ia32".
  45     target_platform: Platform name, e.g. "chromium" or "android".
  46     deps_patch_sha: SHA1 hash which identifies a particular combination of
  47         custom revisions for dependency repositories.
  48     extra_src: Path to a script which can be used to modify the bisect script's
  49         behavior.
  50
  51   Returns:
  52     A pair of strings (bucket, path), where the archive is expected to be.
  53   """
  54   logging.info('Creating BuildArchive, type "%s", arch "%s", platform "%s".',
  55                builder_type, target_arch, target_platform)
  56   build_archive = BuildArchive.Create(
  57       builder_type, target_arch=target_arch, target_platform=target_platform,
  58       extra_src=extra_src)
  59   bucket = build_archive.BucketName()
  60   remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha)
  61   return bucket, remote_path
  62
  63
  64 class BuildArchive(object):
  65   """Represents a place where builds of some type are stored.
  66
  67   There are two pieces of information required to locate a file in Google
  68   Cloud Storage, bucket name and file path. Subclasses of this class contain
  69   specific logic about which bucket names and paths should be used to fetch
  70   a build.
  71   """
  72
  73   @staticmethod
  74   def Create(builder_type, target_arch='ia32', target_platform='chromium',
  75              extra_src=None):
  76     logging.info('Creating BuildArchive, type "%s", arch "%s", platform "%s".',
  77                  builder_type, target_arch, target_platform)
  78     if builder_type == PERF_BUILDER:
  79       return PerfBuildArchive(target_arch, target_platform)
  80     if builder_type == FULL_BUILDER:
  81       return FullBuildArchive(target_arch, target_platform)
  82     if builder_type == ANDROID_CHROME_PERF_BUILDER:
  83       try:
  84         # Load and initialize a module in extra source file and
  85         # return its module object to access android-chrome specific data.
  86         loaded_extra_src = bisect_utils.LoadExtraSrc(extra_src)
  87         return AndroidChromeBuildArchive(
  88             target_arch, target_platform, loaded_extra_src)
  89       except (IOError, TypeError, ImportError):
  90         raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src)
  91     raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
  92
  93   def __init__(self, target_arch='ia32', target_platform='chromium',
  94                extra_src=None):
  95     self._extra_src = extra_src
  96     if bisect_utils.IsLinuxHost() and target_platform == 'android':
  97       self._platform = 'android'
  98     elif bisect_utils.IsLinuxHost():
  99       self._platform = 'linux'
 100     elif bisect_utils.IsMacHost():
 101       self._platform = 'mac'
 102     elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
 103       self._platform = 'win64'
 104     elif bisect_utils.IsWindowsHost():
 105       self._platform = 'win'
 106     else:
 107       raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 108
 109   def BucketName(self):
 110     raise NotImplementedError()
 111
 112   def FilePath(self, revision, deps_patch_sha=None):
 113     """Returns the remote file path to download a build from.
 114
 115     Args:
 116       revision: A Chromium revision; this could be a git commit hash or
 117           commit position or SVN revision number.
 118       deps_patch_sha: The SHA1 hash of a patch to the DEPS file, which
 119           uniquely identifies a change to use a particular revision of
 120           a dependency.
 121
 122     Returns:
 123       A file path, which not does not include a bucket name.
 124     """
 125     raise NotImplementedError()
 126
 127   def _ZipFileName(self, revision, deps_patch_sha=None):
 128     """Gets the file name of a zip archive for a particular revision.
 129
 130     This returns a file name of the form full-build-<platform>_<revision>.zip,
 131     which is a format used by multiple types of builders that store archives.
 132
 133     Args:
 134       revision: A git commit hash or other revision string.
 135       deps_patch_sha: SHA1 hash of a DEPS file patch.
 136
 137     Returns:
 138       The archive file name.
 139     """
 140     base_name = 'full-build-%s' % self._PlatformName()
 141     if deps_patch_sha:
 142       revision = '%s_%s' % (revision, deps_patch_sha)
 143     return '%s_%s.zip' % (base_name, revision)
 144
 145   def _PlatformName(self):
 146     """Return a string to be used in paths for the platform."""
 147     if self._platform in ('win', 'win64'):
 148       # Build archive for win64 is still stored with "win32" in the name.
 149       return 'win32'
 150     if self._platform in ('linux', 'android'):
 151       # Android builds are also stored with "linux" in the name.
 152       return 'linux'
 153     if self._platform == 'mac':
 154       return 'mac'
 155     raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 156
 157
 158 class PerfBuildArchive(BuildArchive):
 159
 160   def BucketName(self):
 161     return 'chrome-perf'
 162
 163   def FilePath(self, revision, deps_patch_sha=None):
 164     return '%s/%s' % (self._ArchiveDirectory(),
 165                       self._ZipFileName(revision, deps_patch_sha))
 166
 167   def _ArchiveDirectory(self):
 168     """Returns the directory name to download builds from."""
 169     platform_to_directory = {
 170         'android': 'android_perf_rel',
 171         'linux': 'Linux Builder',
 172         'mac': 'Mac Builder',
 173         'win64': 'Win x64 Builder',
 174         'win': 'Win Builder',
 175     }
 176     assert self._platform in platform_to_directory
 177     return platform_to_directory.get(self._platform)
 178
 179
 180 class FullBuildArchive(BuildArchive):
 181
 182   def BucketName(self):
 183     platform_to_bucket = {
 184         'android': 'chromium-android',
 185         'linux': 'chromium-linux-archive',
 186         'mac': 'chromium-mac-archive',
 187         'win64': 'chromium-win-archive',
 188         'win': 'chromium-win-archive',
 189     }
 190     assert self._platform in platform_to_bucket
 191     return platform_to_bucket.get(self._platform)
 192
 193   def FilePath(self, revision, deps_patch_sha=None):
 194     return '%s/%s' % (self._ArchiveDirectory(),
 195                       self._ZipFileName(revision, deps_patch_sha))
 196
 197   def _ArchiveDirectory(self):
 198     """Returns the remote directory to download builds from."""
 199     platform_to_directory = {
 200         'android': 'android_main_rel',
 201         'linux': 'chromium.linux/Linux Builder',
 202         'mac': 'chromium.mac/Mac Builder',
 203         'win64': 'chromium.win/Win x64 Builder',
 204         'win': 'chromium.win/Win Builder',
 205     }
 206     assert self._platform in platform_to_directory
 207     return platform_to_directory.get(self._platform)
 208
 209
 210 class AndroidChromeBuildArchive(BuildArchive):
 211   """Represents a place where builds of android-chrome type are stored.
 212
 213   If AndroidChromeBuildArchive is used, it is assumed that the --extra_src
 214   is a valid Python module which contains the module-level functions
 215   GetBucketName and GetArchiveDirectory.
 216   """
 217
 218   def BucketName(self):
 219     return self._extra_src.GetBucketName()
 220
 221   def _ZipFileName(self, revision, deps_patch_sha=None):
 222     """Gets the file name of a zip archive on android-chrome.
 223
 224     This returns a file name of the form build_product_<revision>.zip,
 225     which is a format used by android-chrome.
 226
 227     Args:
 228       revision: A git commit hash or other revision string.
 229       deps_patch_sha: SHA1 hash of a DEPS file patch.
 230
 231     Returns:
 232       The archive file name.
 233     """
 234     if deps_patch_sha:
 235       revision = '%s_%s' % (revision, deps_patch_sha)
 236     return 'build_product_%s.zip' % revision
 237
 238   def FilePath(self, revision, deps_patch_sha=None):
 239     return '%s/%s' % (self._ArchiveDirectory(),
 240                       self._ZipFileName(revision, deps_patch_sha))
 241
 242   def _ArchiveDirectory(self):
 243     """Returns the directory name to download builds from."""
 244     return self._extra_src.GetArchiveDirectory()
 245
 246
 247 def BuildIsAvailable(bucket_name, remote_path):
 248   """Checks whether a build is currently archived at some place."""
 249   logging.info('Checking existance: gs://%s/%s' % (bucket_name, remote_path))
 250   try:
 251     exists = cloud_storage.Exists(bucket_name, remote_path)
 252     logging.info('Exists? %s' % exists)
 253     return exists
 254   except cloud_storage.CloudStorageError:
 255     return False
 256
 257
 258 def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
 259   """Fetches file(s) from the Google Cloud Storage.
 260
 261   As a side-effect, this prints messages to stdout about what's happening.
 262
 263   Args:
 264     bucket_name: Google Storage bucket name.
 265     source_path: Source file path.
 266     destination_dir: Destination file path.
 267
 268   Returns:
 269     Local file path of downloaded file if it was downloaded. If the file does
 270     not exist in the given bucket, or if there was an error while downloading,
 271     None is returned.
 272   """
 273   target_file = os.path.join(destination_dir, os.path.basename(source_path))
 274   gs_url = 'gs://%s/%s' % (bucket_name, source_path)
 275   try:
 276     if cloud_storage.Exists(bucket_name, source_path):
 277       logging.info('Fetching file from %s...', gs_url)
 278       cloud_storage.Get(bucket_name, source_path, target_file)
 279       if os.path.exists(target_file):
 280         return target_file
 281     else:
 282       logging.info('File %s not found in cloud storage.', gs_url)
 283       return None
 284   except Exception as e:
 285     logging.warn('Exception while fetching from cloud storage: %s', e)
 286     if os.path.exists(target_file):
 287       os.remove(target_file)
 288   return None
 289
 290
 291 def Unzip(file_path, output_dir, verbose=True):
 292   """Extracts a zip archive's contents into the given output directory.
 293
 294   This was based on ExtractZip from build/scripts/common/chromium_utils.py.
 295
 296   Args:
 297     file_path: Path of the zip file to extract.
 298     output_dir: Path to the destination directory.
 299     verbose: Whether to print out what is being extracted.
 300
 301   Raises:
 302     IOError: The unzip command had a non-zero exit code.
 303     RuntimeError: Failed to create the output directory.
 304   """
 305   _MakeDirectory(output_dir)
 306
 307   # On Linux and Mac, we use the unzip command because it handles links and
 308   # file permissions bits, so achieving this behavior is easier than with
 309   # ZipInfo options.
 310   #
 311   # The Mac Version of unzip unfortunately does not support Zip64, whereas
 312   # the python module does, so we have to fall back to the python zip module
 313   # on Mac if the file size is greater than 4GB.
 314   mac_zip_size_limit = 2 ** 32  # 4GB
 315   if (bisect_utils.IsLinuxHost() or
 316       (bisect_utils.IsMacHost()
 317        and os.path.getsize(file_path) < mac_zip_size_limit)):
 318     unzip_command = ['unzip', '-o']
 319     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 320     return
 321
 322   # On Windows, try to use 7z if it is installed, otherwise fall back to the
 323   # Python zipfile module. If 7z is not installed, then this may fail if the
 324   # zip file is larger than 512MB.
 325   sevenzip_path = r'C:\Program Files\7-Zip\7z.exe'
 326   if bisect_utils.IsWindowsHost() and os.path.exists(sevenzip_path):
 327     unzip_command = [sevenzip_path, 'x', '-y']
 328     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 329     return
 330
 331   _UnzipUsingZipFile(file_path, output_dir, verbose)
 332
 333
 334 def _UnzipUsingCommand(unzip_command, file_path, output_dir):
 335   """Extracts a zip file using an external command.
 336
 337   Args:
 338     unzip_command: An unzipping command, as a string list, without the filename.
 339     file_path: Path to the zip file.
 340     output_dir: The directory which the contents should be extracted to.
 341
 342   Raises:
 343     IOError: The command had a non-zero exit code.
 344   """
 345   absolute_filepath = os.path.abspath(file_path)
 346   command = unzip_command + [absolute_filepath]
 347   return_code = _RunCommandInDirectory(output_dir, command)
 348   if return_code:
 349     _RemoveDirectoryTree(output_dir)
 350     raise IOError('Unzip failed: %s => %s' % (str(command), return_code))
 351
 352
 353 def _RunCommandInDirectory(directory, command):
 354   """Changes to a directory, runs a command, then changes back."""
 355   saved_dir = os.getcwd()
 356   os.chdir(directory)
 357   return_code = bisect_utils.RunProcess(command)
 358   os.chdir(saved_dir)
 359   return return_code
 360
 361
 362 def _UnzipUsingZipFile(file_path, output_dir, verbose=True):
 363   """Extracts a zip file using the Python zipfile module."""
 364   assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
 365   zf = zipfile.ZipFile(file_path)
 366   for name in zf.namelist():
 367     if verbose:
 368       print 'Extracting %s' % name
 369     zf.extract(name, output_dir)
 370     if bisect_utils.IsMacHost():
 371       # Restore file permission bits.
 372       mode = zf.getinfo(name).external_attr >> 16
 373       os.chmod(os.path.join(output_dir, name), mode)
 374
 375
 376 def _MakeDirectory(path):
 377   try:
 378     os.makedirs(path)
 379   except OSError as e:
 380     if e.errno != errno.EEXIST:
 381       raise
 382
 383
 384 def _RemoveDirectoryTree(path):
 385   try:
 386     if os.path.exists(path):
 387       shutil.rmtree(path)
 388   except OSError, e:
 389     if e.errno != errno.ENOENT:
 390       raise
 391
 392
 393 def Main(argv):
 394   """Downloads and extracts a build based on the command line arguments."""
 395   parser = argparse.ArgumentParser()
 396   parser.add_argument('builder_type')
 397   parser.add_argument('revision')
 398   parser.add_argument('output_dir')
 399   parser.add_argument('--target-arch', default='ia32')
 400   parser.add_argument('--target-platform', default='chromium')
 401   parser.add_argument('--deps-patch-sha')
 402   args = parser.parse_args(argv[1:])
 403
 404   bucket_name, remote_path = GetBucketAndRemotePath(
 405       args.revision, args.builder_type, target_arch=args.target_arch,
 406       target_platform=args.target_platform,
 407       deps_patch_sha=args.deps_patch_sha)
 408   print 'Bucket name: %s, remote path: %s' % (bucket_name, remote_path)
 409
 410   if not BuildIsAvailable(bucket_name, remote_path):
 411     print 'Build is not available.'
 412     return 1
 413
 414   FetchFromCloudStorage(bucket_name, remote_path, args.output_dir)
 415   print 'Build has been downloaded to and extracted in %s.' % args.output_dir
 416   return 0
 417
 418
 419 if __name__ == '__main__':
 420   sys.exit(Main(sys.argv))