tools/auto_bisect/fetch_build.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This module contains functions for fetching and extracting archived builds.
   6
   7 The builds may be stored in different places by different types of builders;
   8 for example, builders on tryserver.chromium.perf stores builds in one place,
   9 while builders on chromium.linux store builds in another.
  10
  11 This module can be either imported or run as a stand-alone script to download
  12 and extract a build.
  13
  14 Usage: fetch_build.py <type> <revision> <output_dir> [options]
  15 """
  16
  17 import argparse
  18 import errno
  19 import os
  20 import shutil
  21 import sys
  22 import zipfile
  23
  24 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH.
  25 from telemetry.util import cloud_storage
  26
  27 import bisect_utils
  28
  29 # Possible builder types.
  30 PERF_BUILDER = 'perf'
  31 FULL_BUILDER = 'full'
  32
  33
  34 def FetchBuild(builder_type, revision, output_dir, target_arch='ia32',
  35                target_platform='chromium', deps_patch_sha=None):
  36   """Downloads and extracts a build for a particular revision.
  37
  38   If the build is successfully downloaded and extracted to |output_dir|, the
  39   downloaded archive file is also deleted.
  40
  41   Args:
  42     revision: Revision string, e.g. a git commit hash or SVN revision.
  43     builder_type: Type of build archive.
  44     target_arch: Architecture, e.g. "ia32".
  45     target_platform: Platform name, e.g. "chromium" or "android".
  46     deps_patch_sha: SHA1 hash of a DEPS file, if we want to fetch a build for
  47         a Chromium revision with custom dependencies.
  48
  49   Raises:
  50     IOError: Unzipping failed.
  51     OSError: Directory creation or deletion failed.
  52   """
  53   build_archive = BuildArchive.Create(
  54       builder_type, target_arch=target_arch, target_platform=target_platform)
  55   bucket = build_archive.BucketName()
  56   remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha)
  57
  58   filename = FetchFromCloudStorage(bucket, remote_path, output_dir)
  59   if not filename:
  60     raise RuntimeError('Failed to fetch gs://%s/%s.' % (bucket, remote_path))
  61
  62   Unzip(filename, output_dir)
  63
  64   if os.path.exists(filename):
  65     os.remove(filename)
  66
  67
  68 class BuildArchive(object):
  69   """Represents a place where builds of some type are stored.
  70
  71   There are two pieces of information required to locate a file in Google
  72   Cloud Storage, bucket name and file path. Subclasses of this class contain
  73   specific logic about which bucket names and paths should be used to fetch
  74   a build.
  75   """
  76
  77   @staticmethod
  78   def Create(builder_type, target_arch='ia32', target_platform='chromium'):
  79     if builder_type == PERF_BUILDER:
  80       return PerfBuildArchive(target_arch, target_platform)
  81     if builder_type == FULL_BUILDER:
  82       return FullBuildArchive(target_arch, target_platform)
  83     raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
  84
  85   def __init__(self, target_arch='ia32', target_platform='chromium'):
  86     if bisect_utils.IsLinuxHost() and target_platform == 'android':
  87       self._platform = 'android'
  88     elif bisect_utils.IsLinuxHost():
  89       self._platform = 'linux'
  90     elif bisect_utils.IsMacHost():
  91       self._platform = 'mac'
  92     elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
  93       self._platform = 'win64'
  94     elif bisect_utils.IsWindowsHost():
  95       self._platform = 'win'
  96     else:
  97       raise NotImplementedError('Unknown platform "%s".' % sys.platform)
  98
  99   def BucketName(self):
 100     raise NotImplementedError()
 101
 102   def FilePath(self, revision, deps_patch_sha=None):
 103     """Returns the remote file path to download a build from.
 104
 105     Args:
 106       revision: A Chromium revision; this could be a git commit hash or
 107           commit position or SVN revision number.
 108       deps_patch_sha: The SHA1 hash of a patch to the DEPS file, which
 109           uniquely identifies a change to use a particular revision of
 110           a dependency.
 111
 112     Returns:
 113       A file path, which not does not include a bucket name.
 114     """
 115     raise NotImplementedError()
 116
 117   def _ZipFileName(self, revision, deps_patch_sha=None):
 118     """Gets the file name of a zip archive for a particular revision.
 119
 120     This returns a file name of the form full-build-<platform>_<revision>.zip,
 121     which is a format used by multiple types of builders that store archives.
 122
 123     Args:
 124       revision: A git commit hash or other revision string.
 125       deps_patch_sha: SHA1 hash of a DEPS file patch.
 126
 127     Returns:
 128       The archive file name.
 129     """
 130     base_name = 'full-build-%s' % self._PlatformName()
 131     if deps_patch_sha:
 132       revision = '%s_%s' % (revision, deps_patch_sha)
 133     return '%s_%s.zip' % (base_name, revision)
 134
 135   def _PlatformName(self):
 136     """Return a string to be used in paths for the platform."""
 137     if self._platform in ('win', 'win64'):
 138       # Build archive for win64 is still stored with "win32" in the name.
 139       return 'win32'
 140     if self._platform in ('linux', 'android'):
 141       # Android builds are also stored with "linux" in the name.
 142       return 'linux'
 143     if self._platform == 'mac':
 144       return 'mac'
 145     raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 146
 147
 148 class PerfBuildArchive(BuildArchive):
 149
 150   def BucketName(self):
 151     return 'chrome-perf'
 152
 153   def FilePath(self, revision, deps_patch_sha=None):
 154     return '%s/%s' % (self._ArchiveDirectory(),
 155                       self._ZipFileName(revision, deps_patch_sha))
 156
 157   def _ArchiveDirectory(self):
 158     """Returns the directory name to download builds from."""
 159     platform_to_directory = {
 160         'android': 'android_perf_rel',
 161         'linux': 'Linux Builder',
 162         'mac': 'Mac Builder',
 163         'win64': 'Win x64 Builder',
 164         'win': 'Win Builder',
 165     }
 166     assert self._platform in platform_to_directory
 167     return platform_to_directory.get(self._platform)
 168
 169
 170 class FullBuildArchive(BuildArchive):
 171
 172   def BucketName(self):
 173     platform_to_bucket = {
 174         'android': 'chromium-android',
 175         'linux': 'chromium-linux-archive',
 176         'mac': 'chromium-mac-archive',
 177         'win64': 'chromium-win-archive',
 178         'win': 'chromium-win-archive',
 179     }
 180     assert self._platform in platform_to_bucket
 181     return platform_to_bucket.get(self._platform)
 182
 183   def FilePath(self, revision, deps_patch_sha=None):
 184     return '%s/%s' % (self._ArchiveDirectory(),
 185                       self._ZipFileName(revision, deps_patch_sha))
 186
 187   def _ArchiveDirectory(self):
 188     """Returns the remote directory to download builds from."""
 189     platform_to_directory = {
 190         'android': 'android_main_rel',
 191         'linux': 'chromium.linux/Linux Builder',
 192         'mac': 'chromium.mac/Mac Builder',
 193         'win64': 'chromium.win/Win x64 Builder',
 194         'win': 'chromium.win/Win Builder',
 195     }
 196     assert self._platform in platform_to_directory
 197     return platform_to_directory.get(self._platform)
 198
 199
 200 def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
 201   """Fetches file(s) from the Google Cloud Storage.
 202
 203   As a side-effect, this prints messages to stdout about what's happening.
 204
 205   Args:
 206     bucket_name: Google Storage bucket name.
 207     source_path: Source file path.
 208     destination_dir: Destination file path.
 209
 210   Returns:
 211     Local file path of downloaded file if it was downloaded. If the file does
 212     not exist in the given bucket, or if there was an error while downloading,
 213     None is returned.
 214   """
 215   target_file = os.path.join(destination_dir, os.path.basename(source_path))
 216   gs_url = 'gs://%s/%s' % (bucket_name, source_path)
 217   try:
 218     if cloud_storage.Exists(bucket_name, source_path):
 219       print 'Fetching file from %s...' % gs_url
 220       cloud_storage.Get(bucket_name, source_path, target_file)
 221       if os.path.exists(target_file):
 222         return target_file
 223     else:
 224       print 'File %s not found in cloud storage.' % gs_url
 225   except Exception as e:
 226     print 'Exception while fetching from cloud storage: %s' % e
 227     if os.path.exists(target_file):
 228       os.remove(target_file)
 229   return None
 230
 231
 232 def Unzip(filename, output_dir, verbose=True):
 233   """Extracts a zip archive's contents into the given output directory.
 234
 235   This was based on ExtractZip from build/scripts/common/chromium_utils.py.
 236
 237   Args:
 238     filename: Name of the zip file to extract.
 239     output_dir: Path to the destination directory.
 240     verbose: Whether to print out what is being extracted.
 241
 242   Raises:
 243     IOError: The unzip command had a non-zero exit code.
 244     RuntimeError: Failed to create the output directory.
 245   """
 246   _MakeDirectory(output_dir)
 247
 248   # On Linux and Mac, we use the unzip command because it handles links and
 249   # file permissions bits, so achieving this behavior is easier than with
 250   # ZipInfo options.
 251   #
 252   # The Mac Version of unzip unfortunately does not support Zip64, whereas
 253   # the python module does, so we have to fall back to the python zip module
 254   # on Mac if the file size is greater than 4GB.
 255   mac_zip_size_limit = 2 ** 32  # 4GB
 256   if (bisect_utils.IsLinuxHost() or
 257       (bisect_utils.IsMacHost()
 258        and os.path.getsize(filename) < mac_zip_size_limit)):
 259     unzip_command = ['unzip', '-o']
 260     _UnzipUsingCommand(unzip_command, filename, output_dir)
 261     return
 262
 263   # On Windows, try to use 7z if it is installed, otherwise fall back to the
 264   # Python zipfile module. If 7z is not installed, then this may fail if the
 265   # zip file is larger than 512MB.
 266   sevenzip_path = r'C:\Program Files\7-Zip\7z.exe'
 267   if bisect_utils.IsWindowsHost() and os.path.exists(sevenzip_path):
 268     unzip_command = [sevenzip_path, 'x', '-y']
 269     _UnzipUsingCommand(unzip_command, filename, output_dir)
 270     return
 271
 272   _UnzipUsingZipFile(filename, output_dir, verbose)
 273
 274
 275 def _UnzipUsingCommand(unzip_command, filename, output_dir):
 276   """Extracts a zip file using an external command.
 277
 278   Args:
 279     unzip_command: An unzipping command, as a string list, without the filename.
 280     filename: Path to the zip file.
 281     output_dir: The directory which the contents should be extracted to.
 282
 283   Raises:
 284     IOError: The command had a non-zero exit code.
 285   """
 286   absolute_filepath = os.path.abspath(filename)
 287   command = unzip_command + [absolute_filepath]
 288   return_code = _RunCommandInDirectory(output_dir, command)
 289   if return_code:
 290     _RemoveDirectoryTree(output_dir)
 291     raise IOError('Unzip failed: %s => %s' % (str(command), return_code))
 292
 293
 294 def _RunCommandInDirectory(directory, command):
 295   """Changes to a directory, runs a command, then changes back."""
 296   saved_dir = os.getcwd()
 297   os.chdir(directory)
 298   return_code = bisect_utils.RunProcess(command)
 299   os.chdir(saved_dir)
 300   return return_code
 301
 302
 303 def _UnzipUsingZipFile(filename, output_dir, verbose=True):
 304   """Extracts a zip file using the Python zipfile module."""
 305   assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
 306   zf = zipfile.ZipFile(filename)
 307   for name in zf.namelist():
 308     if verbose:
 309       print 'Extracting %s' % name
 310     zf.extract(name, output_dir)
 311     if bisect_utils.IsMacHost():
 312       # Restore file permission bits.
 313       mode = zf.getinfo(name).external_attr >> 16
 314       os.chmod(os.path.join(output_dir, name), mode)
 315
 316
 317 def _MakeDirectory(path):
 318   try:
 319     os.makedirs(path)
 320   except OSError as e:
 321     if e.errno != errno.EEXIST:
 322       raise
 323
 324
 325 def _RemoveDirectoryTree(path):
 326   try:
 327     if os.path.exists(path):
 328       shutil.rmtree(path)
 329   except OSError, e:
 330     if e.errno != errno.ENOENT:
 331       raise
 332
 333
 334 def Main(argv):
 335   """Downloads and extracts a build based on the command line arguments."""
 336   parser = argparse.ArgumentParser()
 337   parser.add_argument('builder_type')
 338   parser.add_argument('revision')
 339   parser.add_argument('output_dir')
 340   parser.add_argument('--target-arch', default='ia32')
 341   parser.add_argument('--target-platform', default='chromium')
 342   parser.add_argument('--deps-patch-sha')
 343   args = parser.parse_args(argv[1:])
 344
 345   FetchBuild(
 346       args.builder_type, args.revision, args.output_dir,
 347       target_arch=args.target_arch, target_platform=args.target_platform,
 348       deps_patch_sha=args.deps_patch_sha)
 349
 350   print 'Build has been downloaded to and extracted in %s.' % args.output_dir
 351
 352   return 0
 353
 354
 355 if __name__ == '__main__':
 356   sys.exit(Main(sys.argv))
 357