tools/auto_bisect/fetch_build.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This module contains functions for fetching and extracting archived builds.
   6
   7 The builds may be stored in different places by different types of builders;
   8 for example, builders on tryserver.chromium.perf stores builds in one place,
   9 while builders on chromium.linux store builds in another.
  10
  11 This module can be either imported or run as a stand-alone script to download
  12 and extract a build.
  13
  14 Usage: fetch_build.py <type> <revision> <output_dir> [options]
  15 """
  16
  17 import argparse
  18 import errno
  19 import logging
  20 import os
  21 import shutil
  22 import sys
  23 import zipfile
  24
  25 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH.
  26 from telemetry.util import cloud_storage
  27
  28 import bisect_utils
  29
  30 # Possible builder types.
  31 PERF_BUILDER = 'perf'
  32 FULL_BUILDER = 'full'
  33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf'
  34
  35 # Maximum time in seconds to wait after posting build request to the try server.
  36 MAX_MAC_BUILD_TIME = 14400
  37 MAX_WIN_BUILD_TIME = 14400
  38 MAX_LINUX_BUILD_TIME = 14400
  39
  40
  41 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER,
  42                            target_arch='ia32', target_platform='chromium',
  43                            deps_patch_sha=None, extra_src=None):
  44   """Returns the location where a build archive is expected to be.
  45
  46   Args:
  47     revision: Revision string, e.g. a git commit hash or SVN revision.
  48     builder_type: Type of build archive.
  49     target_arch: Architecture, e.g. "ia32".
  50     target_platform: Platform name, e.g. "chromium" or "android".
  51     deps_patch_sha: SHA1 hash which identifies a particular combination of
  52         custom revisions for dependency repositories.
  53     extra_src: Path to a script which can be used to modify the bisect script's
  54         behavior.
  55
  56   Returns:
  57     A pair of strings (bucket, path), where the archive is expected to be.
  58   """
  59   logging.info('Getting GS URL for archive of builder "%s", "%s", "%s".',
  60                builder_type, target_arch, target_platform)
  61   build_archive = BuildArchive.Create(
  62       builder_type, target_arch=target_arch, target_platform=target_platform,
  63       extra_src=extra_src)
  64   bucket = build_archive.BucketName()
  65   remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha)
  66   return bucket, remote_path
  67
  68
  69 def GetBuilderNameAndBuildTime(builder_type=PERF_BUILDER, target_arch='ia32',
  70                                target_platform='chromium', extra_src=None):
  71   """Gets builder bot name and build time in seconds based on platform."""
  72   logging.info('Getting builder name for builder "%s", "%s", "%s".',
  73                builder_type, target_arch, target_platform)
  74   build_archive = BuildArchive.Create(
  75       builder_type, target_arch=target_arch, target_platform=target_platform,
  76       extra_src=extra_src)
  77   return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime()
  78
  79
  80 class BuildArchive(object):
  81   """Represents a place where builds of some type are stored.
  82
  83   There are two pieces of information required to locate a file in Google
  84   Cloud Storage, bucket name and file path. Subclasses of this class contain
  85   specific logic about which bucket names and paths should be used to fetch
  86   a build.
  87   """
  88
  89   @staticmethod
  90   def Create(builder_type, target_arch='ia32', target_platform='chromium',
  91              extra_src=None):
  92     if builder_type == PERF_BUILDER:
  93       return PerfBuildArchive(target_arch, target_platform)
  94     if builder_type == FULL_BUILDER:
  95       return FullBuildArchive(target_arch, target_platform)
  96     if builder_type == ANDROID_CHROME_PERF_BUILDER:
  97       try:
  98         # Load and initialize a module in extra source file and
  99         # return its module object to access android-chrome specific data.
 100         loaded_extra_src = bisect_utils.LoadExtraSrc(extra_src)
 101         return AndroidChromeBuildArchive(
 102             target_arch, target_platform, loaded_extra_src)
 103       except (IOError, TypeError, ImportError):
 104         raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src)
 105     raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
 106
 107   def __init__(self, target_arch='ia32', target_platform='chromium',
 108                extra_src=None):
 109     self._extra_src = extra_src
 110     if bisect_utils.IsLinuxHost() and target_platform == 'android':
 111       self._platform = 'android'
 112     elif bisect_utils.IsLinuxHost():
 113       self._platform = 'linux'
 114     elif bisect_utils.IsMacHost():
 115       self._platform = 'mac'
 116     elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
 117       self._platform = 'win64'
 118     elif bisect_utils.IsWindowsHost():
 119       self._platform = 'win'
 120     else:
 121       raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 122
 123   def BucketName(self):
 124     raise NotImplementedError()
 125
 126   def FilePath(self, revision, deps_patch_sha=None):
 127     """Returns the remote file path to download a build from.
 128
 129     Args:
 130       revision: A Chromium revision; this could be a git commit hash or
 131           commit position or SVN revision number.
 132       deps_patch_sha: The SHA1 hash of a patch to the DEPS file, which
 133           uniquely identifies a change to use a particular revision of
 134           a dependency.
 135
 136     Returns:
 137       A file path, which not does not include a bucket name.
 138     """
 139     raise NotImplementedError()
 140
 141   def _ZipFileName(self, revision, deps_patch_sha=None):
 142     """Gets the file name of a zip archive for a particular revision.
 143
 144     This returns a file name of the form full-build-<platform>_<revision>.zip,
 145     which is a format used by multiple types of builders that store archives.
 146
 147     Args:
 148       revision: A git commit hash or other revision string.
 149       deps_patch_sha: SHA1 hash of a DEPS file patch.
 150
 151     Returns:
 152       The archive file name.
 153     """
 154     base_name = 'full-build-%s' % self._PlatformName()
 155     if deps_patch_sha:
 156       revision = '%s_%s' % (revision, deps_patch_sha)
 157     return '%s_%s.zip' % (base_name, revision)
 158
 159   def _PlatformName(self):
 160     """Return a string to be used in paths for the platform."""
 161     if self._platform in ('win', 'win64'):
 162       # Build archive for win64 is still stored with "win32" in the name.
 163       return 'win32'
 164     if self._platform in ('linux', 'android'):
 165       # Android builds are also stored with "linux" in the name.
 166       return 'linux'
 167     if self._platform == 'mac':
 168       return 'mac'
 169     raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 170
 171   def GetBuilderName(self):
 172     raise NotImplementedError()
 173
 174   def GetBuilderBuildTime(self):
 175     """Returns the time to wait for a build after requesting one."""
 176     if self._platform in ('win', 'win64'):
 177       return MAX_WIN_BUILD_TIME
 178     if self._platform in ('linux', 'android'):
 179       return MAX_LINUX_BUILD_TIME
 180     if self._platform == 'mac':
 181       return MAX_MAC_BUILD_TIME
 182     raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
 183
 184
 185 class PerfBuildArchive(BuildArchive):
 186
 187   def BucketName(self):
 188     return 'chrome-perf'
 189
 190   def FilePath(self, revision, deps_patch_sha=None):
 191     return '%s/%s' % (self._ArchiveDirectory(),
 192                       self._ZipFileName(revision, deps_patch_sha))
 193
 194   def _ArchiveDirectory(self):
 195     """Returns the directory name to download builds from."""
 196     platform_to_directory = {
 197         'android': 'android_perf_rel',
 198         'linux': 'Linux Builder',
 199         'mac': 'Mac Builder',
 200         'win64': 'Win x64 Builder',
 201         'win': 'Win Builder',
 202     }
 203     assert self._platform in platform_to_directory
 204     return platform_to_directory.get(self._platform)
 205
 206   def GetBuilderName(self):
 207     """Gets builder bot name based on platform."""
 208     if self._platform == 'win64':
 209       return 'win_x64_perf_bisect_builder'
 210     elif self._platform == 'win':
 211       return 'win_perf_bisect_builder'
 212     elif self._platform == 'linux':
 213       return 'linux_perf_bisect_builder'
 214     elif self._platform == 'android':
 215       return 'android_perf_bisect_builder'
 216     elif self._platform == 'mac':
 217       return 'mac_perf_bisect_builder'
 218     raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
 219
 220
 221 class FullBuildArchive(BuildArchive):
 222
 223   def BucketName(self):
 224     platform_to_bucket = {
 225         'android': 'chromium-android',
 226         'linux': 'chromium-linux-archive',
 227         'mac': 'chromium-mac-archive',
 228         'win64': 'chromium-win-archive',
 229         'win': 'chromium-win-archive',
 230     }
 231     assert self._platform in platform_to_bucket
 232     return platform_to_bucket.get(self._platform)
 233
 234   def FilePath(self, revision, deps_patch_sha=None):
 235     return '%s/%s' % (self._ArchiveDirectory(),
 236                       self._ZipFileName(revision, deps_patch_sha))
 237
 238   def _ArchiveDirectory(self):
 239     """Returns the remote directory to download builds from."""
 240     platform_to_directory = {
 241         'android': 'android_main_rel',
 242         'linux': 'chromium.linux/Linux Builder',
 243         'mac': 'chromium.mac/Mac Builder',
 244         'win64': 'chromium.win/Win x64 Builder',
 245         'win': 'chromium.win/Win Builder',
 246     }
 247     assert self._platform in platform_to_directory
 248     return platform_to_directory.get(self._platform)
 249
 250   def GetBuilderName(self):
 251     """Gets builder bot name based on platform."""
 252     if self._platform == 'linux':
 253       return 'linux_full_bisect_builder'
 254     raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
 255
 256 class AndroidChromeBuildArchive(BuildArchive):
 257   """Represents a place where builds of android-chrome type are stored.
 258
 259   If AndroidChromeBuildArchive is used, it is assumed that the --extra_src
 260   is a valid Python module which contains the module-level functions
 261   GetBucketName and GetArchiveDirectory.
 262   """
 263
 264   def BucketName(self):
 265     return self._extra_src.GetBucketName()
 266
 267   def _ZipFileName(self, revision, deps_patch_sha=None):
 268     """Gets the file name of a zip archive on android-chrome.
 269
 270     This returns a file name of the form build_product_<revision>.zip,
 271     which is a format used by android-chrome.
 272
 273     Args:
 274       revision: A git commit hash or other revision string.
 275       deps_patch_sha: SHA1 hash of a DEPS file patch.
 276
 277     Returns:
 278       The archive file name.
 279     """
 280     if deps_patch_sha:
 281       revision = '%s_%s' % (revision, deps_patch_sha)
 282     return 'build_product_%s.zip' % revision
 283
 284   def FilePath(self, revision, deps_patch_sha=None):
 285     return '%s/%s' % (self._ArchiveDirectory(),
 286                       self._ZipFileName(revision, deps_patch_sha))
 287
 288   def _ArchiveDirectory(self):
 289     """Returns the directory name to download builds from."""
 290     return self._extra_src.GetArchiveDirectory()
 291
 292   def GetBuilderName(self):
 293     """Returns the builder name extra source."""
 294     return self._extra_src.GetBuilderName()
 295
 296
 297 def BuildIsAvailable(bucket_name, remote_path):
 298   """Checks whether a build is currently archived at some place."""
 299   logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path))
 300   try:
 301     exists = cloud_storage.Exists(bucket_name, remote_path)
 302     logging.info('Exists? %s' % exists)
 303     return exists
 304   except cloud_storage.CloudStorageError:
 305     return False
 306
 307
 308 def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
 309   """Fetches file(s) from the Google Cloud Storage.
 310
 311   As a side-effect, this prints messages to stdout about what's happening.
 312
 313   Args:
 314     bucket_name: Google Storage bucket name.
 315     source_path: Source file path.
 316     destination_dir: Destination file path.
 317
 318   Returns:
 319     Local file path of downloaded file if it was downloaded. If the file does
 320     not exist in the given bucket, or if there was an error while downloading,
 321     None is returned.
 322   """
 323   target_file = os.path.join(destination_dir, os.path.basename(source_path))
 324   gs_url = 'gs://%s/%s' % (bucket_name, source_path)
 325   try:
 326     if cloud_storage.Exists(bucket_name, source_path):
 327       logging.info('Fetching file from %s...', gs_url)
 328       cloud_storage.Get(bucket_name, source_path, target_file)
 329       if os.path.exists(target_file):
 330         return target_file
 331     else:
 332       logging.info('File %s not found in cloud storage.', gs_url)
 333       return None
 334   except Exception as e:
 335     logging.warn('Exception while fetching from cloud storage: %s', e)
 336     if os.path.exists(target_file):
 337       os.remove(target_file)
 338   return None
 339
 340
 341 def Unzip(file_path, output_dir, verbose=True):
 342   """Extracts a zip archive's contents into the given output directory.
 343
 344   This was based on ExtractZip from build/scripts/common/chromium_utils.py.
 345
 346   Args:
 347     file_path: Path of the zip file to extract.
 348     output_dir: Path to the destination directory.
 349     verbose: Whether to print out what is being extracted.
 350
 351   Raises:
 352     IOError: The unzip command had a non-zero exit code.
 353     RuntimeError: Failed to create the output directory.
 354   """
 355   _MakeDirectory(output_dir)
 356
 357   # On Linux and Mac, we use the unzip command because it handles links and
 358   # file permissions bits, so achieving this behavior is easier than with
 359   # ZipInfo options.
 360   #
 361   # The Mac Version of unzip unfortunately does not support Zip64, whereas
 362   # the python module does, so we have to fall back to the python zip module
 363   # on Mac if the file size is greater than 4GB.
 364   mac_zip_size_limit = 2 ** 32  # 4GB
 365   if (bisect_utils.IsLinuxHost() or
 366       (bisect_utils.IsMacHost()
 367        and os.path.getsize(file_path) < mac_zip_size_limit)):
 368     unzip_command = ['unzip', '-o']
 369     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 370     return
 371
 372   # On Windows, try to use 7z if it is installed, otherwise fall back to the
 373   # Python zipfile module. If 7z is not installed, then this may fail if the
 374   # zip file is larger than 512MB.
 375   sevenzip_path = r'C:\Program Files\7-Zip\7z.exe'
 376   if bisect_utils.IsWindowsHost() and os.path.exists(sevenzip_path):
 377     unzip_command = [sevenzip_path, 'x', '-y']
 378     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 379     return
 380
 381   _UnzipUsingZipFile(file_path, output_dir, verbose)
 382
 383
 384 def _UnzipUsingCommand(unzip_command, file_path, output_dir):
 385   """Extracts a zip file using an external command.
 386
 387   Args:
 388     unzip_command: An unzipping command, as a string list, without the filename.
 389     file_path: Path to the zip file.
 390     output_dir: The directory which the contents should be extracted to.
 391
 392   Raises:
 393     IOError: The command had a non-zero exit code.
 394   """
 395   absolute_filepath = os.path.abspath(file_path)
 396   command = unzip_command + [absolute_filepath]
 397   return_code = _RunCommandInDirectory(output_dir, command)
 398   if return_code:
 399     _RemoveDirectoryTree(output_dir)
 400     raise IOError('Unzip failed: %s => %s' % (str(command), return_code))
 401
 402
 403 def _RunCommandInDirectory(directory, command):
 404   """Changes to a directory, runs a command, then changes back."""
 405   saved_dir = os.getcwd()
 406   os.chdir(directory)
 407   return_code = bisect_utils.RunProcess(command)
 408   os.chdir(saved_dir)
 409   return return_code
 410
 411
 412 def _UnzipUsingZipFile(file_path, output_dir, verbose=True):
 413   """Extracts a zip file using the Python zipfile module."""
 414   assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
 415   zf = zipfile.ZipFile(file_path)
 416   for name in zf.namelist():
 417     if verbose:
 418       print 'Extracting %s' % name
 419     zf.extract(name, output_dir)
 420     if bisect_utils.IsMacHost():
 421       # Restore file permission bits.
 422       mode = zf.getinfo(name).external_attr >> 16
 423       os.chmod(os.path.join(output_dir, name), mode)
 424
 425
 426 def _MakeDirectory(path):
 427   try:
 428     os.makedirs(path)
 429   except OSError as e:
 430     if e.errno != errno.EEXIST:
 431       raise
 432
 433
 434 def _RemoveDirectoryTree(path):
 435   try:
 436     if os.path.exists(path):
 437       shutil.rmtree(path)
 438   except OSError, e:
 439     if e.errno != errno.ENOENT:
 440       raise
 441
 442
 443 def Main(argv):
 444   """Downloads and extracts a build based on the command line arguments."""
 445   parser = argparse.ArgumentParser()
 446   parser.add_argument('builder_type')
 447   parser.add_argument('revision')
 448   parser.add_argument('output_dir')
 449   parser.add_argument('--target-arch', default='ia32')
 450   parser.add_argument('--target-platform', default='chromium')
 451   parser.add_argument('--deps-patch-sha')
 452   args = parser.parse_args(argv[1:])
 453
 454   bucket_name, remote_path = GetBucketAndRemotePath(
 455       args.revision, args.builder_type, target_arch=args.target_arch,
 456       target_platform=args.target_platform,
 457       deps_patch_sha=args.deps_patch_sha)
 458   print 'Bucket name: %s, remote path: %s' % (bucket_name, remote_path)
 459
 460   if not BuildIsAvailable(bucket_name, remote_path):
 461     print 'Build is not available.'
 462     return 1
 463
 464   FetchFromCloudStorage(bucket_name, remote_path, args.output_dir)
 465   print 'Build has been downloaded to and extracted in %s.' % args.output_dir
 466   return 0
 467
 468
 469 if __name__ == '__main__':
 470   sys.exit(Main(sys.argv))