tools/auto_bisect/fetch_build.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This module contains functions for fetching and extracting archived builds.
   6
   7 The builds may be stored in different places by different types of builders;
   8 for example, builders on tryserver.chromium.perf stores builds in one place,
   9 while builders on chromium.linux store builds in another.
  10
  11 This module can be either imported or run as a stand-alone script to download
  12 and extract a build.
  13
  14 Usage: fetch_build.py <type> <revision> <output_dir> [options]
  15 """
  16
  17 import argparse
  18 import errno
  19 import logging
  20 import os
  21 import shutil
  22 import sys
  23 import zipfile
  24
  25 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH.
  26 from catapult_base import cloud_storage
  27
  28 import bisect_utils
  29
  30 # Possible builder types.
  31 PERF_BUILDER = 'perf'
  32 FULL_BUILDER = 'full'
  33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf'
  34
  35 # Maximum time in seconds to wait after posting build request to the try server.
  36 MAX_MAC_BUILD_TIME = 14400
  37 MAX_WIN_BUILD_TIME = 14400
  38 MAX_LINUX_BUILD_TIME = 14400
  39
  40 # Try server status page URLs, used to get build status.
  41 PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf'
  42 LINUX_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.linux'
  43
  44
  45 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER,
  46                            target_arch='ia32', target_platform='chromium',
  47                            deps_patch_sha=None, extra_src=None):
  48   """Returns the location where a build archive is expected to be.
  49
  50   Args:
  51     revision: Revision string, e.g. a git commit hash or SVN revision.
  52     builder_type: Type of build archive.
  53     target_arch: Architecture, e.g. "ia32".
  54     target_platform: Platform name, e.g. "chromium" or "android".
  55     deps_patch_sha: SHA1 hash which identifies a particular combination of
  56         custom revisions for dependency repositories.
  57     extra_src: Path to a script which can be used to modify the bisect script's
  58         behavior.
  59
  60   Returns:
  61     A pair of strings (bucket, path), where the archive is expected to be.
  62   """
  63   logging.info('Getting GS URL for archive of builder "%s", "%s", "%s".',
  64                builder_type, target_arch, target_platform)
  65   build_archive = BuildArchive.Create(
  66       builder_type, target_arch=target_arch, target_platform=target_platform,
  67       extra_src=extra_src)
  68   bucket = build_archive.BucketName()
  69   remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha)
  70   return bucket, remote_path
  71
  72
  73 def GetBuilderNameAndBuildTime(builder_type=PERF_BUILDER, target_arch='ia32',
  74                                target_platform='chromium', extra_src=None):
  75   """Gets builder bot name and build time in seconds based on platform."""
  76   logging.info('Getting builder name for builder "%s", "%s", "%s".',
  77                builder_type, target_arch, target_platform)
  78   build_archive = BuildArchive.Create(
  79       builder_type, target_arch=target_arch, target_platform=target_platform,
  80       extra_src=extra_src)
  81   return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime()
  82
  83
  84 def GetBuildBotUrl(builder_type=PERF_BUILDER, target_arch='ia32',
  85                    target_platform='chromium', extra_src=None):
  86   """Gets buildbot URL for a given builder type."""
  87   logging.info('Getting buildbot URL for "%s", "%s", "%s".',
  88                builder_type, target_arch, target_platform)
  89   build_archive = BuildArchive.Create(
  90       builder_type, target_arch=target_arch, target_platform=target_platform,
  91       extra_src=extra_src)
  92   return build_archive.GetBuildBotUrl()
  93
  94
  95 class BuildArchive(object):
  96   """Represents a place where builds of some type are stored.
  97
  98   There are two pieces of information required to locate a file in Google
  99   Cloud Storage, bucket name and file path. Subclasses of this class contain
 100   specific logic about which bucket names and paths should be used to fetch
 101   a build.
 102   """
 103
 104   @staticmethod
 105   def Create(builder_type, target_arch='ia32', target_platform='chromium',
 106              extra_src=None):
 107     if builder_type == PERF_BUILDER:
 108       return PerfBuildArchive(target_arch, target_platform)
 109     if builder_type == FULL_BUILDER:
 110       return FullBuildArchive(target_arch, target_platform)
 111     if builder_type == ANDROID_CHROME_PERF_BUILDER:
 112       try:
 113         # Load and initialize a module in extra source file and
 114         # return its module object to access android-chrome specific data.
 115         loaded_extra_src = bisect_utils.LoadExtraSrc(extra_src)
 116         return AndroidChromeBuildArchive(
 117             target_arch, target_platform, loaded_extra_src)
 118       except (IOError, TypeError, ImportError):
 119         raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src)
 120     raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
 121
 122   def __init__(self, target_arch='ia32', target_platform='chromium',
 123                extra_src=None):
 124     self._extra_src = extra_src
 125     if bisect_utils.IsLinuxHost() and target_platform == 'android':
 126       if target_arch == 'arm64':
 127         self._platform = 'android_arm64'
 128       else:
 129         self._platform = 'android'
 130     elif bisect_utils.IsLinuxHost() and target_platform == 'android-chrome':
 131       self._platform = 'android-chrome'
 132     elif bisect_utils.IsLinuxHost():
 133       self._platform = 'linux'
 134     elif bisect_utils.IsMacHost():
 135       self._platform = 'mac'
 136     elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
 137       self._platform = 'win64'
 138     elif bisect_utils.IsWindowsHost():
 139       self._platform = 'win'
 140     else:
 141       raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 142
 143   def BucketName(self):
 144     raise NotImplementedError()
 145
 146   def FilePath(self, revision, deps_patch_sha=None):
 147     """Returns the remote file path to download a build from.
 148
 149     Args:
 150       revision: A Chromium revision; this could be a git commit hash or
 151           commit position or SVN revision number.
 152       deps_patch_sha: The SHA1 hash of a patch to the DEPS file, which
 153           uniquely identifies a change to use a particular revision of
 154           a dependency.
 155
 156     Returns:
 157       A file path, which not does not include a bucket name.
 158     """
 159     raise NotImplementedError()
 160
 161   def _ZipFileName(self, revision, deps_patch_sha=None):
 162     """Gets the file name of a zip archive for a particular revision.
 163
 164     This returns a file name of the form full-build-<platform>_<revision>.zip,
 165     which is a format used by multiple types of builders that store archives.
 166
 167     Args:
 168       revision: A git commit hash or other revision string.
 169       deps_patch_sha: SHA1 hash of a DEPS file patch.
 170
 171     Returns:
 172       The archive file name.
 173     """
 174     base_name = 'full-build-%s' % self._PlatformName()
 175     if deps_patch_sha:
 176       revision = '%s_%s' % (revision, deps_patch_sha)
 177     return '%s_%s.zip' % (base_name, revision)
 178
 179   def _PlatformName(self):
 180     """Return a string to be used in paths for the platform."""
 181     if self._platform in ('win', 'win64'):
 182       # Build archive for win64 is still stored with "win32" in the name.
 183       return 'win32'
 184     if self._platform in ('linux', 'android', 'android_arm64'):
 185       # Android builds are also stored with "linux" in the name.
 186       return 'linux'
 187     if self._platform == 'mac':
 188       return 'mac'
 189     raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 190
 191   def GetBuilderName(self):
 192     raise NotImplementedError()
 193
 194   def GetBuilderBuildTime(self):
 195     """Returns the time to wait for a build after requesting one."""
 196     if self._platform in ('win', 'win64'):
 197       return MAX_WIN_BUILD_TIME
 198     if self._platform in ('linux', 'android',
 199                           'android_arm64', 'android-chrome'):
 200       return MAX_LINUX_BUILD_TIME
 201     if self._platform == 'mac':
 202       return MAX_MAC_BUILD_TIME
 203     raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
 204
 205   def GetBuildBotUrl(self):
 206     raise NotImplementedError()
 207
 208
 209 class PerfBuildArchive(BuildArchive):
 210
 211   def BucketName(self):
 212     return 'chrome-perf'
 213
 214   def FilePath(self, revision, deps_patch_sha=None):
 215     return '%s/%s' % (self._ArchiveDirectory(),
 216                       self._ZipFileName(revision, deps_patch_sha))
 217
 218   def _ArchiveDirectory(self):
 219     """Returns the directory name to download builds from."""
 220     platform_to_directory = {
 221         'android': 'android_perf_rel',
 222         'android_arm64': 'android_perf_rel_arm64',
 223         'linux': 'Linux Builder',
 224         'mac': 'Mac Builder',
 225         'win64': 'Win x64 Builder',
 226         'win': 'Win Builder',
 227     }
 228     assert self._platform in platform_to_directory
 229     return platform_to_directory.get(self._platform)
 230
 231   def GetBuilderName(self):
 232     """Gets builder bot name based on platform."""
 233     if self._platform == 'win64':
 234       return 'win_x64_perf_bisect_builder'
 235     elif self._platform == 'win':
 236       return 'win_perf_bisect_builder'
 237     elif self._platform == 'linux':
 238       return 'linux_perf_bisect_builder'
 239     elif self._platform == 'android':
 240       return 'android_perf_bisect_builder'
 241     elif self._platform == 'android_arm64':
 242       return 'android_arm64_perf_bisect_builder'
 243     elif self._platform == 'mac':
 244       return 'mac_perf_bisect_builder'
 245     raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
 246
 247   def GetBuildBotUrl(self):
 248     """Returns buildbot URL for fetching build info."""
 249     return PERF_TRY_SERVER_URL
 250
 251
 252 class FullBuildArchive(BuildArchive):
 253
 254   def BucketName(self):
 255     platform_to_bucket = {
 256         'android': 'chromium-android',
 257         'linux': 'chromium-linux-archive',
 258         'mac': 'chromium-mac-archive',
 259         'win64': 'chromium-win-archive',
 260         'win': 'chromium-win-archive',
 261     }
 262     assert self._platform in platform_to_bucket
 263     return platform_to_bucket.get(self._platform)
 264
 265   def FilePath(self, revision, deps_patch_sha=None):
 266     return '%s/%s' % (self._ArchiveDirectory(),
 267                       self._ZipFileName(revision, deps_patch_sha))
 268
 269   def _ArchiveDirectory(self):
 270     """Returns the remote directory to download builds from."""
 271     platform_to_directory = {
 272         'android': 'android_main_rel',
 273         'linux': 'chromium.linux/Linux Builder',
 274         'mac': 'chromium.mac/Mac Builder',
 275         'win64': 'chromium.win/Win x64 Builder',
 276         'win': 'chromium.win/Win Builder',
 277     }
 278     assert self._platform in platform_to_directory
 279     return platform_to_directory.get(self._platform)
 280
 281   def GetBuilderName(self):
 282     """Gets builder bot name based on platform."""
 283     if self._platform == 'linux':
 284       return 'linux_full_bisect_builder'
 285     raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
 286
 287   def GetBuildBotUrl(self):
 288     """Returns buildbot URL for fetching build info."""
 289     return LINUX_TRY_SERVER_URL
 290
 291
 292 class AndroidChromeBuildArchive(BuildArchive):
 293   """Represents a place where builds of android-chrome type are stored.
 294
 295   If AndroidChromeBuildArchive is used, it is assumed that the --extra_src
 296   is a valid Python module which contains the module-level functions
 297   GetBucketName and GetArchiveDirectory.
 298   """
 299
 300   def BucketName(self):
 301     return self._extra_src.GetBucketName()
 302
 303   def _ZipFileName(self, revision, deps_patch_sha=None):
 304     """Gets the file name of a zip archive on android-chrome.
 305
 306     This returns a file name of the form build_product_<revision>.zip,
 307     which is a format used by android-chrome.
 308
 309     Args:
 310       revision: A git commit hash or other revision string.
 311       deps_patch_sha: SHA1 hash of a DEPS file patch.
 312
 313     Returns:
 314       The archive file name.
 315     """
 316     if deps_patch_sha:
 317       revision = '%s_%s' % (revision, deps_patch_sha)
 318     return 'build_product_%s.zip' % revision
 319
 320   def FilePath(self, revision, deps_patch_sha=None):
 321     return '%s/%s' % (self._ArchiveDirectory(),
 322                       self._ZipFileName(revision, deps_patch_sha))
 323
 324   def _ArchiveDirectory(self):
 325     """Returns the directory name to download builds from."""
 326     return self._extra_src.GetArchiveDirectory()
 327
 328   def GetBuilderName(self):
 329     """Returns the builder name extra source."""
 330     return self._extra_src.GetBuilderName()
 331
 332   def GetBuildBotUrl(self):
 333     """Returns buildbot URL for fetching build info."""
 334     return self._extra_src.GetBuildBotUrl()
 335
 336
 337 def BuildIsAvailable(bucket_name, remote_path):
 338   """Checks whether a build is currently archived at some place."""
 339   logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path))
 340   try:
 341     exists = cloud_storage.Exists(bucket_name, remote_path)
 342     logging.info('Exists? %s' % exists)
 343     return exists
 344   except cloud_storage.CloudStorageError:
 345     return False
 346
 347
 348 def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
 349   """Fetches file(s) from the Google Cloud Storage.
 350
 351   As a side-effect, this prints messages to stdout about what's happening.
 352
 353   Args:
 354     bucket_name: Google Storage bucket name.
 355     source_path: Source file path.
 356     destination_dir: Destination file path.
 357
 358   Returns:
 359     Local file path of downloaded file if it was downloaded. If the file does
 360     not exist in the given bucket, or if there was an error while downloading,
 361     None is returned.
 362   """
 363   target_file = os.path.join(destination_dir, os.path.basename(source_path))
 364   gs_url = 'gs://%s/%s' % (bucket_name, source_path)
 365   try:
 366     if cloud_storage.Exists(bucket_name, source_path):
 367       logging.info('Fetching file from %s...', gs_url)
 368       cloud_storage.Get(bucket_name, source_path, target_file)
 369       if os.path.exists(target_file):
 370         return target_file
 371     else:
 372       logging.info('File %s not found in cloud storage.', gs_url)
 373       return None
 374   except Exception as e:
 375     logging.warn('Exception while fetching from cloud storage: %s', e)
 376     if os.path.exists(target_file):
 377       os.remove(target_file)
 378   return None
 379
 380
 381 def Unzip(file_path, output_dir, verbose=True):
 382   """Extracts a zip archive's contents into the given output directory.
 383
 384   This was based on ExtractZip from build/scripts/common/chromium_utils.py.
 385
 386   Args:
 387     file_path: Path of the zip file to extract.
 388     output_dir: Path to the destination directory.
 389     verbose: Whether to print out what is being extracted.
 390
 391   Raises:
 392     IOError: The unzip command had a non-zero exit code.
 393     RuntimeError: Failed to create the output directory.
 394   """
 395   _MakeDirectory(output_dir)
 396
 397   # On Linux and Mac, we use the unzip command because it handles links and
 398   # file permissions bits, so achieving this behavior is easier than with
 399   # ZipInfo options.
 400   #
 401   # The Mac Version of unzip unfortunately does not support Zip64, whereas
 402   # the python module does, so we have to fall back to the python zip module
 403   # on Mac if the file size is greater than 4GB.
 404   mac_zip_size_limit = 2 ** 32  # 4GB
 405   if (bisect_utils.IsLinuxHost() or
 406       (bisect_utils.IsMacHost()
 407        and os.path.getsize(file_path) < mac_zip_size_limit)):
 408     unzip_command = ['unzip', '-o']
 409     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 410     return
 411
 412   # On Windows, try to use 7z if it is installed, otherwise fall back to the
 413   # Python zipfile module. If 7z is not installed, then this may fail if the
 414   # zip file is larger than 512MB.
 415   sevenzip_path = r'C:\Program Files\7-Zip\7z.exe'
 416   if bisect_utils.IsWindowsHost() and os.path.exists(sevenzip_path):
 417     unzip_command = [sevenzip_path, 'x', '-y']
 418     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 419     return
 420
 421   _UnzipUsingZipFile(file_path, output_dir, verbose)
 422
 423
 424 def _UnzipUsingCommand(unzip_command, file_path, output_dir):
 425   """Extracts a zip file using an external command.
 426
 427   Args:
 428     unzip_command: An unzipping command, as a string list, without the filename.
 429     file_path: Path to the zip file.
 430     output_dir: The directory which the contents should be extracted to.
 431
 432   Raises:
 433     IOError: The command had a non-zero exit code.
 434   """
 435   absolute_filepath = os.path.abspath(file_path)
 436   command = unzip_command + [absolute_filepath]
 437   return_code = _RunCommandInDirectory(output_dir, command)
 438   if return_code:
 439     _RemoveDirectoryTree(output_dir)
 440     raise IOError('Unzip failed: %s => %s' % (str(command), return_code))
 441
 442
 443 def _RunCommandInDirectory(directory, command):
 444   """Changes to a directory, runs a command, then changes back."""
 445   saved_dir = os.getcwd()
 446   os.chdir(directory)
 447   return_code = bisect_utils.RunProcess(command)
 448   os.chdir(saved_dir)
 449   return return_code
 450
 451
 452 def _UnzipUsingZipFile(file_path, output_dir, verbose=True):
 453   """Extracts a zip file using the Python zipfile module."""
 454   assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
 455   zf = zipfile.ZipFile(file_path)
 456   for name in zf.namelist():
 457     if verbose:
 458       print 'Extracting %s' % name
 459     zf.extract(name, output_dir)
 460     if bisect_utils.IsMacHost():
 461       # Restore file permission bits.
 462       mode = zf.getinfo(name).external_attr >> 16
 463       os.chmod(os.path.join(output_dir, name), mode)
 464
 465
 466 def _MakeDirectory(path):
 467   try:
 468     os.makedirs(path)
 469   except OSError as e:
 470     if e.errno != errno.EEXIST:
 471       raise
 472
 473
 474 def _RemoveDirectoryTree(path):
 475   try:
 476     if os.path.exists(path):
 477       shutil.rmtree(path)
 478   except OSError, e:
 479     if e.errno != errno.ENOENT:
 480       raise
 481
 482
 483 def Main(argv):
 484   """Downloads and extracts a build based on the command line arguments."""
 485   parser = argparse.ArgumentParser()
 486   parser.add_argument('builder_type')
 487   parser.add_argument('revision')
 488   parser.add_argument('output_dir')
 489   parser.add_argument('--target-arch', default='ia32')
 490   parser.add_argument('--target-platform', default='chromium')
 491   parser.add_argument('--deps-patch-sha')
 492   args = parser.parse_args(argv[1:])
 493
 494   bucket_name, remote_path = GetBucketAndRemotePath(
 495       args.revision, args.builder_type, target_arch=args.target_arch,
 496       target_platform=args.target_platform,
 497       deps_patch_sha=args.deps_patch_sha)
 498   print 'Bucket name: %s, remote path: %s' % (bucket_name, remote_path)
 499
 500   if not BuildIsAvailable(bucket_name, remote_path):
 501     print 'Build is not available.'
 502     return 1
 503
 504   FetchFromCloudStorage(bucket_name, remote_path, args.output_dir)
 505   print 'Build has been downloaded to and extracted in %s.' % args.output_dir
 506   return 0
 507
 508
 509 if __name__ == '__main__':
 510   sys.exit(Main(sys.argv))