tools/perf/update_reference_build.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Updates the Chrome reference builds.
   8
   9 Before running this script, you should first verify that you are authenticated
  10 for SVN. You can do this by running:
  11   $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
  12 You may need to get your SVN password from https://chromium-access.appspot.com/.
  13
  14 Usage:
  15   $ cd /tmp
  16   $ /path/to/update_reference_build.py
  17   $ cd reference_builds/reference_builds
  18   $ gcl change
  19   $ gcl upload <change>
  20   $ gcl commit <change>
  21 """
  22
  23 import collections
  24 import logging
  25 import os
  26 import shutil
  27 import subprocess
  28 import sys
  29 import time
  30 import urllib
  31 import urllib2
  32 import zipfile
  33
  34
  35 # Google storage location (no public web URL's), example:
  36 # gs://chrome-unsigned/desktop-*/30.0.1595.0/precise32/chrome-precise32.zip
  37 CHROME_GS_URL_FMT = ('gs://chrome-unsigned/desktop-*/%s/%s/%s')
  38
  39
  40 def _ReportValueError(error_string):
  41   #TODO(aiolos): alert sheriffs via email when an error is seen.
  42   #This should be added when alerts are added when updating the build.
  43   raise ValueError(error_string)
  44
  45
  46 class BuildUpdater(object):
  47   # Remove a platform name from this list to disable updating it.
  48   _REF_BUILD_PLATFORMS = ['Mac', 'Win', 'Linux', 'Linux_x64']
  49
  50   # Omaha is Chrome's autoupdate server. It reports the current versions used
  51   # by each platform on each channel.
  52   _OMAHA_PLATFORMS = ['mac', 'linux', 'win']
  53
  54   # All of the information we need to update each platform.
  55   #   omaha: name omaha uses for the plaftorms.
  56   #   zip_name: name of the zip file to be retrieved from cloud storage.
  57   #   gs_build: name of the Chrome build platform used in cloud storage.
  58   #   destination: Name of the folder to download the reference build to.
  59   UpdateInfo = collections.namedtuple('UpdateInfo',
  60       'omaha, gs_build, zip_name, destination')
  61   _PLATFORM_MAP = { 'Mac': UpdateInfo(omaha='mac',
  62                                       gs_build='mac',
  63                                       zip_name='chrome-mac.zip',
  64                                       destination='chrome_mac'),
  65                     'Win': UpdateInfo(omaha='win',
  66                                       gs_build='win',
  67                                       zip_name='chrome-win.zip',
  68                                       destination='chrome_win'),
  69                     'Linux': UpdateInfo(omaha='linux',
  70                                         gs_build='precise32',
  71                                         zip_name='chrome-precise32.zip',
  72                                         destination='chrome_linux'),
  73                     'Linux_x64': UpdateInfo(omaha='linux',
  74                                             gs_build='precise64',
  75                                             zip_name='chrome-precise65.zip',
  76                                             destination='chrome_linux64')}
  77
  78   def __init__(self):
  79     stable_versions = self._StableVersionsMap()
  80     current_versions = self._CurrentRefBuildsMap()
  81     self._platform_to_version_map = {}
  82     for platform in stable_versions:
  83       if (platform not in current_versions or
  84           stable_versions[platform] != current_versions[platform]):
  85         self._platform_to_version_map[platform] = stable_versions[platform]
  86
  87   @classmethod
  88   def _StableVersionsMap(cls):
  89     omaha_versions_map = cls._OmahaVersionsMap()
  90     versions_map = {}
  91     for platform in cls._REF_BUILD_PLATFORMS:
  92       omaha_platform = cls._PLATFORM_MAP[platform].omaha
  93       if omaha_platform in omaha_versions_map:
  94         versions_map[platform] = omaha_versions_map[omaha_platform]
  95     return versions_map
  96
  97   @classmethod
  98   def _OmahaReport(cls):
  99     url ='https://omahaproxy.appspot.com/all?channel=stable'
 100     lines = urllib2.urlopen(url).readlines()
 101     return [l.split(',') for l in lines]
 102
 103   @classmethod
 104   def _OmahaVersionsMap(cls):
 105     platforms = cls._OMAHA_PLATFORMS
 106     rows = cls._OmahaReport()
 107     if (len(rows) < 1 or
 108         not rows[0][0:3] == ['os', 'channel', 'current_version']):
 109       _ReportValueError('Omaha report is not in the expected form: %s.'
 110                         % rows)
 111     versions_map = {}
 112     for row in rows[1:]:
 113       if row[1] != 'stable':
 114         _ReportValueError('Omaha report contains a line with the channel %s'
 115                           % row[1])
 116       if row[0] in platforms:
 117         versions_map[row[0]] = row[2]
 118
 119     if not all(platform in versions_map for platform in platforms):
 120       _ReportValueError('Omaha report did not contain all desired platforms')
 121     return versions_map
 122
 123   @classmethod
 124   def _CurrentRefBuildsMap(cls):
 125     #TODO(aiolos): Add logic for pulling the current reference build versions.
 126     # Return an empty dictionary to force an update until we store the builds in
 127     # in cloud storage.
 128     return {}
 129
 130   @staticmethod
 131   def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
 132     """Executes a subprocess and returns its exit code and output.
 133
 134     Args:
 135       args: A string or a sequence of program arguments.
 136       cwd: If not None, the subprocess's current directory will be changed to
 137         |cwd| before it's executed.
 138       shell: Whether to execute args as a shell command.
 139
 140     Returns:
 141       The tuple (exit code, output).
 142     """
 143     logging.info(str(args) + ' ' + (cwd or ''))
 144     p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
 145                          stderr=subprocess.PIPE, shell=shell)
 146     stdout, stderr = p.communicate()
 147     exit_code = p.returncode
 148     if stderr:
 149       logging.critical(stderr)
 150     logging.info(stdout)
 151     return (exit_code, stdout)
 152
 153   def _GetBuildUrl(self, platform, version, filename):
 154     """Returns the URL for fetching one file.
 155
 156     Args:
 157       platform: Platform name, must be a key in |self._PLATFORM_MAP|.
 158       version: A Chrome version number, e.g. 30.0.1600.1.
 159       filename: Name of the file to fetch.
 160
 161     Returns:
 162       The URL for fetching a file. This may be a GS or HTTP URL.
 163     """
 164     return CHROME_GS_URL_FMT % (
 165         version, self._PLATFORM_MAP[platform].gs_build, filename)
 166
 167   def _FindBuildVersion(self, platform, version, filename):
 168     """Searches for a version where a filename can be found.
 169
 170     Args:
 171       platform: Platform name.
 172       version: A Chrome version number, e.g. 30.0.1600.1.
 173       filename: Filename to look for.
 174
 175     Returns:
 176       A version where the file could be found, or None.
 177     """
 178     # TODO(shadi): Iterate over official versions to find a valid one.
 179     return (version
 180             if self._DoesBuildExist(platform, version, filename) else None)
 181
 182   def _DoesBuildExist(self, platform, version, filename):
 183     """Checks whether a file can be found for the given Chrome version.
 184
 185     Args:
 186       platform: Platform name.
 187       version: Chrome version number, e.g. 30.0.1600.1.
 188       filename: Filename to look for.
 189
 190     Returns:
 191       True if the file could be found, False otherwise.
 192     """
 193     url = self._GetBuildUrl(platform, version, filename)
 194     return self._DoesGSFileExist(url)
 195
 196   def _DoesGSFileExist(self, gs_file_name):
 197     """Returns True if the GS file can be found, False otherwise."""
 198     exit_code = BuildUpdater._GetCmdStatusAndOutput(
 199         ['gsutil', 'ls', gs_file_name])[0]
 200     return not exit_code
 201
 202   def _GetPlatformFiles(self, platform):
 203     """Returns the name of the zip file to fetch for |platform|."""
 204     return BuildUpdater._PLATFORM_MAP[platform].zip_name
 205
 206   def _DownloadBuilds(self):
 207     for platform in self._platform_to_version_map:
 208       version = self._platform_to_version_map[platform]
 209       for filename in self._GetPlatformFiles(platform):
 210         output = os.path.join('dl', platform,
 211                               '%s_%s_%s' % (platform,
 212                                             version,
 213                                             filename))
 214         if os.path.exists(output):
 215           logging.info('%s alread exists, skipping download', output)
 216           continue
 217         build_version = self._FindBuildVersion(platform, version, filename)
 218         if not build_version:
 219           logging.critical('Failed to find %s build for r%s\n', platform,
 220                            version)
 221           sys.exit(1)
 222         dirname = os.path.dirname(output)
 223         if dirname and not os.path.exists(dirname):
 224           os.makedirs(dirname)
 225         url = self._GetBuildUrl(platform, build_version, filename)
 226         self._DownloadFile(url, output)
 227
 228   def _DownloadFile(self, url, output):
 229     logging.info('Downloading %s, saving to %s', url, output)
 230     BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
 231
 232   def _FetchSvnRepos(self):
 233     if not os.path.exists('reference_builds'):
 234       os.makedirs('reference_builds')
 235     BuildUpdater._GetCmdStatusAndOutput(
 236         ['gclient', 'config',
 237          'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
 238         'reference_builds')
 239     BuildUpdater._GetCmdStatusAndOutput(
 240         ['gclient', 'sync'], 'reference_builds')
 241
 242   def _UnzipFile(self, dl_file, dest_dir):
 243     """Unzips a file if it is a zip file.
 244
 245     Args:
 246       dl_file: The downloaded file to unzip.
 247       dest_dir: The destination directory to unzip to.
 248
 249     Returns:
 250       True if the file was unzipped. False if it wasn't a zip file.
 251     """
 252     if not zipfile.is_zipfile(dl_file):
 253       return False
 254     logging.info('Opening %s', dl_file)
 255     with zipfile.ZipFile(dl_file, 'r') as z:
 256       for content in z.namelist():
 257         dest = os.path.join(dest_dir, content[content.find('/')+1:])
 258         # Create dest parent dir if it does not exist.
 259         if not os.path.isdir(os.path.dirname(dest)):
 260           os.makedirs(os.path.dirname(dest))
 261         # If dest is just a dir listing, do nothing.
 262         if not os.path.basename(dest):
 263           continue
 264         if not os.path.isdir(os.path.dirname(dest)):
 265           os.makedirs(os.path.dirname(dest))
 266         with z.open(content) as unzipped_content:
 267           logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
 268           with file(dest, 'wb') as dest_file:
 269             dest_file.write(unzipped_content.read())
 270           permissions = z.getinfo(content).external_attr >> 16
 271           if permissions:
 272             os.chmod(dest, permissions)
 273     return True
 274
 275   def _ClearDir(self, directory):
 276     """Clears all files in |directory| except for hidden files and folders."""
 277     for root, dirs, files in os.walk(directory):
 278       # Skip hidden files and folders (like .svn and .git).
 279       files = [f for f in files if f[0] != '.']
 280       dirs[:] = [d for d in dirs if d[0] != '.']
 281
 282       for f in files:
 283         os.remove(os.path.join(root, f))
 284
 285   def _ExtractBuilds(self):
 286     for platform in self._platform_to_version_map:
 287       if os.path.exists('tmp_unzip'):
 288         os.path.unlink('tmp_unzip')
 289       dest_dir = os.path.join(
 290           'reference_builds', 'reference_builds',
 291           BuildUpdater._PLATFORM_MAP[platform].destination)
 292       self._ClearDir(dest_dir)
 293       for root, _, dl_files in os.walk(os.path.join('dl', platform)):
 294         for dl_file in dl_files:
 295           dl_file = os.path.join(root, dl_file)
 296           if not self._UnzipFile(dl_file, dest_dir):
 297             logging.info('Copying %s to %s', dl_file, dest_dir)
 298             shutil.copy(dl_file, dest_dir)
 299
 300   def _SvnAddAndRemove(self):
 301     svn_dir = os.path.join('reference_builds', 'reference_builds')
 302     # List all changes without ignoring any files.
 303     stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
 304                                                svn_dir)[1]
 305     for line in stat.splitlines():
 306       action, filename = line.split(None, 1)
 307       # Add new and ignored files.
 308       if action == '?' or action == 'I':
 309         BuildUpdater._GetCmdStatusAndOutput(
 310             ['svn', 'add', filename], svn_dir)
 311       elif action == '!':
 312         BuildUpdater._GetCmdStatusAndOutput(
 313             ['svn', 'delete', filename], svn_dir)
 314       filepath = os.path.join(svn_dir, filename)
 315       if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
 316         BuildUpdater._GetCmdStatusAndOutput(
 317             ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
 318
 319   def DownloadAndUpdateBuilds(self):
 320     self._DownloadBuilds()
 321     self._FetchSvnRepos()
 322     self._ExtractBuilds()
 323     self._SvnAddAndRemove()
 324
 325
 326 def main():
 327   logging.getLogger().setLevel(logging.DEBUG)
 328   #TODO(aiolos): check that there are no options passed (argparse).
 329   b = BuildUpdater()
 330   b.DownloadAndUpdateBuilds()
 331   logging.info('Successfully updated reference builds. Move to '
 332                'reference_builds/reference_builds and make a change with gcl.')
 333
 334 if __name__ == '__main__':
 335   main()