tools/roll_webrtc.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import collections
   8 import logging
   9 import os
  10 import re
  11 import subprocess
  12 import sys
  13 import time
  14
  15
  16 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
  17 SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
  18 import find_depot_tools
  19 find_depot_tools.add_depot_tools_to_path()
  20 import rietveld
  21 import roll_dep_svn
  22 from gclient import GClientKeywords
  23 from third_party import upload
  24
  25 # Avoid depot_tools/third_party/upload.py print verbose messages.
  26 upload.verbosity = 0  # Errors only.
  27
  28 CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git'
  29 COMMIT_POSITION_RE = re.compile('^Cr-Original-Commit-Position: .*#([0-9]+).*$')
  30 CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$')
  31 RIETVELD_URL_RE = re.compile('^https?://(.*)/(.*)')
  32 ROLL_BRANCH_NAME = 'special_webrtc_roll_branch'
  33 TRYJOB_STATUS_SLEEP_SECONDS = 30
  34
  35 # Use a shell for subcommands on Windows to get a PATH search.
  36 IS_WIN = sys.platform.startswith('win')
  37 WEBRTC_PATH = os.path.join('third_party', 'webrtc')
  38 LIBJINGLE_PATH = os.path.join('third_party', 'libjingle', 'source', 'talk')
  39 LIBJINGLE_README = os.path.join('third_party', 'libjingle', 'README.chromium')
  40
  41 # Result codes from build/third_party/buildbot_8_4p1/buildbot/status/results.py
  42 # plus the -1 code which is used when there's no result yet.
  43 TRYJOB_STATUS = {
  44   -1: 'RUNNING',
  45   0: 'SUCCESS',
  46   1: 'WARNINGS',
  47   2: 'FAILURE',
  48   3: 'SKIPPED',
  49   4: 'EXCEPTION',
  50   5: 'RETRY',
  51 }
  52 SUCCESS_STATUS = (0, 1, 3)
  53 FAILURE_STATUS = (2, 4, 5)
  54
  55 CommitInfo = collections.namedtuple('CommitInfo', ['commit_position',
  56                                                    'git_commit',
  57                                                    'git_repo_url'])
  58 CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'rietveld_server'])
  59
  60
  61 def _PosixPath(path):
  62   """Convert a possibly-Windows path to a posix-style path."""
  63   (_, path) = os.path.splitdrive(path)
  64   return path.replace(os.sep, '/')
  65
  66
  67 def _ParseGitCommitPosition(description):
  68   for line in reversed(description.splitlines()):
  69     m = COMMIT_POSITION_RE.match(line.strip())
  70     if m:
  71       return m.group(1)
  72   logging.error('Failed to parse svn revision id from:\n%s\n', description)
  73   sys.exit(-1)
  74
  75
  76 def _ParseGitCommitHash(description):
  77   for line in description.splitlines():
  78     if line.startswith('commit '):
  79       return line.split()[1]
  80   logging.error('Failed to parse git commit id from:\n%s\n', description)
  81   sys.exit(-1)
  82   return None
  83
  84
  85 def _ParseDepsFile(filename):
  86   with open(filename, 'rb') as f:
  87     deps_content = f.read()
  88   return _ParseDepsDict(deps_content)
  89
  90
  91 def _ParseDepsDict(deps_content):
  92   local_scope = {}
  93   var = GClientKeywords.VarImpl({}, local_scope)
  94   global_scope = {
  95     'File': GClientKeywords.FileImpl,
  96     'From': GClientKeywords.FromImpl,
  97     'Var': var.Lookup,
  98     'deps_os': {},
  99   }
 100   exec(deps_content, global_scope, local_scope)
 101   return local_scope
 102
 103
 104 def _WaitForTrybots(issue, rietveld_server):
 105   """Wait until all trybots have passed or at least one have failed.
 106
 107   Returns:
 108     An exit code of 0 if all trybots passed or non-zero otherwise.
 109   """
 110   assert type(issue) is int
 111   print 'Trybot status for https://%s/%d:' % (rietveld_server, issue)
 112   remote = rietveld.Rietveld('https://' + rietveld_server, None, None)
 113
 114   attempt = 0
 115   max_tries = 60*60/TRYJOB_STATUS_SLEEP_SECONDS # Max one hour
 116   while attempt < max_tries:
 117     # Get patches for the issue so we can use the latest one.
 118     data = remote.get_issue_properties(issue, messages=False)
 119     patchsets = data['patchsets']
 120
 121     # Get trybot status for the latest patch set.
 122     data = remote.get_patchset_properties(issue, patchsets[-1])
 123
 124     tryjob_results = data['try_job_results']
 125     if len(tryjob_results) == 0:
 126       logging.debug('No trybots have yet been triggered for https://%s/%d' ,
 127                     rietveld_server, issue)
 128     else:
 129       _PrintTrybotsStatus(tryjob_results)
 130       if any(r['result'] in FAILURE_STATUS for r in tryjob_results):
 131         logging.error('Found failing tryjobs (see above)')
 132         return 1
 133       if all(r['result'] in SUCCESS_STATUS for r in tryjob_results):
 134         return 0
 135
 136     logging.debug('Waiting for %d seconds before next check...',
 137                   TRYJOB_STATUS_SLEEP_SECONDS)
 138     time.sleep(TRYJOB_STATUS_SLEEP_SECONDS)
 139     attempt += 1
 140
 141
 142 def _PrintTrybotsStatus(tryjob_results):
 143   status_to_name = {}
 144   for trybot_result in tryjob_results:
 145     status = TRYJOB_STATUS.get(trybot_result['result'], 'UNKNOWN')
 146     status_to_name.setdefault(status, [])
 147     status_to_name[status].append(trybot_result['builder'])
 148
 149   print '\n========== TRYJOBS STATUS =========='
 150   for status,name_list in status_to_name.iteritems():
 151     print '%s: %s' % (status, ','.join(sorted(name_list)))
 152
 153
 154 def _GenerateCLDescriptionCommand(webrtc_current, libjingle_current,
 155                                   webrtc_new, libjingle_new):
 156   delim = ''
 157   webrtc_str = ''
 158   def GetChangeLogURL(git_repo_url, current_hash, new_hash):
 159     return '%s/+log/%s..%s' % (git_repo_url, current_hash[0:7], new_hash[0:7])
 160
 161   if webrtc_current.git_commit != webrtc_new.git_commit:
 162     webrtc_str = 'WebRTC %s:%s' % (webrtc_current.commit_position,
 163                                    webrtc_new.commit_position)
 164     webrtc_changelog_url = GetChangeLogURL(webrtc_current.git_repo_url,
 165                                            webrtc_current.git_commit,
 166                                            webrtc_new.git_commit)
 167
 168   libjingle_str = ''
 169   if libjingle_current.git_commit != libjingle_new.git_commit:
 170     if webrtc_str:
 171       delim += ', '
 172     libjingle_str = 'Libjingle %s:%s' % (libjingle_current.commit_position,
 173                                          libjingle_new.commit_position)
 174     libjingle_changelog_url = GetChangeLogURL(libjingle_current.git_repo_url,
 175                                               libjingle_current.git_commit,
 176                                               libjingle_new.git_commit)
 177
 178   description = [ '-m', 'Roll ' + webrtc_str + delim + libjingle_str ]
 179   if webrtc_str:
 180     description.extend(['-m', webrtc_str])
 181     description.extend(['-m', 'Changes: %s' % webrtc_changelog_url])
 182   if libjingle_str:
 183     description.extend(['-m', libjingle_str])
 184     description.extend(['-m', 'Changes: %s' % libjingle_changelog_url])
 185   description.extend(['-m', 'TBR='])
 186   return description
 187
 188
 189 class AutoRoller(object):
 190   def __init__(self, chromium_src):
 191     self._chromium_src = chromium_src
 192
 193   def _RunCommand(self, command, working_dir=None, ignore_exit_code=False,
 194                   extra_env=None):
 195     """Runs a command and returns the stdout from that command.
 196
 197     If the command fails (exit code != 0), the function will exit the process.
 198     """
 199     working_dir = working_dir or self._chromium_src
 200     logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir)
 201     env = os.environ.copy()
 202     if extra_env:
 203       logging.debug('extra env: %s', extra_env)
 204       env.update(extra_env)
 205     p = subprocess.Popen(command, stdout=subprocess.PIPE,
 206                          stderr=subprocess.PIPE, shell=IS_WIN, env=env,
 207                          cwd=working_dir, universal_newlines=True)
 208     output = p.stdout.read()
 209     p.wait()
 210     p.stdout.close()
 211     p.stderr.close()
 212
 213     if not ignore_exit_code and p.returncode != 0:
 214       logging.error('Command failed: %s\n%s', str(command), output)
 215       sys.exit(p.returncode)
 216     return output
 217
 218   def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None):
 219     working_dir = os.path.join(self._chromium_src, path_below_src)
 220     self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir)
 221     revision_range = git_hash or 'origin'
 222     ret = self._RunCommand(
 223         ['git', '--no-pager', 'log', revision_range, '--pretty=full', '-1'],
 224         working_dir=working_dir)
 225     return CommitInfo(_ParseGitCommitPosition(ret), _ParseGitCommitHash(ret),
 226                       git_repo_url)
 227
 228   def _GetDepsCommitInfo(self, deps_dict, path_below_src):
 229     entry = deps_dict['deps'][_PosixPath('src/%s' % path_below_src)]
 230     at_index = entry.find('@')
 231     git_repo_url = entry[:at_index]
 232     git_hash = entry[at_index + 1:]
 233     return self._GetCommitInfo(path_below_src, git_hash, git_repo_url)
 234
 235   def _GetCLInfo(self):
 236     cl_output = self._RunCommand(['git', 'cl', 'issue'])
 237     m = CL_ISSUE_RE.match(cl_output.strip())
 238     if not m:
 239       logging.error('Cannot find any CL info. Output was:\n%s', cl_output)
 240       sys.exit(-1)
 241     issue_number = int(m.group(1))
 242     url = m.group(2)
 243
 244     # Parse the Rietveld host from the URL.
 245     m = RIETVELD_URL_RE.match(url)
 246     if not m:
 247       logging.error('Cannot parse Rietveld host from URL: %s', url)
 248       sys.exit(-1)
 249     rietveld_server = m.group(1)
 250     return CLInfo(issue_number, url, rietveld_server)
 251
 252   def _GetCurrentBranchName(self):
 253     return self._RunCommand(
 254         ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0]
 255
 256   def _IsTreeClean(self):
 257     lines = self._RunCommand(['git', 'status', '--porcelain']).splitlines()
 258     if len(lines) == 0:
 259       return True
 260
 261     logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines))
 262     return False
 263
 264   def _UpdateReadmeFile(self, readme_path, new_revision):
 265     readme = open(os.path.join(self._chromium_src, readme_path), 'r+')
 266     txt = readme.read()
 267     m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE),
 268         ('Revision: %s' % new_revision), txt)
 269     readme.seek(0)
 270     readme.write(m)
 271     readme.truncate()
 272
 273   def PrepareRoll(self, dry_run, ignore_checks, no_commit, close_previous_roll):
 274     # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for
 275     # cross platform compatibility.
 276
 277     if not ignore_checks:
 278       if self._GetCurrentBranchName() != 'master':
 279         logging.error('Please checkout the master branch.')
 280         return -1
 281       if not self._IsTreeClean():
 282         logging.error('Please make sure you don\'t have any modified files.')
 283         return -1
 284
 285     logging.debug('Checking for a previous roll branch.')
 286     if close_previous_roll:
 287       self.Abort()
 288
 289     logging.debug('Pulling latest changes')
 290     if not ignore_checks:
 291       self._RunCommand(['git', 'pull'])
 292
 293     self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
 294
 295     # Modify Chromium's DEPS file.
 296
 297     # Parse current hashes.
 298     deps_filename = os.path.join(self._chromium_src, 'DEPS')
 299     deps = _ParseDepsFile(deps_filename)
 300     webrtc_current = self._GetDepsCommitInfo(deps, WEBRTC_PATH)
 301     libjingle_current = self._GetDepsCommitInfo(deps, LIBJINGLE_PATH)
 302
 303     # Find ToT revisions.
 304     webrtc_latest = self._GetCommitInfo(WEBRTC_PATH)
 305     libjingle_latest = self._GetCommitInfo(LIBJINGLE_PATH)
 306
 307     if IS_WIN:
 308       # Make sure the roll script doesn't use Windows line endings.
 309       self._RunCommand(['git', 'config', 'core.autocrlf', 'true'])
 310
 311     self._UpdateDep(deps_filename, WEBRTC_PATH, webrtc_latest)
 312     self._UpdateDep(deps_filename, LIBJINGLE_PATH, libjingle_latest)
 313
 314     if self._IsTreeClean():
 315       print 'The latest revision is already rolled for WebRTC and libjingle.'
 316       self._DeleteRollBranch()
 317     else:
 318       self._UpdateReadmeFile(LIBJINGLE_README, libjingle_latest.commit_position)
 319       description = _GenerateCLDescriptionCommand(
 320         webrtc_current, libjingle_current, webrtc_latest, libjingle_latest)
 321       logging.debug('Committing changes locally.')
 322       self._RunCommand(['git', 'add', '--update', '.'])
 323       self._RunCommand(['git', 'commit'] + description)
 324       logging.debug('Uploading changes...')
 325       self._RunCommand(['git', 'cl', 'upload'],
 326                        extra_env={'EDITOR': 'true'})
 327       cl_info = self._GetCLInfo()
 328       logging.debug('Issue: %d URL: %s', cl_info.issue, cl_info.url)
 329
 330       if not dry_run and not no_commit:
 331         logging.debug('Sending the CL to the CQ...')
 332         self._RunCommand(['git', 'cl', 'set_commit'])
 333         logging.debug('Sent the CL to the CQ. Monitor here: %s', cl_info.url)
 334
 335     # TODO(kjellander): Checkout masters/previous branches again.
 336     return 0
 337
 338   def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info):
 339     dep_name = os.path.join('src', dep_relative_to_src)
 340     comment = 'commit position %s' % commit_info.commit_position
 341
 342     # roll_dep_svn.py relies on cwd being the Chromium checkout, so let's
 343     # temporarily change the working directory and then change back.
 344     cwd = os.getcwd()
 345     os.chdir(os.path.dirname(deps_filename))
 346     roll_dep_svn.update_deps(deps_filename, dep_relative_to_src, dep_name,
 347                          commit_info.git_commit, comment)
 348     os.chdir(cwd)
 349
 350   def _DeleteRollBranch(self):
 351     self._RunCommand(['git', 'checkout', 'master'])
 352     self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 353     logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME)
 354
 355
 356   def _GetBranches(self):
 357     """Returns a tuple of active,branches.
 358
 359     The 'active' is the name of the currently active branch and 'branches' is a
 360     list of all branches.
 361     """
 362     lines = self._RunCommand(['git', 'branch']).split('\n')
 363     branches = []
 364     active = ''
 365     for l in lines:
 366       if '*' in l:
 367         # The assumption is that the first char will always be the '*'.
 368         active = l[1:].strip()
 369         branches.append(active)
 370       else:
 371         b = l.strip()
 372         if b:
 373           branches.append(b)
 374     return (active, branches)
 375
 376   def Abort(self):
 377     active_branch, branches = self._GetBranches()
 378     if active_branch == ROLL_BRANCH_NAME:
 379       active_branch = 'master'
 380     if ROLL_BRANCH_NAME in branches:
 381       print 'Aborting pending roll.'
 382       self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
 383       # Ignore an error here in case an issue wasn't created for some reason.
 384       self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True)
 385       self._RunCommand(['git', 'checkout', active_branch])
 386       self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 387     return 0
 388
 389   def WaitForTrybots(self):
 390     active_branch, _ = self._GetBranches()
 391     if active_branch != ROLL_BRANCH_NAME:
 392       self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
 393     cl_info = self._GetCLInfo()
 394     return _WaitForTrybots(cl_info.issue, cl_info.rietveld_server)
 395
 396
 397 def main():
 398   parser = argparse.ArgumentParser(
 399       description='Find webrtc and libjingle revisions for roll.')
 400   parser.add_argument('--abort',
 401     help=('Aborts a previously prepared roll. '
 402           'Closes any associated issues and deletes the roll branches'),
 403     action='store_true')
 404   parser.add_argument('--no-commit',
 405     help=('Don\'t send the CL to the CQ. This is useful if additional changes '
 406           'are needed to the CL (like for API changes).'),
 407     action='store_true')
 408   parser.add_argument('--wait-for-trybots',
 409     help=('Waits until all trybots from a previously created roll are either '
 410           'successful or at least one has failed. This is useful to be able to '
 411           'continuously run this script but not initiating new rolls until a '
 412           'previous one is known to have passed or failed.'),
 413     action='store_true')
 414   parser.add_argument('--close-previous-roll', action='store_true',
 415                       help='Abort a previous roll if one exists.')
 416   parser.add_argument('--dry-run', action='store_true', default=False,
 417       help='Create branches and CLs but doesn\'t send tryjobs or commit.')
 418   parser.add_argument('--ignore-checks', action='store_true', default=False,
 419       help=('Skips checks for being on the master branch, dirty workspaces and '
 420             'the updating of the checkout. Will still delete and create local '
 421             'Git branches.'))
 422   parser.add_argument('-v', '--verbose', action='store_true', default=False,
 423       help='Be extra verbose in printing of log messages.')
 424   args = parser.parse_args()
 425
 426   if args.verbose:
 427     logging.basicConfig(level=logging.DEBUG)
 428   else:
 429     logging.basicConfig(level=logging.ERROR)
 430
 431   autoroller = AutoRoller(SRC_DIR)
 432   if args.abort:
 433     return autoroller.Abort()
 434   elif args.wait_for_trybots:
 435     return autoroller.WaitForTrybots()
 436   else:
 437     return autoroller.PrepareRoll(args.dry_run, args.ignore_checks,
 438                                   args.no_commit, args.close_previous_roll)
 439
 440 if __name__ == '__main__':
 441   sys.exit(main())