tools/roll_webrtc.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import collections
   8 import logging
   9 import os
  10 import re
  11 import subprocess
  12 import sys
  13 import time
  14
  15
  16 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
  17 SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
  18 import find_depot_tools
  19 find_depot_tools.add_depot_tools_to_path()
  20 import rietveld
  21 import roll_dep
  22 from gclient import GClientKeywords
  23 from third_party import upload
  24
  25 # Avoid depot_tools/third_party/upload.py print verbose messages.
  26 upload.verbosity = 0  # Errors only.
  27
  28 CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git'
  29 COMMIT_POSITION_RE = re.compile('^Cr-Original-Commit-Position: .*#([0-9]+).*$')
  30 CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$')
  31 RIETVELD_URL_RE = re.compile('^https?://(.*)/(.*)')
  32 ROLL_BRANCH_NAME = 'special_webrtc_roll_branch'
  33 TRYJOB_STATUS_SLEEP_SECONDS = 30
  34
  35 # Use a shell for subcommands on Windows to get a PATH search.
  36 USE_SHELL = sys.platform.startswith('win')
  37 WEBRTC_PATH = 'third_party/webrtc'
  38 LIBJINGLE_PATH = 'third_party/libjingle/source/talk'
  39 LIBJINGLE_README = 'third_party/libjingle/README.chromium'
  40
  41 # Result codes from build/third_party/buildbot_8_4p1/buildbot/status/results.py
  42 # plus the -1 code which is used when there's no result yet.
  43 TRYJOB_STATUS = {
  44   -1: 'RUNNING',
  45   0: 'SUCCESS',
  46   1: 'WARNINGS',
  47   2: 'FAILURE',
  48   3: 'SKIPPED',
  49   4: 'EXCEPTION',
  50   5: 'RETRY',
  51 }
  52 SUCCESS_STATUS = (0, 1, 3)
  53 FAILURE_STATUS = (2, 4, 5)
  54
  55 CommitInfo = collections.namedtuple('CommitInfo', ['commit_position',
  56                                                    'git_commit',
  57                                                    'git_repo_url'])
  58 CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'rietveld_server'])
  59
  60
  61 def _ParseGitCommitPosition(description):
  62   for line in reversed(description.splitlines()):
  63     m = COMMIT_POSITION_RE.match(line.strip())
  64     if m:
  65       return m.group(1)
  66   logging.error('Failed to parse svn revision id from:\n%s\n', description)
  67   sys.exit(-1)
  68
  69
  70 def _ParseGitCommitHash(description):
  71   for line in description.splitlines():
  72     if line.startswith('commit '):
  73       return line.split()[1]
  74   logging.error('Failed to parse git commit id from:\n%s\n', description)
  75   sys.exit(-1)
  76   return None
  77
  78
  79 def _ParseDepsFile(filename):
  80   with open(filename, 'rb') as f:
  81     deps_content = f.read()
  82   return _ParseDepsDict(deps_content)
  83
  84
  85 def _ParseDepsDict(deps_content):
  86   local_scope = {}
  87   var = GClientKeywords.VarImpl({}, local_scope)
  88   global_scope = {
  89     'File': GClientKeywords.FileImpl,
  90     'From': GClientKeywords.FromImpl,
  91     'Var': var.Lookup,
  92     'deps_os': {},
  93   }
  94   exec(deps_content, global_scope, local_scope)
  95   return local_scope
  96
  97
  98 def _WaitForTrybots(issue, rietveld_server):
  99   """Wait until all trybots have passed or at least one have failed.
 100
 101   Returns:
 102     An exit code of 0 if all trybots passed or non-zero otherwise.
 103   """
 104   assert type(issue) is int
 105   print 'Trybot status for https://%s/%d:' % (rietveld_server, issue)
 106   remote = rietveld.Rietveld('https://' + rietveld_server, None, None)
 107
 108   attempt = 0
 109   max_tries = 60*60/TRYJOB_STATUS_SLEEP_SECONDS # Max one hour
 110   while attempt < max_tries:
 111     # Get patches for the issue so we can use the latest one.
 112     data = remote.get_issue_properties(issue, messages=False)
 113     patchsets = data['patchsets']
 114
 115     # Get trybot status for the latest patch set.
 116     data = remote.get_patchset_properties(issue, patchsets[-1])
 117
 118     tryjob_results = data['try_job_results']
 119     if len(tryjob_results) == 0:
 120       logging.debug('No trybots have yet been triggered for https://%s/%d' ,
 121                     rietveld_server, issue)
 122     else:
 123       _PrintTrybotsStatus(tryjob_results)
 124       if any(r['result'] in FAILURE_STATUS for r in tryjob_results):
 125         logging.error('Found failing tryjobs (see above)')
 126         return 1
 127       if all(r['result'] in SUCCESS_STATUS for r in tryjob_results):
 128         return 0
 129
 130     logging.debug('Waiting for %d seconds before next check...',
 131                   TRYJOB_STATUS_SLEEP_SECONDS)
 132     time.sleep(TRYJOB_STATUS_SLEEP_SECONDS)
 133     attempt += 1
 134
 135
 136 def _PrintTrybotsStatus(tryjob_results):
 137   status_to_name = {}
 138   for trybot_result in tryjob_results:
 139     status = TRYJOB_STATUS.get(trybot_result['result'], 'UNKNOWN')
 140     status_to_name.setdefault(status, [])
 141     status_to_name[status].append(trybot_result['builder'])
 142
 143   print '\n========== TRYJOBS STATUS =========='
 144   for status,name_list in status_to_name.iteritems():
 145     print '%s: %s' % (status, ','.join(sorted(name_list)))
 146
 147
 148 def _GenerateCLDescription(webrtc_current, libjingle_current,
 149                            webrtc_new, libjingle_new):
 150   delim = ''
 151   webrtc_str = ''
 152   def GetChangeLogURL(git_repo_url, current_hash, new_hash):
 153     return '%s/+log/%s..%s' % (git_repo_url, current_hash[0:7], new_hash[0:7])
 154
 155   if webrtc_current.git_commit != webrtc_new.git_commit:
 156     webrtc_str = 'WebRTC %s:%s' % (webrtc_current.commit_position,
 157                                    webrtc_new.commit_position)
 158     webrtc_changelog_url = GetChangeLogURL(webrtc_current.git_repo_url,
 159                                            webrtc_current.git_commit,
 160                                            webrtc_new.git_commit)
 161
 162   libjingle_str = ''
 163   if libjingle_current.git_commit != libjingle_new.git_commit:
 164     if webrtc_str:
 165       delim += ', '
 166     libjingle_str = 'Libjingle %s:%s' % (libjingle_current.commit_position,
 167                                          libjingle_new.commit_position)
 168     libjingle_changelog_url = GetChangeLogURL(libjingle_current.git_repo_url,
 169                                               libjingle_current.git_commit,
 170                                               libjingle_new.git_commit)
 171
 172   description = 'Roll ' + webrtc_str + delim + libjingle_str + '\n\n'
 173   if webrtc_str:
 174     description += webrtc_str + '\n'
 175     description += 'Changes: %s\n\n' % webrtc_changelog_url
 176   if libjingle_str:
 177     description += libjingle_str + '\n'
 178     description += 'Changes: %s\n' % libjingle_changelog_url
 179   description += '\nTBR='
 180   return description
 181
 182
 183 class AutoRoller(object):
 184   def __init__(self, chromium_src):
 185     self._chromium_src = chromium_src
 186
 187   def _RunCommand(self, command, working_dir=None, ignore_exit_code=False,
 188                   extra_env=None):
 189     """Runs a command and returns the stdout from that command.
 190
 191     If the command fails (exit code != 0), the function will exit the process.
 192     """
 193     working_dir = working_dir or self._chromium_src
 194     logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir)
 195     env = os.environ.copy()
 196     if extra_env:
 197       logging.debug('extra env: %s', extra_env)
 198       env.update(extra_env)
 199     p = subprocess.Popen(command, stdout=subprocess.PIPE,
 200                          stderr=subprocess.PIPE, shell=USE_SHELL, env=env,
 201                          cwd=working_dir, universal_newlines=True)
 202     output = p.stdout.read()
 203     p.wait()
 204     p.stdout.close()
 205     p.stderr.close()
 206
 207     if not ignore_exit_code and p.returncode != 0:
 208       logging.error('Command failed: %s\n%s', str(command), output)
 209       sys.exit(p.returncode)
 210     return output
 211
 212   def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None):
 213     working_dir = os.path.join(self._chromium_src, path_below_src)
 214     self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir)
 215     revision_range = git_hash or 'origin'
 216     ret = self._RunCommand(
 217         ['git', '--no-pager', 'log', revision_range, '--pretty=full', '-1'],
 218         working_dir=working_dir)
 219     return CommitInfo(_ParseGitCommitPosition(ret), _ParseGitCommitHash(ret),
 220                       git_repo_url)
 221
 222   def _GetDepsCommitInfo(self, deps_dict, path_below_src):
 223     entry = deps_dict['deps']['src/%s' % path_below_src]
 224     at_index = entry.find('@')
 225     git_repo_url = entry[:at_index]
 226     git_hash = entry[at_index + 1:]
 227     return self._GetCommitInfo(path_below_src, git_hash, git_repo_url)
 228
 229   def _GetCLInfo(self):
 230     cl_output = self._RunCommand(['git', 'cl', 'issue'])
 231     m = CL_ISSUE_RE.match(cl_output.strip())
 232     if not m:
 233       logging.error('Cannot find any CL info. Output was:\n%s', cl_output)
 234       sys.exit(-1)
 235     issue_number = int(m.group(1))
 236     url = m.group(2)
 237
 238     # Parse the Rietveld host from the URL.
 239     m = RIETVELD_URL_RE.match(url)
 240     if not m:
 241       logging.error('Cannot parse Rietveld host from URL: %s', url)
 242       sys.exit(-1)
 243     rietveld_server = m.group(1)
 244     return CLInfo(issue_number, url, rietveld_server)
 245
 246   def _GetCurrentBranchName(self):
 247     return self._RunCommand(
 248         ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0]
 249
 250   def _IsTreeClean(self):
 251     lines = self._RunCommand(['git', 'status', '--porcelain']).splitlines()
 252     if len(lines) == 0:
 253       return True
 254
 255     logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines))
 256     return False
 257
 258   def _UpdateReadmeFile(self, readme_path, new_revision):
 259     readme = open(os.path.join(self._chromium_src, readme_path), 'r+')
 260     txt = readme.read()
 261     m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE),
 262         ('Revision: %s' % new_revision), txt)
 263     readme.seek(0)
 264     readme.write(m)
 265     readme.truncate()
 266
 267   def PrepareRoll(self, dry_run, ignore_checks, no_commit, close_previous_roll):
 268     # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for
 269     # cross platform compatibility.
 270
 271     if not ignore_checks:
 272       if self._GetCurrentBranchName() != 'master':
 273         logging.error('Please checkout the master branch.')
 274         return -1
 275       if not self._IsTreeClean():
 276         logging.error('Please make sure you don\'t have any modified files.')
 277         return -1
 278
 279     logging.debug('Checking for a previous roll branch.')
 280     if close_previous_roll:
 281       self.Abort()
 282
 283     logging.debug('Pulling latest changes')
 284     if not ignore_checks:
 285       self._RunCommand(['git', 'pull'])
 286
 287     self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
 288
 289     # Modify Chromium's DEPS file.
 290
 291     # Parse current hashes.
 292     deps_filename = os.path.join(self._chromium_src, 'DEPS')
 293     deps = _ParseDepsFile(deps_filename)
 294     webrtc_current = self._GetDepsCommitInfo(deps, WEBRTC_PATH)
 295     libjingle_current = self._GetDepsCommitInfo(deps, LIBJINGLE_PATH)
 296
 297     # Find ToT revisions.
 298     webrtc_latest = self._GetCommitInfo(WEBRTC_PATH)
 299     libjingle_latest = self._GetCommitInfo(LIBJINGLE_PATH)
 300
 301     self._UpdateDep(deps_filename, WEBRTC_PATH, webrtc_latest)
 302     self._UpdateDep(deps_filename, LIBJINGLE_PATH, libjingle_latest)
 303
 304     if self._IsTreeClean():
 305       logging.debug('Tree is clean - no changes detected.')
 306       self._DeleteRollBranch()
 307     else:
 308       self._UpdateReadmeFile(LIBJINGLE_README, libjingle_latest.commit_position)
 309       description = _GenerateCLDescription(webrtc_current, libjingle_current,
 310                                            webrtc_latest, libjingle_latest)
 311       logging.debug('Committing changes locally.')
 312       self._RunCommand(['git', 'add', '--update', '.'])
 313       self._RunCommand(['git', 'commit', '-m', description])
 314       logging.debug('Uploading changes...')
 315       self._RunCommand(['git', 'cl', 'upload', '-m', description],
 316                        extra_env={'EDITOR': 'true'})
 317       cl_info = self._GetCLInfo()
 318       logging.debug('Issue: %d URL: %s', cl_info.issue, cl_info.url)
 319
 320       if not dry_run and not no_commit:
 321         logging.debug('Sending the CL to the CQ...')
 322         self._RunCommand(['git', 'cl', 'set_commit'])
 323         logging.debug('Sent the CL to the CQ. Monitor here: %s', cl_info.url)
 324
 325     # TODO(kjellander): Checkout masters/previous branches again.
 326     return 0
 327
 328   def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info):
 329     dep_name = os.path.join('src', dep_relative_to_src)
 330     comment = 'commit position %s' % commit_info.commit_position
 331
 332     # roll_dep.py relies on cwd being the Chromium checkout, so let's
 333     # temporarily change the working directory and then change back.
 334     cwd = os.getcwd()
 335     os.chdir(os.path.dirname(deps_filename))
 336     roll_dep.update_deps(deps_filename, dep_relative_to_src, dep_name,
 337                          commit_info.git_commit, comment)
 338     os.chdir(cwd)
 339
 340   def _DeleteRollBranch(self):
 341     self._RunCommand(['git', 'checkout', 'master'])
 342     self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 343     logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME)
 344
 345
 346   def _GetBranches(self):
 347     """Returns a tuple of active,branches.
 348
 349     The 'active' is the name of the currently active branch and 'branches' is a
 350     list of all branches.
 351     """
 352     lines = self._RunCommand(['git', 'branch']).split('\n')
 353     branches = []
 354     active = ''
 355     for l in lines:
 356       if '*' in l:
 357         # The assumption is that the first char will always be the '*'.
 358         active = l[1:].strip()
 359         branches.append(active)
 360       else:
 361         b = l.strip()
 362         if b:
 363           branches.append(b)
 364     return (active, branches)
 365
 366   def Abort(self):
 367     active_branch, branches = self._GetBranches()
 368     if active_branch == ROLL_BRANCH_NAME:
 369       active_branch = 'master'
 370     if ROLL_BRANCH_NAME in branches:
 371       print 'Aborting pending roll.'
 372       self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
 373       # Ignore an error here in case an issue wasn't created for some reason.
 374       self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True)
 375       self._RunCommand(['git', 'checkout', active_branch])
 376       self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 377     return 0
 378
 379   def WaitForTrybots(self):
 380     active_branch, _ = self._GetBranches()
 381     if active_branch != ROLL_BRANCH_NAME:
 382       self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
 383     cl_info = self._GetCLInfo()
 384     return _WaitForTrybots(cl_info.issue, cl_info.rietveld_server)
 385
 386
 387 def main():
 388   if sys.platform in ('win32', 'cygwin'):
 389     logging.error('Only Linux and Mac platforms are supported right now.')
 390     return -1
 391
 392   parser = argparse.ArgumentParser(
 393       description='Find webrtc and libjingle revisions for roll.')
 394   parser.add_argument('--abort',
 395     help=('Aborts a previously prepared roll. '
 396           'Closes any associated issues and deletes the roll branches'),
 397     action='store_true')
 398   parser.add_argument('--no-commit',
 399     help=('Don\'t send the CL to the CQ. This is useful if additional changes '
 400           'are needed to the CL (like for API changes).'),
 401     action='store_true')
 402   parser.add_argument('--wait-for-trybots',
 403     help=('Waits until all trybots from a previously created roll are either '
 404           'successful or at least one has failed. This is useful to be able to '
 405           'continuously run this script but not initiating new rolls until a '
 406           'previous one is known to have passed or failed.'),
 407     action='store_true')
 408   parser.add_argument('--close-previous-roll', action='store_true',
 409                       help='Abort a previous roll if one exists.')
 410   parser.add_argument('--dry-run', action='store_true', default=False,
 411       help='Create branches and CLs but doesn\'t send tryjobs or commit.')
 412   parser.add_argument('--ignore-checks', action='store_true', default=False,
 413       help=('Skips checks for being on the master branch, dirty workspaces and '
 414             'the updating of the checkout. Will still delete and create local '
 415             'Git branches.'))
 416   parser.add_argument('-v', '--verbose', action='store_true', default=False,
 417       help='Be extra verbose in printing of log messages.')
 418   args = parser.parse_args()
 419
 420   if args.verbose:
 421     logging.basicConfig(level=logging.DEBUG)
 422   else:
 423     logging.basicConfig(level=logging.ERROR)
 424
 425   autoroller = AutoRoller(SRC_DIR)
 426   if args.abort:
 427     return autoroller.Abort()
 428   elif args.wait_for_trybots:
 429     return autoroller.WaitForTrybots()
 430   else:
 431     return autoroller.PrepareRoll(args.dry_run, args.ignore_checks,
 432                                   args.no_commit, args.close_previous_roll)
 433
 434 if __name__ == '__main__':
 435   sys.exit(main())