tools/gn/bin/roll_gn.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """An auto-roller for GN binaries into Chromium.
   6
   7 This script is used to update the GN binaries that a Chromium
   8 checkout uses. In order to update the binaries, one must follow
   9 four steps in order:
  10
  11 1. Trigger try jobs to build a new GN binary at tip-of-tree and upload
  12    the newly-built binaries into the right Google CloudStorage bucket.
  13 2. Wait for the try jobs to complete.
  14 3. Update the buildtools repo with the .sha1 hashes of the newly built
  15    binaries.
  16 4. Update Chromium's DEPS file to the new version of the buildtools repo.
  17
  18 The script has four commands that correspond to the four steps above:
  19 'build', 'wait', 'roll_buildtools', and 'roll_deps'.
  20
  21 The script has a fifth command, 'roll', that runs the four in order.
  22
  23 If given no arguments, the script will run the 'roll' command.
  24
  25 It can only be run on linux in a clean Chromium checkout; it should
  26 error out in most cases if something bad happens, but the error checking
  27 isn't yet foolproof.
  28
  29 """
  30
  31 from __future__ import print_function
  32
  33 import argparse
  34 import json
  35 import os
  36 import re
  37 import subprocess
  38 import sys
  39 import tempfile
  40 import time
  41 import urllib2
  42
  43 depot_tools_path = None
  44 for p in os.environ['PATH'].split(os.pathsep):
  45   if (p.rstrip(os.sep).endswith('depot_tools') and
  46       os.path.isfile(os.path.join(p, 'gclient.py'))):
  47     depot_tools_path = p
  48
  49 assert depot_tools_path
  50 if not depot_tools_path in sys.path:
  51   sys.path.insert(0, depot_tools_path)
  52
  53 third_party_path = os.path.join(depot_tools_path, 'third_party')
  54 if not third_party_path in sys.path:
  55   sys.path.insert(0, third_party_path)
  56
  57 import upload
  58
  59
  60 CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git'
  61
  62 CODE_REVIEW_SERVER = 'https://codereview.chromium.org'
  63
  64 COMMITISH_DIGITS = 10
  65
  66 class GNRoller(object):
  67   def __init__(self):
  68     self.chromium_src_dir = None
  69     self.buildtools_dir = None
  70     self.old_gn_commitish = None
  71     self.new_gn_commitish = None
  72     self.old_gn_version   = None
  73     self.new_gn_version = None
  74     self.reviewer = 'dpranke@chromium.org'
  75     if os.getenv('USER') == 'dpranke':
  76       self.reviewer = 'brettw@chromium.org'
  77
  78   def Roll(self):
  79     parser = argparse.ArgumentParser()
  80     parser.usage = __doc__
  81     parser.add_argument('command', nargs='?', default='roll',
  82                         help='build|roll|roll_buildtools|roll_deps|wait'
  83                              ' (%(default)s is the default)')
  84
  85     args = parser.parse_args()
  86     command = args.command
  87     ret = self.SetUp()
  88     if not ret and command in ('roll', 'build'):
  89       ret = self.TriggerBuild()
  90     if not ret and command in ('roll', 'wait'):
  91       ret = self.WaitForBuildToFinish()
  92     if not ret and command in ('roll', 'roll_buildtools'):
  93       ret = self.RollBuildtools()
  94     if not ret and command in ('roll', 'roll_deps'):
  95       ret = self.RollDEPS()
  96
  97     return ret
  98
  99   def SetUp(self):
 100     if sys.platform != 'linux2':
 101       print('roll_gn is only tested and working on Linux for now.')
 102       return 1
 103
 104     ret, out, _ = self.Call('git config --get remote.origin.url')
 105     origin = out.strip()
 106     if ret or origin != CHROMIUM_REPO:
 107       print('Not in a Chromium repo? git config --get remote.origin.url '
 108             'returned %d: %s' % (ret, origin))
 109       return 1
 110
 111     ret, _, _ = self.Call('git diff -q')
 112     if ret:
 113       print("Checkout is dirty, exiting")
 114       return 1
 115
 116     _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd())
 117     self.chromium_src_dir = out.strip()
 118     self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools')
 119
 120     self.new_gn_commitish, self.new_gn_version = self.GetNewVersions()
 121
 122     _, out, _ = self.Call('gn --version')
 123     self.old_gn_version = out.strip()
 124
 125     _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version)
 126     self.old_gn_commitish = out.strip()
 127     return 0
 128
 129   def GetNewVersions(self):
 130     _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position')
 131     commit_msg = out.splitlines()
 132     first_line = commit_msg[0]
 133     new_gn_commitish = first_line.split()[1]
 134
 135     last_line = commit_msg[-1]
 136     new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line)
 137
 138     return new_gn_commitish, new_gn_version
 139
 140   def TriggerBuild(self):
 141     ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version)
 142     if ret:
 143       print('Failed to create a new branch for build_gn_%s' %
 144             self.new_gn_version)
 145       return 1
 146
 147     self.MakeDummyDepsChange()
 148
 149     ret, out, err = self.Call('git commit -a -m "Build gn at %s"' %
 150                          self.new_gn_version)
 151     if ret:
 152       print('git commit failed: %s' % out + err)
 153       return 1
 154
 155     print('Uploading CL to build GN at {#%s} - %s' %
 156           (self.new_gn_version, self.new_gn_commitish))
 157     ret, out, err = self.Call('git cl upload -f')
 158     if ret:
 159       print('git-cl upload failed: %s' % out + err)
 160       return 1
 161
 162     print('Starting try jobs')
 163     self.Call('git-cl try -m tryserver.chromium.linux '
 164               '-b linux_chromium_gn_upload -r %s' % self.new_gn_commitish)
 165     self.Call('git-cl try -m tryserver.chromium.mac '
 166               '-b mac_chromium_gn_upload -r %s' % self.new_gn_commitish)
 167     self.Call('git-cl try -m tryserver.chromium.win '
 168               '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish)
 169
 170     return 0
 171
 172   def MakeDummyDepsChange(self):
 173     with open('DEPS') as fp:
 174       deps_content = fp.read()
 175       new_deps = deps_content.replace("'buildtools_revision':",
 176                                       "'buildtools_revision':  ")
 177
 178     with open('DEPS', 'w') as fp:
 179       fp.write(new_deps)
 180
 181   def WaitForBuildToFinish(self):
 182     print('Checking build')
 183     results = self.CheckBuild()
 184     while any(r['state'] == 'pending' for r in results.values()):
 185       print()
 186       print('Sleeping for 30 seconds')
 187       time.sleep(30)
 188       print('Checking build')
 189       results = self.CheckBuild()
 190     return 0 if all(r['state'] == 'success' for r in results.values()) else 1
 191
 192   def CheckBuild(self):
 193     _, out, _ = self.Call('git-cl issue')
 194
 195     issue = int(out.split()[2])
 196
 197     _, out, _ = self.Call('git config user.email')
 198     email = ''
 199     rpc_server = upload.GetRpcServer(CODE_REVIEW_SERVER, email)
 200     try:
 201       props = json.loads(rpc_server.Send('/api/%d' % issue))
 202     except Exception as _e:
 203       raise
 204
 205     patchset = int(props['patchsets'][-1])
 206
 207     try:
 208       patchset_data = json.loads(rpc_server.Send('/api/%d/%d' %
 209                                                  (issue, patchset)))
 210     except Exception as _e:
 211       raise
 212
 213     TRY_JOB_RESULT_STATES = ('success', 'warnings', 'failure', 'skipped',
 214                              'exception', 'retry', 'pending')
 215     try_job_results = patchset_data['try_job_results']
 216     if not try_job_results:
 217       print('No try jobs found on most recent patchset')
 218       return 1
 219
 220     results = {}
 221     for job in try_job_results:
 222       builder = job['builder']
 223       if builder == 'linux_chromium_gn_upload':
 224         platform = 'linux64'
 225       elif builder == 'mac_chromium_gn_upload':
 226         platform = 'mac'
 227       elif builder == 'win8_chromium_gn_upload':
 228         platform = 'win'
 229       else:
 230         print('Unexpected builder: %s')
 231         continue
 232
 233       state = TRY_JOB_RESULT_STATES[int(job['result'])]
 234       url_str = ' %s' % job['url']
 235       build = url_str.split('/')[-1]
 236
 237       sha1 = '-'
 238       results.setdefault(platform, {'build': -1, 'sha1': '', 'url': url_str})
 239
 240       if state == 'success':
 241         jsurl = url_str.replace('/builders/', '/json/builders/')
 242         fp = urllib2.urlopen(jsurl)
 243         js = json.loads(fp.read())
 244         fp.close()
 245         for step in js['steps']:
 246           if step['name'] == 'gn sha1':
 247             sha1 = step['text'][1]
 248
 249       if results[platform]['build'] < build:
 250         results[platform]['build'] = build
 251         results[platform]['sha1'] = sha1
 252         results[platform]['state'] = state
 253         results[platform]['url'] = url_str
 254
 255     for platform, r in results.items():
 256       print(platform)
 257       print('  sha1:  %s' % r['sha1'])
 258       print('  state: %s' % r['state'])
 259       print('  build: %s' % r['build'])
 260       print('  url:   %s' % r['url'])
 261       print()
 262
 263     return results
 264
 265   def RollBuildtools(self):
 266     results = self.CheckBuild()
 267     if not all(r['state'] == 'success' for r in results.values()):
 268       print("Roll isn't done or didn't succeed, exiting:")
 269       return 1
 270
 271     desc = self.GetBuildtoolsDesc()
 272
 273     self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version,
 274               cwd=self.buildtools_dir)
 275
 276     for platform in results:
 277       fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1'
 278       path = os.path.join(self.buildtools_dir, platform, fname)
 279       with open(path, 'w') as fp:
 280         fp.write('%s\n' % results[platform]['sha1'])
 281
 282     desc_file = tempfile.NamedTemporaryFile(delete=False)
 283     try:
 284       desc_file.write(desc)
 285       desc_file.close()
 286       self.Call('git commit -a -F %s' % desc_file.name,
 287                 cwd=self.buildtools_dir)
 288       self.Call('git-cl upload -f --send-mail',
 289                 cwd=self.buildtools_dir)
 290     finally:
 291       os.remove(desc_file.name)
 292
 293     ret, out, err = self.Call('git cl land', cwd=self.buildtools_dir)
 294     if ret:
 295         print("buildtools git cl land failed: %d" % ret)
 296         if out:
 297             print(out)
 298         if err:
 299             print(err)
 300         return ret
 301
 302     # Fetch the revision we just committed so that RollDEPS will find it.
 303     self.Call('git fetch', cwd=self.buildtools_dir)
 304
 305     return 0
 306
 307   def RollDEPS(self):
 308     ret, _, _ = self.Call('git new-branch roll_gn_%s' % self.new_gn_version)
 309     if ret:
 310       print('Failed to create a new branch for roll_gn_%s' %
 311             self.new_gn_version)
 312       return 1
 313
 314     _, out, _ = self.Call('git rev-parse origin/master',
 315                           cwd=self.buildtools_dir)
 316     new_buildtools_commitish = out.strip()
 317
 318     new_deps_lines = []
 319     old_buildtools_commitish = ''
 320     with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp:
 321       for l in fp.readlines():
 322         m = re.match(".*'buildtools_revision':.*'(.+)',", l)
 323         if m:
 324           old_buildtools_commitish = m.group(1)
 325           new_deps_lines.append("  'buildtools_revision': '%s'," %
 326                                 new_buildtools_commitish)
 327         else:
 328           new_deps_lines.append(l)
 329
 330     if not old_buildtools_commitish:
 331       print('Could not update DEPS properly, exiting')
 332       return 1
 333
 334     with open('DEPS', 'w') as fp:
 335       fp.write(''.join(new_deps_lines) + '\n')
 336
 337     desc = self.GetDEPSRollDesc(old_buildtools_commitish,
 338                                 new_buildtools_commitish)
 339     desc_file = tempfile.NamedTemporaryFile(delete=False)
 340     try:
 341       desc_file.write(desc)
 342       desc_file.close()
 343       self.Call('git commit -a -F %s' % desc_file.name)
 344       self.Call('git-cl upload -f --send-mail --use-commit-queue')
 345     finally:
 346       os.remove(desc_file.name)
 347     return 0
 348
 349   def GetBuildtoolsDesc(self):
 350     gn_changes = self.GetGNChanges()
 351     return (
 352       'Roll gn %s..%s (r%s:r%s)\n'
 353       '\n'
 354       '%s'
 355       '\n'
 356       'TBR=%s\n' % (
 357         self.old_gn_commitish[:COMMITISH_DIGITS],
 358         self.new_gn_commitish[:COMMITISH_DIGITS],
 359         self.old_gn_version,
 360         self.new_gn_version,
 361         gn_changes,
 362         self.reviewer,
 363       ))
 364
 365   def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish):
 366     gn_changes = self.GetGNChanges()
 367
 368     return (
 369       'Roll DEPS %s..%s\n'
 370       '\n'
 371       '  In order to roll GN %s..%s (r%s:r%s) and pick up\n'
 372       '  the following changes:\n'
 373       '\n'
 374       '%s'
 375       '\n'
 376       'TBR=%s\n'
 377       'CQ_EXTRA_TRYBOTS=tryserver.chromium.mac:mac_chromium_gn_rel,'
 378       'mac_chromium_gn_dbg;'
 379       'tryserver.chromium.win:win8_chromium_gn_dbg,'
 380       'win_chromium_gn_x64_rel\n' % (
 381         old_buildtools_commitish[:COMMITISH_DIGITS],
 382         new_buildtools_commitish[:COMMITISH_DIGITS],
 383         self.old_gn_commitish[:COMMITISH_DIGITS],
 384         self.new_gn_commitish[:COMMITISH_DIGITS],
 385         self.old_gn_version,
 386         self.new_gn_version,
 387         gn_changes,
 388         self.reviewer,
 389       ))
 390
 391   def GetGNChanges(self):
 392     _, out, _ = self.Call(
 393         "git log --pretty='  %h %s' " +
 394         "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish))
 395     return out
 396
 397   def Call(self, cmd, cwd=None):
 398     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
 399                             cwd=(cwd or self.chromium_src_dir))
 400     out, err = proc.communicate()
 401     return proc.returncode, out, err
 402
 403
 404 if __name__ == '__main__':
 405   roller = GNRoller()
 406   sys.exit(roller.Roll())