tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import imp
  11 import os
  12 import shutil
  13 import stat
  14 import subprocess
  15 import sys
  16
  17 DEFAULT_GCLIENT_CUSTOM_DEPS = {
  18     "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  19                             "chrome/data/page_cycler/.git",
  20     "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  21                          "chrome/data/dom_perf/.git",
  22     "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
  23                            "chrome/data/mach_ports/.git",
  24     "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  25                            "chrome/tools/perf/data/.git",
  26     "src/third_party/adobe/flash/binaries/ppapi/linux":
  27         "https://chrome-internal.googlesource.com/"
  28         "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
  29     "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
  30         "https://chrome-internal.googlesource.com/"
  31         "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
  32     "src/third_party/adobe/flash/binaries/ppapi/mac":
  33         "https://chrome-internal.googlesource.com/"
  34         "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
  35     "src/third_party/adobe/flash/binaries/ppapi/mac_64":
  36         "https://chrome-internal.googlesource.com/"
  37         "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
  38     "src/third_party/adobe/flash/binaries/ppapi/win":
  39         "https://chrome-internal.googlesource.com/"
  40         "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
  41     "src/third_party/adobe/flash/binaries/ppapi/win_x64":
  42         "https://chrome-internal.googlesource.com/"
  43         "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",
  44     "src/chrome/tools/test/reference_build/chrome_win": None,
  45     "src/chrome/tools/test/reference_build/chrome_mac": None,
  46     "src/chrome/tools/test/reference_build/chrome_linux": None,
  47     "src/third_party/WebKit/LayoutTests": None,
  48     "src/tools/valgrind": None,}
  49
  50 GCLIENT_SPEC_DATA = [
  51   { "name"        : "src",
  52     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  53     "deps_file"   : ".DEPS.git",
  54     "managed"     : True,
  55     "custom_deps" : {},
  56     "safesync_url": "",
  57   },
  58 ]
  59 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
  60 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  61 FILE_DEPS_GIT = '.DEPS.git'
  62 FILE_DEPS = 'DEPS'
  63
  64 REPO_PARAMS = [
  65   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  66   '--repo-url',
  67   'https://git.chromium.org/external/repo.git'
  68 ]
  69
  70 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  71                     '--before=%d remotes/m/master)'
  72
  73 ORIGINAL_ENV = {}
  74
  75 def OutputAnnotationStepStart(name):
  76   """Outputs appropriate annotation to signal the start of a step to
  77   a trybot.
  78
  79   Args:
  80     name: The name of the step.
  81   """
  82   print
  83   print '@@@SEED_STEP %s@@@' % name
  84   print '@@@STEP_CURSOR %s@@@' % name
  85   print '@@@STEP_STARTED@@@'
  86   print
  87   sys.stdout.flush()
  88
  89
  90 def OutputAnnotationStepClosed():
  91   """Outputs appropriate annotation to signal the closing of a step to
  92   a trybot."""
  93   print
  94   print '@@@STEP_CLOSED@@@'
  95   print
  96   sys.stdout.flush()
  97
  98
  99 def OutputAnnotationStepLink(label, url):
 100   """Outputs appropriate annotation to print a link.
 101
 102   Args:
 103     label: The name to print.
 104     url: The url to print.
 105   """
 106   print
 107   print '@@@STEP_LINK@%s@%s@@@' % (label, url)
 108   print
 109   sys.stdout.flush()
 110
 111
 112 def LoadExtraSrc(path_to_file):
 113   """Attempts to load an extra source file. If this is successful, uses the
 114   new module to override some global values, such as gclient spec data.
 115
 116   Returns:
 117     The loaded src module, or None."""
 118   try:
 119     global GCLIENT_SPEC_DATA
 120     global GCLIENT_SPEC_ANDROID
 121     extra_src = imp.load_source('data', path_to_file)
 122     GCLIENT_SPEC_DATA = extra_src.GetGClientSpec()
 123     GCLIENT_SPEC_ANDROID = extra_src.GetGClientSpecExtraParams()
 124     return extra_src
 125   except ImportError, e:
 126     return None
 127
 128
 129 def IsTelemetryCommand(command):
 130   """Attempts to discern whether or not a given command is running telemetry."""
 131   return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command)
 132
 133
 134 def CreateAndChangeToSourceDirectory(working_directory):
 135   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
 136   the function is successful, the current working directory will change to that
 137   of the new 'bisect' directory.
 138
 139   Returns:
 140     True if the directory was successfully created (or already existed).
 141   """
 142   cwd = os.getcwd()
 143   os.chdir(working_directory)
 144   try:
 145     os.mkdir('bisect')
 146   except OSError, e:
 147     if e.errno != errno.EEXIST:
 148       return False
 149   os.chdir('bisect')
 150   return True
 151
 152
 153 def SubprocessCall(cmd, cwd=None):
 154   """Runs a subprocess with specified parameters.
 155
 156   Args:
 157     params: A list of parameters to pass to gclient.
 158     cwd: Working directory to run from.
 159
 160   Returns:
 161     The return code of the call.
 162   """
 163   if os.name == 'nt':
 164     # "HOME" isn't normally defined on windows, but is needed
 165     # for git to find the user's .netrc file.
 166     if not os.getenv('HOME'):
 167       os.environ['HOME'] = os.environ['USERPROFILE']
 168   shell = os.name == 'nt'
 169   return subprocess.call(cmd, shell=shell, cwd=cwd)
 170
 171
 172 def RunGClient(params, cwd=None):
 173   """Runs gclient with the specified parameters.
 174
 175   Args:
 176     params: A list of parameters to pass to gclient.
 177     cwd: Working directory to run from.
 178
 179   Returns:
 180     The return code of the call.
 181   """
 182   cmd = ['gclient'] + params
 183
 184   return SubprocessCall(cmd, cwd=cwd)
 185
 186
 187 def RunRepo(params):
 188   """Runs cros repo command with specified parameters.
 189
 190   Args:
 191     params: A list of parameters to pass to gclient.
 192
 193   Returns:
 194     The return code of the call.
 195   """
 196   cmd = ['repo'] + params
 197
 198   return SubprocessCall(cmd)
 199
 200
 201 def RunRepoSyncAtTimestamp(timestamp):
 202   """Syncs all git depots to the timestamp specified using repo forall.
 203
 204   Args:
 205     params: Unix timestamp to sync to.
 206
 207   Returns:
 208     The return code of the call.
 209   """
 210   repo_sync = REPO_SYNC_COMMAND % timestamp
 211   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 212   return RunRepo(cmd)
 213
 214
 215 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 216   """Runs gclient and creates a config containing both src and src-internal.
 217
 218   Args:
 219     opts: The options parsed from the command line through parse_args().
 220     custom_deps: A dictionary of additional dependencies to add to .gclient.
 221     cwd: Working directory to run from.
 222
 223   Returns:
 224     The return code of the call.
 225   """
 226   spec = GCLIENT_SPEC_DATA
 227
 228   if custom_deps:
 229     for k, v in custom_deps.iteritems():
 230       spec[0]['custom_deps'][k] = v
 231
 232   # Cannot have newlines in string on windows
 233   spec = 'solutions =' + str(spec)
 234   spec = ''.join([l for l in spec.splitlines()])
 235
 236   if 'android' in opts.target_platform:
 237     spec += GCLIENT_SPEC_ANDROID
 238
 239   return_code = RunGClient(
 240       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 241   return return_code
 242
 243
 244 def IsDepsFileBlink():
 245   """Reads .DEPS.git and returns whether or not we're using blink.
 246
 247   Returns:
 248     True if blink, false if webkit.
 249   """
 250   locals = {'Var': lambda _: locals["vars"][_],
 251             'From': lambda *args: None}
 252   execfile(FILE_DEPS_GIT, {}, locals)
 253   return 'blink.git' in locals['vars']['webkit_url']
 254
 255
 256 def OnAccessError(func, path, exc_info):
 257   """
 258   Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied
 259
 260   Error handler for ``shutil.rmtree``.
 261
 262   If the error is due to an access error (read only file)
 263   it attempts to add write permission and then retries.
 264
 265   If the error is for another reason it re-raises the error.
 266
 267   Args:
 268     func: The function that raised the error.
 269     path: The path name passed to func.
 270     exc_info: Exception information returned by sys.exc_info().
 271   """
 272   if not os.access(path, os.W_OK):
 273     # Is the error an access error ?
 274     os.chmod(path, stat.S_IWUSR)
 275     func(path)
 276   else:
 277     raise
 278
 279
 280 def RemoveThirdPartyDirectory(dir_name):
 281   """Removes third_party directory from the source.
 282
 283   At some point, some of the third_parties were causing issues to changes in
 284   the way they are synced. We remove such folder in order to avoid sync errors
 285   while bisecting.
 286
 287   Returns:
 288     True on success, otherwise False.
 289   """
 290   path_to_dir = os.path.join(os.getcwd(), 'third_party', dir_name)
 291   try:
 292     if os.path.exists(path_to_dir):
 293       shutil.rmtree(path_to_dir, onerror=OnAccessError)
 294   except OSError, e:
 295     print 'Error #%d while running shutil.rmtree(%s): %s' % (
 296         e.errno, path_to_dir, str(e))
 297     if e.errno != errno.ENOENT:
 298       return False
 299   return True
 300
 301
 302 def _CleanupPreviousGitRuns():
 303   """Performs necessary cleanup between runs."""
 304   # If a previous run of git crashed, bot was reset, etc... we
 305   # might end up with leftover index.lock files.
 306   for (path, dir, files) in os.walk(os.getcwd()):
 307     for cur_file in files:
 308       if cur_file.endswith('index.lock'):
 309         path_to_file = os.path.join(path, cur_file)
 310         os.remove(path_to_file)
 311
 312
 313 def RunGClientAndSync(cwd=None):
 314   """Runs gclient and does a normal sync.
 315
 316   Args:
 317     cwd: Working directory to run from.
 318
 319   Returns:
 320     The return code of the call.
 321   """
 322   params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
 323   return RunGClient(params, cwd=cwd)
 324
 325
 326 def SetupGitDepot(opts, custom_deps):
 327   """Sets up the depot for the bisection. The depot will be located in a
 328   subdirectory called 'bisect'.
 329
 330   Args:
 331     opts: The options parsed from the command line through parse_args().
 332     custom_deps: A dictionary of additional dependencies to add to .gclient.
 333
 334   Returns:
 335     True if gclient successfully created the config file and did a sync, False
 336     otherwise.
 337   """
 338   name = 'Setting up Bisection Depot'
 339
 340   if opts.output_buildbot_annotations:
 341     OutputAnnotationStepStart(name)
 342
 343   passed = False
 344
 345   if not RunGClientAndCreateConfig(opts, custom_deps):
 346     passed_deps_check = True
 347     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 348       cwd = os.getcwd()
 349       os.chdir('src')
 350       if not IsDepsFileBlink():
 351         passed_deps_check = RemoveThirdPartyDirectory('Webkit')
 352       else:
 353         passed_deps_check = True
 354       if passed_deps_check:
 355         passed_deps_check = RemoveThirdPartyDirectory('libjingle')
 356       if passed_deps_check:
 357         passed_deps_check = RemoveThirdPartyDirectory('skia')
 358       os.chdir(cwd)
 359
 360     if passed_deps_check:
 361       _CleanupPreviousGitRuns()
 362
 363       RunGClient(['revert'])
 364       if not RunGClientAndSync():
 365         passed = True
 366
 367   if opts.output_buildbot_annotations:
 368     print
 369     OutputAnnotationStepClosed()
 370
 371   return passed
 372
 373
 374 def SetupCrosRepo():
 375   """Sets up cros repo for bisecting chromeos.
 376
 377   Returns:
 378     Returns 0 on success.
 379   """
 380   cwd = os.getcwd()
 381   try:
 382     os.mkdir('cros')
 383   except OSError, e:
 384     if e.errno != errno.EEXIST:
 385       return False
 386   os.chdir('cros')
 387
 388   cmd = ['init', '-u'] + REPO_PARAMS
 389
 390   passed = False
 391
 392   if not RunRepo(cmd):
 393     if not RunRepo(['sync']):
 394       passed = True
 395   os.chdir(cwd)
 396
 397   return passed
 398
 399
 400 def CopyAndSaveOriginalEnvironmentVars():
 401   """Makes a copy of the current environment variables."""
 402   # TODO: Waiting on crbug.com/255689, will remove this after.
 403   vars_to_remove = []
 404   for k, v in os.environ.iteritems():
 405     if 'ANDROID' in k:
 406       vars_to_remove.append(k)
 407   vars_to_remove.append('CHROME_SRC')
 408   vars_to_remove.append('CHROMIUM_GYP_FILE')
 409   vars_to_remove.append('GYP_CROSSCOMPILE')
 410   vars_to_remove.append('GYP_DEFINES')
 411   vars_to_remove.append('GYP_GENERATORS')
 412   vars_to_remove.append('GYP_GENERATOR_FLAGS')
 413   vars_to_remove.append('OBJCOPY')
 414   for k in vars_to_remove:
 415     if os.environ.has_key(k):
 416       del os.environ[k]
 417
 418   global ORIGINAL_ENV
 419   ORIGINAL_ENV = os.environ.copy()
 420
 421
 422 def SetupAndroidBuildEnvironment(opts, path_to_src=None):
 423   """Sets up the android build environment.
 424
 425   Args:
 426     opts: The options parsed from the command line through parse_args().
 427     path_to_src: Path to the src checkout.
 428
 429   Returns:
 430     True if successful.
 431   """
 432
 433   # Revert the environment variables back to default before setting them up
 434   # with envsetup.sh.
 435   env_vars = os.environ.copy()
 436   for k, _ in env_vars.iteritems():
 437     del os.environ[k]
 438   for k, v in ORIGINAL_ENV.iteritems():
 439     os.environ[k] = v
 440
 441   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 442   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 443                            stdout=subprocess.PIPE,
 444                            stderr=subprocess.PIPE,
 445                            cwd=path_to_src)
 446   (out, _) = proc.communicate()
 447
 448   for line in out.splitlines():
 449     (k, _, v) = line.partition('=')
 450     os.environ[k] = v
 451   # envsetup.sh no longer sets OS=android to GYP_DEFINES env variable
 452   # (CL/170273005). Set this variable explicitly inorder to build chrome on
 453   # android.
 454   try:
 455     if 'OS=android' not in os.environ['GYP_DEFINES']:
 456       os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'],
 457                                               'OS=android')
 458   except KeyError:
 459     os.environ['GYP_DEFINES'] = 'OS=android'
 460
 461   if opts.use_goma:
 462     os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'],
 463                                            'use_goma=1')
 464   return not proc.returncode
 465
 466
 467 def SetupPlatformBuildEnvironment(opts):
 468   """Performs any platform specific setup.
 469
 470   Args:
 471     opts: The options parsed from the command line through parse_args().
 472
 473   Returns:
 474     True if successful.
 475   """
 476   if 'android' in opts.target_platform:
 477     CopyAndSaveOriginalEnvironmentVars()
 478     return SetupAndroidBuildEnvironment(opts)
 479   elif opts.target_platform == 'cros':
 480     return SetupCrosRepo()
 481
 482   return True
 483
 484
 485 def CheckIfBisectDepotExists(opts):
 486   """Checks if the bisect directory already exists.
 487
 488   Args:
 489     opts: The options parsed from the command line through parse_args().
 490
 491   Returns:
 492     Returns True if it exists.
 493   """
 494   path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
 495   return os.path.exists(path_to_dir)
 496
 497
 498 def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
 499   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 500   there using gclient.
 501
 502   Args:
 503     opts: The options parsed from the command line through parse_args().
 504     custom_deps: A dictionary of additional dependencies to add to .gclient.
 505   """
 506   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 507     raise RuntimeError('Could not create bisect directory.')
 508
 509   if not SetupGitDepot(opts, custom_deps):
 510     raise RuntimeError('Failed to grab source.')