tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import imp
  11 import os
  12 import shutil
  13 import stat
  14 import subprocess
  15 import sys
  16
  17 DEFAULT_GCLIENT_CUSTOM_DEPS = {
  18     "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  19                             "chrome/data/page_cycler/.git",
  20     "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  21                          "chrome/data/dom_perf/.git",
  22     "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
  23                            "chrome/data/mach_ports/.git",
  24     "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  25                            "chrome/tools/perf/data/.git",
  26     "src/third_party/adobe/flash/binaries/ppapi/linux":
  27         "https://chrome-internal.googlesource.com/"
  28         "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
  29     "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
  30         "https://chrome-internal.googlesource.com/"
  31         "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
  32     "src/third_party/adobe/flash/binaries/ppapi/mac":
  33         "https://chrome-internal.googlesource.com/"
  34         "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
  35     "src/third_party/adobe/flash/binaries/ppapi/mac_64":
  36         "https://chrome-internal.googlesource.com/"
  37         "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
  38     "src/third_party/adobe/flash/binaries/ppapi/win":
  39         "https://chrome-internal.googlesource.com/"
  40         "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
  41     "src/third_party/adobe/flash/binaries/ppapi/win_x64":
  42         "https://chrome-internal.googlesource.com/"
  43         "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",
  44     "src/chrome/tools/test/reference_build/chrome_win": None,
  45     "src/chrome/tools/test/reference_build/chrome_mac": None,
  46     "src/chrome/tools/test/reference_build/chrome_linux": None,
  47     "src/third_party/WebKit/LayoutTests": None,
  48     "src/tools/valgrind": None,}
  49
  50 GCLIENT_SPEC_DATA = [
  51   { "name"        : "src",
  52     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  53     "deps_file"   : ".DEPS.git",
  54     "managed"     : True,
  55     "custom_deps" : {},
  56     "safesync_url": "",
  57   },
  58 ]
  59 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
  60 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  61 FILE_DEPS_GIT = '.DEPS.git'
  62 FILE_DEPS = 'DEPS'
  63
  64 REPO_PARAMS = [
  65   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  66   '--repo-url',
  67   'https://git.chromium.org/external/repo.git'
  68 ]
  69
  70 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  71                     '--before=%d remotes/m/master)'
  72
  73 ORIGINAL_ENV = {}
  74
  75 def OutputAnnotationStepStart(name):
  76   """Outputs appropriate annotation to signal the start of a step to
  77   a trybot.
  78
  79   Args:
  80     name: The name of the step.
  81   """
  82   print
  83   print '@@@SEED_STEP %s@@@' % name
  84   print '@@@STEP_CURSOR %s@@@' % name
  85   print '@@@STEP_STARTED@@@'
  86   print
  87   sys.stdout.flush()
  88
  89
  90 def OutputAnnotationStepClosed():
  91   """Outputs appropriate annotation to signal the closing of a step to
  92   a trybot."""
  93   print
  94   print '@@@STEP_CLOSED@@@'
  95   print
  96   sys.stdout.flush()
  97
  98
  99 def OutputAnnotationStepLink(label, url):
 100   """Outputs appropriate annotation to print a link.
 101
 102   Args:
 103     label: The name to print.
 104     url: The url to print.
 105   """
 106   print
 107   print '@@@STEP_LINK@%s@%s@@@' % (label, url)
 108   print
 109   sys.stdout.flush()
 110
 111
 112 def LoadExtraSrc(path_to_file):
 113   """Attempts to load an extra source file. If this is successful, uses the
 114   new module to override some global values, such as gclient spec data.
 115
 116   Returns:
 117     The loaded src module, or None."""
 118   try:
 119     global GCLIENT_SPEC_DATA
 120     global GCLIENT_SPEC_ANDROID
 121     extra_src = imp.load_source('data', path_to_file)
 122     GCLIENT_SPEC_DATA = extra_src.GetGClientSpec()
 123     GCLIENT_SPEC_ANDROID = extra_src.GetGClientSpecExtraParams()
 124     return extra_src
 125   except ImportError, e:
 126     return None
 127
 128
 129 def IsTelemetryCommand(command):
 130   """Attempts to discern whether or not a given command is running telemetry."""
 131   return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command)
 132
 133
 134 def CreateAndChangeToSourceDirectory(working_directory):
 135   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
 136   the function is successful, the current working directory will change to that
 137   of the new 'bisect' directory.
 138
 139   Returns:
 140     True if the directory was successfully created (or already existed).
 141   """
 142   cwd = os.getcwd()
 143   os.chdir(working_directory)
 144   try:
 145     os.mkdir('bisect')
 146   except OSError, e:
 147     if e.errno != errno.EEXIST:
 148       return False
 149   os.chdir('bisect')
 150   return True
 151
 152
 153 def SubprocessCall(cmd, cwd=None):
 154   """Runs a subprocess with specified parameters.
 155
 156   Args:
 157     params: A list of parameters to pass to gclient.
 158     cwd: Working directory to run from.
 159
 160   Returns:
 161     The return code of the call.
 162   """
 163   if os.name == 'nt':
 164     # "HOME" isn't normally defined on windows, but is needed
 165     # for git to find the user's .netrc file.
 166     if not os.getenv('HOME'):
 167       os.environ['HOME'] = os.environ['USERPROFILE']
 168   shell = os.name == 'nt'
 169   return subprocess.call(cmd, shell=shell, cwd=cwd)
 170
 171
 172 def RunGClient(params, cwd=None):
 173   """Runs gclient with the specified parameters.
 174
 175   Args:
 176     params: A list of parameters to pass to gclient.
 177     cwd: Working directory to run from.
 178
 179   Returns:
 180     The return code of the call.
 181   """
 182   cmd = ['gclient'] + params
 183
 184   return SubprocessCall(cmd, cwd=cwd)
 185
 186
 187 def RunRepo(params):
 188   """Runs cros repo command with specified parameters.
 189
 190   Args:
 191     params: A list of parameters to pass to gclient.
 192
 193   Returns:
 194     The return code of the call.
 195   """
 196   cmd = ['repo'] + params
 197
 198   return SubprocessCall(cmd)
 199
 200
 201 def RunRepoSyncAtTimestamp(timestamp):
 202   """Syncs all git depots to the timestamp specified using repo forall.
 203
 204   Args:
 205     params: Unix timestamp to sync to.
 206
 207   Returns:
 208     The return code of the call.
 209   """
 210   repo_sync = REPO_SYNC_COMMAND % timestamp
 211   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 212   return RunRepo(cmd)
 213
 214
 215 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 216   """Runs gclient and creates a config containing both src and src-internal.
 217
 218   Args:
 219     opts: The options parsed from the command line through parse_args().
 220     custom_deps: A dictionary of additional dependencies to add to .gclient.
 221     cwd: Working directory to run from.
 222
 223   Returns:
 224     The return code of the call.
 225   """
 226   spec = GCLIENT_SPEC_DATA
 227
 228   if custom_deps:
 229     for k, v in custom_deps.iteritems():
 230       spec[0]['custom_deps'][k] = v
 231
 232   # Cannot have newlines in string on windows
 233   spec = 'solutions =' + str(spec)
 234   spec = ''.join([l for l in spec.splitlines()])
 235
 236   if 'android' in opts.target_platform:
 237     spec += GCLIENT_SPEC_ANDROID
 238
 239   return_code = RunGClient(
 240       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 241   return return_code
 242
 243
 244 def IsDepsFileBlink():
 245   """Reads .DEPS.git and returns whether or not we're using blink.
 246
 247   Returns:
 248     True if blink, false if webkit.
 249   """
 250   locals = {'Var': lambda _: locals["vars"][_],
 251             'From': lambda *args: None}
 252   execfile(FILE_DEPS_GIT, {}, locals)
 253   return 'blink.git' in locals['vars']['webkit_url']
 254
 255
 256 def RemoveThirdPartyWebkitDirectory():
 257   """Removes third_party/WebKit.
 258
 259   Returns:
 260     True on success.
 261   """
 262   try:
 263     path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit')
 264     if os.path.exists(path_to_dir):
 265       shutil.rmtree(path_to_dir)
 266   except OSError, e:
 267     if e.errno != errno.ENOENT:
 268       return False
 269   return True
 270
 271
 272 def OnAccessError(func, path, exc_info):
 273   """
 274   Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied
 275
 276   Error handler for ``shutil.rmtree``.
 277
 278   If the error is due to an access error (read only file)
 279   it attempts to add write permission and then retries.
 280
 281   If the error is for another reason it re-raises the error.
 282
 283   Args:
 284     func: The function that raised the error.
 285     path: The path name passed to func.
 286     exc_info: Exception information returned by sys.exc_info().
 287   """
 288   if not os.access(path, os.W_OK):
 289     # Is the error an access error ?
 290     os.chmod(path, stat.S_IWUSR)
 291     func(path)
 292   else:
 293     raise
 294
 295
 296 def RemoveThirdPartyLibjingleDirectory():
 297   """Removes third_party/libjingle. At some point, libjingle was causing issues
 298   syncing when using the git workflow (crbug.com/266324).
 299
 300   Returns:
 301     True on success.
 302   """
 303   path_to_dir = os.path.join(os.getcwd(), 'third_party', 'libjingle')
 304   try:
 305     if os.path.exists(path_to_dir):
 306       shutil.rmtree(path_to_dir, onerror=OnAccessError)
 307   except OSError, e:
 308     print 'Error #%d while running shutil.rmtree(%s): %s' % (
 309         e.errno, path_to_dir, str(e))
 310     if e.errno != errno.ENOENT:
 311       return False
 312   return True
 313
 314
 315 def _CleanupPreviousGitRuns():
 316   """Performs necessary cleanup between runs."""
 317   # If a previous run of git crashed, bot was reset, etc... we
 318   # might end up with leftover index.lock files.
 319   for (path, dir, files) in os.walk(os.getcwd()):
 320     for cur_file in files:
 321       if cur_file.endswith('index.lock'):
 322         path_to_file = os.path.join(path, cur_file)
 323         os.remove(path_to_file)
 324
 325
 326 def RunGClientAndSync(cwd=None):
 327   """Runs gclient and does a normal sync.
 328
 329   Args:
 330     cwd: Working directory to run from.
 331
 332   Returns:
 333     The return code of the call.
 334   """
 335   params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
 336   return RunGClient(params, cwd=cwd)
 337
 338
 339 def SetupGitDepot(opts, custom_deps):
 340   """Sets up the depot for the bisection. The depot will be located in a
 341   subdirectory called 'bisect'.
 342
 343   Args:
 344     opts: The options parsed from the command line through parse_args().
 345     custom_deps: A dictionary of additional dependencies to add to .gclient.
 346
 347   Returns:
 348     True if gclient successfully created the config file and did a sync, False
 349     otherwise.
 350   """
 351   name = 'Setting up Bisection Depot'
 352
 353   if opts.output_buildbot_annotations:
 354     OutputAnnotationStepStart(name)
 355
 356   passed = False
 357
 358   if not RunGClientAndCreateConfig(opts, custom_deps):
 359     passed_deps_check = True
 360     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 361       cwd = os.getcwd()
 362       os.chdir('src')
 363       if not IsDepsFileBlink():
 364         passed_deps_check = RemoveThirdPartyWebkitDirectory()
 365       else:
 366         passed_deps_check = True
 367       if passed_deps_check:
 368         passed_deps_check = RemoveThirdPartyLibjingleDirectory()
 369       os.chdir(cwd)
 370
 371     if passed_deps_check:
 372       _CleanupPreviousGitRuns()
 373
 374       RunGClient(['revert'])
 375       if not RunGClientAndSync():
 376         passed = True
 377
 378   if opts.output_buildbot_annotations:
 379     print
 380     OutputAnnotationStepClosed()
 381
 382   return passed
 383
 384
 385 def SetupCrosRepo():
 386   """Sets up cros repo for bisecting chromeos.
 387
 388   Returns:
 389     Returns 0 on success.
 390   """
 391   cwd = os.getcwd()
 392   try:
 393     os.mkdir('cros')
 394   except OSError, e:
 395     if e.errno != errno.EEXIST:
 396       return False
 397   os.chdir('cros')
 398
 399   cmd = ['init', '-u'] + REPO_PARAMS
 400
 401   passed = False
 402
 403   if not RunRepo(cmd):
 404     if not RunRepo(['sync']):
 405       passed = True
 406   os.chdir(cwd)
 407
 408   return passed
 409
 410
 411 def CopyAndSaveOriginalEnvironmentVars():
 412   """Makes a copy of the current environment variables."""
 413   # TODO: Waiting on crbug.com/255689, will remove this after.
 414   vars_to_remove = []
 415   for k, v in os.environ.iteritems():
 416     if 'ANDROID' in k:
 417       vars_to_remove.append(k)
 418   vars_to_remove.append('CHROME_SRC')
 419   vars_to_remove.append('CHROMIUM_GYP_FILE')
 420   vars_to_remove.append('GYP_CROSSCOMPILE')
 421   vars_to_remove.append('GYP_DEFINES')
 422   vars_to_remove.append('GYP_GENERATORS')
 423   vars_to_remove.append('GYP_GENERATOR_FLAGS')
 424   vars_to_remove.append('OBJCOPY')
 425   for k in vars_to_remove:
 426     if os.environ.has_key(k):
 427       del os.environ[k]
 428
 429   global ORIGINAL_ENV
 430   ORIGINAL_ENV = os.environ.copy()
 431
 432
 433 def SetupAndroidBuildEnvironment(opts, path_to_src=None):
 434   """Sets up the android build environment.
 435
 436   Args:
 437     opts: The options parsed from the command line through parse_args().
 438     path_to_src: Path to the src checkout.
 439
 440   Returns:
 441     True if successful.
 442   """
 443
 444   # Revert the environment variables back to default before setting them up
 445   # with envsetup.sh.
 446   env_vars = os.environ.copy()
 447   for k, _ in env_vars.iteritems():
 448     del os.environ[k]
 449   for k, v in ORIGINAL_ENV.iteritems():
 450     os.environ[k] = v
 451
 452   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 453   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 454                            stdout=subprocess.PIPE,
 455                            stderr=subprocess.PIPE,
 456                            cwd=path_to_src)
 457   (out, _) = proc.communicate()
 458
 459   for line in out.splitlines():
 460     (k, _, v) = line.partition('=')
 461     os.environ[k] = v
 462   # envsetup.sh no longer sets OS=android to GYP_DEFINES env variable
 463   # (CL/170273005). Set this variable explicitly inorder to build chrome on
 464   # android.
 465   try:
 466     if 'OS=android' not in os.environ['GYP_DEFINES']:
 467       os.environ['GYP_DEFINES'] = '%s %s' % (os.environ['GYP_DEFINES'],
 468                                               'OS=android')
 469   except KeyError:
 470     os.environ['GYP_DEFINES'] = 'OS=android'
 471
 472   return not proc.returncode
 473
 474
 475 def SetupPlatformBuildEnvironment(opts):
 476   """Performs any platform specific setup.
 477
 478   Args:
 479     opts: The options parsed from the command line through parse_args().
 480
 481   Returns:
 482     True if successful.
 483   """
 484   if 'android' in opts.target_platform:
 485     CopyAndSaveOriginalEnvironmentVars()
 486     return SetupAndroidBuildEnvironment(opts)
 487   elif opts.target_platform == 'cros':
 488     return SetupCrosRepo()
 489
 490   return True
 491
 492
 493 def CheckIfBisectDepotExists(opts):
 494   """Checks if the bisect directory already exists.
 495
 496   Args:
 497     opts: The options parsed from the command line through parse_args().
 498
 499   Returns:
 500     Returns True if it exists.
 501   """
 502   path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
 503   return os.path.exists(path_to_dir)
 504
 505
 506 def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
 507   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 508   there using gclient.
 509
 510   Args:
 511     opts: The options parsed from the command line through parse_args().
 512     custom_deps: A dictionary of additional dependencies to add to .gclient.
 513   """
 514   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 515     raise RuntimeError('Could not create bisect directory.')
 516
 517   if not SetupGitDepot(opts, custom_deps):
 518     raise RuntimeError('Failed to grab source.')