tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import os
  11 import shutil
  12 import stat
  13 import subprocess
  14 import sys
  15
  16 DEFAULT_GCLIENT_CUSTOM_DEPS = {
  17     "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  18                             "chrome/data/page_cycler/.git",
  19     "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  20                          "chrome/data/dom_perf/.git",
  21     "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
  22                            "chrome/data/mach_ports/.git",
  23     "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  24                            "chrome/tools/perf/data/.git",
  25     "src/third_party/adobe/flash/binaries/ppapi/linux":
  26         "https://chrome-internal.googlesource.com/"
  27         "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
  28     "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
  29         "https://chrome-internal.googlesource.com/"
  30         "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
  31     "src/third_party/adobe/flash/binaries/ppapi/mac":
  32         "https://chrome-internal.googlesource.com/"
  33         "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
  34     "src/third_party/adobe/flash/binaries/ppapi/mac_64":
  35         "https://chrome-internal.googlesource.com/"
  36         "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
  37     "src/third_party/adobe/flash/binaries/ppapi/win":
  38         "https://chrome-internal.googlesource.com/"
  39         "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
  40     "src/third_party/adobe/flash/binaries/ppapi/win_x64":
  41         "https://chrome-internal.googlesource.com/"
  42         "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",}
  43
  44 GCLIENT_SPEC_DATA = [
  45   { "name"        : "src",
  46     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  47     "deps_file"   : ".DEPS.git",
  48     "managed"     : True,
  49     "custom_deps" : {},
  50     "safesync_url": "",
  51   },
  52 ]
  53 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
  54 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  55 FILE_DEPS_GIT = '.DEPS.git'
  56
  57 REPO_PARAMS = [
  58   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  59   '--repo-url',
  60   'https://git.chromium.org/external/repo.git'
  61 ]
  62
  63 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  64                     '--before=%d remotes/m/master)'
  65
  66 ORIGINAL_ENV = {}
  67
  68 def OutputAnnotationStepStart(name):
  69   """Outputs appropriate annotation to signal the start of a step to
  70   a trybot.
  71
  72   Args:
  73     name: The name of the step.
  74   """
  75   print
  76   print '@@@SEED_STEP %s@@@' % name
  77   print '@@@STEP_CURSOR %s@@@' % name
  78   print '@@@STEP_STARTED@@@'
  79   print
  80   sys.stdout.flush()
  81
  82
  83 def OutputAnnotationStepClosed():
  84   """Outputs appropriate annotation to signal the closing of a step to
  85   a trybot."""
  86   print
  87   print '@@@STEP_CLOSED@@@'
  88   print
  89   sys.stdout.flush()
  90
  91
  92 def CreateAndChangeToSourceDirectory(working_directory):
  93   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
  94   the function is successful, the current working directory will change to that
  95   of the new 'bisect' directory.
  96
  97   Returns:
  98     True if the directory was successfully created (or already existed).
  99   """
 100   cwd = os.getcwd()
 101   os.chdir(working_directory)
 102   try:
 103     os.mkdir('bisect')
 104   except OSError, e:
 105     if e.errno != errno.EEXIST:
 106       return False
 107   os.chdir('bisect')
 108   return True
 109
 110
 111 def SubprocessCall(cmd, cwd=None):
 112   """Runs a subprocess with specified parameters.
 113
 114   Args:
 115     params: A list of parameters to pass to gclient.
 116     cwd: Working directory to run from.
 117
 118   Returns:
 119     The return code of the call.
 120   """
 121   if os.name == 'nt':
 122     # "HOME" isn't normally defined on windows, but is needed
 123     # for git to find the user's .netrc file.
 124     if not os.getenv('HOME'):
 125       os.environ['HOME'] = os.environ['USERPROFILE']
 126   shell = os.name == 'nt'
 127   return subprocess.call(cmd, shell=shell, cwd=cwd)
 128
 129
 130 def RunGClient(params, cwd=None):
 131   """Runs gclient with the specified parameters.
 132
 133   Args:
 134     params: A list of parameters to pass to gclient.
 135     cwd: Working directory to run from.
 136
 137   Returns:
 138     The return code of the call.
 139   """
 140   cmd = ['gclient'] + params
 141
 142   return SubprocessCall(cmd, cwd=cwd)
 143
 144
 145 def RunRepo(params):
 146   """Runs cros repo command with specified parameters.
 147
 148   Args:
 149     params: A list of parameters to pass to gclient.
 150
 151   Returns:
 152     The return code of the call.
 153   """
 154   cmd = ['repo'] + params
 155
 156   return SubprocessCall(cmd)
 157
 158
 159 def RunRepoSyncAtTimestamp(timestamp):
 160   """Syncs all git depots to the timestamp specified using repo forall.
 161
 162   Args:
 163     params: Unix timestamp to sync to.
 164
 165   Returns:
 166     The return code of the call.
 167   """
 168   repo_sync = REPO_SYNC_COMMAND % timestamp
 169   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 170   return RunRepo(cmd)
 171
 172
 173 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 174   """Runs gclient and creates a config containing both src and src-internal.
 175
 176   Args:
 177     opts: The options parsed from the command line through parse_args().
 178     custom_deps: A dictionary of additional dependencies to add to .gclient.
 179     cwd: Working directory to run from.
 180
 181   Returns:
 182     The return code of the call.
 183   """
 184   spec = GCLIENT_SPEC_DATA
 185
 186   if custom_deps:
 187     for k, v in custom_deps.iteritems():
 188       spec[0]['custom_deps'][k] = v
 189
 190   # Cannot have newlines in string on windows
 191   spec = 'solutions =' + str(spec)
 192   spec = ''.join([l for l in spec.splitlines()])
 193
 194   if opts.target_platform == 'android':
 195     spec += GCLIENT_SPEC_ANDROID
 196
 197   return_code = RunGClient(
 198       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 199   return return_code
 200
 201
 202 def IsDepsFileBlink():
 203   """Reads .DEPS.git and returns whether or not we're using blink.
 204
 205   Returns:
 206     True if blink, false if webkit.
 207   """
 208   locals = {'Var': lambda _: locals["vars"][_],
 209             'From': lambda *args: None}
 210   execfile(FILE_DEPS_GIT, {}, locals)
 211   return 'blink.git' in locals['vars']['webkit_url']
 212
 213
 214 def RemoveThirdPartyWebkitDirectory():
 215   """Removes third_party/WebKit.
 216
 217   Returns:
 218     True on success.
 219   """
 220   try:
 221     path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit')
 222     if os.path.exists(path_to_dir):
 223       shutil.rmtree(path_to_dir)
 224   except OSError, e:
 225     if e.errno != errno.ENOENT:
 226       return False
 227   return True
 228
 229
 230 def OnAccessError(func, path, exc_info):
 231   """
 232   Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied
 233
 234   Error handler for ``shutil.rmtree``.
 235
 236   If the error is due to an access error (read only file)
 237   it attempts to add write permission and then retries.
 238
 239   If the error is for another reason it re-raises the error.
 240
 241   Args:
 242     func: The function that raised the error.
 243     path: The path name passed to func.
 244     exc_info: Exception information returned by sys.exc_info().
 245   """
 246   if not os.access(path, os.W_OK):
 247     # Is the error an access error ?
 248     os.chmod(path, stat.S_IWUSR)
 249     func(path)
 250   else:
 251     raise
 252
 253
 254 def RemoveThirdPartyLibjingleDirectory():
 255   """Removes third_party/libjingle. At some point, libjingle was causing issues
 256   syncing when using the git workflow (crbug.com/266324).
 257
 258   Returns:
 259     True on success.
 260   """
 261   path_to_dir = os.path.join(os.getcwd(), 'third_party', 'libjingle')
 262   try:
 263     if os.path.exists(path_to_dir):
 264       shutil.rmtree(path_to_dir, onerror=OnAccessError)
 265   except OSError, e:
 266     print 'Error #%d while running shutil.rmtree(%s): %s' % (
 267         e.errno, path_to_dir, str(e))
 268     if e.errno != errno.ENOENT:
 269       return False
 270   return True
 271
 272
 273 def RunGClientAndSync(cwd=None):
 274   """Runs gclient and does a normal sync.
 275
 276   Args:
 277     cwd: Working directory to run from.
 278
 279   Returns:
 280     The return code of the call.
 281   """
 282   params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
 283   return RunGClient(params, cwd=cwd)
 284
 285
 286 def SetupGitDepot(opts, custom_deps):
 287   """Sets up the depot for the bisection. The depot will be located in a
 288   subdirectory called 'bisect'.
 289
 290   Args:
 291     opts: The options parsed from the command line through parse_args().
 292     custom_deps: A dictionary of additional dependencies to add to .gclient.
 293
 294   Returns:
 295     True if gclient successfully created the config file and did a sync, False
 296     otherwise.
 297   """
 298   name = 'Setting up Bisection Depot'
 299
 300   if opts.output_buildbot_annotations:
 301     OutputAnnotationStepStart(name)
 302
 303   passed = False
 304
 305   if not RunGClientAndCreateConfig(opts, custom_deps):
 306     passed_deps_check = True
 307     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 308       cwd = os.getcwd()
 309       os.chdir('src')
 310       if not IsDepsFileBlink():
 311         passed_deps_check = RemoveThirdPartyWebkitDirectory()
 312       else:
 313         passed_deps_check = True
 314       if passed_deps_check:
 315         passed_deps_check = RemoveThirdPartyLibjingleDirectory()
 316       os.chdir(cwd)
 317
 318     if passed_deps_check:
 319       RunGClient(['revert'])
 320       if not RunGClientAndSync():
 321         passed = True
 322
 323   if opts.output_buildbot_annotations:
 324     print
 325     OutputAnnotationStepClosed()
 326
 327   return passed
 328
 329
 330 def SetupCrosRepo():
 331   """Sets up cros repo for bisecting chromeos.
 332
 333   Returns:
 334     Returns 0 on success.
 335   """
 336   cwd = os.getcwd()
 337   try:
 338     os.mkdir('cros')
 339   except OSError, e:
 340     if e.errno != errno.EEXIST:
 341       return False
 342   os.chdir('cros')
 343
 344   cmd = ['init', '-u'] + REPO_PARAMS
 345
 346   passed = False
 347
 348   if not RunRepo(cmd):
 349     if not RunRepo(['sync']):
 350       passed = True
 351   os.chdir(cwd)
 352
 353   return passed
 354
 355
 356 def CopyAndSaveOriginalEnvironmentVars():
 357   """Makes a copy of the current environment variables."""
 358   # TODO: Waiting on crbug.com/255689, will remove this after.
 359   vars_to_remove = []
 360   for k, v in os.environ.iteritems():
 361     if 'ANDROID' in k:
 362       vars_to_remove.append(k)
 363   vars_to_remove.append('CHROME_SRC')
 364   vars_to_remove.append('CHROMIUM_GYP_FILE')
 365   vars_to_remove.append('GYP_CROSSCOMPILE')
 366   vars_to_remove.append('GYP_DEFINES')
 367   vars_to_remove.append('GYP_GENERATORS')
 368   vars_to_remove.append('GYP_GENERATOR_FLAGS')
 369   vars_to_remove.append('OBJCOPY')
 370   for k in vars_to_remove:
 371     if os.environ.has_key(k):
 372       del os.environ[k]
 373
 374   global ORIGINAL_ENV
 375   ORIGINAL_ENV = os.environ.copy()
 376
 377
 378 def SetupAndroidBuildEnvironment(opts):
 379   """Sets up the android build environment.
 380
 381   Args:
 382     opts: The options parsed from the command line through parse_args().
 383     path_to_file: Path to the bisect script's directory.
 384
 385   Returns:
 386     True if successful.
 387   """
 388
 389   # Revert the environment variables back to default before setting them up
 390   # with envsetup.sh.
 391   env_vars = os.environ.copy()
 392   for k, _ in env_vars.iteritems():
 393     del os.environ[k]
 394   for k, v in ORIGINAL_ENV.iteritems():
 395     os.environ[k] = v
 396
 397   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 398   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 399                            stdout=subprocess.PIPE,
 400                            stderr=subprocess.PIPE,
 401                            cwd='src')
 402   (out, _) = proc.communicate()
 403
 404   for line in out.splitlines():
 405     (k, _, v) = line.partition('=')
 406     os.environ[k] = v
 407   return not proc.returncode
 408
 409
 410 def SetupPlatformBuildEnvironment(opts):
 411   """Performs any platform specific setup.
 412
 413   Args:
 414     opts: The options parsed from the command line through parse_args().
 415     path_to_file: Path to the bisect script's directory.
 416
 417   Returns:
 418     True if successful.
 419   """
 420   if opts.target_platform == 'android':
 421     CopyAndSaveOriginalEnvironmentVars()
 422     return SetupAndroidBuildEnvironment(opts)
 423   elif opts.target_platform == 'cros':
 424     return SetupCrosRepo()
 425
 426   return True
 427
 428
 429 def CheckIfBisectDepotExists(opts):
 430   """Checks if the bisect directory already exists.
 431
 432   Args:
 433     opts: The options parsed from the command line through parse_args().
 434
 435   Returns:
 436     Returns True if it exists.
 437   """
 438   path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
 439   return os.path.exists(path_to_dir)
 440
 441
 442 def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
 443   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 444   there using gclient.
 445
 446   Args:
 447     opts: The options parsed from the command line through parse_args().
 448     custom_deps: A dictionary of additional dependencies to add to .gclient.
 449
 450   Returns:
 451     Returns 0 on success, otherwise 1.
 452   """
 453   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 454     print 'Error: Could not create bisect directory.'
 455     print
 456     return 1
 457
 458   if not SetupGitDepot(opts, custom_deps):
 459     print 'Error: Failed to grab source.'
 460     print
 461     return 1
 462
 463   return 0