tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import errno
  17 import json
  18 import os
  19 import pipes
  20 import pprint
  21 import shlex
  22 import shutil
  23 import sys
  24 import subprocess
  25 import tempfile
  26
  27 def main(args):
  28   mbw = MetaBuildWrapper()
  29   mbw.ParseArgs(args)
  30   return mbw.args.func()
  31
  32
  33 class MetaBuildWrapper(object):
  34   def __init__(self):
  35     p = os.path
  36     d = os.path.dirname
  37     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  38     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  39                                  'mb_config.pyl')
  40     self.executable = sys.executable
  41     self.platform = sys.platform
  42     self.sep = os.sep
  43     self.args = argparse.Namespace()
  44     self.configs = {}
  45     self.masters = {}
  46     self.mixins = {}
  47     self.private_configs = []
  48     self.common_dev_configs = []
  49     self.unsupported_configs = []
  50
  51   def ParseArgs(self, argv):
  52     def AddCommonOptions(subp):
  53       subp.add_argument('-b', '--builder',
  54                         help='builder name to look up config from')
  55       subp.add_argument('-m', '--master',
  56                         help='master name to look up config from')
  57       subp.add_argument('-c', '--config',
  58                         help='configuration to analyze')
  59       subp.add_argument('-f', '--config-file', metavar='PATH',
  60                         default=self.default_config,
  61                         help='path to config file '
  62                             '(default is //tools/mb/mb_config.pyl)')
  63       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  64                         help='path to goma directory (default is %(default)s).')
  65       subp.add_argument('-n', '--dryrun', action='store_true',
  66                         help='Do a dry run (i.e., do nothing, just print '
  67                              'the commands that will run)')
  68       subp.add_argument('-v', '--verbose', action='store_true',
  69                         help='verbose logging')
  70
  71     parser = argparse.ArgumentParser(prog='mb')
  72     subps = parser.add_subparsers()
  73
  74     subp = subps.add_parser('analyze',
  75                             help='analyze whether changes to a set of files '
  76                                  'will cause a set of binaries to be rebuilt.')
  77     AddCommonOptions(subp)
  78     subp.add_argument('--swarming-targets-file',
  79                       help='save runtime dependencies for targets listed '
  80                            'in file.')
  81     subp.add_argument('path', nargs=1,
  82                       help='path build was generated into.')
  83     subp.add_argument('input_path', nargs=1,
  84                       help='path to a file containing the input arguments '
  85                            'as a JSON object.')
  86     subp.add_argument('output_path', nargs=1,
  87                       help='path to a file containing the output arguments '
  88                            'as a JSON object.')
  89     subp.set_defaults(func=self.CmdAnalyze)
  90
  91     subp = subps.add_parser('gen',
  92                             help='generate a new set of build files')
  93     AddCommonOptions(subp)
  94     subp.add_argument('--swarming-targets-file',
  95                       help='save runtime dependencies for targets listed '
  96                            'in file.')
  97     subp.add_argument('path', nargs=1,
  98                       help='path to generate build into')
  99     subp.set_defaults(func=self.CmdGen)
 100
 101     subp = subps.add_parser('lookup',
 102                             help='look up the command for a given config or '
 103                                  'builder')
 104     AddCommonOptions(subp)
 105     subp.set_defaults(func=self.CmdLookup)
 106
 107     subp = subps.add_parser('validate',
 108                             help='validate the config file')
 109     subp.add_argument('-f', '--config-file', metavar='PATH',
 110                       default=self.default_config,
 111                       help='path to config file '
 112                           '(default is //tools/mb/mb_config.pyl)')
 113     subp.set_defaults(func=self.CmdValidate)
 114
 115     subp = subps.add_parser('help',
 116                             help='Get help on a subcommand.')
 117     subp.add_argument(nargs='?', action='store', dest='subcommand',
 118                       help='The command to get help for.')
 119     subp.set_defaults(func=self.CmdHelp)
 120
 121     self.args = parser.parse_args(argv)
 122
 123   def CmdAnalyze(self):
 124     vals = self.GetConfig()
 125     if vals['type'] == 'gn':
 126       return self.RunGNAnalyze(vals)
 127     elif vals['type'] == 'gyp':
 128       return self.RunGYPAnalyze(vals)
 129     else:
 130       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 131
 132   def CmdGen(self):
 133     vals = self.GetConfig()
 134
 135     self.ClobberIfNeeded(vals)
 136
 137     if vals['type'] == 'gn':
 138       return self.RunGNGen(vals)
 139     if vals['type'] == 'gyp':
 140       return self.RunGYPGen(vals)
 141
 142     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 143
 144   def CmdLookup(self):
 145     vals = self.GetConfig()
 146     if vals['type'] == 'gn':
 147       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 148     elif vals['type'] == 'gyp':
 149       if vals['gyp_crosscompile']:
 150         self.Print('GYP_CROSSCOMPILE=1')
 151       cmd = self.GYPCmd('<path>', vals['gyp_defines'])
 152     else:
 153       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 154
 155     self.PrintCmd(cmd)
 156     return 0
 157
 158   def CmdHelp(self):
 159     if self.args.subcommand:
 160       self.ParseArgs([self.args.subcommand, '--help'])
 161     else:
 162       self.ParseArgs(['--help'])
 163
 164   def CmdValidate(self):
 165     errs = []
 166
 167     # Read the file to make sure it parses.
 168     self.ReadConfigFile()
 169
 170     # Figure out the whole list of configs and ensure that no config is
 171     # listed in more than one category.
 172     all_configs = {}
 173     for config in self.common_dev_configs:
 174       all_configs[config] = 'common_dev_configs'
 175     for config in self.private_configs:
 176       if config in all_configs:
 177         errs.append('config "%s" listed in "private_configs" also '
 178                     'listed in "%s"' % (config, all_configs['config']))
 179       else:
 180         all_configs[config] = 'private_configs'
 181     for config in self.unsupported_configs:
 182       if config in all_configs:
 183         errs.append('config "%s" listed in "unsupported_configs" also '
 184                     'listed in "%s"' % (config, all_configs['config']))
 185       else:
 186         all_configs[config] = 'unsupported_configs'
 187
 188     for master in self.masters:
 189       for builder in self.masters[master]:
 190         config = self.masters[master][builder]
 191         if config in all_configs and all_configs[config] not in self.masters:
 192           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 193                       (config, all_configs[config]))
 194         else:
 195           all_configs[config] = master
 196
 197     # Check that every referenced config actually exists.
 198     for config, loc in all_configs.items():
 199       if not config in self.configs:
 200         errs.append('Unknown config "%s" referenced from "%s".' %
 201                     (config, loc))
 202
 203     # Check that every actual config is actually referenced.
 204     for config in self.configs:
 205       if not config in all_configs:
 206         errs.append('Unused config "%s".' % config)
 207
 208     # Figure out the whole list of mixins, and check that every mixin
 209     # listed by a config or another mixin actually exists.
 210     referenced_mixins = set()
 211     for config, mixins in self.configs.items():
 212       for mixin in mixins:
 213         if not mixin in self.mixins:
 214           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 215                       (mixin, config))
 216         referenced_mixins.add(mixin)
 217
 218     for mixin in self.mixins:
 219       for sub_mixin in self.mixins[mixin].get('mixins', []):
 220         if not sub_mixin in self.mixins:
 221           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 222                       (sub_mixin, mixin))
 223         referenced_mixins.add(sub_mixin)
 224
 225     # Check that every mixin defined is actually referenced somewhere.
 226     for mixin in self.mixins:
 227       if not mixin in referenced_mixins:
 228         errs.append('Unreferenced mixin "%s".' % mixin)
 229
 230     if errs:
 231       raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
 232                     '\n  ' + '\n  '.join(errs))
 233
 234     self.Print('mb config file %s looks ok.' % self.args.config_file)
 235     return 0
 236
 237   def GetConfig(self):
 238     self.ReadConfigFile()
 239     config = self.ConfigFromArgs()
 240     if not config in self.configs:
 241       raise MBErr('Config "%s" not found in %s' %
 242                   (config, self.args.config_file))
 243
 244     return self.FlattenConfig(config)
 245
 246   def ReadConfigFile(self):
 247     if not self.Exists(self.args.config_file):
 248       raise MBErr('config file not found at %s' % self.args.config_file)
 249
 250     try:
 251       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 252     except SyntaxError as e:
 253       raise MBErr('Failed to parse config file "%s": %s' %
 254                  (self.args.config_file, e))
 255
 256     self.common_dev_configs = contents['common_dev_configs']
 257     self.configs = contents['configs']
 258     self.masters = contents['masters']
 259     self.mixins = contents['mixins']
 260     self.private_configs = contents['private_configs']
 261     self.unsupported_configs = contents['unsupported_configs']
 262
 263   def ConfigFromArgs(self):
 264     if self.args.config:
 265       if self.args.master or self.args.builder:
 266         raise MBErr('Can not specific both -c/--config and -m/--master or '
 267                     '-b/--builder')
 268
 269       return self.args.config
 270
 271     if not self.args.master or not self.args.builder:
 272       raise MBErr('Must specify either -c/--config or '
 273                   '(-m/--master and -b/--builder)')
 274
 275     if not self.args.master in self.masters:
 276       raise MBErr('Master name "%s" not found in "%s"' %
 277                   (self.args.master, self.args.config_file))
 278
 279     if not self.args.builder in self.masters[self.args.master]:
 280       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 281                   (self.args.builder, self.args.master, self.args.config_file))
 282
 283     return self.masters[self.args.master][self.args.builder]
 284
 285   def FlattenConfig(self, config):
 286     mixins = self.configs[config]
 287     vals = {
 288       'type': None,
 289       'gn_args': [],
 290       'gyp_defines': '',
 291       'gyp_crosscompile': False,
 292     }
 293
 294     visited = []
 295     self.FlattenMixins(mixins, vals, visited)
 296     return vals
 297
 298   def FlattenMixins(self, mixins, vals, visited):
 299     for m in mixins:
 300       if m not in self.mixins:
 301         raise MBErr('Unknown mixin "%s"' % m)
 302
 303       # TODO: check for cycles in mixins.
 304
 305       visited.append(m)
 306
 307       mixin_vals = self.mixins[m]
 308       if 'type' in mixin_vals:
 309         vals['type'] = mixin_vals['type']
 310       if 'gn_args' in mixin_vals:
 311         if vals['gn_args']:
 312           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 313         else:
 314           vals['gn_args'] = mixin_vals['gn_args']
 315       if 'gyp_crosscompile' in mixin_vals:
 316         vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
 317       if 'gyp_defines' in mixin_vals:
 318         if vals['gyp_defines']:
 319           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 320         else:
 321           vals['gyp_defines'] = mixin_vals['gyp_defines']
 322       if 'mixins' in mixin_vals:
 323         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 324     return vals
 325
 326   def ClobberIfNeeded(self, vals):
 327     path = self.args.path[0]
 328     build_dir = self.ToAbsPath(path)
 329     mb_type_path = self.PathJoin(build_dir, 'mb_type')
 330     needs_clobber = False
 331     new_mb_type = vals['type']
 332     if self.Exists(build_dir):
 333       if self.Exists(mb_type_path):
 334         old_mb_type = self.ReadFile(mb_type_path)
 335         if old_mb_type != new_mb_type:
 336           self.Print("Build type mismatch: was %s, will be %s, clobbering %s" %
 337                      (old_mb_type, new_mb_type, path))
 338           needs_clobber = True
 339       else:
 340         # There is no 'mb_type' file in the build directory, so this probably
 341         # means that the prior build(s) were not done through mb, and we
 342         # have no idea if this was a GYP build or a GN build. Clobber it
 343         # to be safe.
 344         self.Print("%s/mb_type missing, clobbering to be safe" % path)
 345         needs_clobber = True
 346
 347     if needs_clobber:
 348       self.RemoveDirectory(build_dir)
 349
 350     self.MaybeMakeDirectory(build_dir)
 351     self.WriteFile(mb_type_path, new_mb_type)
 352
 353   def RunGNGen(self, vals):
 354     path = self.args.path[0]
 355
 356     cmd = self.GNCmd('gen', path, vals['gn_args'], extra_args=['--check'])
 357
 358     swarming_targets = []
 359     if self.args.swarming_targets_file:
 360       # We need GN to generate the list of runtime dependencies for
 361       # the compile targets listed (one per line) in the file so
 362       # we can run them via swarming. We use ninja_to_gn.pyl to convert
 363       # the compile targets to the matching GN labels.
 364       contents = self.ReadFile(self.args.swarming_targets_file)
 365       swarming_targets = contents.splitlines()
 366       gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin(
 367           self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
 368       gn_labels = []
 369       for target in swarming_targets:
 370         if not target in gn_isolate_map:
 371           raise MBErr('test target "%s"  not found in %s' %
 372                       (target, '//testing/buildbot/gn_isolate_map.pyl'))
 373         gn_labels.append(gn_isolate_map[target]['label'])
 374
 375       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 376
 377       # Since GN hasn't run yet, the build directory may not even exist.
 378       self.MaybeMakeDirectory(self.ToAbsPath(path))
 379
 380       self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
 381       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 382
 383     ret, _, _ = self.Run(cmd)
 384     if ret:
 385         # If `gn gen` failed, we should exit early rather than trying to
 386         # generate isolates. Run() will have already logged any error output.
 387         self.Print('GN gen failed: %d' % ret)
 388         return ret
 389
 390     for target in swarming_targets:
 391       if gn_isolate_map[target]['type'] == 'gpu_browser_test':
 392         runtime_deps_target = 'browser_tests'
 393       elif gn_isolate_map[target]['type'] == 'script':
 394         # For script targets, the build target is usually a group,
 395         # for which gn generates the runtime_deps next to the stamp file
 396         # for the label, which lives under the obj/ directory.
 397         label = gn_isolate_map[target]['label']
 398         runtime_deps_target = 'obj/%s.stamp' % label.replace(':', '/')
 399       else:
 400         runtime_deps_target = target
 401       if self.platform == 'win32':
 402         deps_path = self.ToAbsPath(path,
 403                                    runtime_deps_target + '.exe.runtime_deps')
 404       else:
 405         deps_path = self.ToAbsPath(path,
 406                                    runtime_deps_target + '.runtime_deps')
 407       if not self.Exists(deps_path):
 408           raise MBErr('did not generate %s' % deps_path)
 409
 410       command, extra_files = self.GetIsolateCommand(target, vals,
 411                                                     gn_isolate_map)
 412
 413       runtime_deps = self.ReadFile(deps_path).splitlines()
 414
 415       isolate_path = self.ToAbsPath(path, target + '.isolate')
 416       self.WriteFile(isolate_path,
 417         pprint.pformat({
 418           'variables': {
 419             'command': command,
 420             'files': sorted(runtime_deps + extra_files),
 421           }
 422         }) + '\n')
 423
 424       self.WriteJSON(
 425         {
 426           'args': [
 427             '--isolated',
 428             self.ToSrcRelPath('%s%s%s.isolated' % (path, self.sep, target)),
 429             '--isolate',
 430             self.ToSrcRelPath('%s%s%s.isolate' % (path, self.sep, target)),
 431           ],
 432           'dir': self.chromium_src_dir,
 433           'version': 1,
 434         },
 435         isolate_path + 'd.gen.json',
 436       )
 437
 438     return ret
 439
 440   def GNCmd(self, subcommand, path, gn_args='', extra_args=None):
 441     if self.platform == 'linux2':
 442       subdir = 'linux64'
 443     elif self.platform == 'darwin':
 444       subdir = 'mac'
 445     else:
 446       subdir = 'win'
 447     gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, 'gn')
 448
 449     cmd = [gn_path, subcommand, path]
 450     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 451     if gn_args:
 452       cmd.append('--args=%s' % gn_args)
 453     if extra_args:
 454       cmd.extend(extra_args)
 455     return cmd
 456
 457   def RunGYPGen(self, vals):
 458     path = self.args.path[0]
 459
 460     output_dir = self.ParseGYPConfigPath(path)
 461     cmd = self.GYPCmd(output_dir, vals['gyp_defines'])
 462     env = None
 463     if vals['gyp_crosscompile']:
 464       if self.args.verbose:
 465         self.Print('Setting GYP_CROSSCOMPILE=1 in the environment')
 466       env = os.environ.copy()
 467       env['GYP_CROSSCOMPILE'] = '1'
 468     ret, _, _ = self.Run(cmd, env=env)
 469     return ret
 470
 471   def RunGYPAnalyze(self, vals):
 472     output_dir = self.ParseGYPConfigPath(self.args.path[0])
 473     if self.args.verbose:
 474       inp = self.ReadInputJSON(['files', 'targets'])
 475       self.Print()
 476       self.Print('analyze input:')
 477       self.PrintJSON(inp)
 478       self.Print()
 479
 480     cmd = self.GYPCmd(output_dir, vals['gyp_defines'])
 481     cmd.extend(['-f', 'analyzer',
 482                 '-G', 'config_path=%s' % self.args.input_path[0],
 483                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 484     ret, _, _ = self.Run(cmd)
 485     if not ret and self.args.verbose:
 486       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 487       self.Print()
 488       self.Print('analyze output:')
 489       self.PrintJSON(outp)
 490       self.Print()
 491
 492     return ret
 493
 494   def GetIsolateCommand(self, target, vals, gn_isolate_map):
 495     # This needs to mirror the settings in //build/config/ui.gni:
 496     # use_x11 = is_linux && !use_ozone.
 497     # TODO(dpranke): Figure out how to keep this in sync better.
 498     use_x11 = (self.platform == 'linux2' and
 499                not 'target_os="android"' in vals['gn_args'] and
 500                not 'use_ozone=true' in vals['gn_args'])
 501
 502     asan = 'is_asan=true' in vals['gn_args']
 503     msan = 'is_msan=true' in vals['gn_args']
 504     tsan = 'is_tsan=true' in vals['gn_args']
 505
 506     executable_suffix = '.exe' if self.platform == 'win32' else ''
 507
 508     test_type = gn_isolate_map[target]['type']
 509     cmdline = []
 510     extra_files = []
 511
 512     if use_x11 and test_type == 'windowed_test_launcher':
 513       extra_files = [
 514           'xdisplaycheck',
 515           '../../testing/test_env.py',
 516           '../../testing/xvfb.py',
 517       ]
 518       cmdline = [
 519         '../../testing/xvfb.py',
 520         '.',
 521         './' + str(target),
 522         '--brave-new-test-launcher',
 523         '--test-launcher-bot-mode',
 524         '--asan=%d' % asan,
 525         '--msan=%d' % msan,
 526         '--tsan=%d' % tsan,
 527       ]
 528     elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
 529       extra_files = [
 530           '../../testing/test_env.py'
 531       ]
 532       cmdline = [
 533           '../../testing/test_env.py',
 534           './' + str(target) + executable_suffix,
 535           '--brave-new-test-launcher',
 536           '--test-launcher-bot-mode',
 537           '--asan=%d' % asan,
 538           '--msan=%d' % msan,
 539           '--tsan=%d' % tsan,
 540       ]
 541     elif test_type == 'gpu_browser_test':
 542       extra_files = [
 543           '../../testing/test_env.py'
 544       ]
 545       gtest_filter = gn_isolate_map[target]['gtest_filter']
 546       cmdline = [
 547           '../../testing/test_env.py',
 548           './browser_tests' + executable_suffix,
 549           '--test-launcher-bot-mode',
 550           '--enable-gpu',
 551           '--test-launcher-jobs=1',
 552           '--gtest_filter=%s' % gtest_filter,
 553       ]
 554     elif test_type == 'script':
 555       extra_files = [
 556           '../../testing/test_env.py'
 557       ]
 558       cmdline = [
 559           '../../testing/test_env.py',
 560       ] + ['../../' + self.ToSrcRelPath(gn_isolate_map[target]['script'])]
 561     elif test_type in ('raw'):
 562       extra_files = []
 563       cmdline = [
 564           './' + str(target) + executable_suffix,
 565       ] + gn_isolate_map[target].get('args')
 566
 567     else:
 568       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
 569                                 % (target, test_type), output_path=None)
 570
 571     return cmdline, extra_files
 572
 573   def ToAbsPath(self, build_path, *comps):
 574     return self.PathJoin(self.chromium_src_dir,
 575                          self.ToSrcRelPath(build_path),
 576                          *comps)
 577
 578   def ToSrcRelPath(self, path):
 579     """Returns a relative path from the top of the repo."""
 580     # TODO: Support normal paths in addition to source-absolute paths.
 581     assert(path.startswith('//'))
 582     return path[2:].replace('/', self.sep)
 583
 584   def ParseGYPConfigPath(self, path):
 585     rpath = self.ToSrcRelPath(path)
 586     output_dir, _, _ = rpath.rpartition(self.sep)
 587     return output_dir
 588
 589   def GYPCmd(self, output_dir, gyp_defines):
 590     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 591     cmd = [
 592         self.executable,
 593         self.PathJoin('build', 'gyp_chromium'),
 594         '-G',
 595         'output_dir=' + output_dir,
 596     ]
 597     for d in shlex.split(gyp_defines):
 598       cmd += ['-D', d]
 599     return cmd
 600
 601   def RunGNAnalyze(self, vals):
 602     # analyze runs before 'gn gen' now, so we need to run gn gen
 603     # in order to ensure that we have a build directory.
 604     ret = self.RunGNGen(vals)
 605     if ret:
 606       return ret
 607
 608     inp = self.ReadInputJSON(['files', 'targets'])
 609     if self.args.verbose:
 610       self.Print()
 611       self.Print('analyze input:')
 612       self.PrintJSON(inp)
 613       self.Print()
 614
 615     output_path = self.args.output_path[0]
 616
 617     # Bail out early if a GN file was modified, since 'gn refs' won't know
 618     # what to do about it.
 619     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 620       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 621       return 0
 622
 623     # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
 624     if 'all' in inp['targets']:
 625       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 626       return 0
 627
 628     # This shouldn't normally happen, but could due to unusual race conditions,
 629     # like a try job that gets scheduled before a patch lands but runs after
 630     # the patch has landed.
 631     if not inp['files']:
 632       self.Print('Warning: No files modified in patch, bailing out early.')
 633       self.WriteJSON({'targets': [],
 634                       'build_targets': [],
 635                       'status': 'No dependency'}, output_path)
 636       return 0
 637
 638     ret = 0
 639     response_file = self.TempFile()
 640     response_file.write('\n'.join(inp['files']) + '\n')
 641     response_file.close()
 642
 643     matching_targets = []
 644     try:
 645       cmd = self.GNCmd('refs', self.args.path[0]) + [
 646           '@%s' % response_file.name, '--all', '--as=output']
 647       ret, out, _ = self.Run(cmd, force_verbose=False)
 648       if ret and not 'The input matches no targets' in out:
 649         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 650                                   output_path)
 651       build_dir = self.ToSrcRelPath(self.args.path[0]) + self.sep
 652       for output in out.splitlines():
 653         build_output = output.replace(build_dir, '')
 654         if build_output in inp['targets']:
 655           matching_targets.append(build_output)
 656
 657       cmd = self.GNCmd('refs', self.args.path[0]) + [
 658           '@%s' % response_file.name, '--all']
 659       ret, out, _ = self.Run(cmd, force_verbose=False)
 660       if ret and not 'The input matches no targets' in out:
 661         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 662                                   output_path)
 663       for label in out.splitlines():
 664         build_target = label[2:]
 665         # We want to accept 'chrome/android:chrome_public_apk' and
 666         # just 'chrome_public_apk'. This may result in too many targets
 667         # getting built, but we can adjust that later if need be.
 668         for input_target in inp['targets']:
 669           if (input_target == build_target or
 670               build_target.endswith(':' + input_target)):
 671             matching_targets.append(input_target)
 672     finally:
 673       self.RemoveFile(response_file.name)
 674
 675     if matching_targets:
 676       # TODO: it could be that a target X might depend on a target Y
 677       # and both would be listed in the input, but we would only need
 678       # to specify target X as a build_target (whereas both X and Y are
 679       # targets). I'm not sure if that optimization is generally worth it.
 680       self.WriteJSON({'targets': sorted(set(matching_targets)),
 681                       'build_targets': sorted(set(matching_targets)),
 682                       'status': 'Found dependency'}, output_path)
 683     else:
 684       self.WriteJSON({'targets': [],
 685                       'build_targets': [],
 686                       'status': 'No dependency'}, output_path)
 687
 688     if self.args.verbose:
 689       outp = json.loads(self.ReadFile(output_path))
 690       self.Print()
 691       self.Print('analyze output:')
 692       self.PrintJSON(outp)
 693       self.Print()
 694
 695     return 0
 696
 697   def ReadInputJSON(self, required_keys):
 698     path = self.args.input_path[0]
 699     output_path = self.args.output_path[0]
 700     if not self.Exists(path):
 701       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 702
 703     try:
 704       inp = json.loads(self.ReadFile(path))
 705     except Exception as e:
 706       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 707                                 (path, e), output_path)
 708
 709     for k in required_keys:
 710       if not k in inp:
 711         self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
 712                                   output_path)
 713
 714     return inp
 715
 716   def WriteFailureAndRaise(self, msg, output_path):
 717     if output_path:
 718       self.WriteJSON({'error': msg}, output_path, force_verbose=True)
 719     raise MBErr(msg)
 720
 721   def WriteJSON(self, obj, path, force_verbose=False):
 722     try:
 723       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n',
 724                      force_verbose=force_verbose)
 725     except Exception as e:
 726       raise MBErr('Error %s writing to the output path "%s"' %
 727                  (e, path))
 728
 729   def PrintCmd(self, cmd):
 730     if cmd[0] == self.executable:
 731       cmd = ['python'] + cmd[1:]
 732     self.Print(*[pipes.quote(c) for c in cmd])
 733
 734   def PrintJSON(self, obj):
 735     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 736
 737   def Print(self, *args, **kwargs):
 738     # This function largely exists so it can be overridden for testing.
 739     print(*args, **kwargs)
 740
 741   def Run(self, cmd, env=None, force_verbose=True):
 742     # This function largely exists so it can be overridden for testing.
 743     if self.args.dryrun or self.args.verbose or force_verbose:
 744       self.PrintCmd(cmd)
 745     if self.args.dryrun:
 746       return 0, '', ''
 747
 748     ret, out, err = self.Call(cmd, env=env)
 749     if self.args.verbose or force_verbose:
 750       if out:
 751         self.Print(out, end='')
 752       if err:
 753         self.Print(err, end='', file=sys.stderr)
 754     return ret, out, err
 755
 756   def Call(self, cmd, env=None):
 757     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 758                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 759                          env=env)
 760     out, err = p.communicate()
 761     return p.returncode, out, err
 762
 763   def ExpandUser(self, path):
 764     # This function largely exists so it can be overridden for testing.
 765     return os.path.expanduser(path)
 766
 767   def Exists(self, path):
 768     # This function largely exists so it can be overridden for testing.
 769     return os.path.exists(path)
 770
 771   def MaybeMakeDirectory(self, path):
 772     try:
 773       os.makedirs(path)
 774     except OSError, e:
 775       if e.errno != errno.EEXIST:
 776         raise
 777
 778   def PathJoin(self, *comps):
 779     # This function largely exists so it can be overriden for testing.
 780     return os.path.join(*comps)
 781
 782   def ReadFile(self, path):
 783     # This function largely exists so it can be overriden for testing.
 784     with open(path) as fp:
 785       return fp.read()
 786
 787   def RemoveFile(self, path):
 788     # This function largely exists so it can be overriden for testing.
 789     os.remove(path)
 790
 791   def RemoveDirectory(self, abs_path):
 792     if self.platform == 'win32':
 793       # In other places in chromium, we often have to retry this command
 794       # because we're worried about other processes still holding on to
 795       # file handles, but when MB is invoked, it will be early enough in the
 796       # build that their should be no other processes to interfere. We
 797       # can change this if need be.
 798       self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
 799     else:
 800       shutil.rmtree(abs_path, ignore_errors=True)
 801
 802   def TempFile(self, mode='w'):
 803     # This function largely exists so it can be overriden for testing.
 804     return tempfile.NamedTemporaryFile(mode=mode, delete=False)
 805
 806   def WriteFile(self, path, contents, force_verbose=False):
 807     # This function largely exists so it can be overriden for testing.
 808     if self.args.dryrun or self.args.verbose or force_verbose:
 809       self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
 810     with open(path, 'w') as fp:
 811       return fp.write(contents)
 812
 813
 814 class MBErr(Exception):
 815   pass
 816
 817
 818 if __name__ == '__main__':
 819   try:
 820     sys.exit(main(sys.argv[1:]))
 821   except MBErr as e:
 822     print(e)
 823     sys.exit(1)
 824   except KeyboardInterrupt:
 825     print("interrupted, exiting", stream=sys.stderr)
 826     sys.exit(130)