tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import errno
  17 import json
  18 import os
  19 import pipes
  20 import pprint
  21 import shlex
  22 import shutil
  23 import sys
  24 import subprocess
  25 import tempfile
  26
  27
  28 def main(args):
  29   mbw = MetaBuildWrapper()
  30   mbw.ParseArgs(args)
  31   return mbw.args.func()
  32
  33
  34 class MetaBuildWrapper(object):
  35   def __init__(self):
  36     p = os.path
  37     d = os.path.dirname
  38     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  39     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  40                                  'mb_config.pyl')
  41     self.platform = sys.platform
  42     self.args = argparse.Namespace()
  43     self.configs = {}
  44     self.masters = {}
  45     self.mixins = {}
  46     self.private_configs = []
  47     self.common_dev_configs = []
  48     self.unsupported_configs = []
  49
  50   def ParseArgs(self, argv):
  51     def AddCommonOptions(subp):
  52       subp.add_argument('-b', '--builder',
  53                         help='builder name to look up config from')
  54       subp.add_argument('-m', '--master',
  55                         help='master name to look up config from')
  56       subp.add_argument('-c', '--config',
  57                         help='configuration to analyze')
  58       subp.add_argument('-f', '--config-file', metavar='PATH',
  59                         default=self.default_config,
  60                         help='path to config file '
  61                             '(default is //tools/mb/mb_config.pyl)')
  62       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  63                         help='path to goma directory (default is %(default)s).')
  64       subp.add_argument('-n', '--dryrun', action='store_true',
  65                         help='Do a dry run (i.e., do nothing, just print '
  66                              'the commands that will run)')
  67       subp.add_argument('-q', '--quiet', action='store_true',
  68                         help='Do not print anything on success, '
  69                              'just return an exit code.')
  70       subp.add_argument('-v', '--verbose', action='count',
  71                         help='verbose logging (may specify multiple times).')
  72
  73     parser = argparse.ArgumentParser(prog='mb')
  74     subps = parser.add_subparsers()
  75
  76     subp = subps.add_parser('analyze',
  77                             help='analyze whether changes to a set of files '
  78                                  'will cause a set of binaries to be rebuilt.')
  79     AddCommonOptions(subp)
  80     subp.add_argument('--swarming-targets-file',
  81                       help='save runtime dependencies for targets listed '
  82                            'in file.')
  83     subp.add_argument('path', nargs=1,
  84                       help='path build was generated into.')
  85     subp.add_argument('input_path', nargs=1,
  86                       help='path to a file containing the input arguments '
  87                            'as a JSON object.')
  88     subp.add_argument('output_path', nargs=1,
  89                       help='path to a file containing the output arguments '
  90                            'as a JSON object.')
  91     subp.set_defaults(func=self.CmdAnalyze)
  92
  93     subp = subps.add_parser('gen',
  94                             help='generate a new set of build files')
  95     AddCommonOptions(subp)
  96     subp.add_argument('--swarming-targets-file',
  97                       help='save runtime dependencies for targets listed '
  98                            'in file.')
  99     subp.add_argument('path', nargs=1,
 100                       help='path to generate build into')
 101     subp.set_defaults(func=self.CmdGen)
 102
 103     subp = subps.add_parser('lookup',
 104                             help='look up the command for a given config or '
 105                                  'builder')
 106     AddCommonOptions(subp)
 107     subp.set_defaults(func=self.CmdLookup)
 108
 109     subp = subps.add_parser('validate',
 110                             help='validate the config file')
 111     subp.add_argument('-f', '--config-file', metavar='PATH',
 112                       default=self.default_config,
 113                       help='path to config file '
 114                           '(default is //tools/mb/mb_config.pyl)')
 115     subp.add_argument('-q', '--quiet', action='store_true',
 116                       help='Do not print anything on success, '
 117                            'just return an exit code.')
 118     subp.set_defaults(func=self.CmdValidate)
 119
 120     subp = subps.add_parser('help',
 121                             help='Get help on a subcommand.')
 122     subp.add_argument(nargs='?', action='store', dest='subcommand',
 123                       help='The command to get help for.')
 124     subp.set_defaults(func=self.CmdHelp)
 125
 126     self.args = parser.parse_args(argv)
 127
 128   def CmdAnalyze(self):
 129     vals = self.GetConfig()
 130     if vals['type'] == 'gn':
 131       return self.RunGNAnalyze(vals)
 132     elif vals['type'] == 'gyp':
 133       return self.RunGYPAnalyze(vals)
 134     else:
 135       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 136
 137   def CmdGen(self):
 138     vals = self.GetConfig()
 139     if vals['type'] == 'gn':
 140       return self.RunGNGen(vals)
 141     if vals['type'] == 'gyp':
 142       return self.RunGYPGen(vals)
 143
 144     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 145
 146   def CmdLookup(self):
 147     vals = self.GetConfig()
 148     if vals['type'] == 'gn':
 149       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 150     elif vals['type'] == 'gyp':
 151       cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
 152     else:
 153       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 154
 155     self.PrintCmd(cmd)
 156     return 0
 157
 158   def CmdHelp(self):
 159     if self.args.subcommand:
 160       self.ParseArgs([self.args.subcommand, '--help'])
 161     else:
 162       self.ParseArgs(['--help'])
 163
 164   def CmdValidate(self):
 165     errs = []
 166
 167     # Read the file to make sure it parses.
 168     self.ReadConfigFile()
 169
 170     # Figure out the whole list of configs and ensure that no config is
 171     # listed in more than one category.
 172     all_configs = {}
 173     for config in self.common_dev_configs:
 174       all_configs[config] = 'common_dev_configs'
 175     for config in self.private_configs:
 176       if config in all_configs:
 177         errs.append('config "%s" listed in "private_configs" also '
 178                     'listed in "%s"' % (config, all_configs['config']))
 179       else:
 180         all_configs[config] = 'private_configs'
 181     for config in self.unsupported_configs:
 182       if config in all_configs:
 183         errs.append('config "%s" listed in "unsupported_configs" also '
 184                     'listed in "%s"' % (config, all_configs['config']))
 185       else:
 186         all_configs[config] = 'unsupported_configs'
 187
 188     for master in self.masters:
 189       for builder in self.masters[master]:
 190         config = self.masters[master][builder]
 191         if config in all_configs and all_configs[config] not in self.masters:
 192           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 193                       (config, all_configs[config]))
 194         else:
 195           all_configs[config] = master
 196
 197     # Check that every referenced config actually exists.
 198     for config, loc in all_configs.items():
 199       if not config in self.configs:
 200         errs.append('Unknown config "%s" referenced from "%s".' %
 201                     (config, loc))
 202
 203     # Check that every actual config is actually referenced.
 204     for config in self.configs:
 205       if not config in all_configs:
 206         errs.append('Unused config "%s".' % config)
 207
 208     # Figure out the whole list of mixins, and check that every mixin
 209     # listed by a config or another mixin actually exists.
 210     referenced_mixins = set()
 211     for config, mixins in self.configs.items():
 212       for mixin in mixins:
 213         if not mixin in self.mixins:
 214           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 215                       (mixin, config))
 216         referenced_mixins.add(mixin)
 217
 218     for mixin in self.mixins:
 219       for sub_mixin in self.mixins[mixin].get('mixins', []):
 220         if not sub_mixin in self.mixins:
 221           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 222                       (sub_mixin, mixin))
 223         referenced_mixins.add(sub_mixin)
 224
 225     # Check that every mixin defined is actually referenced somewhere.
 226     for mixin in self.mixins:
 227       if not mixin in referenced_mixins:
 228         errs.append('Unreferenced mixin "%s".' % mixin)
 229
 230     if errs:
 231       raise MBErr('mb config file %s has problems:\n  ' + '\n  '.join(errs))
 232
 233     if not self.args.quiet:
 234       self.Print('mb config file %s looks ok.' % self.args.config_file)
 235     return 0
 236
 237   def GetConfig(self):
 238     self.ReadConfigFile()
 239     config = self.ConfigFromArgs()
 240     if not config in self.configs:
 241       raise MBErr('Config "%s" not found in %s' %
 242                   (config, self.args.config_file))
 243
 244     return self.FlattenConfig(config)
 245
 246   def ReadConfigFile(self):
 247     if not self.Exists(self.args.config_file):
 248       raise MBErr('config file not found at %s' % self.args.config_file)
 249
 250     try:
 251       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 252     except SyntaxError as e:
 253       raise MBErr('Failed to parse config file "%s": %s' %
 254                  (self.args.config_file, e))
 255
 256     self.common_dev_configs = contents['common_dev_configs']
 257     self.configs = contents['configs']
 258     self.masters = contents['masters']
 259     self.mixins = contents['mixins']
 260     self.private_configs = contents['private_configs']
 261     self.unsupported_configs = contents['unsupported_configs']
 262
 263   def ConfigFromArgs(self):
 264     if self.args.config:
 265       if self.args.master or self.args.builder:
 266         raise MBErr('Can not specific both -c/--config and -m/--master or '
 267                     '-b/--builder')
 268
 269       return self.args.config
 270
 271     if not self.args.master or not self.args.builder:
 272       raise MBErr('Must specify either -c/--config or '
 273                   '(-m/--master and -b/--builder)')
 274
 275     if not self.args.master in self.masters:
 276       raise MBErr('Master name "%s" not found in "%s"' %
 277                   (self.args.master, self.args.config_file))
 278
 279     if not self.args.builder in self.masters[self.args.master]:
 280       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 281                   (self.args.builder, self.args.master, self.args.config_file))
 282
 283     return self.masters[self.args.master][self.args.builder]
 284
 285   def FlattenConfig(self, config):
 286     mixins = self.configs[config]
 287     vals = {
 288       'type': None,
 289       'gn_args': [],
 290       'gyp_config': [],
 291       'gyp_defines': [],
 292     }
 293
 294     visited = []
 295     self.FlattenMixins(mixins, vals, visited)
 296     return vals
 297
 298   def FlattenMixins(self, mixins, vals, visited):
 299     for m in mixins:
 300       if m not in self.mixins:
 301         raise MBErr('Unknown mixin "%s"' % m)
 302
 303       # TODO: check for cycles in mixins.
 304
 305       visited.append(m)
 306
 307       mixin_vals = self.mixins[m]
 308       if 'type' in mixin_vals:
 309         vals['type'] = mixin_vals['type']
 310       if 'gn_args' in mixin_vals:
 311         if vals['gn_args']:
 312           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 313         else:
 314           vals['gn_args'] = mixin_vals['gn_args']
 315       if 'gyp_config' in mixin_vals:
 316         vals['gyp_config'] = mixin_vals['gyp_config']
 317       if 'gyp_defines' in mixin_vals:
 318         if vals['gyp_defines']:
 319           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 320         else:
 321           vals['gyp_defines'] = mixin_vals['gyp_defines']
 322       if 'mixins' in mixin_vals:
 323         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 324     return vals
 325
 326   def RunGNGen(self, vals):
 327     path = self.args.path[0]
 328
 329     cmd = self.GNCmd('gen', path, vals['gn_args'])
 330
 331     swarming_targets = []
 332     if self.args.swarming_targets_file:
 333       # We need GN to generate the list of runtime dependencies for
 334       # the compile targets listed (one per line) in the file so
 335       # we can run them via swarming. We use ninja_to_gn.pyl to convert
 336       # the compile targets to the matching GN labels.
 337       contents = self.ReadFile(self.args.swarming_targets_file)
 338       swarming_targets = contents.splitlines()
 339       gn_isolate_map = ast.literal_eval(self.ReadFile(os.path.join(
 340           self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
 341       gn_labels = []
 342       for target in swarming_targets:
 343         if not target in gn_isolate_map:
 344           raise MBErr('test target "%s"  not found in %s' %
 345                       (target, '//testing/buildbot/gn_isolate_map.pyl'))
 346         gn_labels.append(gn_isolate_map[target]['label'])
 347
 348       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 349
 350       # Since GN hasn't run yet, the build directory may not even exist.
 351       self.MaybeMakeDirectory(self.ToAbsPath(path))
 352
 353       self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
 354       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 355
 356     ret, _, _ = self.Run(cmd)
 357
 358     for target in swarming_targets:
 359       if sys.platform == 'win32':
 360         deps_path = self.ToAbsPath(path, target + '.exe.runtime_deps')
 361       else:
 362         deps_path = self.ToAbsPath(path, target + '.runtime_deps')
 363       if not self.Exists(deps_path):
 364           raise MBErr('did not generate %s' % deps_path)
 365
 366       command, extra_files = self.GetIsolateCommand(target, vals,
 367                                                     gn_isolate_map)
 368
 369       runtime_deps = self.ReadFile(deps_path).splitlines()
 370
 371       isolate_path = self.ToAbsPath(path, target + '.isolate')
 372       self.WriteFile(isolate_path,
 373         pprint.pformat({
 374           'variables': {
 375             'command': command,
 376             'files': sorted(runtime_deps + extra_files),
 377           }
 378         }) + '\n')
 379
 380       self.WriteJSON(
 381         {
 382           'args': [
 383             '--isolated',
 384             self.ToSrcRelPath('%s%s%s.isolated' % (path, os.sep, target)),
 385             '--isolate',
 386             self.ToSrcRelPath('%s%s%s.isolate' % (path, os.sep, target)),
 387           ],
 388           'dir': self.chromium_src_dir,
 389           'version': 1,
 390         },
 391         isolate_path + 'd.gen.json',
 392       )
 393
 394
 395     return ret
 396
 397   def GNCmd(self, subcommand, path, gn_args=''):
 398     if self.platform == 'linux2':
 399       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
 400                              'gn')
 401     elif self.platform == 'darwin':
 402       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
 403                              'gn')
 404     else:
 405       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
 406                              'gn.exe')
 407
 408     cmd = [gn_path, subcommand, path]
 409     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 410     if gn_args:
 411       cmd.append('--args=%s' % gn_args)
 412     return cmd
 413
 414   def RunGYPGen(self, vals):
 415     path = self.args.path[0]
 416
 417     output_dir, gyp_config = self.ParseGYPConfigPath(path)
 418     if gyp_config != vals['gyp_config']:
 419       raise MBErr('The last component of the path (%s) must match the '
 420                   'GYP configuration specified in the config (%s), and '
 421                   'it does not.' % (gyp_config, vals['gyp_config']))
 422     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 423     ret, _, _ = self.Run(cmd)
 424     return ret
 425
 426   def RunGYPAnalyze(self, vals):
 427     output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
 428     if gyp_config != vals['gyp_config']:
 429       raise MBErr('The last component of the path (%s) must match the '
 430                   'GYP configuration specified in the config (%s), and '
 431                   'it does not.' % (gyp_config, vals['gyp_config']))
 432     if self.args.verbose:
 433       inp = self.GetAnalyzeInput()
 434       self.Print()
 435       self.Print('analyze input:')
 436       self.PrintJSON(inp)
 437       self.Print()
 438
 439     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 440     cmd.extend(['-G', 'config_path=%s' % self.args.input_path[0],
 441                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 442     ret, _, _ = self.Run(cmd)
 443     if not ret and self.args.verbose:
 444       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 445       self.Print()
 446       self.Print('analyze output:')
 447       self.PrintJSON(outp)
 448       self.Print()
 449
 450     return ret
 451
 452   def RunGNIsolate(self, vals):
 453     build_path = self.args.path[0]
 454     inp = self.ReadInputJSON(['targets'])
 455     if self.args.verbose:
 456       self.Print()
 457       self.Print('isolate input:')
 458       self.PrintJSON(inp)
 459       self.Print()
 460     output_path = self.args.output_path[0]
 461
 462     for target in inp['targets']:
 463       runtime_deps_path = self.ToAbsPath(build_path, target + '.runtime_deps')
 464
 465       if not self.Exists(runtime_deps_path):
 466         self.WriteFailureAndRaise('"%s" does not exist' % runtime_deps_path,
 467                                   output_path)
 468
 469       command, extra_files = self.GetIsolateCommand(target, vals, None)
 470
 471       runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
 472
 473
 474       isolate_path = self.ToAbsPath(build_path, target + '.isolate')
 475       self.WriteFile(isolate_path,
 476         pprint.pformat({
 477           'variables': {
 478             'command': command,
 479             'files': sorted(runtime_deps + extra_files),
 480           }
 481         }) + '\n')
 482
 483       self.WriteJSON(
 484         {
 485           'args': [
 486             '--isolated',
 487             self.ToSrcRelPath('%s/%s.isolated' % (build_path, target)),
 488             '--isolate',
 489             self.ToSrcRelPath('%s/%s.isolate' % (build_path, target)),
 490           ],
 491           'dir': self.chromium_src_dir,
 492           'version': 1,
 493         },
 494         isolate_path + 'd.gen.json',
 495       )
 496
 497     return 0
 498
 499   def GetIsolateCommand(self, target, vals, gn_isolate_map):
 500     # This needs to mirror the settings in //build/config/ui.gni:
 501     # use_x11 = is_linux && !use_ozone.
 502     # TODO(dpranke): Figure out how to keep this in sync better.
 503     use_x11 = (sys.platform == 'linux2' and
 504                not 'target_os="android"' in vals['gn_args'] and
 505                not 'use_ozone=true' in vals['gn_args'])
 506
 507     asan = 'is_asan=true' in vals['gn_args']
 508     msan = 'is_msan=true' in vals['gn_args']
 509     tsan = 'is_tsan=true' in vals['gn_args']
 510
 511     executable_suffix = '.exe' if sys.platform == 'win32' else ''
 512
 513     test_type = gn_isolate_map[target]['type']
 514     cmdline = []
 515     extra_files = []
 516
 517     if use_x11 and test_type == 'windowed_test_launcher':
 518       extra_files = [
 519           'xdisplaycheck',
 520           '../../testing/test_env.py',
 521           '../../testing/xvfb.py',
 522       ]
 523       cmdline = [
 524         '../../testing/xvfb.py',
 525         '.',
 526         './' + str(target),
 527         '--brave-new-test-launcher',
 528         '--test-launcher-bot-mode',
 529         '--asan=%d' % asan,
 530         '--msan=%d' % msan,
 531         '--tsan=%d' % tsan,
 532       ]
 533     elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
 534       extra_files = [
 535           '../../testing/test_env.py'
 536       ]
 537       cmdline = [
 538           '../../testing/test_env.py',
 539           './' + str(target) + executable_suffix,
 540           '--brave-new-test-launcher',
 541           '--test-launcher-bot-mode',
 542           '--asan=%d' % asan,
 543           '--msan=%d' % msan,
 544           '--tsan=%d' % tsan,
 545       ]
 546     elif test_type in ('raw'):
 547       extra_files = []
 548       cmdline = [
 549           './' + str(target) + executable_suffix,
 550       ] + gn_isolate_map[target].get('args')
 551
 552     else:
 553       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
 554                                 % (target, test_type), output_path=None)
 555
 556     return cmdline, extra_files
 557
 558   def ToAbsPath(self, build_path, *comps):
 559     return os.path.join(self.chromium_src_dir,
 560                         self.ToSrcRelPath(build_path),
 561                         *comps)
 562
 563   def ToSrcRelPath(self, path):
 564     """Returns a relative path from the top of the repo."""
 565     # TODO: Support normal paths in addition to source-absolute paths.
 566     assert(path.startswith('//'))
 567     return path[2:].replace('/', os.sep)
 568
 569   def ParseGYPConfigPath(self, path):
 570     rpath = self.ToSrcRelPath(path)
 571     output_dir, _, config = rpath.rpartition('/')
 572     self.CheckGYPConfigIsSupported(config, path)
 573     return output_dir, config
 574
 575   def CheckGYPConfigIsSupported(self, config, path):
 576     if config not in ('Debug', 'Release'):
 577       if (sys.platform in ('win32', 'cygwin') and
 578           config not in ('Debug_x64', 'Release_x64')):
 579         raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
 580                     config, path)
 581
 582   def GYPCmd(self, output_dir, gyp_defines, config):
 583     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 584     cmd = [
 585         sys.executable,
 586         os.path.join('build', 'gyp_chromium'),
 587         '-G',
 588         'output_dir=' + output_dir,
 589         '-G',
 590         'config=' + config,
 591     ]
 592     for d in shlex.split(gyp_defines):
 593       cmd += ['-D', d]
 594     return cmd
 595
 596   def RunGNAnalyze(self, vals):
 597     # analyze runs before 'gn gen' now, so we need to run gn gen
 598     # in order to ensure that we have a build directory.
 599     ret = self.RunGNGen(vals)
 600     if ret:
 601       return ret
 602
 603     inp = self.ReadInputJSON(['files', 'targets'])
 604     if self.args.verbose:
 605       self.Print()
 606       self.Print('analyze input:')
 607       self.PrintJSON(inp)
 608       self.Print()
 609
 610     output_path = self.args.output_path[0]
 611
 612     # Bail out early if a GN file was modified, since 'gn refs' won't know
 613     # what to do about it.
 614     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 615       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 616       return 0
 617
 618     # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
 619     if 'all' in inp['targets']:
 620       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 621       return 0
 622
 623     # This shouldn't normally happen, but could due to unusual race conditions,
 624     # like a try job that gets scheduled before a patch lands but runs after
 625     # the patch has landed.
 626     if not inp['files']:
 627       self.Print('Warning: No files modified in patch, bailing out early.')
 628       self.WriteJSON({'targets': [],
 629                       'build_targets': [],
 630                       'status': 'No dependency'}, output_path)
 631       return 0
 632
 633     ret = 0
 634     response_file = self.TempFile()
 635     response_file.write('\n'.join(inp['files']) + '\n')
 636     response_file.close()
 637
 638     matching_targets = []
 639     try:
 640       cmd = self.GNCmd('refs', self.args.path[0]) + [
 641           '@%s' % response_file.name, '--all', '--as=output']
 642       ret, out, _ = self.Run(cmd)
 643       if ret and not 'The input matches no targets' in out:
 644         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 645                                   output_path)
 646       build_dir = self.ToSrcRelPath(self.args.path[0]) + os.sep
 647       for output in out.splitlines():
 648         build_output = output.replace(build_dir, '')
 649         if build_output in inp['targets']:
 650           matching_targets.append(build_output)
 651
 652       cmd = self.GNCmd('refs', self.args.path[0]) + [
 653           '@%s' % response_file.name, '--all']
 654       ret, out, _ = self.Run(cmd)
 655       if ret and not 'The input matches no targets' in out:
 656         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 657                                   output_path)
 658       for label in out.splitlines():
 659         build_target = label[2:]
 660         # We want to accept 'chrome/android:chrome_shell_apk' and
 661         # just 'chrome_shell_apk'. This may result in too many targets
 662         # getting built, but we can adjust that later if need be.
 663         for input_target in inp['targets']:
 664           if (input_target == build_target or
 665               build_target.endswith(':' + input_target)):
 666             matching_targets.append(input_target)
 667     finally:
 668       self.RemoveFile(response_file.name)
 669
 670     if matching_targets:
 671       # TODO: it could be that a target X might depend on a target Y
 672       # and both would be listed in the input, but we would only need
 673       # to specify target X as a build_target (whereas both X and Y are
 674       # targets). I'm not sure if that optimization is generally worth it.
 675       self.WriteJSON({'targets': sorted(matching_targets),
 676                       'build_targets': sorted(matching_targets),
 677                       'status': 'Found dependency'}, output_path)
 678     else:
 679       self.WriteJSON({'targets': [],
 680                       'build_targets': [],
 681                       'status': 'No dependency'}, output_path)
 682
 683     if not ret and self.args.verbose:
 684       outp = json.loads(self.ReadFile(output_path))
 685       self.Print()
 686       self.Print('analyze output:')
 687       self.PrintJSON(outp)
 688       self.Print()
 689
 690     return 0
 691
 692   def ReadInputJSON(self, required_keys):
 693     path = self.args.input_path[0]
 694     output_path = self.args.output_path[0]
 695     if not self.Exists(path):
 696       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 697
 698     try:
 699       inp = json.loads(self.ReadFile(path))
 700     except Exception as e:
 701       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 702                                 (path, e), output_path)
 703
 704     for k in required_keys:
 705       if not k in inp:
 706         self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
 707                                   output_path)
 708
 709     return inp
 710
 711   def WriteFailureAndRaise(self, msg, output_path):
 712     if output_path:
 713       self.WriteJSON({'error': msg}, output_path)
 714     raise MBErr(msg)
 715
 716   def WriteJSON(self, obj, path):
 717     try:
 718       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n')
 719     except Exception as e:
 720       raise MBErr('Error %s writing to the output path "%s"' %
 721                  (e, path))
 722
 723   def PrintCmd(self, cmd):
 724     if cmd[0] == sys.executable:
 725       cmd = ['python'] + cmd[1:]
 726     self.Print(*[pipes.quote(c) for c in cmd])
 727
 728   def PrintJSON(self, obj):
 729     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 730
 731   def Print(self, *args, **kwargs):
 732     # This function largely exists so it can be overridden for testing.
 733     print(*args, **kwargs)
 734
 735   def Run(self, cmd):
 736     # This function largely exists so it can be overridden for testing.
 737     if self.args.dryrun or self.args.verbose:
 738       self.PrintCmd(cmd)
 739     if self.args.dryrun:
 740       return 0, '', ''
 741     ret, out, err = self.Call(cmd)
 742     if self.args.verbose:
 743       if out:
 744         self.Print(out, end='')
 745       if err:
 746         self.Print(err, end='', file=sys.stderr)
 747     return ret, out, err
 748
 749   def Call(self, cmd):
 750     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 751                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 752     out, err = p.communicate()
 753     return p.returncode, out, err
 754
 755   def ExpandUser(self, path):
 756     # This function largely exists so it can be overridden for testing.
 757     return os.path.expanduser(path)
 758
 759   def Exists(self, path):
 760     # This function largely exists so it can be overridden for testing.
 761     return os.path.exists(path)
 762
 763   def MaybeMakeDirectory(self, path):
 764     try:
 765       os.makedirs(path)
 766     except OSError, e:
 767       if e.errno != errno.EEXIST:
 768         raise
 769
 770   def ReadFile(self, path):
 771     # This function largely exists so it can be overriden for testing.
 772     with open(path) as fp:
 773       return fp.read()
 774
 775   def RemoveFile(self, path):
 776     # This function largely exists so it can be overriden for testing.
 777     os.remove(path)
 778
 779   def TempFile(self, mode='w'):
 780     # This function largely exists so it can be overriden for testing.
 781     return tempfile.NamedTemporaryFile(mode=mode, delete=False)
 782
 783   def WriteFile(self, path, contents):
 784     # This function largely exists so it can be overriden for testing.
 785     if self.args.dryrun or self.args.verbose:
 786       self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
 787     with open(path, 'w') as fp:
 788       return fp.write(contents)
 789
 790
 791 class MBErr(Exception):
 792   pass
 793
 794
 795 if __name__ == '__main__':
 796   try:
 797     sys.exit(main(sys.argv[1:]))
 798   except MBErr as e:
 799     print(e)
 800     sys.exit(1)
 801   except KeyboardInterrupt:
 802     print("interrupted, exiting", stream=sys.stderr)
 803     sys.exit(130)