tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import json
  17 import os
  18 import pipes
  19 import shlex
  20 import shutil
  21 import sys
  22 import subprocess
  23
  24
  25 def main(args):
  26   mbw = MetaBuildWrapper()
  27   mbw.ParseArgs(args)
  28   return mbw.args.func()
  29
  30
  31 class MetaBuildWrapper(object):
  32   def __init__(self):
  33     p = os.path
  34     d = os.path.dirname
  35     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  36     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  37                                  'mb_config.pyl')
  38     self.platform = sys.platform
  39     self.args = argparse.Namespace()
  40     self.configs = {}
  41     self.masters = {}
  42     self.mixins = {}
  43     self.private_configs = []
  44     self.common_dev_configs = []
  45     self.unsupported_configs = []
  46
  47   def ParseArgs(self, argv):
  48     def AddCommonOptions(subp):
  49       subp.add_argument('-b', '--builder',
  50                         help='builder name to look up config from')
  51       subp.add_argument('-m', '--master',
  52                         help='master name to look up config from')
  53       subp.add_argument('-c', '--config',
  54                         help='configuration to analyze')
  55       subp.add_argument('-f', '--config-file', metavar='PATH',
  56                         default=self.default_config,
  57                         help='path to config file '
  58                             '(default is //tools/mb/mb_config.pyl)')
  59       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  60                         help='path to goma directory (default is %(default)s).')
  61       subp.add_argument('-n', '--dryrun', action='store_true',
  62                         help='Do a dry run (i.e., do nothing, just print '
  63                              'the commands that will run)')
  64       subp.add_argument('-q', '--quiet', action='store_true',
  65                         help='Do not print anything, just return an exit '
  66                              'code.')
  67       subp.add_argument('-v', '--verbose', action='count',
  68                         help='verbose logging (may specify multiple times).')
  69
  70     parser = argparse.ArgumentParser(prog='mb')
  71     subps = parser.add_subparsers()
  72
  73     subp = subps.add_parser('analyze',
  74                             help='analyze whether changes to a set of files '
  75                                  'will cause a set of binaries to be rebuilt.')
  76     AddCommonOptions(subp)
  77     subp.add_argument('path', type=str, nargs=1,
  78                       help='path build was generated into.')
  79     subp.add_argument('input_path', nargs=1,
  80                       help='path to a file containing the input arguments '
  81                            'as a JSON object.')
  82     subp.add_argument('output_path', nargs=1,
  83                       help='path to a file containing the output arguments '
  84                            'as a JSON object.')
  85     subp.set_defaults(func=self.CmdAnalyze)
  86
  87     subp = subps.add_parser('gen',
  88                             help='generate a new set of build files')
  89     AddCommonOptions(subp)
  90     subp.add_argument('path', type=str, nargs=1,
  91                       help='path to generate build into')
  92     subp.set_defaults(func=self.CmdGen)
  93
  94     subp = subps.add_parser('lookup',
  95                             help='look up the command for a given config or '
  96                                  'builder')
  97     AddCommonOptions(subp)
  98     subp.set_defaults(func=self.CmdLookup)
  99
 100     subp = subps.add_parser('validate',
 101                             help='validate the config file')
 102     AddCommonOptions(subp)
 103     subp.set_defaults(func=self.CmdValidate)
 104
 105     subp = subps.add_parser('help',
 106                             help='Get help on a subcommand.')
 107     subp.add_argument(nargs='?', action='store', dest='subcommand',
 108                       help='The command to get help for.')
 109     subp.set_defaults(func=self.CmdHelp)
 110
 111     self.args = parser.parse_args(argv)
 112
 113   def CmdAnalyze(self):
 114     vals = self.GetConfig()
 115     if vals['type'] == 'gn':
 116       return self.RunGNAnalyze(vals)
 117     elif vals['type'] == 'gyp':
 118       return self.RunGYPAnalyze(vals)
 119     else:
 120       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 121
 122   def CmdGen(self):
 123     vals = self.GetConfig()
 124     if vals['type'] == 'gn':
 125       return self.RunGNGen(self.args.path[0], vals)
 126     if vals['type'] == 'gyp':
 127       return self.RunGYPGen(self.args.path[0], vals)
 128
 129     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 130
 131   def CmdLookup(self):
 132     vals = self.GetConfig()
 133     if vals['type'] == 'gn':
 134       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 135     elif vals['type'] == 'gyp':
 136       cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
 137     else:
 138       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 139
 140     self.PrintCmd(cmd)
 141     return 0
 142
 143   def CmdHelp(self):
 144     if self.args.subcommand:
 145       self.ParseArgs([self.args.subcommand, '--help'])
 146     else:
 147       self.ParseArgs(['--help'])
 148
 149   def CmdValidate(self):
 150     errs = []
 151
 152     # Read the file to make sure it parses.
 153     self.ReadConfigFile()
 154
 155     # Figure out the whole list of configs and ensure that no config is
 156     # listed in more than one category.
 157     all_configs = {}
 158     for config in self.common_dev_configs:
 159       all_configs[config] = 'common_dev_configs'
 160     for config in self.private_configs:
 161       if config in all_configs:
 162         errs.append('config "%s" listed in "private_configs" also '
 163                     'listed in "%s"' % (config, all_configs['config']))
 164       else:
 165         all_configs[config] = 'private_configs'
 166     for config in self.unsupported_configs:
 167       if config in all_configs:
 168         errs.append('config "%s" listed in "unsupported_configs" also '
 169                     'listed in "%s"' % (config, all_configs['config']))
 170       else:
 171         all_configs[config] = 'unsupported_configs'
 172
 173     for master in self.masters:
 174       for builder in self.masters[master]:
 175         config = self.masters[master][builder]
 176         if config in all_configs and all_configs[config] not in self.masters:
 177           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 178                       (config, all_configs[config]))
 179         else:
 180           all_configs[config] = master
 181
 182     # Check that every referenced config actually exists.
 183     for config, loc in all_configs.items():
 184       if not config in self.configs:
 185         errs.append('Unknown config "%s" referenced from "%s".' %
 186                     (config, loc))
 187
 188     # Check that every actual config is actually referenced.
 189     for config in self.configs:
 190       if not config in all_configs:
 191         errs.append('Unused config "%s".' % config)
 192
 193     # Figure out the whole list of mixins, and check that every mixin
 194     # listed by a config or another mixin actually exists.
 195     referenced_mixins = set()
 196     for config, mixins in self.configs.items():
 197       for mixin in mixins:
 198         if not mixin in self.mixins:
 199           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 200                       (mixin, config))
 201         referenced_mixins.add(mixin)
 202
 203     for mixin in self.mixins:
 204       for sub_mixin in self.mixins[mixin].get('mixins', []):
 205         if not sub_mixin in self.mixins:
 206           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 207                       (sub_mixin, mixin))
 208         referenced_mixins.add(sub_mixin)
 209
 210     # Check that every mixin defined is actually referenced somewhere.
 211     for mixin in self.mixins:
 212       if not mixin in referenced_mixins:
 213         errs.append('Unreferenced mixin "%s".' % mixin)
 214
 215     if errs:
 216       raise MBErr('mb config file %s has problems:\n  ' + '\n  '.join(errs))
 217
 218     if not self.args.quiet:
 219       self.Print('mb config file %s looks ok.' % self.args.config_file)
 220     return 0
 221
 222   def GetConfig(self):
 223     self.ReadConfigFile()
 224     config = self.ConfigFromArgs()
 225     if not config in self.configs:
 226       raise MBErr('Config "%s" not found in %s' %
 227                   (config, self.args.config_file))
 228
 229     return self.FlattenConfig(config)
 230
 231   def ReadConfigFile(self):
 232     if not self.Exists(self.args.config_file):
 233       raise MBErr('config file not found at %s' % self.args.config_file)
 234
 235     try:
 236       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 237     except SyntaxError as e:
 238       raise MBErr('Failed to parse config file "%s": %s' %
 239                  (self.args.config_file, e))
 240
 241     self.common_dev_configs = contents['common_dev_configs']
 242     self.configs = contents['configs']
 243     self.masters = contents['masters']
 244     self.mixins = contents['mixins']
 245     self.private_configs = contents['private_configs']
 246     self.unsupported_configs = contents['unsupported_configs']
 247
 248   def ConfigFromArgs(self):
 249     if self.args.config:
 250       if self.args.master or self.args.builder:
 251         raise MBErr('Can not specific both -c/--config and -m/--master or '
 252                     '-b/--builder')
 253
 254       return self.args.config
 255
 256     if not self.args.master or not self.args.builder:
 257       raise MBErr('Must specify either -c/--config or '
 258                   '(-m/--master and -b/--builder)')
 259
 260     if not self.args.master in self.masters:
 261       raise MBErr('Master name "%s" not found in "%s"' %
 262                   (self.args.master, self.args.config_file))
 263
 264     if not self.args.builder in self.masters[self.args.master]:
 265       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 266                   (self.args.builder, self.args.master, self.args.config_file))
 267
 268     return self.masters[self.args.master][self.args.builder]
 269
 270   def FlattenConfig(self, config):
 271     mixins = self.configs[config]
 272     vals = {
 273       'type': None,
 274       'gn_args': [],
 275       'gyp_config': [],
 276       'gyp_defines': [],
 277     }
 278
 279     visited = []
 280     self.FlattenMixins(mixins, vals, visited)
 281     return vals
 282
 283   def FlattenMixins(self, mixins, vals, visited):
 284     for m in mixins:
 285       if m not in self.mixins:
 286         raise MBErr('Unknown mixin "%s"' % m)
 287
 288       # TODO: check for cycles in mixins.
 289
 290       visited.append(m)
 291
 292       mixin_vals = self.mixins[m]
 293       if 'type' in mixin_vals:
 294         vals['type'] = mixin_vals['type']
 295       if 'gn_args' in mixin_vals:
 296         if vals['gn_args']:
 297           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 298         else:
 299           vals['gn_args'] = mixin_vals['gn_args']
 300       if 'gyp_config' in mixin_vals:
 301         vals['gyp_config'] = mixin_vals['gyp_config']
 302       if 'gyp_defines' in mixin_vals:
 303         if vals['gyp_defines']:
 304           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 305         else:
 306           vals['gyp_defines'] = mixin_vals['gyp_defines']
 307       if 'mixins' in mixin_vals:
 308         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 309     return vals
 310
 311   def RunGNGen(self, path, vals):
 312     cmd = self.GNCmd('gen', path, vals['gn_args'])
 313     ret, _, _ = self.Run(cmd)
 314     return ret
 315
 316   def GNCmd(self, subcommand, path, gn_args=''):
 317     if self.platform == 'linux2':
 318       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
 319                              'gn')
 320     elif self.platform == 'darwin':
 321       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
 322                              'gn')
 323     else:
 324       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
 325                              'gn.exe')
 326
 327     cmd = [gn_path, subcommand, path]
 328     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 329     if gn_args:
 330       cmd.append('--args=%s' % gn_args)
 331     return cmd
 332
 333   def RunGYPGen(self, path, vals):
 334     output_dir, gyp_config = self.ParseGYPConfigPath(path)
 335     if gyp_config != vals['gyp_config']:
 336       raise MBErr('The last component of the path (%s) must match the '
 337                   'GYP configuration specified in the config (%s), and '
 338                   'it does not.' % (gyp_config, vals['gyp_config']))
 339     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 340     ret, _, _ = self.Run(cmd)
 341     return ret
 342
 343   def RunGYPAnalyze(self, vals):
 344     output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
 345     if gyp_config != vals['gyp_config']:
 346       raise MBErr('The last component of the path (%s) must match the '
 347                   'GYP configuration specified in the config (%s), and '
 348                   'it does not.' % (gyp_config, vals['gyp_config']))
 349     if self.args.verbose:
 350       inp = self.GetAnalyzeInput()
 351       self.Print()
 352       self.Print('analyze input:')
 353       self.PrintJSON(inp)
 354       self.Print()
 355
 356     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 357     cmd.extend(['-G', 'config_path=%s' % self.args.input_path[0],
 358                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 359     ret, _, _ = self.Run(cmd)
 360     if not ret and self.args.verbose:
 361       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 362       self.Print()
 363       self.Print('analyze output:')
 364       self.PrintJSON(inp)
 365       self.Print()
 366
 367     return ret
 368
 369   def ToSrcRelPath(self, path):
 370     """Returns a relative path from the top of the repo."""
 371     # TODO: Support normal paths in addition to source-absolute paths.
 372     assert(path.startswith('//'))
 373     return path[2:]
 374
 375   def ParseGYPConfigPath(self, path):
 376     rpath = self.ToSrcRelPath(path)
 377     output_dir, _, config = rpath.rpartition('/')
 378     self.CheckGYPConfigIsSupported(config, path)
 379     return output_dir, config
 380
 381   def CheckGYPConfigIsSupported(self, config, path):
 382     if config not in ('Debug', 'Release'):
 383       if (sys.platform in ('win32', 'cygwin') and
 384           config not in ('Debug_x64', 'Release_x64')):
 385         raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
 386                     config, path)
 387
 388   def GYPCmd(self, output_dir, gyp_defines, config):
 389     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 390     cmd = [
 391         sys.executable,
 392         os.path.join('build', 'gyp_chromium'),
 393         '-G',
 394         'output_dir=' + output_dir,
 395         '-G',
 396         'config=' + config,
 397     ]
 398     for d in shlex.split(gyp_defines):
 399       cmd += ['-D', d]
 400     return cmd
 401
 402   def RunGNAnalyze(self, _vals):
 403     inp = self.GetAnalyzeInput()
 404     if self.args.verbose:
 405       self.Print()
 406       self.Print('analyze input:')
 407       self.PrintJSON(inp)
 408       self.Print()
 409
 410     output_path = self.args.output_path[0]
 411
 412     # Bail out early if a GN file was modified, since 'gn refs' won't know
 413     # what to do about it.
 414     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 415       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 416       return 0
 417
 418     # TODO: Because of the --type=executable filter below, we don't detect
 419     # when files will cause 'all' or 'gn_all' or similar targets to be
 420     # dirty. We need to figure out how to handle that properly, but for
 421     # now we can just bail out early.
 422     if 'gn_all' in inp['targets'] or 'all' in inp['targets']:
 423       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 424       return 0
 425
 426     all_needed_targets = set()
 427     ret = 0
 428     for f in inp['files']:
 429       cmd = self.GNCmd('refs', self.args.path[0]) + [
 430              '//' + f, '--type=executable', '--all', '--as=output']
 431       ret, out, _ = self.Run(cmd)
 432       if ret and not 'The input matches no targets' in out:
 433         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 434                                   output_path)
 435
 436       rpath = self.ToSrcRelPath(self.args.path[0]) + os.sep
 437       needed_targets = [t.replace(rpath, '') for t in out.splitlines()]
 438       needed_targets = [nt for nt in needed_targets if nt in inp['targets']]
 439       all_needed_targets.update(set(needed_targets))
 440
 441     if all_needed_targets:
 442       # TODO: it could be that a target X might depend on a target Y
 443       # and both would be listed in the input, but we would only need
 444       # to specify target X as a build_target (whereas both X and Y are
 445       # targets). I'm not sure if that optimization is generally worth it.
 446       self.WriteJSON({'targets': sorted(all_needed_targets),
 447                       'build_targets': sorted(all_needed_targets),
 448                       'status': 'Found dependency'}, output_path)
 449     else:
 450       self.WriteJSON({'targets': [],
 451                       'build_targets': [],
 452                       'status': 'No dependency'}, output_path)
 453
 454     if not ret and self.args.verbose:
 455       outp = json.loads(self.ReadFile(output_path))
 456       self.Print()
 457       self.Print('analyze output:')
 458       self.PrintJSON(outp)
 459       self.Print()
 460
 461     return 0
 462
 463   def GetAnalyzeInput(self):
 464     path = self.args.input_path[0]
 465     output_path = self.args.output_path[0]
 466     if not self.Exists(path):
 467       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 468
 469     try:
 470       inp = json.loads(self.ReadFile(path))
 471     except Exception as e:
 472       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 473                                 (path, e), output_path)
 474     if not 'files' in inp:
 475       self.WriteFailureAndRaise('input file is missing a "files" key',
 476                                 output_path)
 477     if not 'targets' in inp:
 478       self.WriteFailureAndRaise('input file is missing a "targets" key',
 479                                 output_path)
 480
 481     return inp
 482
 483   def WriteFailureAndRaise(self, msg, path):
 484     self.WriteJSON({'error': msg}, path)
 485     raise MBErr(msg)
 486
 487   def WriteJSON(self, obj, path):
 488     try:
 489       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n')
 490     except Exception as e:
 491       raise MBErr('Error %s writing to the output path "%s"' %
 492                  (e, path))
 493
 494   def PrintCmd(self, cmd):
 495     if cmd[0] == sys.executable:
 496       cmd = ['python'] + cmd[1:]
 497     self.Print(*[pipes.quote(c) for c in cmd])
 498
 499   def PrintJSON(self, obj):
 500     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 501
 502   def Print(self, *args, **kwargs):
 503     # This function largely exists so it can be overridden for testing.
 504     print(*args, **kwargs)
 505
 506   def Run(self, cmd):
 507     # This function largely exists so it can be overridden for testing.
 508     if self.args.dryrun or self.args.verbose:
 509       self.PrintCmd(cmd)
 510     if self.args.dryrun:
 511       return 0, '', ''
 512     ret, out, err = self.Call(cmd)
 513     if self.args.verbose:
 514       if out:
 515         self.Print(out, end='')
 516       if err:
 517         self.Print(err, end='', file=sys.stderr)
 518     return ret, out, err
 519
 520   def Call(self, cmd):
 521     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 522                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 523     out, err = p.communicate()
 524     return p.returncode, out, err
 525
 526   def ExpandUser(self, path):
 527     # This function largely exists so it can be overridden for testing.
 528     return os.path.expanduser(path)
 529
 530   def Exists(self, path):
 531     # This function largely exists so it can be overridden for testing.
 532     return os.path.exists(path)
 533
 534   def ReadFile(self, path):
 535     # This function largely exists so it can be overriden for testing.
 536     with open(path) as fp:
 537       return fp.read()
 538
 539   def WriteFile(self, path, contents):
 540     # This function largely exists so it can be overriden for testing.
 541     with open(path, 'w') as fp:
 542       return fp.write(contents)
 543
 544 class MBErr(Exception):
 545   pass
 546
 547
 548 if __name__ == '__main__':
 549   try:
 550     sys.exit(main(sys.argv[1:]))
 551   except MBErr as e:
 552     print(e)
 553     sys.exit(1)
 554   except KeyboardInterrupt:
 555     print("interrupted, exiting", stream=sys.stderr)
 556     sys.exit(130)