chrome/browser/resources/chromeos/chromevox/tools/jsbundler.py

   1 #!/usr/bin/env python
   2
   3 # Copyright 2014 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 '''Produces various output formats from a set of JavaScript files with
   8 closure style require/provide calls.
   9
  10 Scans one or more directory trees for JavaScript files.  Then, from a
  11 given list of top-level files, sorts all required input files topologically.
  12 The top-level files are appended to the sorted list in the order specified
  13 on the command line.  If no root directories are specified, the source
  14 files are assumed to be ordered already and no dependency analysis is
  15 performed.  The resulting file list can then be used in one of the following
  16 ways:
  17
  18 - list: a plain list of files, one per line is output.
  19
  20 - html: a series of html <script> tags with src attributes containing paths
  21   is output.
  22
  23 - bundle: a concatenation of all the files, separated by newlines is output.
  24
  25 - compressed_bundle: A bundle where non-significant whitespace, including
  26   comments, has been stripped is output.
  27
  28 - copy: the files are copied, or hard linked if possible, to the destination
  29   directory.  In this case, no output is generated.
  30 '''
  31
  32
  33 import optparse
  34 import os
  35 import re
  36 import shutil
  37 import sys
  38
  39 _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
  40 _CHROME_SOURCE = os.path.realpath(
  41     os.path.join(_SCRIPT_DIR, *[os.path.pardir] * 6))
  42 sys.path.insert(0, os.path.join(
  43     _CHROME_SOURCE, 'third_party/WebKit/Source/build/scripts'))
  44 sys.path.insert(0, os.path.join(
  45     _CHROME_SOURCE, ('chrome/third_party/chromevox/third_party/' +
  46                      'closure-library/closure/bin/build')))
  47 import depstree
  48 import rjsmin
  49 import source
  50 import treescan
  51
  52
  53 def Die(message):
  54   '''Prints an error message and exit the program.'''
  55   print >>sys.stderr, message
  56   sys.exit(1)
  57
  58
  59 class SourceWithPaths(source.Source):
  60   '''A source.Source object with its relative input and output paths'''
  61
  62   def __init__(self, content, in_path, out_path):
  63     super(SourceWithPaths, self).__init__(content)
  64     self._in_path = in_path
  65     self._out_path = out_path
  66
  67   def GetInPath(self):
  68     return self._in_path
  69
  70   def GetOutPath(self):
  71     return self._out_path
  72
  73
  74 class Bundle():
  75   '''An ordered list of sources without duplicates.'''
  76
  77   def __init__(self):
  78     self._added_paths = set()
  79     self._added_sources = []
  80
  81   def Add(self, sources):
  82     '''Appends one or more source objects the list if it doesn't already
  83     exist.
  84
  85     Args:
  86       sources: A SourceWithPath or an iterable of such objects.
  87     '''
  88     if isinstance(sources, SourceWithPaths):
  89       sources = [sources]
  90     for source in sources:
  91       path = source.GetInPath()
  92       if path not in self._added_paths:
  93         self._added_paths.add(path)
  94         self._added_sources.append(source)
  95
  96   def GetInPaths(self):
  97     return (source.GetInPath() for source in self._added_sources)
  98
  99   def GetOutPaths(self):
 100     return (source.GetOutPath() for source in self._added_sources)
 101
 102   def GetSources(self):
 103     return self._added_sources
 104
 105   def GetUncompressedSource(self):
 106     return '\n'.join((s.GetSource() for s in self._added_sources))
 107
 108   def GetCompressedSource(self):
 109     return rjsmin.jsmin(self.GetUncompressedSource())
 110
 111
 112 class PathRewriter():
 113   '''A list of simple path rewrite rules to map relative input paths to
 114   relative output paths.
 115   '''
 116
 117   def __init__(self, specs=[]):
 118     '''Args:
 119       specs: A list of mappings, each consisting of the input prefix and
 120         the corresponding output prefix separated by colons.
 121     '''
 122     self._prefix_map = []
 123     for spec in specs:
 124       parts = spec.split(':')
 125       if len(parts) != 2:
 126         Die('Invalid prefix rewrite spec %s' % spec)
 127       if not parts[0].endswith('/') and parts[0] != '':
 128         parts[0] += '/'
 129       self._prefix_map.append(parts)
 130     self._prefix_map.sort(reverse=True)
 131
 132   def RewritePath(self, in_path):
 133     '''Rewrites an input path according to the list of rules.
 134
 135     Args:
 136       in_path, str: The input path to rewrite.
 137     Returns:
 138       str: The corresponding output path.
 139     '''
 140     for in_prefix, out_prefix in self._prefix_map:
 141       if in_path.startswith(in_prefix):
 142         return os.path.join(out_prefix, in_path[len(in_prefix):])
 143     return in_path
 144
 145
 146 def ReadSources(roots=[], source_files=[], need_source_text=False,
 147                 path_rewriter=PathRewriter(), exclude=[]):
 148   '''Reads all source specified on the command line, including sources
 149   included by --root options.
 150   '''
 151
 152   def EnsureSourceLoaded(in_path, sources):
 153     if in_path not in sources:
 154       out_path = path_rewriter.RewritePath(in_path)
 155       sources[in_path] = SourceWithPaths(source.GetFileContents(in_path),
 156                                          in_path, out_path)
 157
 158   # Only read the actual source file if we will do a dependency analysis or
 159   # the caller asks for it.
 160   need_source_text = need_source_text or len(roots) > 0
 161   sources = {}
 162   for root in roots:
 163     for name in treescan.ScanTreeForJsFiles(root):
 164       if any((r.search(name) for r in exclude)):
 165         continue
 166       EnsureSourceLoaded(name, sources)
 167   for path in source_files:
 168     if need_source_text:
 169       EnsureSourceLoaded(path, sources)
 170     else:
 171       # Just add an empty representation of the source.
 172       sources[path] = SourceWithPaths(
 173           '', path, path_rewriter.RewritePath(path))
 174   return sources
 175
 176
 177 def _GetBase(sources):
 178   '''Gets the closure base.js file if present among the sources.
 179
 180   Args:
 181     sources: Dictionary with input path names as keys and SourceWithPaths
 182       as values.
 183   Returns:
 184     SourceWithPath: The source file providing the goog namespace.
 185   '''
 186   for source in sources.itervalues():
 187     if (os.path.basename(source.GetInPath()) == 'base.js' and
 188         'goog' in source.provides):
 189       return source
 190   Die('goog.base not provided by any file.')
 191
 192
 193 def CalcDeps(bundle, sources, top_level):
 194   '''Calculates dependencies for a set of top-level files.
 195
 196   Args:
 197     bundle: Bundle to add the sources to.
 198     sources, dict: Mapping from input path to SourceWithPaths objects.
 199     top_level, list: List of top-level input paths to calculate dependencies
 200       for.
 201   '''
 202   providers = [s for s in sources.itervalues() if len(s.provides) > 0]
 203   deps = depstree.DepsTree(providers)
 204   namespaces = []
 205   for path in top_level:
 206     namespaces.extend(sources[path].requires)
 207   # base.js is an implicit dependency that always goes first.
 208   bundle.Add(_GetBase(sources))
 209   bundle.Add(deps.GetDependencies(namespaces))
 210
 211
 212 def _MarkAsCompiled(sources):
 213   '''Sets COMPILED to true in the Closure base.js source.
 214
 215   Args:
 216     sources: Dictionary with input paths names as keys and SourcWithPaths
 217       objects as values.
 218   '''
 219   base = _GetBase(sources)
 220   new_content, count = re.subn('^var COMPILED = false;$',
 221                                'var COMPILED = true;',
 222                                base.GetSource(),
 223                                count=1,
 224                                flags=re.MULTILINE)
 225   if count != 1:
 226     Die('COMPILED var assignment not found in %s' % base.GetInPath())
 227   sources[base.GetInPath()] = SourceWithPaths(
 228       new_content,
 229       base.GetInPath(),
 230       base.GetOutPath())
 231
 232 def LinkOrCopyFiles(sources, dest_dir):
 233   '''Copies a list of sources to a destination directory.'''
 234
 235   def LinkOrCopyOneFile(src, dst):
 236     if not os.path.exists(os.path.dirname(dst)):
 237       os.makedirs(os.path.dirname(dst))
 238     if os.path.exists(dst):
 239       os.unlink(dst)
 240     try:
 241       os.link(src, dst)
 242     except:
 243       shutil.copy(src, dst)
 244
 245   for source in sources:
 246     LinkOrCopyOneFile(source.GetInPath(),
 247                       os.path.join(dest_dir, source.GetOutPath()))
 248
 249
 250 def WriteOutput(bundle, format, out_file, dest_dir):
 251   '''Writes output in the specified format.
 252
 253   Args:
 254     bundle: The ordered bundle iwth all sources already added.
 255     format: Output format, one of list, html, bundle, compressed_bundle.
 256     out_file: File object to receive the output.
 257     dest_dir: Prepended to each path mentioned in the output, if applicable.
 258   '''
 259   if format == 'list':
 260     paths = bundle.GetOutPaths()
 261     if dest_dir:
 262       paths = (os.path.join(dest_dir, p) for p in paths)
 263     paths = (os.path.normpath(p) for p in paths)
 264     out_file.write('\n'.join(paths))
 265   elif format == 'html':
 266     HTML_TEMPLATE = '<script src=\'%s\'>'
 267     script_lines = (HTML_TEMPLATE % p for p in bundle.GetOutPaths())
 268     out_file.write('\n'.join(script_lines))
 269   elif format == 'bundle':
 270     out_file.write(bundle.GetUncompressedSource())
 271   elif format == 'compressed_bundle':
 272     out_file.write(bundle.GetCompressedSource())
 273   out_file.write('\n')
 274
 275
 276 def WriteStampfile(stampfile):
 277   '''Writes a stamp file.
 278
 279   Args:
 280     stampfile, string: name of stamp file to touch
 281   '''
 282   with open(stampfile, 'w') as file:
 283     os.utime(stampfile, None)
 284
 285
 286 def WriteDepfile(depfile, outfile, infiles):
 287   '''Writes a depfile.
 288
 289   Args:
 290     depfile, string: name of dep file to write
 291     outfile, string: Name of output file to use as the target in the generated
 292       .d file.
 293     infiles, list: File names to list as dependencies in the .d file.
 294   '''
 295   content = '%s: %s' % (outfile, ' '.join(infiles))
 296   open(depfile, 'w').write(content)
 297
 298
 299 def CreateOptionParser():
 300   parser = optparse.OptionParser(description=__doc__)
 301   parser.usage = '%prog [options] <top_level_file>...'
 302   parser.add_option('-d', '--dest_dir', action='store', metavar='DIR',
 303                     help=('Destination directory.  Used when translating ' +
 304                           'input paths to output paths and when copying '
 305                           'files.'))
 306   parser.add_option('-o', '--output_file', action='store', metavar='FILE',
 307                     help=('File to output result to for modes that output '
 308                           'a single file.'))
 309   parser.add_option('-r', '--root', dest='roots', action='append', default=[],
 310                     metavar='ROOT',
 311                     help='Roots of directory trees to scan for sources.')
 312   parser.add_option('-M', '--module', dest='modules', action='append',
 313                     default=[], metavar='FILENAME',
 314                     help='Source modules to load')
 315   parser.add_option('-w', '--rewrite_prefix', action='append', default=[],
 316                     dest='prefix_map', metavar='SPEC',
 317                     help=('Two path prefixes, separated by colons ' +
 318                           'specifying that a file whose (relative) path ' +
 319                           'name starts with the first prefix should have ' +
 320                           'that prefix replaced by the second prefix to ' +
 321                           'form a path relative to the output directory.'))
 322   parser.add_option('-m', '--mode', type='choice', action='store',
 323                     choices=['list', 'html', 'bundle',
 324                              'compressed_bundle', 'copy'],
 325                     default='list', metavar='MODE',
 326                     help=("Otput mode. One of 'list', 'html', 'bundle', " +
 327                           "'compressed_bundle' or 'copy'."))
 328   parser.add_option('-x', '--exclude', action='append', default=[],
 329                     help=('Exclude files whose full path contains a match for '
 330                           'the given regular expression.  Does not apply to '
 331                           'filenames given as arguments or with the '
 332                           '-m option.'))
 333   parser.add_option('--depfile', metavar='FILENAME',
 334                     help='Store .d style dependencies in FILENAME')
 335   parser.add_option('--stampfile', metavar='FILENAME',
 336                     help='Write empty stamp file')
 337   return parser
 338
 339
 340 def main():
 341   options, args = CreateOptionParser().parse_args()
 342   if len(args) < 1:
 343     Die('At least one top-level source file must be specified.')
 344   if options.depfile and not options.output_file:
 345     Die('--depfile requires an output file')
 346   will_output_source_text = options.mode in ('bundle', 'compressed_bundle')
 347   path_rewriter = PathRewriter(options.prefix_map)
 348   exclude = [re.compile(r) for r in options.exclude]
 349   sources = ReadSources(options.roots, options.modules + args,
 350                         will_output_source_text or len(options.modules) > 0,
 351                         path_rewriter, exclude)
 352   if will_output_source_text:
 353     _MarkAsCompiled(sources)
 354   bundle = Bundle()
 355   if len(options.roots) > 0 or len(options.modules) > 0:
 356     CalcDeps(bundle, sources, args)
 357   bundle.Add((sources[name] for name in args))
 358   if options.mode == 'copy':
 359     if options.dest_dir is None:
 360       Die('Must specify --dest_dir when copying.')
 361     LinkOrCopyFiles(bundle.GetSources(), options.dest_dir)
 362   else:
 363     if options.output_file:
 364       out_file = open(options.output_file, 'w')
 365     else:
 366       out_file = sys.stdout
 367     try:
 368       WriteOutput(bundle, options.mode, out_file, options.dest_dir)
 369     finally:
 370       if options.output_file:
 371         out_file.close()
 372   if options.stampfile:
 373     WriteStampfile(options.stampfile)
 374   if options.depfile:
 375     WriteDepfile(options.depfile, options.output_file, bundle.GetInPaths())
 376
 377 if __name__ == '__main__':
 378   main()