chrome/browser/resources/chromeos/chromevox/tools/jsbundler.py

   1 #!/usr/bin/env python
   2
   3 # Copyright 2014 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 '''Produces various output formats from a set of JavaScript files with
   8 closure style require/provide calls.
   9
  10 Scans one or more directory trees for JavaScript files.  Then, from a
  11 given list of top-level files, sorts all required input files topologically.
  12 The top-level files are appended to the sorted list in the order specified
  13 on the command line.  If no root directories are specified, the source
  14 files are assumed to be ordered already and no dependency analysis is
  15 performed.  The resulting file list can then be used in one of the following
  16 ways:
  17
  18 - list: a plain list of files, one per line is output.
  19
  20 - html: a series of html <script> tags with src attributes containing paths
  21   is output.
  22
  23 - bundle: a concatenation of all the files, separated by newlines is output.
  24
  25 - compressed_bundle: A bundle where non-significant whitespace, including
  26   comments, has been stripped is output.
  27
  28 - copy: the files are copied, or hard linked if possible, to the destination
  29   directory.  In this case, no output is generated.
  30 '''
  31
  32
  33 import optparse
  34 import os
  35 import re
  36 import shutil
  37 import sys
  38
  39 _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
  40 _CHROME_SOURCE = os.path.realpath(
  41     os.path.join(_SCRIPT_DIR, *[os.path.pardir] * 6))
  42 sys.path.insert(0, os.path.join(
  43     _CHROME_SOURCE, 'third_party/WebKit/Source/build/scripts'))
  44 sys.path.insert(0, os.path.join(
  45     _CHROME_SOURCE, ('chrome/third_party/chromevox/third_party/' +
  46                      'closure-library/closure/bin/build')))
  47 import depstree
  48 import rjsmin
  49 import source
  50 import treescan
  51
  52
  53 def Die(message):
  54   '''Prints an error message and exit the program.'''
  55   print >>sys.stderr, message
  56   sys.exit(1)
  57
  58
  59 class SourceWithPaths(source.Source):
  60   '''A source.Source object with its relative input and output paths'''
  61
  62   def __init__(self, content, in_path, out_path):
  63     super(SourceWithPaths, self).__init__(content)
  64     self._in_path = in_path
  65     self._out_path = out_path
  66
  67   def GetInPath(self):
  68     return self._in_path
  69
  70   def GetOutPath(self):
  71     return self._out_path
  72
  73
  74 class Bundle():
  75   '''An ordered list of sources without duplicates.'''
  76
  77   def __init__(self):
  78     self._added_paths = set()
  79     self._added_sources = []
  80
  81   def Add(self, sources):
  82     '''Appends one or more source objects the list if it doesn't already
  83     exist.
  84
  85     Args:
  86       sources: A SourceWithPath or an iterable of such objects.
  87     '''
  88     if isinstance(sources, SourceWithPaths):
  89       sources = [sources]
  90     for source in sources:
  91       path = source.GetInPath()
  92       if path not in self._added_paths:
  93         self._added_paths.add(path)
  94         self._added_sources.append(source)
  95
  96   def GetInPaths(self):
  97     return (source.GetInPath() for source in self._added_sources)
  98
  99   def GetOutPaths(self):
 100     return (source.GetOutPath() for source in self._added_sources)
 101
 102   def GetSources(self):
 103     return self._added_sources
 104
 105   def GetUncompressedSource(self):
 106     return '\n'.join((s.GetSource() for s in self._added_sources))
 107
 108   def GetCompressedSource(self):
 109     return rjsmin.jsmin(self.GetUncompressedSource())
 110
 111
 112 class PathRewriter():
 113   '''A list of simple path rewrite rules to map relative input paths to
 114   relative output paths.
 115   '''
 116
 117   def __init__(self, specs=[]):
 118     '''Args:
 119       specs: A list of mappings, each consisting of the input prefix and
 120         the corresponding output prefix separated by colons.
 121     '''
 122     self._prefix_map = []
 123     for spec in specs:
 124       parts = spec.split(':')
 125       if len(parts) != 2:
 126         Die('Invalid prefix rewrite spec %s' % spec)
 127       if not parts[0].endswith('/') and parts[0] != '':
 128         parts[0] += '/'
 129       self._prefix_map.append(parts)
 130     self._prefix_map.sort(reverse=True)
 131
 132   def RewritePath(self, in_path):
 133     '''Rewrites an input path according to the list of rules.
 134
 135     Args:
 136       in_path, str: The input path to rewrite.
 137     Returns:
 138       str: The corresponding output path.
 139     '''
 140     for in_prefix, out_prefix in self._prefix_map:
 141       if in_path.startswith(in_prefix):
 142         return os.path.join(out_prefix, in_path[len(in_prefix):])
 143     return in_path
 144
 145
 146 def ReadSources(roots=[], source_files=[], need_source_text=False,
 147                 path_rewriter=PathRewriter(), exclude=[]):
 148   '''Reads all source specified on the command line, including sources
 149   included by --root options.
 150   '''
 151
 152   def EnsureSourceLoaded(in_path, sources):
 153     if in_path not in sources:
 154       out_path = path_rewriter.RewritePath(in_path)
 155       sources[in_path] = SourceWithPaths(source.GetFileContents(in_path),
 156                                          in_path, out_path)
 157
 158   # Only read the actual source file if we will do a dependency analysis or
 159   # the caller asks for it.
 160   need_source_text = need_source_text or len(roots) > 0
 161   sources = {}
 162   for root in roots:
 163     for name in treescan.ScanTreeForJsFiles(root):
 164       if any((r.search(name) for r in exclude)):
 165         continue
 166       EnsureSourceLoaded(name, sources)
 167   for path in source_files:
 168     if need_source_text:
 169       EnsureSourceLoaded(path, sources)
 170     else:
 171       # Just add an empty representation of the source.
 172       sources[path] = SourceWithPaths(
 173           '', path, path_rewriter.RewritePath(path))
 174   return sources
 175
 176
 177 def _GetBase(sources):
 178   '''Gets the closure base.js file if present among the sources.
 179
 180   Args:
 181     sources: Dictionary with input path names as keys and SourceWithPaths
 182       as values.
 183   Returns:
 184     SourceWithPath: The source file providing the goog namespace.
 185   '''
 186   for source in sources.itervalues():
 187     if (os.path.basename(source.GetInPath()) == 'base.js' and
 188         'goog' in source.provides):
 189       return source
 190   Die('goog.base not provided by any file.')
 191
 192
 193 def CalcDeps(bundle, sources, top_level):
 194   '''Calculates dependencies for a set of top-level files.
 195
 196   Args:
 197     bundle: Bundle to add the sources to.
 198     sources, dict: Mapping from input path to SourceWithPaths objects.
 199     top_level, list: List of top-level input paths to calculate dependencies
 200       for.
 201   '''
 202   providers = [s for s in sources.itervalues() if len(s.provides) > 0]
 203   deps = depstree.DepsTree(providers)
 204   namespaces = []
 205   for path in top_level:
 206     namespaces.extend(sources[path].requires)
 207   # base.js is an implicit dependency that always goes first.
 208   bundle.Add(_GetBase(sources))
 209   bundle.Add(deps.GetDependencies(namespaces))
 210
 211
 212 def _MarkAsCompiled(sources):
 213   '''Sets COMPILED to true in the Closure base.js source.
 214
 215   Args:
 216     sources: Dictionary with input paths names as keys and SourcWithPaths
 217       objects as values.
 218   '''
 219   base = _GetBase(sources)
 220   new_content, count = re.subn('^var COMPILED = false;$',
 221                                'var COMPILED = true;',
 222                                base.GetSource(),
 223                                count=1,
 224                                flags=re.MULTILINE)
 225   if count != 1:
 226     Die('COMPILED var assignment not found in %s' % base.GetInPath())
 227   sources[base.GetInPath()] = SourceWithPaths(
 228       new_content,
 229       base.GetInPath(),
 230       base.GetOutPath())
 231
 232 def LinkOrCopyFiles(sources, dest_dir):
 233   '''Copies a list of sources to a destination directory.'''
 234
 235   def LinkOrCopyOneFile(src, dst):
 236     if not os.path.exists(os.path.dirname(dst)):
 237       os.makedirs(os.path.dirname(dst))
 238     if os.path.exists(dst):
 239       # Avoid clobbering the inode if source and destination refer to the
 240       # same file already.
 241       if os.path.samefile(src, dst):
 242         return
 243       os.unlink(dst)
 244     try:
 245       os.link(src, dst)
 246     except:
 247       shutil.copy(src, dst)
 248
 249   for source in sources:
 250     LinkOrCopyOneFile(source.GetInPath(),
 251                       os.path.join(dest_dir, source.GetOutPath()))
 252
 253
 254 def WriteOutput(bundle, format, out_file, dest_dir):
 255   '''Writes output in the specified format.
 256
 257   Args:
 258     bundle: The ordered bundle iwth all sources already added.
 259     format: Output format, one of list, html, bundle, compressed_bundle.
 260     out_file: File object to receive the output.
 261     dest_dir: Prepended to each path mentioned in the output, if applicable.
 262   '''
 263   if format == 'list':
 264     paths = bundle.GetOutPaths()
 265     if dest_dir:
 266       paths = (os.path.join(dest_dir, p) for p in paths)
 267     paths = (os.path.normpath(p) for p in paths)
 268     out_file.write('\n'.join(paths))
 269   elif format == 'html':
 270     HTML_TEMPLATE = '<script src=\'%s\'>'
 271     script_lines = (HTML_TEMPLATE % p for p in bundle.GetOutPaths())
 272     out_file.write('\n'.join(script_lines))
 273   elif format == 'bundle':
 274     out_file.write(bundle.GetUncompressedSource())
 275   elif format == 'compressed_bundle':
 276     out_file.write(bundle.GetCompressedSource())
 277   out_file.write('\n')
 278
 279
 280 def CreateOptionParser():
 281   parser = optparse.OptionParser(description=__doc__)
 282   parser.usage = '%prog [options] <top_level_file>...'
 283   parser.add_option('-d', '--dest_dir', action='store', metavar='DIR',
 284                     help=('Destination directory.  Used when translating ' +
 285                           'input paths to output paths and when copying '
 286                           'files.'))
 287   parser.add_option('-o', '--output_file', action='store', metavar='FILE',
 288                     help=('File to output result to for modes that output '
 289                           'a single file.'))
 290   parser.add_option('-r', '--root', dest='roots', action='append', default=[],
 291                     metavar='ROOT',
 292                     help='Roots of directory trees to scan for sources.')
 293   parser.add_option('-w', '--rewrite_prefix', action='append', default=[],
 294                     dest='prefix_map', metavar='SPEC',
 295                     help=('Two path prefixes, separated by colons ' +
 296                           'specifying that a file whose (relative) path ' +
 297                           'name starts with the first prefix should have ' +
 298                           'that prefix replaced by the second prefix to ' +
 299                           'form a path relative to the output directory.'))
 300   parser.add_option('-m', '--mode', type='choice', action='store',
 301                     choices=['list', 'html', 'bundle',
 302                              'compressed_bundle', 'copy'],
 303                     default='list', metavar='MODE',
 304                     help=("Otput mode. One of 'list', 'html', 'bundle', " +
 305                           "'compressed_bundle' or 'copy'."))
 306   parser.add_option('-x', '--exclude', action='append', default=[],
 307                     help=('Exclude files whose full path contains a match for '
 308                           'the given regular expression.  Does not apply to '
 309                           'filenames given as arguments.'))
 310   return parser
 311
 312
 313 def main():
 314   options, args = CreateOptionParser().parse_args()
 315   if len(args) < 1:
 316     Die('At least one top-level source file must be specified.')
 317   will_output_source_text = options.mode in ('bundle', 'compressed_bundle')
 318   path_rewriter = PathRewriter(options.prefix_map)
 319   exclude = [re.compile(r) for r in options.exclude]
 320   sources = ReadSources(options.roots, args, will_output_source_text,
 321                         path_rewriter, exclude)
 322   if will_output_source_text:
 323     _MarkAsCompiled(sources)
 324   bundle = Bundle()
 325   if len(options.roots) > 0:
 326     CalcDeps(bundle, sources, args)
 327   bundle.Add((sources[name] for name in args))
 328   if options.mode == 'copy':
 329     if options.dest_dir is None:
 330       Die('Must specify --dest_dir when copying.')
 331     LinkOrCopyFiles(bundle.GetSources(), options.dest_dir)
 332   else:
 333     if options.output_file:
 334       out_file = open(options.output_file, 'w')
 335     else:
 336       out_file = sys.stdout
 337     try:
 338       WriteOutput(bundle, options.mode, out_file, options.dest_dir)
 339     finally:
 340       if options.output_file:
 341         out_file.close()
 342
 343
 344 if __name__ == '__main__':
 345   main()