dom/canvas/test/webgl-conf/checkout/closure-library/closure/bin/calcdeps.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2006 The Closure Library Authors. All Rights Reserved.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS-IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17
  18 """Calculates JavaScript dependencies without requiring Google's build system.
  19
  20 This tool is deprecated and is provided for legacy users.
  21 See build/closurebuilder.py and build/depswriter.py for the current tools.
  22
  23 It iterates over a number of search paths and builds a dependency tree.  With
  24 the inputs provided, it walks the dependency tree and outputs all the files
  25 required for compilation.
  26 """
  27
  28
  29
  30
  31
  32 try:
  33   import distutils.version
  34 except ImportError:
  35   # distutils is not available in all environments
  36   distutils = None
  37
  38 import logging
  39 import optparse
  40 import os
  41 import re
  42 import subprocess
  43 import sys
  44
  45
  46 _BASE_REGEX_STRING = '^\s*goog\.%s\(\s*[\'"](.+)[\'"]\s*\)'
  47 req_regex = re.compile(_BASE_REGEX_STRING % 'require')
  48 prov_regex = re.compile(_BASE_REGEX_STRING % 'provide')
  49 ns_regex = re.compile('^ns:((\w+\.)*(\w+))$')
  50 version_regex = re.compile('[\.0-9]+')
  51
  52
  53 def IsValidFile(ref):
  54   """Returns true if the provided reference is a file and exists."""
  55   return os.path.isfile(ref)
  56
  57
  58 def IsJsFile(ref):
  59   """Returns true if the provided reference is a Javascript file."""
  60   return ref.endswith('.js')
  61
  62
  63 def IsNamespace(ref):
  64   """Returns true if the provided reference is a namespace."""
  65   return re.match(ns_regex, ref) is not None
  66
  67
  68 def IsDirectory(ref):
  69   """Returns true if the provided reference is a directory."""
  70   return os.path.isdir(ref)
  71
  72
  73 def ExpandDirectories(refs):
  74   """Expands any directory references into inputs.
  75
  76   Description:
  77     Looks for any directories in the provided references.  Found directories
  78     are recursively searched for .js files, which are then added to the result
  79     list.
  80
  81   Args:
  82     refs: a list of references such as files, directories, and namespaces
  83
  84   Returns:
  85     A list of references with directories removed and replaced by any
  86     .js files that are found in them. Also, the paths will be normalized.
  87   """
  88   result = []
  89   for ref in refs:
  90     if IsDirectory(ref):
  91       # Disable 'Unused variable' for subdirs
  92       # pylint: disable=unused-variable
  93       for (directory, subdirs, filenames) in os.walk(ref):
  94         for filename in filenames:
  95           if IsJsFile(filename):
  96             result.append(os.path.join(directory, filename))
  97     else:
  98       result.append(ref)
  99   return map(os.path.normpath, result)
 100
 101
 102 class DependencyInfo(object):
 103   """Represents a dependency that is used to build and walk a tree."""
 104
 105   def __init__(self, filename):
 106     self.filename = filename
 107     self.provides = []
 108     self.requires = []
 109
 110   def __str__(self):
 111     return '%s Provides: %s Requires: %s' % (self.filename,
 112                                              repr(self.provides),
 113                                              repr(self.requires))
 114
 115
 116 def BuildDependenciesFromFiles(files):
 117   """Build a list of dependencies from a list of files.
 118
 119   Description:
 120     Takes a list of files, extracts their provides and requires, and builds
 121     out a list of dependency objects.
 122
 123   Args:
 124     files: a list of files to be parsed for goog.provides and goog.requires.
 125
 126   Returns:
 127     A list of dependency objects, one for each file in the files argument.
 128   """
 129   result = []
 130   filenames = set()
 131   for filename in files:
 132     if filename in filenames:
 133       continue
 134
 135     # Python 3 requires the file encoding to be specified
 136     if (sys.version_info[0] < 3):
 137       file_handle = open(filename, 'r')
 138     else:
 139       file_handle = open(filename, 'r', encoding='utf8')
 140
 141     try:
 142       dep = CreateDependencyInfo(filename, file_handle)
 143       result.append(dep)
 144     finally:
 145       file_handle.close()
 146
 147     filenames.add(filename)
 148
 149   return result
 150
 151
 152 def CreateDependencyInfo(filename, source):
 153   """Create dependency info.
 154
 155   Args:
 156     filename: Filename for source.
 157     source: File-like object containing source.
 158
 159   Returns:
 160     A DependencyInfo object with provides and requires filled.
 161   """
 162   dep = DependencyInfo(filename)
 163   for line in source:
 164     if re.match(req_regex, line):
 165       dep.requires.append(re.search(req_regex, line).group(1))
 166     if re.match(prov_regex, line):
 167       dep.provides.append(re.search(prov_regex, line).group(1))
 168   return dep
 169
 170
 171 def BuildDependencyHashFromDependencies(deps):
 172   """Builds a hash for searching dependencies by the namespaces they provide.
 173
 174   Description:
 175     Dependency objects can provide multiple namespaces.  This method enumerates
 176     the provides of each dependency and adds them to a hash that can be used
 177     to easily resolve a given dependency by a namespace it provides.
 178
 179   Args:
 180     deps: a list of dependency objects used to build the hash.
 181
 182   Raises:
 183     Exception: If a multiple files try to provide the same namepace.
 184
 185   Returns:
 186     A hash table { namespace: dependency } that can be used to resolve a
 187     dependency by a namespace it provides.
 188   """
 189   dep_hash = {}
 190   for dep in deps:
 191     for provide in dep.provides:
 192       if provide in dep_hash:
 193         raise Exception('Duplicate provide (%s) in (%s, %s)' % (
 194             provide,
 195             dep_hash[provide].filename,
 196             dep.filename))
 197       dep_hash[provide] = dep
 198   return dep_hash
 199
 200
 201 def CalculateDependencies(paths, inputs):
 202   """Calculates the dependencies for given inputs.
 203
 204   Description:
 205     This method takes a list of paths (files, directories) and builds a
 206     searchable data structure based on the namespaces that each .js file
 207     provides.  It then parses through each input, resolving dependencies
 208     against this data structure.  The final output is a list of files,
 209     including the inputs, that represent all of the code that is needed to
 210     compile the given inputs.
 211
 212   Args:
 213     paths: the references (files, directories) that are used to build the
 214       dependency hash.
 215     inputs: the inputs (files, directories, namespaces) that have dependencies
 216       that need to be calculated.
 217
 218   Raises:
 219     Exception: if a provided input is invalid.
 220
 221   Returns:
 222     A list of all files, including inputs, that are needed to compile the given
 223     inputs.
 224   """
 225   deps = BuildDependenciesFromFiles(paths + inputs)
 226   search_hash = BuildDependencyHashFromDependencies(deps)
 227   result_list = []
 228   seen_list = []
 229   for input_file in inputs:
 230     if IsNamespace(input_file):
 231       namespace = re.search(ns_regex, input_file).group(1)
 232       if namespace not in search_hash:
 233         raise Exception('Invalid namespace (%s)' % namespace)
 234       input_file = search_hash[namespace].filename
 235     if not IsValidFile(input_file) or not IsJsFile(input_file):
 236       raise Exception('Invalid file (%s)' % input_file)
 237     seen_list.append(input_file)
 238     file_handle = open(input_file, 'r')
 239     try:
 240       for line in file_handle:
 241         if re.match(req_regex, line):
 242           require = re.search(req_regex, line).group(1)
 243           ResolveDependencies(require, search_hash, result_list, seen_list)
 244     finally:
 245       file_handle.close()
 246     result_list.append(input_file)
 247
 248   # All files depend on base.js, so put it first.
 249   base_js_path = FindClosureBasePath(paths)
 250   if base_js_path:
 251     result_list.insert(0, base_js_path)
 252   else:
 253     logging.warning('Closure Library base.js not found.')
 254
 255   return result_list
 256
 257
 258 def FindClosureBasePath(paths):
 259   """Given a list of file paths, return Closure base.js path, if any.
 260
 261   Args:
 262     paths: A list of paths.
 263
 264   Returns:
 265     The path to Closure's base.js file including filename, if found.
 266   """
 267
 268   for path in paths:
 269     pathname, filename = os.path.split(path)
 270
 271     if filename == 'base.js':
 272       f = open(path)
 273
 274       is_base = False
 275
 276       # Sanity check that this is the Closure base file.  Check that this
 277       # is where goog is defined.  This is determined by the @provideGoog
 278       # flag.
 279       for line in f:
 280         if '@provideGoog' in line:
 281           is_base = True
 282           break
 283
 284       f.close()
 285
 286       if is_base:
 287         return path
 288
 289 def ResolveDependencies(require, search_hash, result_list, seen_list):
 290   """Takes a given requirement and resolves all of the dependencies for it.
 291
 292   Description:
 293     A given requirement may require other dependencies.  This method
 294     recursively resolves all dependencies for the given requirement.
 295
 296   Raises:
 297     Exception: when require does not exist in the search_hash.
 298
 299   Args:
 300     require: the namespace to resolve dependencies for.
 301     search_hash: the data structure used for resolving dependencies.
 302     result_list: a list of filenames that have been calculated as dependencies.
 303       This variable is the output for this function.
 304     seen_list: a list of filenames that have been 'seen'.  This is required
 305       for the dependency->dependant ordering.
 306   """
 307   if require not in search_hash:
 308     raise Exception('Missing provider for (%s)' % require)
 309
 310   dep = search_hash[require]
 311   if not dep.filename in seen_list:
 312     seen_list.append(dep.filename)
 313     for sub_require in dep.requires:
 314       ResolveDependencies(sub_require, search_hash, result_list, seen_list)
 315     result_list.append(dep.filename)
 316
 317
 318 def GetDepsLine(dep, base_path):
 319   """Returns a JS string for a dependency statement in the deps.js file.
 320
 321   Args:
 322     dep: The dependency that we're printing.
 323     base_path: The path to Closure's base.js including filename.
 324   """
 325   return 'goog.addDependency("%s", %s, %s);' % (
 326       GetRelpath(dep.filename, base_path), dep.provides, dep.requires)
 327
 328
 329 def GetRelpath(path, start):
 330   """Return a relative path to |path| from |start|."""
 331   # NOTE: Python 2.6 provides os.path.relpath, which has almost the same
 332   # functionality as this function. Since we want to support 2.4, we have
 333   # to implement it manually. :(
 334   path_list = os.path.abspath(os.path.normpath(path)).split(os.sep)
 335   start_list = os.path.abspath(
 336       os.path.normpath(os.path.dirname(start))).split(os.sep)
 337
 338   common_prefix_count = 0
 339   for i in range(0, min(len(path_list), len(start_list))):
 340     if path_list[i] != start_list[i]:
 341       break
 342     common_prefix_count += 1
 343
 344   # Always use forward slashes, because this will get expanded to a url,
 345   # not a file path.
 346   return '/'.join(['..'] * (len(start_list) - common_prefix_count) +
 347                   path_list[common_prefix_count:])
 348
 349
 350 def PrintLine(msg, out):
 351   out.write(msg)
 352   out.write('\n')
 353
 354
 355 def PrintDeps(source_paths, deps, out):
 356   """Print out a deps.js file from a list of source paths.
 357
 358   Args:
 359     source_paths: Paths that we should generate dependency info for.
 360     deps: Paths that provide dependency info. Their dependency info should
 361         not appear in the deps file.
 362     out: The output file.
 363
 364   Returns:
 365     True on success, false if it was unable to find the base path
 366     to generate deps relative to.
 367   """
 368   base_path = FindClosureBasePath(source_paths + deps)
 369   if not base_path:
 370     return False
 371
 372   PrintLine('// This file was autogenerated by calcdeps.py', out)
 373   excludesSet = set(deps)
 374
 375   for dep in BuildDependenciesFromFiles(source_paths + deps):
 376     if not dep.filename in excludesSet:
 377       PrintLine(GetDepsLine(dep, base_path), out)
 378
 379   return True
 380
 381
 382 def PrintScript(source_paths, out):
 383   for index, dep in enumerate(source_paths):
 384     PrintLine('// Input %d' % index, out)
 385     f = open(dep, 'r')
 386     PrintLine(f.read(), out)
 387     f.close()
 388
 389
 390 def GetJavaVersion():
 391   """Returns the string for the current version of Java installed."""
 392   proc = subprocess.Popen(['java', '-version'], stderr=subprocess.PIPE)
 393   proc.wait()
 394   version_line = proc.stderr.read().splitlines()[0]
 395   return version_regex.search(version_line).group()
 396
 397
 398 def FilterByExcludes(options, files):
 399   """Filters the given files by the exlusions specified at the command line.
 400
 401   Args:
 402     options: The flags to calcdeps.
 403     files: The files to filter.
 404   Returns:
 405     A list of files.
 406   """
 407   excludes = []
 408   if options.excludes:
 409     excludes = ExpandDirectories(options.excludes)
 410
 411   excludesSet = set(excludes)
 412   return [i for i in files if not i in excludesSet]
 413
 414
 415 def GetPathsFromOptions(options):
 416   """Generates the path files from flag options.
 417
 418   Args:
 419     options: The flags to calcdeps.
 420   Returns:
 421     A list of files in the specified paths. (strings).
 422   """
 423
 424   search_paths = options.paths
 425   if not search_paths:
 426     search_paths = ['.']  # Add default folder if no path is specified.
 427
 428   search_paths = ExpandDirectories(search_paths)
 429   return FilterByExcludes(options, search_paths)
 430
 431
 432 def GetInputsFromOptions(options):
 433   """Generates the inputs from flag options.
 434
 435   Args:
 436     options: The flags to calcdeps.
 437   Returns:
 438     A list of inputs (strings).
 439   """
 440   inputs = options.inputs
 441   if not inputs:  # Parse stdin
 442     logging.info('No inputs specified. Reading from stdin...')
 443     inputs = filter(None, [line.strip('\n') for line in sys.stdin.readlines()])
 444
 445   logging.info('Scanning files...')
 446   inputs = ExpandDirectories(inputs)
 447
 448   return FilterByExcludes(options, inputs)
 449
 450
 451 def Compile(compiler_jar_path, source_paths, out, flags=None):
 452   """Prepares command-line call to Closure compiler.
 453
 454   Args:
 455     compiler_jar_path: Path to the Closure compiler .jar file.
 456     source_paths: Source paths to build, in order.
 457     flags: A list of additional flags to pass on to Closure compiler.
 458   """
 459   args = ['java', '-jar', compiler_jar_path]
 460   for path in source_paths:
 461     args += ['--js', path]
 462
 463   if flags:
 464     args += flags
 465
 466   logging.info('Compiling with the following command: %s', ' '.join(args))
 467   proc = subprocess.Popen(args, stdout=subprocess.PIPE)
 468   (stdoutdata, stderrdata) = proc.communicate()
 469   if proc.returncode != 0:
 470     logging.error('JavaScript compilation failed.')
 471     sys.exit(1)
 472   else:
 473     out.write(stdoutdata)
 474
 475
 476 def main():
 477   """The entrypoint for this script."""
 478
 479   logging.basicConfig(format='calcdeps.py: %(message)s', level=logging.INFO)
 480
 481   usage = 'usage: %prog [options] arg'
 482   parser = optparse.OptionParser(usage)
 483   parser.add_option('-i',
 484                     '--input',
 485                     dest='inputs',
 486                     action='append',
 487                     help='The inputs to calculate dependencies for. Valid '
 488                     'values can be files, directories, or namespaces '
 489                     '(ns:goog.net.XhrIo).  Only relevant to "list" and '
 490                     '"script" output.')
 491   parser.add_option('-p',
 492                     '--path',
 493                     dest='paths',
 494                     action='append',
 495                     help='The paths that should be traversed to build the '
 496                     'dependencies.')
 497   parser.add_option('-d',
 498                     '--dep',
 499                     dest='deps',
 500                     action='append',
 501                     help='Directories or files that should be traversed to '
 502                     'find required dependencies for the deps file. '
 503                     'Does not generate dependency information for names '
 504                     'provided by these files. Only useful in "deps" mode.')
 505   parser.add_option('-e',
 506                     '--exclude',
 507                     dest='excludes',
 508                     action='append',
 509                     help='Files or directories to exclude from the --path '
 510                     'and --input flags')
 511   parser.add_option('-o',
 512                     '--output_mode',
 513                     dest='output_mode',
 514                     action='store',
 515                     default='list',
 516                     help='The type of output to generate from this script. '
 517                     'Options are "list" for a list of filenames, "script" '
 518                     'for a single script containing the contents of all the '
 519                     'file, "deps" to generate a deps.js file for all '
 520                     'paths, or "compiled" to produce compiled output with '
 521                     'the Closure compiler.')
 522   parser.add_option('-c',
 523                     '--compiler_jar',
 524                     dest='compiler_jar',
 525                     action='store',
 526                     help='The location of the Closure compiler .jar file.')
 527   parser.add_option('-f',
 528                     '--compiler_flag',
 529                     '--compiler_flags', # for backwards compatability
 530                     dest='compiler_flags',
 531                     action='append',
 532                     help='Additional flag to pass to the Closure compiler. '
 533                     'May be specified multiple times to pass multiple flags.')
 534   parser.add_option('--output_file',
 535                     dest='output_file',
 536                     action='store',
 537                     help=('If specified, write output to this path instead of '
 538                           'writing to standard output.'))
 539
 540   (options, args) = parser.parse_args()
 541
 542   search_paths = GetPathsFromOptions(options)
 543
 544   if options.output_file:
 545     out = open(options.output_file, 'w')
 546   else:
 547     out = sys.stdout
 548
 549   if options.output_mode == 'deps':
 550     result = PrintDeps(search_paths, ExpandDirectories(options.deps or []), out)
 551     if not result:
 552       logging.error('Could not find Closure Library in the specified paths')
 553       sys.exit(1)
 554
 555     return
 556
 557   inputs = GetInputsFromOptions(options)
 558
 559   logging.info('Finding Closure dependencies...')
 560   deps = CalculateDependencies(search_paths, inputs)
 561   output_mode = options.output_mode
 562
 563   if output_mode == 'script':
 564     PrintScript(deps, out)
 565   elif output_mode == 'list':
 566     # Just print out a dep per line
 567     for dep in deps:
 568       PrintLine(dep, out)
 569   elif output_mode == 'compiled':
 570     # Make sure a .jar is specified.
 571     if not options.compiler_jar:
 572       logging.error('--compiler_jar flag must be specified if --output is '
 573                     '"compiled"')
 574       sys.exit(1)
 575
 576     # User friendly version check.
 577     if distutils and not (distutils.version.LooseVersion(GetJavaVersion()) >
 578         distutils.version.LooseVersion('1.6')):
 579       logging.error('Closure Compiler requires Java 1.6 or higher.')
 580       logging.error('Please visit http://www.java.com/getjava')
 581       sys.exit(1)
 582
 583     Compile(options.compiler_jar, deps, out, options.compiler_flags)
 584
 585   else:
 586     logging.error('Invalid value for --output flag.')
 587     sys.exit(1)
 588
 589 if __name__ == '__main__':
 590   main()