third_party/closure_compiler/compile2.py

   1 #!/usr/bin/python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs Closure compiler on JavaScript files to check for errors and produce
   7 minified output."""
   8
   9 import argparse
  10 import os
  11 import re
  12 import subprocess
  13 import sys
  14 import tempfile
  15
  16 import build.inputs
  17 import processor
  18 import error_filter
  19
  20
  21 _CURRENT_DIR = os.path.join(os.path.dirname(__file__))
  22
  23
  24 class Checker(object):
  25   """Runs the Closure compiler on given source files to typecheck them
  26   and produce minified output."""
  27
  28   _JAR_COMMAND = [
  29     "java",
  30     "-jar",
  31     "-Xms1024m",
  32     "-client",
  33     "-XX:+TieredCompilation"
  34   ]
  35
  36   _MAP_FILE_FORMAT = "%s.map"
  37
  38   _POLYMER_EXTERNS = os.path.join(_CURRENT_DIR, "..", "polymer",
  39                                   "v1_0", "components-chromium",
  40                                   "polymer-externs", "polymer.externs.js")
  41
  42   def __init__(self, verbose=False):
  43     """
  44     Args:
  45       verbose: Whether this class should output diagnostic messages.
  46       strict: Whether the Closure Compiler should be invoked more strictly.
  47     """
  48     self._runner_jar = os.path.join(_CURRENT_DIR, "runner", "runner.jar")
  49     self._target = None
  50     self._temp_files = []
  51     self._verbose = verbose
  52     self._error_filter = error_filter.PromiseErrorFilter()
  53
  54   def _nuke_temp_files(self):
  55     """Deletes any temp files this class knows about."""
  56     if not self._temp_files:
  57       return
  58
  59     self._log_debug("Deleting temp files: %s" % ", ".join(self._temp_files))
  60     for f in self._temp_files:
  61       os.remove(f)
  62     self._temp_files = []
  63
  64   def _log_debug(self, msg, error=False):
  65     """Logs |msg| to stdout if --verbose/-v is passed when invoking this script.
  66
  67     Args:
  68       msg: A debug message to log.
  69     """
  70     if self._verbose:
  71       print "(INFO) %s" % msg
  72
  73   def _log_error(self, msg):
  74     """Logs |msg| to stderr regardless of --flags.
  75
  76     Args:
  77       msg: An error message to log.
  78     """
  79     print >> sys.stderr, "(ERROR) %s" % msg
  80
  81   def _run_jar(self, jar, args):
  82     """Runs a .jar from the command line with arguments.
  83
  84     Args:
  85       jar: A file path to a .jar file
  86       args: A list of command line arguments to be passed when running the .jar.
  87
  88     Return:
  89       (exit_code, stderr) The exit code of the command (e.g. 0 for success) and
  90           the stderr collected while running |jar| (as a string).
  91     """
  92     shell_command = " ".join(self._JAR_COMMAND + [jar] + args)
  93     self._log_debug("Running jar: %s" % shell_command)
  94
  95     devnull = open(os.devnull, "w")
  96     kwargs = {"stdout": devnull, "stderr": subprocess.PIPE, "shell": True}
  97     process = subprocess.Popen(shell_command, **kwargs)
  98     _, stderr = process.communicate()
  99     return process.returncode, stderr
 100
 101   def _get_line_number(self, match):
 102     """When chrome is built, it preprocesses its JavaScript from:
 103
 104       <include src="blah.js">
 105       alert(1);
 106
 107     to:
 108
 109       /* contents of blah.js inlined */
 110       alert(1);
 111
 112     Because Closure Compiler requires this inlining already be done (as
 113     <include> isn't valid JavaScript), this script creates temporary files to
 114     expand all the <include>s.
 115
 116     When type errors are hit in temporary files, a developer doesn't know the
 117     original source location to fix. This method maps from /tmp/file:300 back to
 118     /original/source/file:100 so fixing errors is faster for developers.
 119
 120     Args:
 121       match: A re.MatchObject from matching against a line number regex.
 122
 123     Returns:
 124       The fixed up /file and :line number.
 125     """
 126     real_file = self._processor.get_file_from_line(match.group(1))
 127     return "%s:%d" % (os.path.abspath(real_file.file), real_file.line_number)
 128
 129   def _filter_errors(self, errors):
 130     """Removes some extraneous errors. For example, we ignore:
 131
 132       Variable x first declared in /tmp/expanded/file
 133
 134     Because it's just a duplicated error (it'll only ever show up 2+ times).
 135     We also ignore Promise-based errors:
 136
 137       found   : function (VolumeInfo): (Promise<(DirectoryEntry|null)>|null)
 138       required: (function (Promise<VolumeInfo>): ?|null|undefined)
 139
 140     as templates don't work with Promises in all cases yet. See
 141     https://github.com/google/closure-compiler/issues/715 for details.
 142
 143     Args:
 144       errors: A list of string errors extracted from Closure Compiler output.
 145
 146     Return:
 147       A slimmer, sleeker list of relevant errors (strings).
 148     """
 149     first_declared_in = lambda e: " first declared in " not in e
 150     return self._error_filter.filter(filter(first_declared_in, errors))
 151
 152   def _clean_up_error(self, error):
 153     """Reverse the effects that funky <include> preprocessing steps have on
 154     errors messages.
 155
 156     Args:
 157       error: A Closure compiler error (2 line string with error and source).
 158
 159     Return:
 160       The fixed up error string.
 161     """
 162     assert self._target
 163     assert self._expanded_file
 164     expanded_file = self._expanded_file
 165     fixed = re.sub("%s:(\d+)" % expanded_file, self._get_line_number, error)
 166     return fixed.replace(expanded_file, os.path.abspath(self._target))
 167
 168   def _format_errors(self, errors):
 169     """Formats Closure compiler errors to easily spot compiler output.
 170
 171     Args:
 172       errors: A list of strings extracted from the Closure compiler's output.
 173
 174     Returns:
 175       A formatted output string.
 176     """
 177     contents = "\n## ".join("\n\n".join(errors).splitlines())
 178     return "## %s" % contents if contents else ""
 179
 180   def _create_temp_file(self, contents):
 181     """Creates an owned temporary file with |contents|.
 182
 183     Args:
 184       content: A string of the file contens to write to a temporary file.
 185
 186     Return:
 187       The filepath of the newly created, written, and closed temporary file.
 188     """
 189     with tempfile.NamedTemporaryFile(mode="wt", delete=False) as tmp_file:
 190       self._temp_files.append(tmp_file.name)
 191       tmp_file.write(contents)
 192     return tmp_file.name
 193
 194   def check(self, sources, out_file=None, closure_args=None,
 195             custom_sources=True):
 196     """Closure compile |sources| while checking for errors.
 197
 198     Args:
 199       sources: Files to check. sources[0] is the typically the target file.
 200           sources[1:] are externs and dependencies in topological order. Order
 201           is not guaranteed if custom_sources is True.
 202       out_file: A file where the compiled output is written to.
 203       closure_args: Arguments passed directly to the Closure compiler.
 204       custom_sources: Whether |sources| was customized by the target (e.g. not
 205           in GYP dependency order).
 206
 207     Returns:
 208       (found_errors, stderr) A boolean indicating whether errors were found and
 209           the raw Closure compiler stderr (as a string).
 210     """
 211     is_extern = lambda f: 'extern' in f
 212     externs_and_deps = [self._POLYMER_EXTERNS]
 213
 214     if custom_sources:
 215       externs_and_deps += sources
 216     else:
 217       self._target = sources[0]
 218       externs_and_deps += sources[1:]
 219
 220     externs = filter(is_extern, externs_and_deps)
 221     deps = filter(lambda f: not is_extern(f), externs_and_deps)
 222
 223     assert externs or deps or self._target
 224
 225     self._log_debug("Externs: %s" % externs)
 226     self._log_debug("Dependencies: %s" % deps)
 227     self._log_debug("Target: %s" % self._target)
 228
 229     js_args = deps + [self._target] if self._target else []
 230
 231     if not custom_sources:
 232       # TODO(dbeam): compiler.jar automatically detects "@externs" in a --js arg
 233       # and moves these files to a different AST tree. However, because we use
 234       # one big funky <include> meta-file, it thinks all the code is one big
 235       # externs. Just use --js when <include> dies.
 236
 237       cwd, tmp_dir = os.getcwd(), tempfile.gettempdir()
 238       rel_path = lambda f: os.path.join(os.path.relpath(cwd, tmp_dir), f)
 239
 240       contents = ['<include src="%s">' % rel_path(f) for f in js_args]
 241       meta_file = self._create_temp_file("\n".join(contents))
 242       self._log_debug("Meta file: %s" % meta_file)
 243
 244       self._processor = processor.Processor(meta_file)
 245       self._expanded_file = self._create_temp_file(self._processor.contents)
 246       self._log_debug("Expanded file: %s" % self._expanded_file)
 247
 248       js_args = [self._expanded_file]
 249
 250     args = ["--externs=%s" % e for e in externs] + \
 251            ["--js=%s" % s for s in js_args] + \
 252            ["--%s" % arg for arg in closure_args or []]
 253
 254     if out_file:
 255       out_dir = os.path.dirname(out_file)
 256       if not os.path.exists(out_dir):
 257         os.makedirs(out_dir)
 258       args += ["--js_output_file=%s" % out_file]
 259       args += ["--create_source_map=%s" % (self._MAP_FILE_FORMAT % out_file)]
 260
 261     args_file_content = " %s" % " ".join(args)
 262     self._log_debug("Args: %s" % args_file_content.strip())
 263
 264     args_file = self._create_temp_file(args_file_content)
 265     self._log_debug("Args file: %s" % args_file)
 266
 267     runner_args = ["--compiler-args-file=%s" % args_file]
 268     _, stderr = self._run_jar(self._runner_jar, runner_args)
 269
 270     errors = stderr.strip().split("\n\n")
 271     maybe_summary = errors.pop()
 272
 273     if re.search(".*error.*warning.*typed", maybe_summary):
 274       self._log_debug("Summary: %s" % maybe_summary)
 275     else:
 276       # Not a summary. Running the jar failed. Bail.
 277       self._log_error(stderr)
 278       self._nuke_temp_files()
 279       sys.exit(1)
 280
 281     if errors and out_file:
 282       if os.path.exists(out_file):
 283         os.remove(out_file)
 284       if os.path.exists(self._MAP_FILE_FORMAT % out_file):
 285         os.remove(self._MAP_FILE_FORMAT % out_file)
 286
 287     if not custom_sources:
 288       filtered_errors = self._filter_errors(errors)
 289       errors = map(self._clean_up_error, filtered_errors)
 290       output = self._format_errors(errors)
 291
 292       if errors:
 293         prefix = "\n" if output else ""
 294         self._log_error("Error in: %s%s%s" % (self._target, prefix, output))
 295       elif output:
 296         self._log_debug("Output: %s" % output)
 297
 298     self._nuke_temp_files()
 299     return bool(errors), stderr
 300
 301
 302 if __name__ == "__main__":
 303   parser = argparse.ArgumentParser(
 304       description="Typecheck JavaScript using Closure compiler")
 305   parser.add_argument("sources", nargs=argparse.ONE_OR_MORE,
 306                       help="Path to a source file to typecheck")
 307   parser.add_argument("--custom_sources", action="store_true",
 308                       help="Whether this rules has custom sources.")
 309   parser.add_argument("-o", "--out_file",
 310                       help="A file where the compiled output is written to")
 311   parser.add_argument("-c", "--closure_args", nargs=argparse.ZERO_OR_MORE,
 312                       help="Arguments passed directly to the Closure compiler")
 313   parser.add_argument("-v", "--verbose", action="store_true",
 314                       help="Show more information as this script runs")
 315   opts = parser.parse_args()
 316
 317   checker = Checker(verbose=opts.verbose)
 318
 319   found_errors, stderr = checker.check(opts.sources, out_file=opts.out_file,
 320                                        closure_args=opts.closure_args,
 321                                        custom_sources=opts.custom_sources)
 322
 323   if found_errors:
 324     if opts.custom_sources:
 325       print stderr
 326     sys.exit(1)