third_party/closure_compiler/compile.py

   1 #!/usr/bin/python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs Closure compiler on JavaScript files to check for errors and produce
   7 minified output."""
   8
   9 import argparse
  10 import os
  11 import re
  12 import subprocess
  13 import sys
  14 import tempfile
  15
  16 import build.inputs
  17 import processor
  18 import error_filter
  19
  20
  21 _CURRENT_DIR = os.path.join(os.path.dirname(__file__))
  22
  23
  24 class Checker(object):
  25   """Runs the Closure compiler on given source files to typecheck them
  26   and produce minified output."""
  27
  28   _JAR_COMMAND = [
  29     "java",
  30     "-jar",
  31     "-Xms1024m",
  32     "-client",
  33     "-XX:+TieredCompilation"
  34   ]
  35
  36   _MAP_FILE_FORMAT = "%s.map"
  37
  38   def __init__(self, verbose=False, strict=False):
  39     """
  40     Args:
  41       verbose: Whether this class should output diagnostic messages.
  42       strict: Whether the Closure Compiler should be invoked more strictly.
  43     """
  44     self._runner_jar = os.path.join(_CURRENT_DIR, "runner", "runner.jar")
  45     self._temp_files = []
  46     self._verbose = verbose
  47     self._strict = strict
  48     self._error_filter = error_filter.PromiseErrorFilter()
  49
  50   def _nuke_temp_files(self):
  51     """Deletes any temp files this class knows about."""
  52     if not self._temp_files:
  53       return
  54
  55     self._log_debug("Deleting temp files: %s" % ", ".join(self._temp_files))
  56     for f in self._temp_files:
  57       os.remove(f)
  58     self._temp_files = []
  59
  60   def _log_debug(self, msg, error=False):
  61     """Logs |msg| to stdout if --verbose/-v is passed when invoking this script.
  62
  63     Args:
  64       msg: A debug message to log.
  65     """
  66     if self._verbose:
  67       print "(INFO) %s" % msg
  68
  69   def _log_error(self, msg):
  70     """Logs |msg| to stderr regardless of --flags.
  71
  72     Args:
  73       msg: An error message to log.
  74     """
  75     print >> sys.stderr, "(ERROR) %s" % msg
  76
  77   def _run_jar(self, jar, args):
  78     """Runs a .jar from the command line with arguments.
  79
  80     Args:
  81       jar: A file path to a .jar file
  82       args: A list of command line arguments to be passed when running the .jar.
  83
  84     Return:
  85       (exit_code, stderr) The exit code of the command (e.g. 0 for success) and
  86           the stderr collected while running |jar| (as a string).
  87     """
  88     shell_command = " ".join(self._JAR_COMMAND + [jar] + args)
  89     self._log_debug("Running jar: %s" % shell_command)
  90
  91     devnull = open(os.devnull, "w")
  92     kwargs = {"stdout": devnull, "stderr": subprocess.PIPE, "shell": True}
  93     process = subprocess.Popen(shell_command, **kwargs)
  94     _, stderr = process.communicate()
  95     return process.returncode, stderr
  96
  97   def _get_line_number(self, match):
  98     """When chrome is built, it preprocesses its JavaScript from:
  99
 100       <include src="blah.js">
 101       alert(1);
 102
 103     to:
 104
 105       /* contents of blah.js inlined */
 106       alert(1);
 107
 108     Because Closure Compiler requires this inlining already be done (as
 109     <include> isn't valid JavaScript), this script creates temporary files to
 110     expand all the <include>s.
 111
 112     When type errors are hit in temporary files, a developer doesn't know the
 113     original source location to fix. This method maps from /tmp/file:300 back to
 114     /original/source/file:100 so fixing errors is faster for developers.
 115
 116     Args:
 117       match: A re.MatchObject from matching against a line number regex.
 118
 119     Returns:
 120       The fixed up /file and :line number.
 121     """
 122     real_file = self._processor.get_file_from_line(match.group(1))
 123     return "%s:%d" % (os.path.abspath(real_file.file), real_file.line_number)
 124
 125   def _filter_errors(self, errors):
 126     """Removes some extraneous errors. For example, we ignore:
 127
 128       Variable x first declared in /tmp/expanded/file
 129
 130     Because it's just a duplicated error (it'll only ever show up 2+ times).
 131     We also ignore Promose-based errors:
 132
 133       found   : function (VolumeInfo): (Promise<(DirectoryEntry|null)>|null)
 134       required: (function (Promise<VolumeInfo>): ?|null|undefined)
 135
 136     as templates don't work with Promises in all cases yet. See
 137     https://github.com/google/closure-compiler/issues/715 for details.
 138
 139     Args:
 140       errors: A list of string errors extracted from Closure Compiler output.
 141
 142     Return:
 143       A slimmer, sleeker list of relevant errors (strings).
 144     """
 145     first_declared_in = lambda e: " first declared in " not in e
 146     return self._error_filter.filter(filter(first_declared_in, errors))
 147
 148   def _clean_up_error(self, error):
 149     """Reverse the effects that funky <include> preprocessing steps have on
 150     errors messages.
 151
 152     Args:
 153       error: A Closure compiler error (2 line string with error and source).
 154
 155     Return:
 156       The fixed up error string.
 157     """
 158     expanded_file = self._expanded_file
 159     fixed = re.sub("%s:(\d+)" % expanded_file, self._get_line_number, error)
 160     return fixed.replace(expanded_file, os.path.abspath(self._file_arg))
 161
 162   def _format_errors(self, errors):
 163     """Formats Closure compiler errors to easily spot compiler output.
 164
 165     Args:
 166       errors: A list of strings extracted from the Closure compiler's output.
 167
 168     Returns:
 169       A formatted output string.
 170     """
 171     contents = "\n## ".join("\n\n".join(errors).splitlines())
 172     return "## %s" % contents if contents else ""
 173
 174   def _create_temp_file(self, contents):
 175     """Creates an owned temporary file with |contents|.
 176
 177     Args:
 178       content: A string of the file contens to write to a temporary file.
 179
 180     Return:
 181       The filepath of the newly created, written, and closed temporary file.
 182     """
 183     with tempfile.NamedTemporaryFile(mode="wt", delete=False) as tmp_file:
 184       self._temp_files.append(tmp_file.name)
 185       tmp_file.write(contents)
 186     return tmp_file.name
 187
 188   def _run_js_check(self, sources, out_file=None, externs=None,
 189                     closure_args=None):
 190     """Check |sources| for type errors.
 191
 192     Args:
 193       sources: Files to check.
 194       out_file: A file where the compiled output is written to.
 195       externs: @extern files that inform the compiler about custom globals.
 196       closure_args: Arguments passed directly to the Closure compiler.
 197
 198     Returns:
 199       (errors, stderr) A parsed list of errors (strings) found by the compiler
 200           and the raw stderr (as a string).
 201     """
 202     args = ["--js=%s" % s for s in sources]
 203
 204     if out_file:
 205       out_dir = os.path.dirname(out_file)
 206       if not os.path.exists(out_dir):
 207         os.makedirs(out_dir)
 208       args += ["--js_output_file=%s" % out_file]
 209       args += ["--create_source_map=%s" % (self._MAP_FILE_FORMAT % out_file)]
 210
 211     if externs:
 212       args += ["--externs=%s" % e for e in externs]
 213
 214     if closure_args:
 215       args += ["--%s" % arg for arg in closure_args]
 216
 217     args_file_content = " %s" % " ".join(args)
 218     self._log_debug("Args: %s" % args_file_content.strip())
 219
 220     args_file = self._create_temp_file(args_file_content)
 221     self._log_debug("Args file: %s" % args_file)
 222
 223     runner_args = ["--compiler-args-file=%s" % args_file]
 224     _, stderr = self._run_jar(self._runner_jar, runner_args)
 225
 226     errors = stderr.strip().split("\n\n")
 227     maybe_summary = errors.pop()
 228
 229     if re.search(".*error.*warning.*typed", maybe_summary):
 230       self._log_debug("Summary: %s" % maybe_summary)
 231     else:
 232       # Not a summary. Running the jar failed. Bail.
 233       self._log_error(stderr)
 234       self._nuke_temp_files()
 235       sys.exit(1)
 236
 237     if errors and out_file:
 238       if os.path.exists(out_file):
 239         os.remove(out_file)
 240       if os.path.exists(self._MAP_FILE_FORMAT % out_file):
 241         os.remove(self._MAP_FILE_FORMAT % out_file)
 242
 243     return errors, stderr
 244
 245   def check(self, source_file, out_file=None, depends=None, externs=None,
 246             closure_args=None):
 247     """Closure compiler |source_file| while checking for errors.
 248
 249     Args:
 250       source_file: A file to check.
 251       out_file: A file where the compiled output is written to.
 252       depends: Files that |source_file| requires to run (e.g. earlier <script>).
 253       externs: @extern files that inform the compiler about custom globals.
 254       closure_args: Arguments passed directly to the Closure compiler.
 255
 256     Returns:
 257       (found_errors, stderr) A boolean indicating whether errors were found and
 258           the raw Closure compiler stderr (as a string).
 259     """
 260     self._log_debug("FILE: %s" % source_file)
 261
 262     if source_file.endswith("_externs.js"):
 263       self._log_debug("Skipping externs: %s" % source_file)
 264       return
 265
 266     self._file_arg = source_file
 267
 268     cwd, tmp_dir = os.getcwd(), tempfile.gettempdir()
 269     rel_path = lambda f: os.path.join(os.path.relpath(cwd, tmp_dir), f)
 270
 271     depends = depends or []
 272     includes = [rel_path(f) for f in depends + [source_file]]
 273     contents = ['<include src="%s">' % i for i in includes]
 274     meta_file = self._create_temp_file("\n".join(contents))
 275     self._log_debug("Meta file: %s" % meta_file)
 276
 277     self._processor = processor.Processor(meta_file)
 278     self._expanded_file = self._create_temp_file(self._processor.contents)
 279     self._log_debug("Expanded file: %s" % self._expanded_file)
 280
 281     errors, stderr = self._run_js_check([self._expanded_file],
 282                                         out_file=out_file, externs=externs,
 283                                         closure_args=closure_args)
 284     filtered_errors = self._filter_errors(errors)
 285     cleaned_errors = map(self._clean_up_error, filtered_errors)
 286     output = self._format_errors(cleaned_errors)
 287
 288     if cleaned_errors:
 289       prefix = "\n" if output else ""
 290       self._log_error("Error in: %s%s%s" % (source_file, prefix, output))
 291     elif output:
 292       self._log_debug("Output: %s" % output)
 293
 294     self._nuke_temp_files()
 295     return bool(cleaned_errors), stderr
 296
 297   def check_multiple(self, sources, out_file=None, externs=None,
 298                      closure_args=None):
 299     """Closure compile a set of files and check for errors.
 300
 301     Args:
 302       sources: An array of files to check.
 303       out_file: A file where the compiled output is written to.
 304       externs: @extern files that inform the compiler about custom globals.
 305       closure_args: Arguments passed directly to the Closure compiler.
 306
 307     Returns:
 308       (found_errors, stderr) A boolean indicating whether errors were found and
 309           the raw Closure Compiler stderr (as a string).
 310     """
 311     errors, stderr = self._run_js_check(sources, out_file=out_file,
 312                                         externs=externs,
 313                                         closure_args=closure_args)
 314     self._nuke_temp_files()
 315     return bool(errors), stderr
 316
 317
 318 if __name__ == "__main__":
 319   parser = argparse.ArgumentParser(
 320       description="Typecheck JavaScript using Closure compiler")
 321   parser.add_argument("sources", nargs=argparse.ONE_OR_MORE,
 322                       help="Path to a source file to typecheck")
 323   single_file_group = parser.add_mutually_exclusive_group()
 324   single_file_group.add_argument("--single-file", dest="single_file",
 325                                  action="store_true",
 326                                  help="Process each source file individually")
 327   # TODO(twellington): remove --no-single-file and use len(opts.sources).
 328   single_file_group.add_argument("--no-single-file", dest="single_file",
 329                                  action="store_false",
 330                                  help="Process all source files as a group")
 331   parser.add_argument("-d", "--depends", nargs=argparse.ZERO_OR_MORE)
 332   parser.add_argument("-e", "--externs", nargs=argparse.ZERO_OR_MORE)
 333   parser.add_argument("-o", "--out-file", dest="out_file",
 334                       help="A file where the compiled output is written to")
 335   parser.add_argument("-c", "--closure-args", dest="closure_args",
 336                       nargs=argparse.ZERO_OR_MORE,
 337                       help="Arguments passed directly to the Closure compiler")
 338   parser.add_argument("-v", "--verbose", action="store_true",
 339                       help="Show more information as this script runs")
 340
 341   parser.set_defaults(single_file=True, strict=False)
 342   opts = parser.parse_args()
 343
 344   depends = opts.depends or []
 345   externs = set(opts.externs or [])
 346
 347   polymer_externs = os.path.join(os.path.dirname(_CURRENT_DIR), 'polymer',
 348                                  'v1_0', 'components-chromium',
 349                                  'polymer-externs', 'polymer.externs.js')
 350   externs.add(polymer_externs)
 351
 352   checker = Checker(verbose=opts.verbose, strict=opts.strict)
 353   if opts.single_file:
 354     for source in opts.sources:
 355       # Normalize source to the current directory.
 356       source = os.path.normpath(os.path.join(os.getcwd(), source))
 357       depends, externs = build.inputs.resolve_recursive_dependencies(
 358           source, depends, externs)
 359
 360       found_errors, _ = checker.check(source, out_file=opts.out_file,
 361                                       depends=depends, externs=externs,
 362                                       closure_args=opts.closure_args)
 363       if found_errors:
 364         sys.exit(1)
 365   else:
 366     found_errors, stderr = checker.check_multiple(
 367         opts.sources,
 368         out_file=opts.out_file,
 369         externs=externs,
 370         closure_args=opts.closure_args)
 371     if found_errors:
 372       print stderr
 373       sys.exit(1)