third_party/closure_compiler/checker.py

   1 #!/usr/bin/python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs Closure compiler on a JavaScript file to check for errors."""
   7
   8 import argparse
   9 import os
  10 import re
  11 import subprocess
  12 import sys
  13 import tempfile
  14
  15 import build.inputs
  16 import processor
  17 import error_filter
  18
  19
  20 class Checker(object):
  21   """Runs the Closure compiler on a given source file to typecheck it."""
  22
  23   _COMMON_CLOSURE_ARGS = [
  24     "--accept_const_keyword",
  25     "--jscomp_error=accessControls",
  26     "--jscomp_error=ambiguousFunctionDecl",
  27     "--jscomp_error=checkStructDictInheritance",
  28     "--jscomp_error=checkTypes",
  29     "--jscomp_error=checkVars",
  30     "--jscomp_error=constantProperty",
  31     "--jscomp_error=deprecated",
  32     "--jscomp_error=externsValidation",
  33     "--jscomp_error=globalThis",
  34     "--jscomp_error=invalidCasts",
  35     "--jscomp_error=missingProperties",
  36     "--jscomp_error=missingReturn",
  37     "--jscomp_error=nonStandardJsDocs",
  38     "--jscomp_error=suspiciousCode",
  39     "--jscomp_error=undefinedNames",
  40     "--jscomp_error=undefinedVars",
  41     "--jscomp_error=unknownDefines",
  42     "--jscomp_error=uselessCode",
  43     "--jscomp_error=visibility",
  44     "--language_in=ECMASCRIPT5_STRICT",
  45     "--summary_detail_level=3",
  46     "--compilation_level=SIMPLE_OPTIMIZATIONS",
  47     "--source_map_format=V3",
  48   ]
  49
  50   # These are the extra flags used when compiling in strict mode.
  51   # Flags that are normally disabled are turned on for strict mode.
  52   _STRICT_CLOSURE_ARGS = [
  53     "--jscomp_error=reportUnknownTypes",
  54     "--jscomp_error=duplicate",
  55     "--jscomp_error=misplacedTypeAnnotation",
  56   ]
  57
  58   _DISABLED_CLOSURE_ARGS = [
  59     # TODO(dbeam): happens when the same file is <include>d multiple times.
  60     "--jscomp_off=duplicate",
  61     # TODO(fukino): happens when cr.defineProperty() has a type annotation.
  62     # Avoiding parse-time warnings needs 2 pass compiling. crbug.com/421562.
  63     "--jscomp_off=misplacedTypeAnnotation",
  64   ]
  65
  66   _JAR_COMMAND = [
  67     "java",
  68     "-jar",
  69     "-Xms1024m",
  70     "-client",
  71     "-XX:+TieredCompilation"
  72   ]
  73
  74   def __init__(self, verbose=False, strict=False):
  75     """
  76     Args:
  77       verbose: Whether this class should output diagnostic messages.
  78       strict: Whether the Closure Compiler should be invoked more strictly.
  79     """
  80     current_dir = os.path.join(os.path.dirname(__file__))
  81     self._runner_jar = os.path.join(current_dir, "runner", "runner.jar")
  82     self._temp_files = []
  83     self._verbose = verbose
  84     self._strict = strict
  85     self._error_filter = error_filter.PromiseErrorFilter()
  86
  87   def _clean_up(self):
  88     """Deletes any temp files this class knows about."""
  89     if not self._temp_files:
  90       return
  91
  92     self._log_debug("Deleting temp files: %s" % ", ".join(self._temp_files))
  93     for f in self._temp_files:
  94       os.remove(f)
  95     self._temp_files = []
  96
  97   def _log_debug(self, msg, error=False):
  98     """Logs |msg| to stdout if --verbose/-v is passed when invoking this script.
  99
 100     Args:
 101       msg: A debug message to log.
 102     """
 103     if self._verbose:
 104       print "(INFO) %s" % msg
 105
 106   def _log_error(self, msg):
 107     """Logs |msg| to stderr regardless of --flags.
 108
 109     Args:
 110       msg: An error message to log.
 111     """
 112     print >> sys.stderr, "(ERROR) %s" % msg
 113
 114   def _common_args(self):
 115     """Returns an array of the common closure compiler args."""
 116     if self._strict:
 117       return self._COMMON_CLOSURE_ARGS + self._STRICT_CLOSURE_ARGS
 118     return self._COMMON_CLOSURE_ARGS + self._DISABLED_CLOSURE_ARGS
 119
 120   def _run_jar(self, jar, args):
 121     """Runs a .jar from the command line with arguments.
 122
 123     Args:
 124       jar: A file path to a .jar file
 125       args: A list of command line arguments to be passed when running the .jar.
 126
 127     Return:
 128       (exit_code, stderr) The exit code of the command (e.g. 0 for success) and
 129           the stderr collected while running |jar| (as a string).
 130     """
 131     shell_command = " ".join(self._JAR_COMMAND + [jar] + args)
 132     self._log_debug("Running jar: %s" % shell_command)
 133
 134     devnull = open(os.devnull, "w")
 135     kwargs = {"stdout": devnull, "stderr": subprocess.PIPE, "shell": True}
 136     process = subprocess.Popen(shell_command, **kwargs)
 137     _, stderr = process.communicate()
 138     return process.returncode, stderr
 139
 140   def _get_line_number(self, match):
 141     """When chrome is built, it preprocesses its JavaScript from:
 142
 143       <include src="blah.js">
 144       alert(1);
 145
 146     to:
 147
 148       /* contents of blah.js inlined */
 149       alert(1);
 150
 151     Because Closure Compiler requires this inlining already be done (as
 152     <include> isn't valid JavaScript), this script creates temporary files to
 153     expand all the <include>s.
 154
 155     When type errors are hit in temporary files, a developer doesn't know the
 156     original source location to fix. This method maps from /tmp/file:300 back to
 157     /original/source/file:100 so fixing errors is faster for developers.
 158
 159     Args:
 160       match: A re.MatchObject from matching against a line number regex.
 161
 162     Returns:
 163       The fixed up /file and :line number.
 164     """
 165     real_file = self._processor.get_file_from_line(match.group(1))
 166     return "%s:%d" % (os.path.abspath(real_file.file), real_file.line_number)
 167
 168   def _filter_errors(self, errors):
 169     """Removes some extraneous errors. For example, we ignore:
 170
 171       Variable x first declared in /tmp/expanded/file
 172
 173     Because it's just a duplicated error (it'll only ever show up 2+ times).
 174     We also ignore Promose-based errors:
 175
 176       found   : function (VolumeInfo): (Promise<(DirectoryEntry|null)>|null)
 177       required: (function (Promise<VolumeInfo>): ?|null|undefined)
 178
 179     as templates don't work with Promises in all cases yet. See
 180     https://github.com/google/closure-compiler/issues/715 for details.
 181
 182     Args:
 183       errors: A list of string errors extracted from Closure Compiler output.
 184
 185     Return:
 186       A slimmer, sleeker list of relevant errors (strings).
 187     """
 188     first_declared_in = lambda e: " first declared in " not in e
 189     return self._error_filter.filter(filter(first_declared_in, errors))
 190
 191   def _fix_up_error(self, error):
 192     """Reverse the effects that funky <include> preprocessing steps have on
 193     errors messages.
 194
 195     Args:
 196       error: A Closure compiler error (2 line string with error and source).
 197
 198     Return:
 199       The fixed up error string.
 200     """
 201     expanded_file = self._expanded_file
 202     fixed = re.sub("%s:(\d+)" % expanded_file, self._get_line_number, error)
 203     return fixed.replace(expanded_file, os.path.abspath(self._file_arg))
 204
 205   def _format_errors(self, errors):
 206     """Formats Closure compiler errors to easily spot compiler output.
 207
 208     Args:
 209       errors: A list of strings extracted from the Closure compiler's output.
 210
 211     Returns:
 212       A formatted output string.
 213     """
 214     contents = "\n## ".join("\n\n".join(errors).splitlines())
 215     return "## %s" % contents if contents else ""
 216
 217   def _create_temp_file(self, contents):
 218     """Creates an owned temporary file with |contents|.
 219
 220     Args:
 221       content: A string of the file contens to write to a temporary file.
 222
 223     Return:
 224       The filepath of the newly created, written, and closed temporary file.
 225     """
 226     with tempfile.NamedTemporaryFile(mode="wt", delete=False) as tmp_file:
 227       self._temp_files.append(tmp_file.name)
 228       tmp_file.write(contents)
 229     return tmp_file.name
 230
 231   def _run_js_check(self, sources, out_file=None, externs=None):
 232     """Check |sources| for type errors.
 233
 234     Args:
 235       sources: Files to check.
 236       externs: @extern files that inform the compiler about custom globals.
 237
 238     Returns:
 239       (errors, stderr) A parsed list of errors (strings) found by the compiler
 240           and the raw stderr (as a string).
 241     """
 242     args = ["--js=%s" % s for s in sources]
 243
 244     if out_file:
 245       args += ["--js_output_file=%s" % out_file]
 246       args += ["--create_source_map=%s.map" % out_file]
 247
 248     if externs:
 249       args += ["--externs=%s" % e for e in externs]
 250
 251     args_file_content = " %s" % " ".join(self._common_args() + args)
 252     self._log_debug("Args: %s" % args_file_content.strip())
 253
 254     args_file = self._create_temp_file(args_file_content)
 255     self._log_debug("Args file: %s" % args_file)
 256
 257     runner_args = ["--compiler-args-file=%s" % args_file]
 258     _, stderr = self._run_jar(self._runner_jar, runner_args)
 259
 260     errors = stderr.strip().split("\n\n")
 261     maybe_summary = errors.pop()
 262
 263     if re.search(".*error.*warning.*typed", maybe_summary):
 264       self._log_debug("Summary: %s" % maybe_summary)
 265     else:
 266       # Not a summary. Running the jar failed. Bail.
 267       self._log_error(stderr)
 268       self._clean_up()
 269       sys.exit(1)
 270
 271     return errors, stderr
 272
 273   def check(self, source_file, out_file=None, depends=None, externs=None):
 274     """Closure compiler |source_file| while checking for errors.
 275
 276     Args:
 277       source_file: A file to check.
 278       out_file: A file where the compiled output is written to.
 279       depends: Files that |source_file| requires to run (e.g. earlier <script>).
 280       externs: @extern files that inform the compiler about custom globals.
 281
 282     Returns:
 283       (found_errors, stderr) A boolean indicating whether errors were found and
 284           the raw Closure compiler stderr (as a string).
 285     """
 286     self._log_debug("FILE: %s" % source_file)
 287
 288     if source_file.endswith("_externs.js"):
 289       self._log_debug("Skipping externs: %s" % source_file)
 290       return
 291
 292     self._file_arg = source_file
 293
 294     cwd, tmp_dir = os.getcwd(), tempfile.gettempdir()
 295     rel_path = lambda f: os.path.join(os.path.relpath(cwd, tmp_dir), f)
 296
 297     depends = depends or []
 298     includes = [rel_path(f) for f in depends + [source_file]]
 299     contents = ['<include src="%s">' % i for i in includes]
 300     meta_file = self._create_temp_file("\n".join(contents))
 301     self._log_debug("Meta file: %s" % meta_file)
 302
 303     self._processor = processor.Processor(meta_file)
 304     self._expanded_file = self._create_temp_file(self._processor.contents)
 305     self._log_debug("Expanded file: %s" % self._expanded_file)
 306
 307     errors, stderr = self._run_js_check([self._expanded_file],
 308                                         out_file=out_file, externs=externs)
 309     filtered_errors = self._filter_errors(errors)
 310     fixed_errors = map(self._fix_up_error, filtered_errors)
 311     output = self._format_errors(fixed_errors)
 312
 313     if fixed_errors:
 314       prefix = "\n" if output else ""
 315       self._log_error("Error in: %s%s%s" % (source_file, prefix, output))
 316     elif output:
 317       self._log_debug("Output: %s" % output)
 318
 319     self._clean_up()
 320     return bool(fixed_errors), stderr
 321
 322   def check_multiple(self, sources):
 323     """Closure compile a set of files and check for errors.
 324
 325     Args:
 326       sources: An array of files to check.
 327
 328     Returns:
 329       (found_errors, stderr) A boolean indicating whether errors were found and
 330           the raw Closure Compiler stderr (as a string).
 331     """
 332     errors, stderr = self._run_js_check(sources, [])
 333     self._clean_up()
 334     return bool(errors), stderr
 335
 336
 337 if __name__ == "__main__":
 338   parser = argparse.ArgumentParser(
 339       description="Typecheck JavaScript using Closure compiler")
 340   parser.add_argument("sources", nargs=argparse.ONE_OR_MORE,
 341                       help="Path to a source file to typecheck")
 342   single_file_group = parser.add_mutually_exclusive_group()
 343   single_file_group.add_argument("--single-file", dest="single_file",
 344                                  action="store_true",
 345                                  help="Process each source file individually")
 346   single_file_group.add_argument("--no-single-file", dest="single_file",
 347                                  action="store_false",
 348                                  help="Process all source files as a group")
 349   parser.add_argument("-d", "--depends", nargs=argparse.ZERO_OR_MORE)
 350   parser.add_argument("-e", "--externs", nargs=argparse.ZERO_OR_MORE)
 351   parser.add_argument("-o", "--out_file",
 352                       help="A file where the compiled output is written to")
 353   parser.add_argument("-v", "--verbose", action="store_true",
 354                       help="Show more information as this script runs")
 355   parser.add_argument("--strict", action="store_true",
 356                       help="Enable strict type checking")
 357   parser.add_argument("--success-stamp",
 358                       help="Timestamp file to update upon success")
 359
 360   parser.set_defaults(single_file=True, strict=False)
 361   opts = parser.parse_args()
 362
 363   depends = opts.depends or []
 364   externs = opts.externs or set()
 365
 366   if opts.out_file:
 367     out_dir = os.path.dirname(opts.out_file)
 368     if not os.path.exists(out_dir):
 369       os.makedirs(out_dir)
 370
 371   checker = Checker(verbose=opts.verbose, strict=opts.strict)
 372   if opts.single_file:
 373     for source in opts.sources:
 374       depends, externs = build.inputs.resolve_recursive_dependencies(
 375           source, depends, externs)
 376       found_errors, _ = checker.check(source, out_file=opts.out_file,
 377                                       depends=depends, externs=externs)
 378       if found_errors:
 379         sys.exit(1)
 380   else:
 381     found_errors, stderr = checker.check_multiple(opts.sources)
 382     if found_errors:
 383       print stderr
 384       sys.exit(1)
 385
 386   if opts.success_stamp:
 387     with open(opts.success_stamp, "w"):
 388       os.utime(opts.success_stamp, None)