third_party/closure_compiler/compile.py

   1 #!/usr/bin/python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Runs Closure compiler on JavaScript files to check for errors and produce
   7 minified output."""
   8
   9 import argparse
  10 import os
  11 import re
  12 import subprocess
  13 import sys
  14 import tempfile
  15
  16 import build.inputs
  17 import processor
  18 import error_filter
  19
  20
  21 _CURRENT_DIR = os.path.join(os.path.dirname(__file__))
  22
  23
  24 class Checker(object):
  25   """Runs the Closure compiler on given source files to typecheck them
  26   and produce minified output."""
  27
  28   _COMMON_JSCOMP_ERRORS = [
  29     "accessControls",
  30     "ambiguousFunctionDecl",
  31     "checkStructDictInheritance",
  32     "checkTypes",
  33     "checkVars",
  34     "constantProperty",
  35     "deprecated",
  36     "externsValidation",
  37     "globalThis",
  38     "invalidCasts",
  39     "missingProperties",
  40     "missingReturn",
  41     "nonStandardJsDocs",
  42     "suspiciousCode",
  43     "undefinedNames",
  44     "undefinedVars",
  45     "unknownDefines",
  46     "uselessCode",
  47     "visibility",
  48   ]
  49
  50   # Extra @jsDocAnnotations used when compiling polymer code.
  51   _POLYMER_EXTRA_ANNOTATIONS = [
  52     "attribute",
  53     "status",
  54     "element",
  55     "homepage",
  56     "submodule",
  57     "group",
  58   ]
  59
  60   _COMMON_CLOSURE_ARGS = [
  61     "--accept_const_keyword",
  62     "--language_in=ECMASCRIPT5_STRICT",
  63     "--summary_detail_level=3",
  64     "--compilation_level=SIMPLE_OPTIMIZATIONS",
  65     "--source_map_format=V3",
  66     "--polymer_pass",
  67   ] + [
  68     "--jscomp_error=%s" % err for err in _COMMON_JSCOMP_ERRORS
  69   ] + [
  70     "--extra_annotation_name=%s" % a for a in _POLYMER_EXTRA_ANNOTATIONS
  71   ]
  72
  73   # These are the extra flags used when compiling in strict mode.
  74   # Flags that are normally disabled are turned on for strict mode.
  75   _STRICT_CLOSURE_ARGS = [
  76     "--jscomp_error=reportUnknownTypes",
  77     "--jscomp_error=duplicate",
  78     "--jscomp_error=misplacedTypeAnnotation",
  79   ]
  80
  81   _DISABLED_CLOSURE_ARGS = [
  82     # TODO(dbeam): happens when the same file is <include>d multiple times.
  83     "--jscomp_off=duplicate",
  84     # TODO(fukino): happens when cr.defineProperty() has a type annotation.
  85     # Avoiding parse-time warnings needs 2 pass compiling. crbug.com/421562.
  86     "--jscomp_off=misplacedTypeAnnotation",
  87   ]
  88
  89   _JAR_COMMAND = [
  90     "java",
  91     "-jar",
  92     "-Xms1024m",
  93     "-client",
  94     "-XX:+TieredCompilation"
  95   ]
  96
  97   _MAP_FILE_FORMAT = "%s.map"
  98
  99   def __init__(self, verbose=False, strict=False):
 100     """
 101     Args:
 102       verbose: Whether this class should output diagnostic messages.
 103       strict: Whether the Closure Compiler should be invoked more strictly.
 104     """
 105     self._runner_jar = os.path.join(_CURRENT_DIR, "runner", "runner.jar")
 106     self._temp_files = []
 107     self._verbose = verbose
 108     self._strict = strict
 109     self._error_filter = error_filter.PromiseErrorFilter()
 110
 111   def _nuke_temp_files(self):
 112     """Deletes any temp files this class knows about."""
 113     if not self._temp_files:
 114       return
 115
 116     self._log_debug("Deleting temp files: %s" % ", ".join(self._temp_files))
 117     for f in self._temp_files:
 118       os.remove(f)
 119     self._temp_files = []
 120
 121   def _log_debug(self, msg, error=False):
 122     """Logs |msg| to stdout if --verbose/-v is passed when invoking this script.
 123
 124     Args:
 125       msg: A debug message to log.
 126     """
 127     if self._verbose:
 128       print "(INFO) %s" % msg
 129
 130   def _log_error(self, msg):
 131     """Logs |msg| to stderr regardless of --flags.
 132
 133     Args:
 134       msg: An error message to log.
 135     """
 136     print >> sys.stderr, "(ERROR) %s" % msg
 137
 138   def _common_args(self):
 139     """Returns an array of the common closure compiler args."""
 140     if self._strict:
 141       return self._COMMON_CLOSURE_ARGS + self._STRICT_CLOSURE_ARGS
 142     return self._COMMON_CLOSURE_ARGS + self._DISABLED_CLOSURE_ARGS
 143
 144   def _run_jar(self, jar, args):
 145     """Runs a .jar from the command line with arguments.
 146
 147     Args:
 148       jar: A file path to a .jar file
 149       args: A list of command line arguments to be passed when running the .jar.
 150
 151     Return:
 152       (exit_code, stderr) The exit code of the command (e.g. 0 for success) and
 153           the stderr collected while running |jar| (as a string).
 154     """
 155     shell_command = " ".join(self._JAR_COMMAND + [jar] + args)
 156     self._log_debug("Running jar: %s" % shell_command)
 157
 158     devnull = open(os.devnull, "w")
 159     kwargs = {"stdout": devnull, "stderr": subprocess.PIPE, "shell": True}
 160     process = subprocess.Popen(shell_command, **kwargs)
 161     _, stderr = process.communicate()
 162     return process.returncode, stderr
 163
 164   def _get_line_number(self, match):
 165     """When chrome is built, it preprocesses its JavaScript from:
 166
 167       <include src="blah.js">
 168       alert(1);
 169
 170     to:
 171
 172       /* contents of blah.js inlined */
 173       alert(1);
 174
 175     Because Closure Compiler requires this inlining already be done (as
 176     <include> isn't valid JavaScript), this script creates temporary files to
 177     expand all the <include>s.
 178
 179     When type errors are hit in temporary files, a developer doesn't know the
 180     original source location to fix. This method maps from /tmp/file:300 back to
 181     /original/source/file:100 so fixing errors is faster for developers.
 182
 183     Args:
 184       match: A re.MatchObject from matching against a line number regex.
 185
 186     Returns:
 187       The fixed up /file and :line number.
 188     """
 189     real_file = self._processor.get_file_from_line(match.group(1))
 190     return "%s:%d" % (os.path.abspath(real_file.file), real_file.line_number)
 191
 192   def _filter_errors(self, errors):
 193     """Removes some extraneous errors. For example, we ignore:
 194
 195       Variable x first declared in /tmp/expanded/file
 196
 197     Because it's just a duplicated error (it'll only ever show up 2+ times).
 198     We also ignore Promose-based errors:
 199
 200       found   : function (VolumeInfo): (Promise<(DirectoryEntry|null)>|null)
 201       required: (function (Promise<VolumeInfo>): ?|null|undefined)
 202
 203     as templates don't work with Promises in all cases yet. See
 204     https://github.com/google/closure-compiler/issues/715 for details.
 205
 206     Args:
 207       errors: A list of string errors extracted from Closure Compiler output.
 208
 209     Return:
 210       A slimmer, sleeker list of relevant errors (strings).
 211     """
 212     first_declared_in = lambda e: " first declared in " not in e
 213     return self._error_filter.filter(filter(first_declared_in, errors))
 214
 215   def _clean_up_error(self, error):
 216     """Reverse the effects that funky <include> preprocessing steps have on
 217     errors messages.
 218
 219     Args:
 220       error: A Closure compiler error (2 line string with error and source).
 221
 222     Return:
 223       The fixed up error string.
 224     """
 225     expanded_file = self._expanded_file
 226     fixed = re.sub("%s:(\d+)" % expanded_file, self._get_line_number, error)
 227     return fixed.replace(expanded_file, os.path.abspath(self._file_arg))
 228
 229   def _format_errors(self, errors):
 230     """Formats Closure compiler errors to easily spot compiler output.
 231
 232     Args:
 233       errors: A list of strings extracted from the Closure compiler's output.
 234
 235     Returns:
 236       A formatted output string.
 237     """
 238     contents = "\n## ".join("\n\n".join(errors).splitlines())
 239     return "## %s" % contents if contents else ""
 240
 241   def _create_temp_file(self, contents):
 242     """Creates an owned temporary file with |contents|.
 243
 244     Args:
 245       content: A string of the file contens to write to a temporary file.
 246
 247     Return:
 248       The filepath of the newly created, written, and closed temporary file.
 249     """
 250     with tempfile.NamedTemporaryFile(mode="wt", delete=False) as tmp_file:
 251       self._temp_files.append(tmp_file.name)
 252       tmp_file.write(contents)
 253     return tmp_file.name
 254
 255   def _run_js_check(self, sources, out_file=None, externs=None,
 256                     output_wrapper=None):
 257     """Check |sources| for type errors.
 258
 259     Args:
 260       sources: Files to check.
 261       out_file: A file where the compiled output is written to.
 262       externs: @extern files that inform the compiler about custom globals.
 263       output_wrapper: Wraps output into this string at the place denoted by the
 264           marker token %output%.
 265
 266     Returns:
 267       (errors, stderr) A parsed list of errors (strings) found by the compiler
 268           and the raw stderr (as a string).
 269     """
 270     args = ["--js=%s" % s for s in sources]
 271
 272     if out_file:
 273       out_dir = os.path.dirname(out_file)
 274       if not os.path.exists(out_dir):
 275         os.makedirs(out_dir)
 276       args += ["--js_output_file=%s" % out_file]
 277       args += ["--create_source_map=%s" % (self._MAP_FILE_FORMAT % out_file)]
 278
 279     if externs:
 280       args += ["--externs=%s" % e for e in externs]
 281
 282     if output_wrapper:
 283       args += ['--output_wrapper="%s"' % output_wrapper]
 284
 285     args_file_content = " %s" % " ".join(self._common_args() + args)
 286     self._log_debug("Args: %s" % args_file_content.strip())
 287
 288     args_file = self._create_temp_file(args_file_content)
 289     self._log_debug("Args file: %s" % args_file)
 290
 291     runner_args = ["--compiler-args-file=%s" % args_file]
 292     _, stderr = self._run_jar(self._runner_jar, runner_args)
 293
 294     errors = stderr.strip().split("\n\n")
 295     maybe_summary = errors.pop()
 296
 297     if re.search(".*error.*warning.*typed", maybe_summary):
 298       self._log_debug("Summary: %s" % maybe_summary)
 299     else:
 300       # Not a summary. Running the jar failed. Bail.
 301       self._log_error(stderr)
 302       self._nuke_temp_files()
 303       sys.exit(1)
 304
 305     if errors and out_file:
 306       if os.path.exists(out_file):
 307         os.remove(out_file)
 308       if os.path.exists(self._MAP_FILE_FORMAT % out_file):
 309         os.remove(self._MAP_FILE_FORMAT % out_file)
 310
 311     return errors, stderr
 312
 313   def check(self, source_file, out_file=None, depends=None, externs=None,
 314             output_wrapper=None):
 315     """Closure compiler |source_file| while checking for errors.
 316
 317     Args:
 318       source_file: A file to check.
 319       out_file: A file where the compiled output is written to.
 320       depends: Files that |source_file| requires to run (e.g. earlier <script>).
 321       externs: @extern files that inform the compiler about custom globals.
 322       output_wrapper: Wraps output into this string at the place denoted by the
 323           marker token %output%.
 324
 325     Returns:
 326       (found_errors, stderr) A boolean indicating whether errors were found and
 327           the raw Closure compiler stderr (as a string).
 328     """
 329     self._log_debug("FILE: %s" % source_file)
 330
 331     if source_file.endswith("_externs.js"):
 332       self._log_debug("Skipping externs: %s" % source_file)
 333       return
 334
 335     self._file_arg = source_file
 336
 337     cwd, tmp_dir = os.getcwd(), tempfile.gettempdir()
 338     rel_path = lambda f: os.path.join(os.path.relpath(cwd, tmp_dir), f)
 339
 340     depends = depends or []
 341     includes = [rel_path(f) for f in depends + [source_file]]
 342     contents = ['<include src="%s">' % i for i in includes]
 343     meta_file = self._create_temp_file("\n".join(contents))
 344     self._log_debug("Meta file: %s" % meta_file)
 345
 346     self._processor = processor.Processor(meta_file)
 347     self._expanded_file = self._create_temp_file(self._processor.contents)
 348     self._log_debug("Expanded file: %s" % self._expanded_file)
 349
 350     errors, stderr = self._run_js_check([self._expanded_file],
 351                                         out_file=out_file, externs=externs,
 352                                         output_wrapper=output_wrapper)
 353     filtered_errors = self._filter_errors(errors)
 354     cleaned_errors = map(self._clean_up_error, filtered_errors)
 355     output = self._format_errors(cleaned_errors)
 356
 357     if cleaned_errors:
 358       prefix = "\n" if output else ""
 359       self._log_error("Error in: %s%s%s" % (source_file, prefix, output))
 360     elif output:
 361       self._log_debug("Output: %s" % output)
 362
 363     self._nuke_temp_files()
 364     return bool(cleaned_errors), stderr
 365
 366   def check_multiple(self, sources, out_file=None, output_wrapper=None,
 367                      externs=None):
 368     """Closure compile a set of files and check for errors.
 369
 370     Args:
 371       sources: An array of files to check.
 372       out_file: A file where the compiled output is written to.
 373       output_wrapper: Wraps output into this string at the place denoted by the
 374           marker token %output%.
 375       externs: @extern files that inform the compiler about custom globals.
 376
 377     Returns:
 378       (found_errors, stderr) A boolean indicating whether errors were found and
 379           the raw Closure Compiler stderr (as a string).
 380     """
 381     errors, stderr = self._run_js_check(sources, out_file=out_file,
 382                                         output_wrapper=output_wrapper,
 383                                         externs=externs)
 384     self._nuke_temp_files()
 385     return bool(errors), stderr
 386
 387
 388 if __name__ == "__main__":
 389   parser = argparse.ArgumentParser(
 390       description="Typecheck JavaScript using Closure compiler")
 391   parser.add_argument("sources", nargs=argparse.ONE_OR_MORE,
 392                       help="Path to a source file to typecheck")
 393   single_file_group = parser.add_mutually_exclusive_group()
 394   single_file_group.add_argument("--single-file", dest="single_file",
 395                                  action="store_true",
 396                                  help="Process each source file individually")
 397   single_file_group.add_argument("--no-single-file", dest="single_file",
 398                                  action="store_false",
 399                                  help="Process all source files as a group")
 400   parser.add_argument("-d", "--depends", nargs=argparse.ZERO_OR_MORE)
 401   parser.add_argument("-e", "--externs", nargs=argparse.ZERO_OR_MORE)
 402   parser.add_argument("-o", "--out_file",
 403                       help="A file where the compiled output is written to")
 404   parser.add_argument("-w", "--output_wrapper",
 405                       help="Wraps output into this string at the place"
 406                          + " denoted by the marker token %output%")
 407   parser.add_argument("-v", "--verbose", action="store_true",
 408                       help="Show more information as this script runs")
 409   parser.add_argument("--strict", action="store_true",
 410                       help="Enable strict type checking")
 411   parser.add_argument("--success-stamp",
 412                       help="Timestamp file to update upon success")
 413
 414   parser.set_defaults(single_file=True, strict=False)
 415   opts = parser.parse_args()
 416
 417   depends = opts.depends or []
 418   externs = set(opts.externs or [])
 419
 420   polymer_externs = os.path.join(os.path.dirname(_CURRENT_DIR), 'polymer',
 421                                  'v0_8', 'components-chromium',
 422                                  'polymer-externs', 'polymer.externs.js')
 423   externs.add(polymer_externs)
 424
 425   checker = Checker(verbose=opts.verbose, strict=opts.strict)
 426   if opts.single_file:
 427     for source in opts.sources:
 428       depends, externs = build.inputs.resolve_recursive_dependencies(
 429           source, depends, externs)
 430       found_errors, _ = checker.check(source, out_file=opts.out_file,
 431                                       depends=depends, externs=externs,
 432                                       output_wrapper=opts.output_wrapper)
 433       if found_errors:
 434         sys.exit(1)
 435   else:
 436     found_errors, stderr = checker.check_multiple(
 437         opts.sources,
 438         out_file=opts.out_file,
 439         output_wrapper=opts.output_wrapper,
 440         externs=externs)
 441     if found_errors:
 442       print stderr
 443       sys.exit(1)
 444
 445   if opts.success_stamp:
 446     with open(opts.success_stamp, "w"):
 447       os.utime(opts.success_stamp, None)