third_party/WebKit/Source/devtools/scripts/compile_frontend.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 from modular_build import read_file, write_file
  31 import os
  32 import os.path as path
  33 import generate_injected_script_externs
  34 import generate_protocol_externs
  35 import modular_build
  36 import re
  37 import shutil
  38 import subprocess
  39 import sys
  40 import tempfile
  41 try:
  42     import simplejson as json
  43 except ImportError:
  44     import json
  45
  46
  47 if len(sys.argv) == 2 and sys.argv[1] == '--help':
  48     print("Usage: %s [module_names]" % path.basename(sys.argv[0]))
  49     print("  module_names    list of modules for which the Closure compilation should run.")
  50     print("                  If absent, the entire frontend will be compiled.")
  51     sys.exit(0)
  52
  53 is_cygwin = sys.platform == 'cygwin'
  54
  55
  56 def popen(arguments):
  57     return subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  58
  59 def to_platform_path(filepath):
  60     if not is_cygwin:
  61         return filepath
  62     return re.sub(r'^/cygdrive/(\w)', '\\1:', filepath)
  63
  64
  65 def to_platform_path_exact(filepath):
  66     if not is_cygwin:
  67         return filepath
  68     output, _ = popen(['cygpath', '-w', filepath]).communicate()
  69     # pylint: disable=E1103
  70     return output.strip().replace('\\', '\\\\')
  71
  72 scripts_path = path.dirname(path.abspath(__file__))
  73 devtools_path = path.dirname(scripts_path)
  74 inspector_path = path.join(path.dirname(devtools_path), 'core', 'inspector')
  75 devtools_frontend_path = path.join(devtools_path, 'front_end')
  76 patched_es6_externs_file = to_platform_path(path.join(devtools_frontend_path, 'es6.js'))
  77 global_externs_file = to_platform_path(path.join(devtools_frontend_path, 'externs.js'))
  78 protocol_externs_file = path.join(devtools_frontend_path, 'protocol_externs.js')
  79 injected_script_source_name = path.join(inspector_path, 'InjectedScriptSource.js')
  80 injected_script_externs_file = path.join(inspector_path, 'injected_script_externs.js')
  81
  82 jsmodule_name_prefix = 'jsmodule_'
  83 runtime_module_name = '_runtime'
  84
  85 type_checked_jsdoc_tags_list = ['param', 'return', 'type', 'enum']
  86 type_checked_jsdoc_tags_or = '|'.join(type_checked_jsdoc_tags_list)
  87
  88 # Basic regex for invalid JsDoc types: an object type name ([A-Z][A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
  89 invalid_type_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*\{.*(?<![!?:.A-Za-z0-9])([A-Z][A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}')
  90 invalid_type_designator_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*.*(?<![{: ])([?!])=?\}')
  91 invalid_non_object_type_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*\{.*(![a-z]+)[^/]*\}')
  92 error_warning_regex = re.compile(r'WARNING|ERROR')
  93 loaded_css_regex = re.compile(r'(?:registerRequiredCSS|WebInspector\.View\.createStyleElement)\s*\(\s*"(.+)"\s*\)')
  94
  95 java_build_regex = re.compile(r'^\w+ version "(\d+)\.(\d+)')
  96 errors_found = False
  97
  98 generate_protocol_externs.generate_protocol_externs(protocol_externs_file, path.join(devtools_path, 'protocol.json'))
  99
 100
 101 # Based on http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python.
 102 def which(program):
 103     def is_exe(fpath):
 104         return path.isfile(fpath) and os.access(fpath, os.X_OK)
 105
 106     fpath, fname = path.split(program)
 107     if fpath:
 108         if is_exe(program):
 109             return program
 110     else:
 111         for part in os.environ["PATH"].split(os.pathsep):
 112             part = part.strip('"')
 113             exe_file = path.join(part, program)
 114             if is_exe(exe_file):
 115                 return exe_file
 116     return None
 117
 118
 119 def log_error(message):
 120     print 'ERROR: ' + message
 121
 122 def error_excepthook(exctype, value, traceback):
 123     print 'ERROR:'
 124     sys.__excepthook__(exctype, value, traceback)
 125 sys.excepthook = error_excepthook
 126
 127 application_descriptors = ['inspector.json', 'toolbox.json']
 128 loader = modular_build.DescriptorLoader(devtools_frontend_path)
 129 descriptors = loader.load_applications(application_descriptors)
 130 modules_by_name = descriptors.modules
 131
 132
 133 def hasErrors(output):
 134     return re.search(error_warning_regex, output) != None
 135
 136
 137 def verify_jsdoc_extra(additional_files):
 138     files = [to_platform_path(file) for file in descriptors.all_compiled_files() + additional_files]
 139     file_list = tempfile.NamedTemporaryFile(mode='wt', delete=False)
 140     try:
 141         file_list.write('\n'.join(files))
 142     finally:
 143         file_list.close()
 144     return popen(java_exec + ['-jar', jsdoc_validator_jar, '--files-list-name', to_platform_path_exact(file_list.name)]), file_list
 145
 146
 147 def verify_jsdoc(additional_files):
 148     def file_list():
 149         return descriptors.all_compiled_files() + additional_files
 150
 151     errors_found = False
 152     for full_file_name in file_list():
 153         lineIndex = 0
 154         with open(full_file_name, 'r') as sourceFile:
 155             for line in sourceFile:
 156                 line = line.rstrip()
 157                 lineIndex += 1
 158                 if not line:
 159                     continue
 160                 if verify_jsdoc_line(full_file_name, lineIndex, line):
 161                     errors_found = True
 162     return errors_found
 163
 164
 165 def verify_jsdoc_line(fileName, lineIndex, line):
 166     def print_error(message, errorPosition):
 167         print '%s:%s: ERROR - %s%s%s%s%s%s' % (fileName, lineIndex, message, os.linesep, line, os.linesep, ' ' * errorPosition + '^', os.linesep)
 168
 169     known_css = {}
 170     errors_found = False
 171     match = re.search(invalid_type_regex, line)
 172     if match:
 173         print_error('Type "%s" nullability not marked explicitly with "?" (nullable) or "!" (non-nullable)' % match.group(1), match.start(1))
 174         errors_found = True
 175
 176     match = re.search(invalid_non_object_type_regex, line)
 177     if match:
 178         print_error('Non-object type explicitly marked with "!" (non-nullable), which is the default and should be omitted', match.start(1))
 179         errors_found = True
 180
 181     match = re.search(invalid_type_designator_regex, line)
 182     if match:
 183         print_error('Type nullability indicator misplaced, should precede type', match.start(1))
 184         errors_found = True
 185
 186     match = re.search(loaded_css_regex, line)
 187     if match:
 188         file = path.join(devtools_frontend_path, match.group(1))
 189         exists = known_css.get(file)
 190         if exists is None:
 191             exists = path.isfile(file)
 192             known_css[file] = exists
 193         if not exists:
 194             print_error('Dynamically loaded CSS stylesheet is missing in the source tree', match.start(1))
 195             errors_found = True
 196     return errors_found
 197
 198
 199 def find_java():
 200     required_major = 1
 201     required_minor = 7
 202     exec_command = None
 203     has_server_jvm = True
 204     java_path = which('java')
 205     if not java_path:
 206         java_path = which('java.exe')
 207
 208     if not java_path:
 209         print 'NOTE: No Java executable found in $PATH.'
 210         sys.exit(1)
 211
 212     is_ok = False
 213     java_version_out, _ = popen([java_path, '-version']).communicate()
 214     # pylint: disable=E1103
 215     match = re.search(java_build_regex, java_version_out)
 216     if match:
 217         major = int(match.group(1))
 218         minor = int(match.group(2))
 219         is_ok = major >= required_major and minor >= required_minor
 220     if is_ok:
 221         exec_command = [java_path, '-Xms1024m', '-server', '-XX:+TieredCompilation']
 222         check_server_proc = popen(exec_command + ['-version'])
 223         check_server_proc.communicate()
 224         if check_server_proc.returncode != 0:
 225             # Not all Java installs have server JVMs.
 226             exec_command = exec_command.remove('-server')
 227             has_server_jvm = False
 228
 229     if not is_ok:
 230         print 'NOTE: Java executable version %d.%d or above not found in $PATH.' % (required_major, required_minor)
 231         sys.exit(1)
 232     print 'Java executable: %s%s' % (java_path, '' if has_server_jvm else ' (no server JVM)')
 233     return exec_command
 234
 235 java_exec = find_java()
 236
 237 closure_compiler_jar = to_platform_path(path.join(scripts_path, 'closure', 'compiler.jar'))
 238 closure_runner_jar = to_platform_path(path.join(scripts_path, 'compiler-runner', 'closure-runner.jar'))
 239 jsdoc_validator_jar = to_platform_path(path.join(scripts_path, 'jsdoc-validator', 'jsdoc-validator.jar'))
 240
 241 modules_dir = tempfile.mkdtemp()
 242 common_closure_args = [
 243     '--summary_detail_level', '3',
 244     '--jscomp_error', 'visibility',
 245     '--compilation_level', 'SIMPLE_OPTIMIZATIONS',
 246     '--warning_level', 'VERBOSE',
 247     '--language_in=ES6_STRICT',
 248     '--language_out=ES5_STRICT',
 249     '--extra_annotation_name', 'suppressReceiverCheck',
 250     '--extra_annotation_name', 'suppressGlobalPropertiesCheck',
 251     '--module_output_path_prefix', to_platform_path_exact(modules_dir + path.sep)
 252 ]
 253
 254 worker_modules_by_name = {}
 255 dependents_by_module_name = {}
 256
 257 for module_name in descriptors.application:
 258     module = descriptors.modules[module_name]
 259     if descriptors.application[module_name].get('type', None) == 'worker':
 260         worker_modules_by_name[module_name] = module
 261     for dep in module.get('dependencies', []):
 262         list = dependents_by_module_name.get(dep)
 263         if not list:
 264             list = []
 265             dependents_by_module_name[dep] = list
 266         list.append(module_name)
 267
 268
 269 def check_conditional_dependencies():
 270     for name in modules_by_name:
 271         for dep_name in modules_by_name[name].get('dependencies', []):
 272             dependency = modules_by_name[dep_name]
 273             if dependency.get('experiment') or dependency.get('condition'):
 274                 log_error('Module "%s" may not depend on the conditional module "%s"' % (name, dep_name))
 275                 errors_found = True
 276
 277 check_conditional_dependencies()
 278
 279
 280 def verify_worker_modules():
 281     for name in modules_by_name:
 282         for dependency in modules_by_name[name].get('dependencies', []):
 283             if dependency in worker_modules_by_name:
 284                 log_error('Module "%s" may not depend on the worker module "%s"' % (name, dependency))
 285                 errors_found = True
 286
 287 verify_worker_modules()
 288
 289
 290 def check_duplicate_files():
 291
 292     def check_module(module, seen_files, seen_modules):
 293         name = module['name']
 294         seen_modules[name] = True
 295         for dep_name in module.get('dependencies', []):
 296             if not dep_name in seen_modules:
 297                 check_module(modules_by_name[dep_name], seen_files, seen_modules)
 298         for source in module.get('scripts', []):
 299             referencing_module = seen_files.get(source)
 300             if referencing_module:
 301                 log_error('Duplicate use of %s in "%s" (previously seen in "%s")' % (source, name, referencing_module))
 302             seen_files[source] = name
 303
 304     for module_name in worker_modules_by_name:
 305         check_module(worker_modules_by_name[module_name], {}, {})
 306
 307 print 'Checking duplicate files across modules...'
 308 check_duplicate_files()
 309
 310
 311 def module_arg(module_name):
 312     return ' --module ' + jsmodule_name_prefix + module_name
 313
 314
 315 def modules_to_check():
 316     if len(sys.argv) == 1:
 317         return descriptors.sorted_modules()
 318     print 'Compiling only these modules: %s' % sys.argv[1:]
 319     return [module for module in descriptors.sorted_modules() if module in set(sys.argv[1:])]
 320
 321
 322 def dump_module(name, recursively, processed_modules):
 323     if name in processed_modules:
 324         return ''
 325     processed_modules[name] = True
 326     module = modules_by_name[name]
 327     skipped_scripts = set(module.get('skip_compilation', []))
 328
 329     command = ''
 330     dependencies = module.get('dependencies', [])
 331     if recursively:
 332         for dependency in dependencies:
 333             command += dump_module(dependency, recursively, processed_modules)
 334     command += module_arg(name) + ':'
 335     filtered_scripts = descriptors.module_compiled_files(name)
 336     command += str(len(filtered_scripts))
 337     firstDependency = True
 338     for dependency in dependencies + [runtime_module_name]:
 339         if firstDependency:
 340             command += ':'
 341         else:
 342             command += ','
 343         firstDependency = False
 344         command += jsmodule_name_prefix + dependency
 345     for script in filtered_scripts:
 346         command += ' --js ' + to_platform_path(path.join(devtools_frontend_path, name, script))
 347     return command
 348
 349 print 'Compiling frontend...'
 350
 351 compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
 352 try:
 353     platform_protocol_externs_file = to_platform_path(protocol_externs_file)
 354     runtime_js_path = to_platform_path(path.join(devtools_frontend_path, 'Runtime.js'))
 355     checked_modules = modules_to_check()
 356     for name in checked_modules:
 357         closure_args = ' '.join(common_closure_args)
 358         closure_args += ' --externs ' + to_platform_path(patched_es6_externs_file)
 359         closure_args += ' --externs ' + to_platform_path(global_externs_file)
 360         closure_args += ' --externs ' + platform_protocol_externs_file
 361         runtime_module = module_arg(runtime_module_name) + ':1 --js ' + runtime_js_path
 362         closure_args += runtime_module + dump_module(name, True, {})
 363         compiler_args_file.write('%s %s%s' % (name, closure_args, os.linesep))
 364 finally:
 365     compiler_args_file.close()
 366
 367 modular_compiler_proc = popen(java_exec + ['-jar', closure_runner_jar, '--compiler-args-file', to_platform_path_exact(compiler_args_file.name)])
 368
 369
 370 def unclosure_injected_script(sourceFileName, outFileName):
 371
 372     source = read_file(sourceFileName)
 373
 374     def replace_function(matchobj):
 375         return re.sub(r'@param', 'param', matchobj.group(1) or '') + '\n//' + matchobj.group(2)
 376
 377     # Comment out the closure function and its jsdocs
 378     source = re.sub(r'(/\*\*(?:[\s\n]*\*\s*@param[^\n]+\n)+\s*\*/\s*)?\n(\(function)', replace_function, source, count=1)
 379
 380     # Comment out its return statement
 381     source = re.sub(r'\n(\s*return\s+[^;]+;\s*\n\}\)\s*)$', '\n/*\\1*/', source)
 382
 383     # Replace the "var Object" override with a "self.Object" one
 384     source = re.sub(r'\nvar Object =', '\nself.Object =', source, count=1)
 385
 386     write_file(outFileName, source)
 387
 388 injectedScriptSourceTmpFile = to_platform_path(path.join(inspector_path, 'InjectedScriptSourceTmp.js'))
 389
 390 unclosure_injected_script(injected_script_source_name, injectedScriptSourceTmpFile)
 391
 392 print 'Compiling InjectedScriptSource.js...'
 393
 394 spawned_compiler_command = java_exec + [
 395     '-jar',
 396     closure_compiler_jar
 397 ] + common_closure_args
 398
 399 command = spawned_compiler_command + [
 400     '--externs', to_platform_path_exact(injected_script_externs_file),
 401     '--externs', to_platform_path_exact(protocol_externs_file),
 402     '--module', jsmodule_name_prefix + 'injected_script' + ':1',
 403     '--js', to_platform_path(injectedScriptSourceTmpFile)
 404 ]
 405
 406 injectedScriptCompileProc = popen(command)
 407
 408 print 'Compiling devtools.js...'
 409
 410 command = spawned_compiler_command + [
 411     '--externs', to_platform_path(global_externs_file),
 412     '--externs', to_platform_path(path.join(devtools_frontend_path, 'host', 'InspectorFrontendHostAPI.js')),
 413     '--module', jsmodule_name_prefix + 'devtools_js' + ':1',
 414     '--js', to_platform_path(path.join(devtools_frontend_path, 'devtools.js'))
 415 ]
 416 devtoolsJSCompileProc = popen(command)
 417
 418 print 'Verifying JSDoc comments...'
 419 additional_jsdoc_check_files = [injectedScriptSourceTmpFile]
 420 errors_found |= verify_jsdoc(additional_jsdoc_check_files)
 421 jsdocValidatorProc, jsdocValidatorFileList = verify_jsdoc_extra(additional_jsdoc_check_files)
 422
 423 print 'Validating InjectedScriptSource.js...'
 424 injectedscript_check_script_path = path.join(scripts_path, "check_injected_script_source.py")
 425 validateInjectedScriptProc = popen([sys.executable, injectedscript_check_script_path, injected_script_source_name])
 426
 427 print
 428
 429 (jsdocValidatorOut, _) = jsdocValidatorProc.communicate()
 430 if jsdocValidatorOut:
 431     print ('JSDoc validator output:%s%s' % (os.linesep, jsdocValidatorOut))
 432     errors_found = True
 433
 434 os.remove(jsdocValidatorFileList.name)
 435
 436 (moduleCompileOut, _) = modular_compiler_proc.communicate()
 437 print 'Modular compilation output:'
 438
 439 start_module_regex = re.compile(r'^@@ START_MODULE:(.+) @@$')
 440 end_module_regex = re.compile(r'^@@ END_MODULE @@$')
 441
 442 in_module = False
 443 skipped_modules = {}
 444 error_count = 0
 445
 446 def skip_dependents(module_name):
 447     for skipped_module in dependents_by_module_name.get(module_name, []):
 448         skipped_modules[skipped_module] = True
 449
 450 has_module_output = False
 451
 452 # pylint: disable=E1103
 453 for line in moduleCompileOut.splitlines():
 454     if not in_module:
 455         match = re.search(start_module_regex, line)
 456         if not match:
 457             continue
 458         in_module = True
 459         has_module_output = True
 460         module_error_count = 0
 461         module_output = []
 462         module_name = match.group(1)
 463         skip_module = skipped_modules.get(module_name)
 464         if skip_module:
 465             skip_dependents(module_name)
 466     else:
 467         match = re.search(end_module_regex, line)
 468         if not match:
 469             if not skip_module:
 470                 module_output.append(line)
 471                 if hasErrors(line):
 472                     error_count += 1
 473                     module_error_count += 1
 474                     skip_dependents(module_name)
 475             continue
 476
 477         in_module = False
 478         if skip_module:
 479             print 'Skipping module %s...' % module_name
 480         elif not module_error_count:
 481             print 'Module %s compiled successfully: %s' % (module_name, module_output[0])
 482         else:
 483             print 'Module %s compile failed: %s errors%s' % (module_name, module_error_count, os.linesep)
 484             print os.linesep.join(module_output)
 485
 486 if not has_module_output:
 487     print moduleCompileOut
 488
 489 if error_count:
 490     print 'Total Closure errors: %d%s' % (error_count, os.linesep)
 491     errors_found = True
 492
 493 (injectedScriptCompileOut, _) = injectedScriptCompileProc.communicate()
 494 print 'InjectedScriptSource.js compilation output:%s' % os.linesep, injectedScriptCompileOut
 495 errors_found |= hasErrors(injectedScriptCompileOut)
 496
 497 (devtoolsJSCompileOut, _) = devtoolsJSCompileProc.communicate()
 498 print 'devtools.js compilation output:%s' % os.linesep, devtoolsJSCompileOut
 499 errors_found |= hasErrors(devtoolsJSCompileOut)
 500
 501 (validateInjectedScriptOut, _) = validateInjectedScriptProc.communicate()
 502 print 'Validate InjectedScriptSource.js output:%s' % os.linesep, (validateInjectedScriptOut if validateInjectedScriptOut else '<empty>')
 503 errors_found |= hasErrors(validateInjectedScriptOut)
 504
 505 if errors_found:
 506     print 'ERRORS DETECTED'
 507
 508 os.remove(injectedScriptSourceTmpFile)
 509 os.remove(compiler_args_file.name)
 510 os.remove(protocol_externs_file)
 511 shutil.rmtree(modules_dir, True)