PRESUBMIT.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Top-level presubmit script for Chromium.
   6
   7 See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
   8 for more details about the presubmit API built into gcl.
   9 """
  10
  11
  12 _EXCLUDED_PATHS = (
  13     r"^breakpad[\\\/].*",
  14     r"^native_client_sdk[\\\/]src[\\\/]build_tools[\\\/]make_rules.py",
  15     r"^native_client_sdk[\\\/]src[\\\/]build_tools[\\\/]make_simple.py",
  16     r"^native_client_sdk[\\\/]src[\\\/]tools[\\\/].*.mk",
  17     r"^net[\\\/]tools[\\\/]spdyshark[\\\/].*",
  18     r"^skia[\\\/].*",
  19     r"^v8[\\\/].*",
  20     r".*MakeFile$",
  21     r".+_autogen\.h$",
  22     r".+[\\\/]pnacl_shim\.c$",
  23     r"^gpu[\\\/]config[\\\/].*_list_json\.cc$",
  24     r"^chrome[\\\/]browser[\\\/]resources[\\\/]pdf[\\\/]index.js"
  25 )
  26
  27 # The NetscapePlugIn library is excluded from pan-project as it will soon
  28 # be deleted together with the rest of the NPAPI and it's not worthwhile to
  29 # update the coding style until then.
  30 _TESTRUNNER_PATHS = (
  31     r"^content[\\\/]shell[\\\/]tools[\\\/]plugin[\\\/].*",
  32 )
  33
  34 # Fragment of a regular expression that matches C++ and Objective-C++
  35 # implementation files.
  36 _IMPLEMENTATION_EXTENSIONS = r'\.(cc|cpp|cxx|mm)$'
  37
  38 # Regular expression that matches code only used for test binaries
  39 # (best effort).
  40 _TEST_CODE_EXCLUDED_PATHS = (
  41     r'.*[\\\/](fake_|test_|mock_).+%s' % _IMPLEMENTATION_EXTENSIONS,
  42     r'.+_test_(base|support|util)%s' % _IMPLEMENTATION_EXTENSIONS,
  43     r'.+_(api|browser|kif|perf|pixel|unit|ui)?test(_[a-z]+)?%s' %
  44         _IMPLEMENTATION_EXTENSIONS,
  45     r'.+profile_sync_service_harness%s' % _IMPLEMENTATION_EXTENSIONS,
  46     r'.*[\\\/](test|tool(s)?)[\\\/].*',
  47     # content_shell is used for running layout tests.
  48     r'content[\\\/]shell[\\\/].*',
  49     # At request of folks maintaining this folder.
  50     r'chrome[\\\/]browser[\\\/]automation[\\\/].*',
  51     # Non-production example code.
  52     r'mojo[\\\/]examples[\\\/].*',
  53     # Launcher for running iOS tests on the simulator.
  54     r'testing[\\\/]iossim[\\\/]iossim\.mm$',
  55 )
  56
  57 _TEST_ONLY_WARNING = (
  58     'You might be calling functions intended only for testing from\n'
  59     'production code.  It is OK to ignore this warning if you know what\n'
  60     'you are doing, as the heuristics used to detect the situation are\n'
  61     'not perfect.  The commit queue will not block on this warning.')
  62
  63
  64 _INCLUDE_ORDER_WARNING = (
  65     'Your #include order seems to be broken. Send mail to\n'
  66     'marja@chromium.org if this is not the case.')
  67
  68
  69 _BANNED_OBJC_FUNCTIONS = (
  70     (
  71       'addTrackingRect:',
  72       (
  73        'The use of -[NSView addTrackingRect:owner:userData:assumeInside:] is'
  74        'prohibited. Please use CrTrackingArea instead.',
  75        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
  76       ),
  77       False,
  78     ),
  79     (
  80       r'/NSTrackingArea\W',
  81       (
  82        'The use of NSTrackingAreas is prohibited. Please use CrTrackingArea',
  83        'instead.',
  84        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
  85       ),
  86       False,
  87     ),
  88     (
  89       'convertPointFromBase:',
  90       (
  91        'The use of -[NSView convertPointFromBase:] is almost certainly wrong.',
  92        'Please use |convertPoint:(point) fromView:nil| instead.',
  93        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
  94       ),
  95       True,
  96     ),
  97     (
  98       'convertPointToBase:',
  99       (
 100        'The use of -[NSView convertPointToBase:] is almost certainly wrong.',
 101        'Please use |convertPoint:(point) toView:nil| instead.',
 102        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 103       ),
 104       True,
 105     ),
 106     (
 107       'convertRectFromBase:',
 108       (
 109        'The use of -[NSView convertRectFromBase:] is almost certainly wrong.',
 110        'Please use |convertRect:(point) fromView:nil| instead.',
 111        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 112       ),
 113       True,
 114     ),
 115     (
 116       'convertRectToBase:',
 117       (
 118        'The use of -[NSView convertRectToBase:] is almost certainly wrong.',
 119        'Please use |convertRect:(point) toView:nil| instead.',
 120        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 121       ),
 122       True,
 123     ),
 124     (
 125       'convertSizeFromBase:',
 126       (
 127        'The use of -[NSView convertSizeFromBase:] is almost certainly wrong.',
 128        'Please use |convertSize:(point) fromView:nil| instead.',
 129        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 130       ),
 131       True,
 132     ),
 133     (
 134       'convertSizeToBase:',
 135       (
 136        'The use of -[NSView convertSizeToBase:] is almost certainly wrong.',
 137        'Please use |convertSize:(point) toView:nil| instead.',
 138        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 139       ),
 140       True,
 141     ),
 142 )
 143
 144
 145 _BANNED_CPP_FUNCTIONS = (
 146     # Make sure that gtest's FRIEND_TEST() macro is not used; the
 147     # FRIEND_TEST_ALL_PREFIXES() macro from base/gtest_prod_util.h should be
 148     # used instead since that allows for FLAKY_ and DISABLED_ prefixes.
 149     (
 150       'FRIEND_TEST(',
 151       (
 152        'Chromium code should not use gtest\'s FRIEND_TEST() macro. Include',
 153        'base/gtest_prod_util.h and use FRIEND_TEST_ALL_PREFIXES() instead.',
 154       ),
 155       False,
 156       (),
 157     ),
 158     (
 159       'ScopedAllowIO',
 160       (
 161        'New code should not use ScopedAllowIO. Post a task to the blocking',
 162        'pool or the FILE thread instead.',
 163       ),
 164       True,
 165       (
 166         r"^base[\\\/]process[\\\/]process_metrics_linux\.cc$",
 167         r"^chrome[\\\/]browser[\\\/]chromeos[\\\/]boot_times_recorder\.cc$",
 168         r"^chrome[\\\/]browser[\\\/]chromeos[\\\/]"
 169             "customization_document_browsertest\.cc$",
 170         r"^components[\\\/]crash[\\\/]app[\\\/]breakpad_mac\.mm$",
 171         r"^content[\\\/]shell[\\\/]browser[\\\/]shell_browser_main\.cc$",
 172         r"^content[\\\/]shell[\\\/]browser[\\\/]shell_message_filter\.cc$",
 173         r"^mojo[\\\/]edk[\\\/]embedder[\\\/]" +
 174             r"simple_platform_shared_buffer_posix\.cc$",
 175         r"^net[\\\/]disk_cache[\\\/]cache_util\.cc$",
 176         r"^net[\\\/]url_request[\\\/]test_url_fetcher_factory\.cc$",
 177       ),
 178     ),
 179     (
 180       'SkRefPtr',
 181       (
 182         'The use of SkRefPtr is prohibited. ',
 183         'Please use skia::RefPtr instead.'
 184       ),
 185       True,
 186       (),
 187     ),
 188     (
 189       'SkAutoRef',
 190       (
 191         'The indirect use of SkRefPtr via SkAutoRef is prohibited. ',
 192         'Please use skia::RefPtr instead.'
 193       ),
 194       True,
 195       (),
 196     ),
 197     (
 198       'SkAutoTUnref',
 199       (
 200         'The use of SkAutoTUnref is dangerous because it implicitly ',
 201         'converts to a raw pointer. Please use skia::RefPtr instead.'
 202       ),
 203       True,
 204       (),
 205     ),
 206     (
 207       'SkAutoUnref',
 208       (
 209         'The indirect use of SkAutoTUnref through SkAutoUnref is dangerous ',
 210         'because it implicitly converts to a raw pointer. ',
 211         'Please use skia::RefPtr instead.'
 212       ),
 213       True,
 214       (),
 215     ),
 216     (
 217       r'/HANDLE_EINTR\(.*close',
 218       (
 219        'HANDLE_EINTR(close) is invalid. If close fails with EINTR, the file',
 220        'descriptor will be closed, and it is incorrect to retry the close.',
 221        'Either call close directly and ignore its return value, or wrap close',
 222        'in IGNORE_EINTR to use its return value. See http://crbug.com/269623'
 223       ),
 224       True,
 225       (),
 226     ),
 227     (
 228       r'/IGNORE_EINTR\((?!.*close)',
 229       (
 230        'IGNORE_EINTR is only valid when wrapping close. To wrap other system',
 231        'calls, use HANDLE_EINTR. See http://crbug.com/269623',
 232       ),
 233       True,
 234       (
 235         # Files that #define IGNORE_EINTR.
 236         r'^base[\\\/]posix[\\\/]eintr_wrapper\.h$',
 237         r'^ppapi[\\\/]tests[\\\/]test_broker\.cc$',
 238       ),
 239     ),
 240     (
 241       r'/v8::Extension\(',
 242       (
 243         'Do not introduce new v8::Extensions into the code base, use',
 244         'gin::Wrappable instead. See http://crbug.com/334679',
 245       ),
 246       True,
 247       (
 248         r'extensions[\\\/]renderer[\\\/]safe_builtins\.*',
 249       ),
 250     ),
 251 )
 252
 253 _IPC_ENUM_TRAITS_DEPRECATED = (
 254     'You are using IPC_ENUM_TRAITS() in your code. It has been deprecated.\n'
 255     'See http://www.chromium.org/Home/chromium-security/education/security-tips-for-ipc')
 256
 257
 258 _VALID_OS_MACROS = (
 259     # Please keep sorted.
 260     'OS_ANDROID',
 261     'OS_ANDROID_HOST',
 262     'OS_BSD',
 263     'OS_CAT',       # For testing.
 264     'OS_CHROMEOS',
 265     'OS_FREEBSD',
 266     'OS_IOS',
 267     'OS_LINUX',
 268     'OS_MACOSX',
 269     'OS_NACL',
 270     'OS_NACL_NONSFI',
 271     'OS_NACL_SFI',
 272     'OS_OPENBSD',
 273     'OS_POSIX',
 274     'OS_QNX',
 275     'OS_SOLARIS',
 276     'OS_WIN',
 277 )
 278
 279
 280 def _CheckNoProductionCodeUsingTestOnlyFunctions(input_api, output_api):
 281   """Attempts to prevent use of functions intended only for testing in
 282   non-testing code. For now this is just a best-effort implementation
 283   that ignores header files and may have some false positives. A
 284   better implementation would probably need a proper C++ parser.
 285   """
 286   # We only scan .cc files and the like, as the declaration of
 287   # for-testing functions in header files are hard to distinguish from
 288   # calls to such functions without a proper C++ parser.
 289   file_inclusion_pattern = r'.+%s' % _IMPLEMENTATION_EXTENSIONS
 290
 291   base_function_pattern = r'[ :]test::[^\s]+|ForTest(ing)?|for_test(ing)?'
 292   inclusion_pattern = input_api.re.compile(r'(%s)\s*\(' % base_function_pattern)
 293   comment_pattern = input_api.re.compile(r'//.*(%s)' % base_function_pattern)
 294   exclusion_pattern = input_api.re.compile(
 295     r'::[A-Za-z0-9_]+(%s)|(%s)[^;]+\{' % (
 296       base_function_pattern, base_function_pattern))
 297
 298   def FilterFile(affected_file):
 299     black_list = (_EXCLUDED_PATHS +
 300                   _TEST_CODE_EXCLUDED_PATHS +
 301                   input_api.DEFAULT_BLACK_LIST)
 302     return input_api.FilterSourceFile(
 303       affected_file,
 304       white_list=(file_inclusion_pattern, ),
 305       black_list=black_list)
 306
 307   problems = []
 308   for f in input_api.AffectedSourceFiles(FilterFile):
 309     local_path = f.LocalPath()
 310     for line_number, line in f.ChangedContents():
 311       if (inclusion_pattern.search(line) and
 312           not comment_pattern.search(line) and
 313           not exclusion_pattern.search(line)):
 314         problems.append(
 315           '%s:%d\n    %s' % (local_path, line_number, line.strip()))
 316
 317   if problems:
 318     return [output_api.PresubmitPromptOrNotify(_TEST_ONLY_WARNING, problems)]
 319   else:
 320     return []
 321
 322
 323 def _CheckNoIOStreamInHeaders(input_api, output_api):
 324   """Checks to make sure no .h files include <iostream>."""
 325   files = []
 326   pattern = input_api.re.compile(r'^#include\s*<iostream>',
 327                                  input_api.re.MULTILINE)
 328   for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
 329     if not f.LocalPath().endswith('.h'):
 330       continue
 331     contents = input_api.ReadFile(f)
 332     if pattern.search(contents):
 333       files.append(f)
 334
 335   if len(files):
 336     return [ output_api.PresubmitError(
 337         'Do not #include <iostream> in header files, since it inserts static '
 338         'initialization into every file including the header. Instead, '
 339         '#include <ostream>. See http://crbug.com/94794',
 340         files) ]
 341   return []
 342
 343
 344 def _CheckNoUNIT_TESTInSourceFiles(input_api, output_api):
 345   """Checks to make sure no source files use UNIT_TEST"""
 346   problems = []
 347   for f in input_api.AffectedFiles():
 348     if (not f.LocalPath().endswith(('.cc', '.mm'))):
 349       continue
 350
 351     for line_num, line in f.ChangedContents():
 352       if 'UNIT_TEST ' in line or line.endswith('UNIT_TEST'):
 353         problems.append('    %s:%d' % (f.LocalPath(), line_num))
 354
 355   if not problems:
 356     return []
 357   return [output_api.PresubmitPromptWarning('UNIT_TEST is only for headers.\n' +
 358       '\n'.join(problems))]
 359
 360
 361 def _CheckNoNewWStrings(input_api, output_api):
 362   """Checks to make sure we don't introduce use of wstrings."""
 363   problems = []
 364   for f in input_api.AffectedFiles():
 365     if (not f.LocalPath().endswith(('.cc', '.h')) or
 366         f.LocalPath().endswith(('test.cc', '_win.cc', '_win.h')) or
 367         '/win/' in f.LocalPath()):
 368       continue
 369
 370     allowWString = False
 371     for line_num, line in f.ChangedContents():
 372       if 'presubmit: allow wstring' in line:
 373         allowWString = True
 374       elif not allowWString and 'wstring' in line:
 375         problems.append('    %s:%d' % (f.LocalPath(), line_num))
 376         allowWString = False
 377       else:
 378         allowWString = False
 379
 380   if not problems:
 381     return []
 382   return [output_api.PresubmitPromptWarning('New code should not use wstrings.'
 383       '  If you are calling a cross-platform API that accepts a wstring, '
 384       'fix the API.\n' +
 385       '\n'.join(problems))]
 386
 387
 388 def _CheckNoDEPSGIT(input_api, output_api):
 389   """Make sure .DEPS.git is never modified manually."""
 390   if any(f.LocalPath().endswith('.DEPS.git') for f in
 391       input_api.AffectedFiles()):
 392     return [output_api.PresubmitError(
 393       'Never commit changes to .DEPS.git. This file is maintained by an\n'
 394       'automated system based on what\'s in DEPS and your changes will be\n'
 395       'overwritten.\n'
 396       'See https://sites.google.com/a/chromium.org/dev/developers/how-tos/get-the-code#Rolling_DEPS\n'
 397       'for more information')]
 398   return []
 399
 400
 401 def _CheckValidHostsInDEPS(input_api, output_api):
 402   """Checks that DEPS file deps are from allowed_hosts."""
 403   # Run only if DEPS file has been modified to annoy fewer bystanders.
 404   if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()):
 405     return []
 406   # Outsource work to gclient verify
 407   try:
 408     input_api.subprocess.check_output(['gclient', 'verify'])
 409     return []
 410   except input_api.subprocess.CalledProcessError, error:
 411     return [output_api.PresubmitError(
 412         'DEPS file must have only git dependencies.',
 413         long_text=error.output)]
 414
 415
 416 def _CheckNoBannedFunctions(input_api, output_api):
 417   """Make sure that banned functions are not used."""
 418   warnings = []
 419   errors = []
 420
 421   file_filter = lambda f: f.LocalPath().endswith(('.mm', '.m', '.h'))
 422   for f in input_api.AffectedFiles(file_filter=file_filter):
 423     for line_num, line in f.ChangedContents():
 424       for func_name, message, error in _BANNED_OBJC_FUNCTIONS:
 425         matched = False
 426         if func_name[0:1] == '/':
 427           regex = func_name[1:]
 428           if input_api.re.search(regex, line):
 429             matched = True
 430         elif func_name in line:
 431             matched = True
 432         if matched:
 433           problems = warnings;
 434           if error:
 435             problems = errors;
 436           problems.append('    %s:%d:' % (f.LocalPath(), line_num))
 437           for message_line in message:
 438             problems.append('      %s' % message_line)
 439
 440   file_filter = lambda f: f.LocalPath().endswith(('.cc', '.mm', '.h'))
 441   for f in input_api.AffectedFiles(file_filter=file_filter):
 442     for line_num, line in f.ChangedContents():
 443       for func_name, message, error, excluded_paths in _BANNED_CPP_FUNCTIONS:
 444         def IsBlacklisted(affected_file, blacklist):
 445           local_path = affected_file.LocalPath()
 446           for item in blacklist:
 447             if input_api.re.match(item, local_path):
 448               return True
 449           return False
 450         if IsBlacklisted(f, excluded_paths):
 451           continue
 452         matched = False
 453         if func_name[0:1] == '/':
 454           regex = func_name[1:]
 455           if input_api.re.search(regex, line):
 456             matched = True
 457         elif func_name in line:
 458             matched = True
 459         if matched:
 460           problems = warnings;
 461           if error:
 462             problems = errors;
 463           problems.append('    %s:%d:' % (f.LocalPath(), line_num))
 464           for message_line in message:
 465             problems.append('      %s' % message_line)
 466
 467   result = []
 468   if (warnings):
 469     result.append(output_api.PresubmitPromptWarning(
 470         'Banned functions were used.\n' + '\n'.join(warnings)))
 471   if (errors):
 472     result.append(output_api.PresubmitError(
 473         'Banned functions were used.\n' + '\n'.join(errors)))
 474   return result
 475
 476
 477 def _CheckNoPragmaOnce(input_api, output_api):
 478   """Make sure that banned functions are not used."""
 479   files = []
 480   pattern = input_api.re.compile(r'^#pragma\s+once',
 481                                  input_api.re.MULTILINE)
 482   for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
 483     if not f.LocalPath().endswith('.h'):
 484       continue
 485     contents = input_api.ReadFile(f)
 486     if pattern.search(contents):
 487       files.append(f)
 488
 489   if files:
 490     return [output_api.PresubmitError(
 491         'Do not use #pragma once in header files.\n'
 492         'See http://www.chromium.org/developers/coding-style#TOC-File-headers',
 493         files)]
 494   return []
 495
 496
 497 def _CheckNoTrinaryTrueFalse(input_api, output_api):
 498   """Checks to make sure we don't introduce use of foo ? true : false."""
 499   problems = []
 500   pattern = input_api.re.compile(r'\?\s*(true|false)\s*:\s*(true|false)')
 501   for f in input_api.AffectedFiles():
 502     if not f.LocalPath().endswith(('.cc', '.h', '.inl', '.m', '.mm')):
 503       continue
 504
 505     for line_num, line in f.ChangedContents():
 506       if pattern.match(line):
 507         problems.append('    %s:%d' % (f.LocalPath(), line_num))
 508
 509   if not problems:
 510     return []
 511   return [output_api.PresubmitPromptWarning(
 512       'Please consider avoiding the "? true : false" pattern if possible.\n' +
 513       '\n'.join(problems))]
 514
 515
 516 def _CheckUnwantedDependencies(input_api, output_api):
 517   """Runs checkdeps on #include statements added in this
 518   change. Breaking - rules is an error, breaking ! rules is a
 519   warning.
 520   """
 521   import sys
 522   # We need to wait until we have an input_api object and use this
 523   # roundabout construct to import checkdeps because this file is
 524   # eval-ed and thus doesn't have __file__.
 525   original_sys_path = sys.path
 526   try:
 527     sys.path = sys.path + [input_api.os_path.join(
 528         input_api.PresubmitLocalPath(), 'buildtools', 'checkdeps')]
 529     import checkdeps
 530     from cpp_checker import CppChecker
 531     from rules import Rule
 532   finally:
 533     # Restore sys.path to what it was before.
 534     sys.path = original_sys_path
 535
 536   added_includes = []
 537   for f in input_api.AffectedFiles():
 538     if not CppChecker.IsCppFile(f.LocalPath()):
 539       continue
 540
 541     changed_lines = [line for line_num, line in f.ChangedContents()]
 542     added_includes.append([f.LocalPath(), changed_lines])
 543
 544   deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath())
 545
 546   error_descriptions = []
 547   warning_descriptions = []
 548   for path, rule_type, rule_description in deps_checker.CheckAddedCppIncludes(
 549       added_includes):
 550     description_with_path = '%s\n    %s' % (path, rule_description)
 551     if rule_type == Rule.DISALLOW:
 552       error_descriptions.append(description_with_path)
 553     else:
 554       warning_descriptions.append(description_with_path)
 555
 556   results = []
 557   if error_descriptions:
 558     results.append(output_api.PresubmitError(
 559         'You added one or more #includes that violate checkdeps rules.',
 560         error_descriptions))
 561   if warning_descriptions:
 562     results.append(output_api.PresubmitPromptOrNotify(
 563         'You added one or more #includes of files that are temporarily\n'
 564         'allowed but being removed. Can you avoid introducing the\n'
 565         '#include? See relevant DEPS file(s) for details and contacts.',
 566         warning_descriptions))
 567   return results
 568
 569
 570 def _CheckFilePermissions(input_api, output_api):
 571   """Check that all files have their permissions properly set."""
 572   if input_api.platform == 'win32':
 573     return []
 574   args = [input_api.python_executable, 'tools/checkperms/checkperms.py',
 575           '--root', input_api.change.RepositoryRoot()]
 576   for f in input_api.AffectedFiles():
 577     args += ['--file', f.LocalPath()]
 578   checkperms = input_api.subprocess.Popen(args,
 579                                           stdout=input_api.subprocess.PIPE)
 580   errors = checkperms.communicate()[0].strip()
 581   if errors:
 582     return [output_api.PresubmitError('checkperms.py failed.',
 583                                       errors.splitlines())]
 584   return []
 585
 586
 587 def _CheckNoAuraWindowPropertyHInHeaders(input_api, output_api):
 588   """Makes sure we don't include ui/aura/window_property.h
 589   in header files.
 590   """
 591   pattern = input_api.re.compile(r'^#include\s*"ui/aura/window_property.h"')
 592   errors = []
 593   for f in input_api.AffectedFiles():
 594     if not f.LocalPath().endswith('.h'):
 595       continue
 596     for line_num, line in f.ChangedContents():
 597       if pattern.match(line):
 598         errors.append('    %s:%d' % (f.LocalPath(), line_num))
 599
 600   results = []
 601   if errors:
 602     results.append(output_api.PresubmitError(
 603       'Header files should not include ui/aura/window_property.h', errors))
 604   return results
 605
 606
 607 def _CheckIncludeOrderForScope(scope, input_api, file_path, changed_linenums):
 608   """Checks that the lines in scope occur in the right order.
 609
 610   1. C system files in alphabetical order
 611   2. C++ system files in alphabetical order
 612   3. Project's .h files
 613   """
 614
 615   c_system_include_pattern = input_api.re.compile(r'\s*#include <.*\.h>')
 616   cpp_system_include_pattern = input_api.re.compile(r'\s*#include <.*>')
 617   custom_include_pattern = input_api.re.compile(r'\s*#include ".*')
 618
 619   C_SYSTEM_INCLUDES, CPP_SYSTEM_INCLUDES, CUSTOM_INCLUDES = range(3)
 620
 621   state = C_SYSTEM_INCLUDES
 622
 623   previous_line = ''
 624   previous_line_num = 0
 625   problem_linenums = []
 626   for line_num, line in scope:
 627     if c_system_include_pattern.match(line):
 628       if state != C_SYSTEM_INCLUDES:
 629         problem_linenums.append((line_num, previous_line_num))
 630       elif previous_line and previous_line > line:
 631         problem_linenums.append((line_num, previous_line_num))
 632     elif cpp_system_include_pattern.match(line):
 633       if state == C_SYSTEM_INCLUDES:
 634         state = CPP_SYSTEM_INCLUDES
 635       elif state == CUSTOM_INCLUDES:
 636         problem_linenums.append((line_num, previous_line_num))
 637       elif previous_line and previous_line > line:
 638         problem_linenums.append((line_num, previous_line_num))
 639     elif custom_include_pattern.match(line):
 640       if state != CUSTOM_INCLUDES:
 641         state = CUSTOM_INCLUDES
 642       elif previous_line and previous_line > line:
 643         problem_linenums.append((line_num, previous_line_num))
 644     else:
 645       problem_linenums.append(line_num)
 646     previous_line = line
 647     previous_line_num = line_num
 648
 649   warnings = []
 650   for (line_num, previous_line_num) in problem_linenums:
 651     if line_num in changed_linenums or previous_line_num in changed_linenums:
 652       warnings.append('    %s:%d' % (file_path, line_num))
 653   return warnings
 654
 655
 656 def _CheckIncludeOrderInFile(input_api, f, changed_linenums):
 657   """Checks the #include order for the given file f."""
 658
 659   system_include_pattern = input_api.re.compile(r'\s*#include \<.*')
 660   # Exclude the following includes from the check:
 661   # 1) #include <.../...>, e.g., <sys/...> includes often need to appear in a
 662   # specific order.
 663   # 2) <atlbase.h>, "build/build_config.h"
 664   excluded_include_pattern = input_api.re.compile(
 665       r'\s*#include (\<.*/.*|\<atlbase\.h\>|"build/build_config.h")')
 666   custom_include_pattern = input_api.re.compile(r'\s*#include "(?P<FILE>.*)"')
 667   # Match the final or penultimate token if it is xxxtest so we can ignore it
 668   # when considering the special first include.
 669   test_file_tag_pattern = input_api.re.compile(
 670     r'_[a-z]+test(?=(_[a-zA-Z0-9]+)?\.)')
 671   if_pattern = input_api.re.compile(
 672       r'\s*#\s*(if|elif|else|endif|define|undef).*')
 673   # Some files need specialized order of includes; exclude such files from this
 674   # check.
 675   uncheckable_includes_pattern = input_api.re.compile(
 676       r'\s*#include '
 677       '("ipc/.*macros\.h"|<windows\.h>|".*gl.*autogen.h")\s*')
 678
 679   contents = f.NewContents()
 680   warnings = []
 681   line_num = 0
 682
 683   # Handle the special first include. If the first include file is
 684   # some/path/file.h, the corresponding including file can be some/path/file.cc,
 685   # some/other/path/file.cc, some/path/file_platform.cc, some/path/file-suffix.h
 686   # etc. It's also possible that no special first include exists.
 687   # If the included file is some/path/file_platform.h the including file could
 688   # also be some/path/file_xxxtest_platform.h.
 689   including_file_base_name = test_file_tag_pattern.sub(
 690     '', input_api.os_path.basename(f.LocalPath()))
 691
 692   for line in contents:
 693     line_num += 1
 694     if system_include_pattern.match(line):
 695       # No special first include -> process the line again along with normal
 696       # includes.
 697       line_num -= 1
 698       break
 699     match = custom_include_pattern.match(line)
 700     if match:
 701       match_dict = match.groupdict()
 702       header_basename = test_file_tag_pattern.sub(
 703         '', input_api.os_path.basename(match_dict['FILE'])).replace('.h', '')
 704
 705       if header_basename not in including_file_base_name:
 706         # No special first include -> process the line again along with normal
 707         # includes.
 708         line_num -= 1
 709       break
 710
 711   # Split into scopes: Each region between #if and #endif is its own scope.
 712   scopes = []
 713   current_scope = []
 714   for line in contents[line_num:]:
 715     line_num += 1
 716     if uncheckable_includes_pattern.match(line):
 717       continue
 718     if if_pattern.match(line):
 719       scopes.append(current_scope)
 720       current_scope = []
 721     elif ((system_include_pattern.match(line) or
 722            custom_include_pattern.match(line)) and
 723           not excluded_include_pattern.match(line)):
 724       current_scope.append((line_num, line))
 725   scopes.append(current_scope)
 726
 727   for scope in scopes:
 728     warnings.extend(_CheckIncludeOrderForScope(scope, input_api, f.LocalPath(),
 729                                                changed_linenums))
 730   return warnings
 731
 732
 733 def _CheckIncludeOrder(input_api, output_api):
 734   """Checks that the #include order is correct.
 735
 736   1. The corresponding header for source files.
 737   2. C system files in alphabetical order
 738   3. C++ system files in alphabetical order
 739   4. Project's .h files in alphabetical order
 740
 741   Each region separated by #if, #elif, #else, #endif, #define and #undef follows
 742   these rules separately.
 743   """
 744   def FileFilterIncludeOrder(affected_file):
 745     black_list = (_EXCLUDED_PATHS + input_api.DEFAULT_BLACK_LIST)
 746     return input_api.FilterSourceFile(affected_file, black_list=black_list)
 747
 748   warnings = []
 749   for f in input_api.AffectedFiles(file_filter=FileFilterIncludeOrder):
 750     if f.LocalPath().endswith(('.cc', '.h')):
 751       changed_linenums = set(line_num for line_num, _ in f.ChangedContents())
 752       warnings.extend(_CheckIncludeOrderInFile(input_api, f, changed_linenums))
 753
 754   results = []
 755   if warnings:
 756     results.append(output_api.PresubmitPromptOrNotify(_INCLUDE_ORDER_WARNING,
 757                                                       warnings))
 758   return results
 759
 760
 761 def _CheckForVersionControlConflictsInFile(input_api, f):
 762   pattern = input_api.re.compile('^(?:<<<<<<<|>>>>>>>) |^=======$')
 763   errors = []
 764   for line_num, line in f.ChangedContents():
 765     if pattern.match(line):
 766       errors.append('    %s:%d %s' % (f.LocalPath(), line_num, line))
 767   return errors
 768
 769
 770 def _CheckForVersionControlConflicts(input_api, output_api):
 771   """Usually this is not intentional and will cause a compile failure."""
 772   errors = []
 773   for f in input_api.AffectedFiles():
 774     errors.extend(_CheckForVersionControlConflictsInFile(input_api, f))
 775
 776   results = []
 777   if errors:
 778     results.append(output_api.PresubmitError(
 779       'Version control conflict markers found, please resolve.', errors))
 780   return results
 781
 782
 783 def _CheckHardcodedGoogleHostsInLowerLayers(input_api, output_api):
 784   def FilterFile(affected_file):
 785     """Filter function for use with input_api.AffectedSourceFiles,
 786     below.  This filters out everything except non-test files from
 787     top-level directories that generally speaking should not hard-code
 788     service URLs (e.g. src/android_webview/, src/content/ and others).
 789     """
 790     return input_api.FilterSourceFile(
 791       affected_file,
 792       white_list=(r'^(android_webview|base|content|net)[\\\/].*', ),
 793       black_list=(_EXCLUDED_PATHS +
 794                   _TEST_CODE_EXCLUDED_PATHS +
 795                   input_api.DEFAULT_BLACK_LIST))
 796
 797   base_pattern = '"[^"]*google\.com[^"]*"'
 798   comment_pattern = input_api.re.compile('//.*%s' % base_pattern)
 799   pattern = input_api.re.compile(base_pattern)
 800   problems = []  # items are (filename, line_number, line)
 801   for f in input_api.AffectedSourceFiles(FilterFile):
 802     for line_num, line in f.ChangedContents():
 803       if not comment_pattern.search(line) and pattern.search(line):
 804         problems.append((f.LocalPath(), line_num, line))
 805
 806   if problems:
 807     return [output_api.PresubmitPromptOrNotify(
 808         'Most layers below src/chrome/ should not hardcode service URLs.\n'
 809         'Are you sure this is correct?',
 810         ['  %s:%d:  %s' % (
 811             problem[0], problem[1], problem[2]) for problem in problems])]
 812   else:
 813     return []
 814
 815
 816 def _CheckNoAbbreviationInPngFileName(input_api, output_api):
 817   """Makes sure there are no abbreviations in the name of PNG files.
 818   The native_client_sdk directory is excluded because it has auto-generated PNG
 819   files for documentation.
 820   """
 821   errors = []
 822   white_list = (r'.*_[a-z]_.*\.png$|.*_[a-z]\.png$',)
 823   black_list = (r'^native_client_sdk[\\\/]',)
 824   file_filter = lambda f: input_api.FilterSourceFile(
 825       f, white_list=white_list, black_list=black_list)
 826   for f in input_api.AffectedFiles(include_deletes=False,
 827                                    file_filter=file_filter):
 828     errors.append('    %s' % f.LocalPath())
 829
 830   results = []
 831   if errors:
 832     results.append(output_api.PresubmitError(
 833         'The name of PNG files should not have abbreviations. \n'
 834         'Use _hover.png, _center.png, instead of _h.png, _c.png.\n'
 835         'Contact oshima@chromium.org if you have questions.', errors))
 836   return results
 837
 838
 839 def _FilesToCheckForIncomingDeps(re, changed_lines):
 840   """Helper method for _CheckAddedDepsHaveTargetApprovals. Returns
 841   a set of DEPS entries that we should look up.
 842
 843   For a directory (rather than a specific filename) we fake a path to
 844   a specific filename by adding /DEPS. This is chosen as a file that
 845   will seldom or never be subject to per-file include_rules.
 846   """
 847   # We ignore deps entries on auto-generated directories.
 848   AUTO_GENERATED_DIRS = ['grit', 'jni']
 849
 850   # This pattern grabs the path without basename in the first
 851   # parentheses, and the basename (if present) in the second. It
 852   # relies on the simple heuristic that if there is a basename it will
 853   # be a header file ending in ".h".
 854   pattern = re.compile(
 855       r"""['"]\+([^'"]+?)(/[a-zA-Z0-9_]+\.h)?['"].*""")
 856   results = set()
 857   for changed_line in changed_lines:
 858     m = pattern.match(changed_line)
 859     if m:
 860       path = m.group(1)
 861       if path.split('/')[0] not in AUTO_GENERATED_DIRS:
 862         if m.group(2):
 863           results.add('%s%s' % (path, m.group(2)))
 864         else:
 865           results.add('%s/DEPS' % path)
 866   return results
 867
 868
 869 def _CheckAddedDepsHaveTargetApprovals(input_api, output_api):
 870   """When a dependency prefixed with + is added to a DEPS file, we
 871   want to make sure that the change is reviewed by an OWNER of the
 872   target file or directory, to avoid layering violations from being
 873   introduced. This check verifies that this happens.
 874   """
 875   changed_lines = set()
 876   for f in input_api.AffectedFiles():
 877     filename = input_api.os_path.basename(f.LocalPath())
 878     if filename == 'DEPS':
 879       changed_lines |= set(line.strip()
 880                            for line_num, line
 881                            in f.ChangedContents())
 882   if not changed_lines:
 883     return []
 884
 885   virtual_depended_on_files = _FilesToCheckForIncomingDeps(input_api.re,
 886                                                            changed_lines)
 887   if not virtual_depended_on_files:
 888     return []
 889
 890   if input_api.is_committing:
 891     if input_api.tbr:
 892       return [output_api.PresubmitNotifyResult(
 893           '--tbr was specified, skipping OWNERS check for DEPS additions')]
 894     if not input_api.change.issue:
 895       return [output_api.PresubmitError(
 896           "DEPS approval by OWNERS check failed: this change has "
 897           "no Rietveld issue number, so we can't check it for approvals.")]
 898     output = output_api.PresubmitError
 899   else:
 900     output = output_api.PresubmitNotifyResult
 901
 902   owners_db = input_api.owners_db
 903   owner_email, reviewers = input_api.canned_checks._RietveldOwnerAndReviewers(
 904       input_api,
 905       owners_db.email_regexp,
 906       approval_needed=input_api.is_committing)
 907
 908   owner_email = owner_email or input_api.change.author_email
 909
 910   reviewers_plus_owner = set(reviewers)
 911   if owner_email:
 912     reviewers_plus_owner.add(owner_email)
 913   missing_files = owners_db.files_not_covered_by(virtual_depended_on_files,
 914                                                  reviewers_plus_owner)
 915
 916   # We strip the /DEPS part that was added by
 917   # _FilesToCheckForIncomingDeps to fake a path to a file in a
 918   # directory.
 919   def StripDeps(path):
 920     start_deps = path.rfind('/DEPS')
 921     if start_deps != -1:
 922       return path[:start_deps]
 923     else:
 924       return path
 925   unapproved_dependencies = ["'+%s'," % StripDeps(path)
 926                              for path in missing_files]
 927
 928   if unapproved_dependencies:
 929     output_list = [
 930       output('Missing LGTM from OWNERS of dependencies added to DEPS:\n    %s' %
 931              '\n    '.join(sorted(unapproved_dependencies)))]
 932     if not input_api.is_committing:
 933       suggested_owners = owners_db.reviewers_for(missing_files, owner_email)
 934       output_list.append(output(
 935           'Suggested missing target path OWNERS:\n    %s' %
 936           '\n    '.join(suggested_owners or [])))
 937     return output_list
 938
 939   return []
 940
 941
 942 def _CheckSpamLogging(input_api, output_api):
 943   file_inclusion_pattern = r'.+%s' % _IMPLEMENTATION_EXTENSIONS
 944   black_list = (_EXCLUDED_PATHS +
 945                 _TEST_CODE_EXCLUDED_PATHS +
 946                 input_api.DEFAULT_BLACK_LIST +
 947                 (r"^base[\\\/]logging\.h$",
 948                  r"^base[\\\/]logging\.cc$",
 949                  r"^chrome[\\\/]app[\\\/]chrome_main_delegate\.cc$",
 950                  r"^chrome[\\\/]browser[\\\/]chrome_browser_main\.cc$",
 951                  r"^chrome[\\\/]browser[\\\/]ui[\\\/]startup[\\\/]"
 952                      r"startup_browser_creator\.cc$",
 953                  r"^chrome[\\\/]installer[\\\/]setup[\\\/].*",
 954                  r"chrome[\\\/]browser[\\\/]diagnostics[\\\/]" +
 955                      r"diagnostics_writer\.cc$",
 956                  r"^chrome_elf[\\\/]dll_hash[\\\/]dll_hash_main\.cc$",
 957                  r"^chromecast[\\\/]",
 958                  r"^cloud_print[\\\/]",
 959                  r"^content[\\\/]common[\\\/]gpu[\\\/]client[\\\/]"
 960                      r"gl_helper_benchmark\.cc$",
 961                  r"^courgette[\\\/]courgette_tool\.cc$",
 962                  r"^extensions[\\\/]renderer[\\\/]logging_native_handler\.cc$",
 963                  r"^ipc[\\\/]ipc_logging\.cc$",
 964                  r"^native_client_sdk[\\\/]",
 965                  r"^remoting[\\\/]base[\\\/]logging\.h$",
 966                  r"^remoting[\\\/]host[\\\/].*",
 967                  r"^sandbox[\\\/]linux[\\\/].*",
 968                  r"^tools[\\\/]",
 969                  r"^ui[\\\/]aura[\\\/]bench[\\\/]bench_main\.cc$",
 970                  r"^storage[\\\/]browser[\\\/]fileapi[\\\/]" +
 971                      r"dump_file_system.cc$",))
 972   source_file_filter = lambda x: input_api.FilterSourceFile(
 973       x, white_list=(file_inclusion_pattern,), black_list=black_list)
 974
 975   log_info = []
 976   printf = []
 977
 978   for f in input_api.AffectedSourceFiles(source_file_filter):
 979     contents = input_api.ReadFile(f, 'rb')
 980     if input_api.re.search(r"\bD?LOG\s*\(\s*INFO\s*\)", contents):
 981       log_info.append(f.LocalPath())
 982     elif input_api.re.search(r"\bD?LOG_IF\s*\(\s*INFO\s*,", contents):
 983       log_info.append(f.LocalPath())
 984
 985     if input_api.re.search(r"\bprintf\(", contents):
 986       printf.append(f.LocalPath())
 987     elif input_api.re.search(r"\bfprintf\((stdout|stderr)", contents):
 988       printf.append(f.LocalPath())
 989
 990   if log_info:
 991     return [output_api.PresubmitError(
 992       'These files spam the console log with LOG(INFO):',
 993       items=log_info)]
 994   if printf:
 995     return [output_api.PresubmitError(
 996       'These files spam the console log with printf/fprintf:',
 997       items=printf)]
 998   return []
 999
1000
1001 def _CheckForAnonymousVariables(input_api, output_api):
1002   """These types are all expected to hold locks while in scope and
1003      so should never be anonymous (which causes them to be immediately
1004      destroyed)."""
1005   they_who_must_be_named = [
1006     'base::AutoLock',
1007     'base::AutoReset',
1008     'base::AutoUnlock',
1009     'SkAutoAlphaRestore',
1010     'SkAutoBitmapShaderInstall',
1011     'SkAutoBlitterChoose',
1012     'SkAutoBounderCommit',
1013     'SkAutoCallProc',
1014     'SkAutoCanvasRestore',
1015     'SkAutoCommentBlock',
1016     'SkAutoDescriptor',
1017     'SkAutoDisableDirectionCheck',
1018     'SkAutoDisableOvalCheck',
1019     'SkAutoFree',
1020     'SkAutoGlyphCache',
1021     'SkAutoHDC',
1022     'SkAutoLockColors',
1023     'SkAutoLockPixels',
1024     'SkAutoMalloc',
1025     'SkAutoMaskFreeImage',
1026     'SkAutoMutexAcquire',
1027     'SkAutoPathBoundsUpdate',
1028     'SkAutoPDFRelease',
1029     'SkAutoRasterClipValidate',
1030     'SkAutoRef',
1031     'SkAutoTime',
1032     'SkAutoTrace',
1033     'SkAutoUnref',
1034   ]
1035   anonymous = r'(%s)\s*[({]' % '|'.join(they_who_must_be_named)
1036   # bad: base::AutoLock(lock.get());
1037   # not bad: base::AutoLock lock(lock.get());
1038   bad_pattern = input_api.re.compile(anonymous)
1039   # good: new base::AutoLock(lock.get())
1040   good_pattern = input_api.re.compile(r'\bnew\s*' + anonymous)
1041   errors = []
1042
1043   for f in input_api.AffectedFiles():
1044     if not f.LocalPath().endswith(('.cc', '.h', '.inl', '.m', '.mm')):
1045       continue
1046     for linenum, line in f.ChangedContents():
1047       if bad_pattern.search(line) and not good_pattern.search(line):
1048         errors.append('%s:%d' % (f.LocalPath(), linenum))
1049
1050   if errors:
1051     return [output_api.PresubmitError(
1052       'These lines create anonymous variables that need to be named:',
1053       items=errors)]
1054   return []
1055
1056
1057 def _CheckCygwinShell(input_api, output_api):
1058   source_file_filter = lambda x: input_api.FilterSourceFile(
1059       x, white_list=(r'.+\.(gyp|gypi)$',))
1060   cygwin_shell = []
1061
1062   for f in input_api.AffectedSourceFiles(source_file_filter):
1063     for linenum, line in f.ChangedContents():
1064       if 'msvs_cygwin_shell' in line:
1065         cygwin_shell.append(f.LocalPath())
1066         break
1067
1068   if cygwin_shell:
1069     return [output_api.PresubmitError(
1070       'These files should not use msvs_cygwin_shell (the default is 0):',
1071       items=cygwin_shell)]
1072   return []
1073
1074
1075 def _CheckUserActionUpdate(input_api, output_api):
1076   """Checks if any new user action has been added."""
1077   if any('actions.xml' == input_api.os_path.basename(f) for f in
1078          input_api.LocalPaths()):
1079     # If actions.xml is already included in the changelist, the PRESUBMIT
1080     # for actions.xml will do a more complete presubmit check.
1081     return []
1082
1083   file_filter = lambda f: f.LocalPath().endswith(('.cc', '.mm'))
1084   action_re = r'[^a-zA-Z]UserMetricsAction\("([^"]*)'
1085   current_actions = None
1086   for f in input_api.AffectedFiles(file_filter=file_filter):
1087     for line_num, line in f.ChangedContents():
1088       match = input_api.re.search(action_re, line)
1089       if match:
1090         # Loads contents in tools/metrics/actions/actions.xml to memory. It's
1091         # loaded only once.
1092         if not current_actions:
1093           with open('tools/metrics/actions/actions.xml') as actions_f:
1094             current_actions = actions_f.read()
1095         # Search for the matched user action name in |current_actions|.
1096         for action_name in match.groups():
1097           action = 'name="{0}"'.format(action_name)
1098           if action not in current_actions:
1099             return [output_api.PresubmitPromptWarning(
1100               'File %s line %d: %s is missing in '
1101               'tools/metrics/actions/actions.xml. Please run '
1102               'tools/metrics/actions/extract_actions.py to update.'
1103               % (f.LocalPath(), line_num, action_name))]
1104   return []
1105
1106
1107 def _GetJSONParseError(input_api, filename, eat_comments=True):
1108   try:
1109     contents = input_api.ReadFile(filename)
1110     if eat_comments:
1111       json_comment_eater = input_api.os_path.join(
1112           input_api.PresubmitLocalPath(),
1113           'tools', 'json_comment_eater', 'json_comment_eater.py')
1114       process = input_api.subprocess.Popen(
1115           [input_api.python_executable, json_comment_eater],
1116           stdin=input_api.subprocess.PIPE,
1117           stdout=input_api.subprocess.PIPE,
1118           universal_newlines=True)
1119       (contents, _) = process.communicate(input=contents)
1120
1121     input_api.json.loads(contents)
1122   except ValueError as e:
1123     return e
1124   return None
1125
1126
1127 def _GetIDLParseError(input_api, filename):
1128   try:
1129     contents = input_api.ReadFile(filename)
1130     idl_schema = input_api.os_path.join(
1131         input_api.PresubmitLocalPath(),
1132         'tools', 'json_schema_compiler', 'idl_schema.py')
1133     process = input_api.subprocess.Popen(
1134         [input_api.python_executable, idl_schema],
1135         stdin=input_api.subprocess.PIPE,
1136         stdout=input_api.subprocess.PIPE,
1137         stderr=input_api.subprocess.PIPE,
1138         universal_newlines=True)
1139     (_, error) = process.communicate(input=contents)
1140     return error or None
1141   except ValueError as e:
1142     return e
1143
1144
1145 def _CheckParseErrors(input_api, output_api):
1146   """Check that IDL and JSON files do not contain syntax errors."""
1147   actions = {
1148     '.idl': _GetIDLParseError,
1149     '.json': _GetJSONParseError,
1150   }
1151   # These paths contain test data and other known invalid JSON files.
1152   excluded_patterns = [
1153     r'test[\\\/]data[\\\/]',
1154     r'^components[\\\/]policy[\\\/]resources[\\\/]policy_templates\.json$',
1155   ]
1156   # Most JSON files are preprocessed and support comments, but these do not.
1157   json_no_comments_patterns = [
1158     r'^testing[\\\/]',
1159   ]
1160   # Only run IDL checker on files in these directories.
1161   idl_included_patterns = [
1162     r'^chrome[\\\/]common[\\\/]extensions[\\\/]api[\\\/]',
1163     r'^extensions[\\\/]common[\\\/]api[\\\/]',
1164   ]
1165
1166   def get_action(affected_file):
1167     filename = affected_file.LocalPath()
1168     return actions.get(input_api.os_path.splitext(filename)[1])
1169
1170   def MatchesFile(patterns, path):
1171     for pattern in patterns:
1172       if input_api.re.search(pattern, path):
1173         return True
1174     return False
1175
1176   def FilterFile(affected_file):
1177     action = get_action(affected_file)
1178     if not action:
1179       return False
1180     path = affected_file.LocalPath()
1181
1182     if MatchesFile(excluded_patterns, path):
1183       return False
1184
1185     if (action == _GetIDLParseError and
1186         not MatchesFile(idl_included_patterns, path)):
1187       return False
1188     return True
1189
1190   results = []
1191   for affected_file in input_api.AffectedFiles(
1192       file_filter=FilterFile, include_deletes=False):
1193     action = get_action(affected_file)
1194     kwargs = {}
1195     if (action == _GetJSONParseError and
1196         MatchesFile(json_no_comments_patterns, affected_file.LocalPath())):
1197       kwargs['eat_comments'] = False
1198     parse_error = action(input_api,
1199                          affected_file.AbsoluteLocalPath(),
1200                          **kwargs)
1201     if parse_error:
1202       results.append(output_api.PresubmitError('%s could not be parsed: %s' %
1203           (affected_file.LocalPath(), parse_error)))
1204   return results
1205
1206
1207 def _CheckJavaStyle(input_api, output_api):
1208   """Runs checkstyle on changed java files and returns errors if any exist."""
1209   import sys
1210   original_sys_path = sys.path
1211   try:
1212     sys.path = sys.path + [input_api.os_path.join(
1213         input_api.PresubmitLocalPath(), 'tools', 'android', 'checkstyle')]
1214     import checkstyle
1215   finally:
1216     # Restore sys.path to what it was before.
1217     sys.path = original_sys_path
1218
1219   return checkstyle.RunCheckstyle(
1220       input_api, output_api, 'tools/android/checkstyle/chromium-style-5.0.xml')
1221
1222
1223 def _CheckForCopyrightedCode(input_api, output_api):
1224   """Verifies that newly added code doesn't contain copyrighted material
1225   and is properly licensed under the standard Chromium license.
1226
1227   As there can be false positives, we maintain a whitelist file. This check
1228   also verifies that the whitelist file is up to date.
1229   """
1230   import sys
1231   original_sys_path = sys.path
1232   try:
1233     sys.path = sys.path + [input_api.os_path.join(
1234         input_api.PresubmitLocalPath(), 'android_webview', 'tools')]
1235     import copyright_scanner
1236   finally:
1237     # Restore sys.path to what it was before.
1238     sys.path = original_sys_path
1239
1240   return copyright_scanner.ScanAtPresubmit(input_api, output_api)
1241
1242
1243 _DEPRECATED_CSS = [
1244   # Values
1245   ( "-webkit-box", "flex" ),
1246   ( "-webkit-inline-box", "inline-flex" ),
1247   ( "-webkit-flex", "flex" ),
1248   ( "-webkit-inline-flex", "inline-flex" ),
1249   ( "-webkit-min-content", "min-content" ),
1250   ( "-webkit-max-content", "max-content" ),
1251
1252   # Properties
1253   ( "-webkit-background-clip", "background-clip" ),
1254   ( "-webkit-background-origin", "background-origin" ),
1255   ( "-webkit-background-size", "background-size" ),
1256   ( "-webkit-box-shadow", "box-shadow" ),
1257
1258   # Functions
1259   ( "-webkit-gradient", "gradient" ),
1260   ( "-webkit-repeating-gradient", "repeating-gradient" ),
1261   ( "-webkit-linear-gradient", "linear-gradient" ),
1262   ( "-webkit-repeating-linear-gradient", "repeating-linear-gradient" ),
1263   ( "-webkit-radial-gradient", "radial-gradient" ),
1264   ( "-webkit-repeating-radial-gradient", "repeating-radial-gradient" ),
1265 ]
1266
1267 def _CheckNoDeprecatedCSS(input_api, output_api):
1268   """ Make sure that we don't use deprecated CSS
1269       properties, functions or values. Our external
1270       documentation is ignored by the hooks as it
1271       needs to be consumed by WebKit. """
1272   results = []
1273   file_inclusion_pattern = (r".+\.css$",)
1274   black_list = (_EXCLUDED_PATHS +
1275                 _TEST_CODE_EXCLUDED_PATHS +
1276                 input_api.DEFAULT_BLACK_LIST +
1277                 (r"^chrome/common/extensions/docs",
1278                  r"^chrome/docs",
1279                  r"^native_client_sdk"))
1280   file_filter = lambda f: input_api.FilterSourceFile(
1281       f, white_list=file_inclusion_pattern, black_list=black_list)
1282   for fpath in input_api.AffectedFiles(file_filter=file_filter):
1283     for line_num, line in fpath.ChangedContents():
1284       for (deprecated_value, value) in _DEPRECATED_CSS:
1285         if deprecated_value in line:
1286           results.append(output_api.PresubmitError(
1287               "%s:%d: Use of deprecated CSS %s, use %s instead" %
1288               (fpath.LocalPath(), line_num, deprecated_value, value)))
1289   return results
1290
1291
1292 _DEPRECATED_JS = [
1293   ( "__lookupGetter__", "Object.getOwnPropertyDescriptor" ),
1294   ( "__defineGetter__", "Object.defineProperty" ),
1295   ( "__defineSetter__", "Object.defineProperty" ),
1296 ]
1297
1298 def _CheckNoDeprecatedJS(input_api, output_api):
1299   """Make sure that we don't use deprecated JS in Chrome code."""
1300   results = []
1301   file_inclusion_pattern = (r".+\.js$",)  # TODO(dbeam): .html?
1302   black_list = (_EXCLUDED_PATHS + _TEST_CODE_EXCLUDED_PATHS +
1303                 input_api.DEFAULT_BLACK_LIST)
1304   file_filter = lambda f: input_api.FilterSourceFile(
1305       f, white_list=file_inclusion_pattern, black_list=black_list)
1306   for fpath in input_api.AffectedFiles(file_filter=file_filter):
1307     for lnum, line in fpath.ChangedContents():
1308       for (deprecated, replacement) in _DEPRECATED_JS:
1309         if deprecated in line:
1310           results.append(output_api.PresubmitError(
1311               "%s:%d: Use of deprecated JS %s, use %s instead" %
1312               (fpath.LocalPath(), lnum, deprecated, replacement)))
1313   return results
1314
1315
1316 def _CommonChecks(input_api, output_api):
1317   """Checks common to both upload and commit."""
1318   results = []
1319   results.extend(input_api.canned_checks.PanProjectChecks(
1320       input_api, output_api,
1321       excluded_paths=_EXCLUDED_PATHS + _TESTRUNNER_PATHS))
1322   results.extend(_CheckAuthorizedAuthor(input_api, output_api))
1323   results.extend(
1324       _CheckNoProductionCodeUsingTestOnlyFunctions(input_api, output_api))
1325   results.extend(_CheckNoIOStreamInHeaders(input_api, output_api))
1326   results.extend(_CheckNoUNIT_TESTInSourceFiles(input_api, output_api))
1327   results.extend(_CheckNoNewWStrings(input_api, output_api))
1328   results.extend(_CheckNoDEPSGIT(input_api, output_api))
1329   results.extend(_CheckNoBannedFunctions(input_api, output_api))
1330   results.extend(_CheckNoPragmaOnce(input_api, output_api))
1331   results.extend(_CheckNoTrinaryTrueFalse(input_api, output_api))
1332   results.extend(_CheckUnwantedDependencies(input_api, output_api))
1333   results.extend(_CheckFilePermissions(input_api, output_api))
1334   results.extend(_CheckNoAuraWindowPropertyHInHeaders(input_api, output_api))
1335   results.extend(_CheckIncludeOrder(input_api, output_api))
1336   results.extend(_CheckForVersionControlConflicts(input_api, output_api))
1337   results.extend(_CheckPatchFiles(input_api, output_api))
1338   results.extend(_CheckHardcodedGoogleHostsInLowerLayers(input_api, output_api))
1339   results.extend(_CheckNoAbbreviationInPngFileName(input_api, output_api))
1340   results.extend(_CheckForInvalidOSMacros(input_api, output_api))
1341   results.extend(_CheckForInvalidIfDefinedMacros(input_api, output_api))
1342   # TODO(danakj): Remove this when base/move.h is removed.
1343   results.extend(_CheckForUsingSideEffectsOfPass(input_api, output_api))
1344   results.extend(_CheckAddedDepsHaveTargetApprovals(input_api, output_api))
1345   results.extend(
1346       input_api.canned_checks.CheckChangeHasNoTabs(
1347           input_api,
1348           output_api,
1349           source_file_filter=lambda x: x.LocalPath().endswith('.grd')))
1350   results.extend(_CheckSpamLogging(input_api, output_api))
1351   results.extend(_CheckForAnonymousVariables(input_api, output_api))
1352   results.extend(_CheckCygwinShell(input_api, output_api))
1353   results.extend(_CheckUserActionUpdate(input_api, output_api))
1354   results.extend(_CheckNoDeprecatedCSS(input_api, output_api))
1355   results.extend(_CheckNoDeprecatedJS(input_api, output_api))
1356   results.extend(_CheckParseErrors(input_api, output_api))
1357   results.extend(_CheckForIPCRules(input_api, output_api))
1358   results.extend(_CheckForCopyrightedCode(input_api, output_api))
1359   results.extend(_CheckForWindowsLineEndings(input_api, output_api))
1360
1361   if any('PRESUBMIT.py' == f.LocalPath() for f in input_api.AffectedFiles()):
1362     results.extend(input_api.canned_checks.RunUnitTestsInDirectory(
1363         input_api, output_api,
1364         input_api.PresubmitLocalPath(),
1365         whitelist=[r'^PRESUBMIT_test\.py$']))
1366   return results
1367
1368
1369 def _CheckAuthorizedAuthor(input_api, output_api):
1370   """For non-googler/chromites committers, verify the author's email address is
1371   in AUTHORS.
1372   """
1373   # TODO(maruel): Add it to input_api?
1374   import fnmatch
1375
1376   author = input_api.change.author_email
1377   if not author:
1378     input_api.logging.info('No author, skipping AUTHOR check')
1379     return []
1380   authors_path = input_api.os_path.join(
1381       input_api.PresubmitLocalPath(), 'AUTHORS')
1382   valid_authors = (
1383       input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line)
1384       for line in open(authors_path))
1385   valid_authors = [item.group(1).lower() for item in valid_authors if item]
1386   if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors):
1387     input_api.logging.info('Valid authors are %s', ', '.join(valid_authors))
1388     return [output_api.PresubmitPromptWarning(
1389         ('%s is not in AUTHORS file. If you are a new contributor, please visit'
1390         '\n'
1391         'http://www.chromium.org/developers/contributing-code and read the '
1392         '"Legal" section\n'
1393         'If you are a chromite, verify the contributor signed the CLA.') %
1394         author)]
1395   return []
1396
1397
1398 def _CheckPatchFiles(input_api, output_api):
1399   problems = [f.LocalPath() for f in input_api.AffectedFiles()
1400       if f.LocalPath().endswith(('.orig', '.rej'))]
1401   if problems:
1402     return [output_api.PresubmitError(
1403         "Don't commit .rej and .orig files.", problems)]
1404   else:
1405     return []
1406
1407
1408 def _DidYouMeanOSMacro(bad_macro):
1409   try:
1410     return {'A': 'OS_ANDROID',
1411             'B': 'OS_BSD',
1412             'C': 'OS_CHROMEOS',
1413             'F': 'OS_FREEBSD',
1414             'L': 'OS_LINUX',
1415             'M': 'OS_MACOSX',
1416             'N': 'OS_NACL',
1417             'O': 'OS_OPENBSD',
1418             'P': 'OS_POSIX',
1419             'S': 'OS_SOLARIS',
1420             'W': 'OS_WIN'}[bad_macro[3].upper()]
1421   except KeyError:
1422     return ''
1423
1424
1425 def _CheckForInvalidOSMacrosInFile(input_api, f):
1426   """Check for sensible looking, totally invalid OS macros."""
1427   preprocessor_statement = input_api.re.compile(r'^\s*#')
1428   os_macro = input_api.re.compile(r'defined\((OS_[^)]+)\)')
1429   results = []
1430   for lnum, line in f.ChangedContents():
1431     if preprocessor_statement.search(line):
1432       for match in os_macro.finditer(line):
1433         if not match.group(1) in _VALID_OS_MACROS:
1434           good = _DidYouMeanOSMacro(match.group(1))
1435           did_you_mean = ' (did you mean %s?)' % good if good else ''
1436           results.append('    %s:%d %s%s' % (f.LocalPath(),
1437                                              lnum,
1438                                              match.group(1),
1439                                              did_you_mean))
1440   return results
1441
1442
1443 def _CheckForInvalidOSMacros(input_api, output_api):
1444   """Check all affected files for invalid OS macros."""
1445   bad_macros = []
1446   for f in input_api.AffectedFiles():
1447     if not f.LocalPath().endswith(('.py', '.js', '.html', '.css')):
1448       bad_macros.extend(_CheckForInvalidOSMacrosInFile(input_api, f))
1449
1450   if not bad_macros:
1451     return []
1452
1453   return [output_api.PresubmitError(
1454       'Possibly invalid OS macro[s] found. Please fix your code\n'
1455       'or add your macro to src/PRESUBMIT.py.', bad_macros)]
1456
1457
1458 def _CheckForInvalidIfDefinedMacrosInFile(input_api, f):
1459   """Check all affected files for invalid "if defined" macros."""
1460   ALWAYS_DEFINED_MACROS = (
1461       "TARGET_CPU_PPC",
1462       "TARGET_CPU_PPC64",
1463       "TARGET_CPU_68K",
1464       "TARGET_CPU_X86",
1465       "TARGET_CPU_ARM",
1466       "TARGET_CPU_MIPS",
1467       "TARGET_CPU_SPARC",
1468       "TARGET_CPU_ALPHA",
1469       "TARGET_IPHONE_SIMULATOR",
1470       "TARGET_OS_EMBEDDED",
1471       "TARGET_OS_IPHONE",
1472       "TARGET_OS_MAC",
1473       "TARGET_OS_UNIX",
1474       "TARGET_OS_WIN32",
1475   )
1476   ifdef_macro = input_api.re.compile(r'^\s*#.*(?:ifdef\s|defined\()([^\s\)]+)')
1477   results = []
1478   for lnum, line in f.ChangedContents():
1479     for match in ifdef_macro.finditer(line):
1480       if match.group(1) in ALWAYS_DEFINED_MACROS:
1481         always_defined = ' %s is always defined. ' % match.group(1)
1482         did_you_mean = 'Did you mean \'#if %s\'?' % match.group(1)
1483         results.append('    %s:%d %s\n\t%s' % (f.LocalPath(),
1484                                                lnum,
1485                                                always_defined,
1486                                                did_you_mean))
1487   return results
1488
1489
1490 def _CheckForInvalidIfDefinedMacros(input_api, output_api):
1491   """Check all affected files for invalid "if defined" macros."""
1492   bad_macros = []
1493   for f in input_api.AffectedFiles():
1494     if f.LocalPath().endswith(('.h', '.c', '.cc', '.m', '.mm')):
1495       bad_macros.extend(_CheckForInvalidIfDefinedMacrosInFile(input_api, f))
1496
1497   if not bad_macros:
1498     return []
1499
1500   return [output_api.PresubmitError(
1501       'Found ifdef check on always-defined macro[s]. Please fix your code\n'
1502       'or check the list of ALWAYS_DEFINED_MACROS in src/PRESUBMIT.py.',
1503       bad_macros)]
1504
1505
1506 def _CheckForUsingSideEffectsOfPass(input_api, output_api):
1507   """Check all affected files for using side effects of Pass."""
1508   errors = []
1509   for f in input_api.AffectedFiles():
1510     if f.LocalPath().endswith(('.h', '.c', '.cc', '.m', '.mm')):
1511       for lnum, line in f.ChangedContents():
1512         # Disallow Foo(*my_scoped_thing.Pass()); See crbug.com/418297.
1513         if input_api.re.search(r'\*[a-zA-Z0-9_]+\.Pass\(\)', line):
1514           errors.append(output_api.PresubmitError(
1515             ('%s:%d uses *foo.Pass() to delete the contents of scoped_ptr. ' +
1516              'See crbug.com/418297.') % (f.LocalPath(), lnum)))
1517   return errors
1518
1519
1520 def _CheckForIPCRules(input_api, output_api):
1521   """Check for same IPC rules described in
1522   http://www.chromium.org/Home/chromium-security/education/security-tips-for-ipc
1523   """
1524   base_pattern = r'IPC_ENUM_TRAITS\('
1525   inclusion_pattern = input_api.re.compile(r'(%s)' % base_pattern)
1526   comment_pattern = input_api.re.compile(r'//.*(%s)' % base_pattern)
1527
1528   problems = []
1529   for f in input_api.AffectedSourceFiles(None):
1530     local_path = f.LocalPath()
1531     if not local_path.endswith('.h'):
1532       continue
1533     for line_number, line in f.ChangedContents():
1534       if inclusion_pattern.search(line) and not comment_pattern.search(line):
1535         problems.append(
1536           '%s:%d\n    %s' % (local_path, line_number, line.strip()))
1537
1538   if problems:
1539     return [output_api.PresubmitPromptWarning(
1540         _IPC_ENUM_TRAITS_DEPRECATED, problems)]
1541   else:
1542     return []
1543
1544
1545 def _CheckForWindowsLineEndings(input_api, output_api):
1546   """Check source code and known ascii text files for Windows style line
1547   endings.
1548   """
1549   known_text_files = r'.*\.(txt|html|htm|mhtml|py)$'
1550
1551   file_inclusion_pattern = (
1552     known_text_files,
1553     r'.+%s' % _IMPLEMENTATION_EXTENSIONS
1554   )
1555
1556   filter = lambda f: input_api.FilterSourceFile(
1557     f, white_list=file_inclusion_pattern, black_list=None)
1558   files = [f.LocalPath() for f in
1559            input_api.AffectedSourceFiles(filter)]
1560
1561   problems = []
1562
1563   for file in files:
1564     fp = open(file, 'r')
1565     for line in fp:
1566       if line.endswith('\r\n'):
1567         problems.append(file)
1568         break
1569     fp.close()
1570
1571   if problems:
1572     return [output_api.PresubmitPromptWarning('Are you sure that you want '
1573         'these files to contain Windows style line endings?\n' +
1574         '\n'.join(problems))]
1575
1576   return []
1577
1578
1579 def CheckChangeOnUpload(input_api, output_api):
1580   results = []
1581   results.extend(_CommonChecks(input_api, output_api))
1582   results.extend(_CheckValidHostsInDEPS(input_api, output_api))
1583   results.extend(_CheckJavaStyle(input_api, output_api))
1584   results.extend(
1585       input_api.canned_checks.CheckGNFormatted(input_api, output_api))
1586   return results
1587
1588
1589 def GetTryServerMasterForBot(bot):
1590   """Returns the Try Server master for the given bot.
1591
1592   It tries to guess the master from the bot name, but may still fail
1593   and return None.  There is no longer a default master.
1594   """
1595   # Potentially ambiguous bot names are listed explicitly.
1596   master_map = {
1597       'linux_gpu': 'tryserver.chromium.gpu',
1598       'win_gpu': 'tryserver.chromium.gpu',
1599       'chromium_presubmit': 'tryserver.chromium.linux',
1600       'blink_presubmit': 'tryserver.chromium.linux',
1601       'tools_build_presubmit': 'tryserver.chromium.linux',
1602   }
1603   master = master_map.get(bot)
1604   if not master:
1605     if 'gpu' in bot:
1606       master = 'tryserver.chromium.gpu'
1607     elif 'linux' in bot or 'android' in bot or 'presubmit' in bot:
1608       master = 'tryserver.chromium.linux'
1609     elif 'win' in bot:
1610       master = 'tryserver.chromium.win'
1611     elif 'mac' in bot or 'ios' in bot:
1612       master = 'tryserver.chromium.mac'
1613   return master
1614
1615
1616 def GetDefaultTryConfigs(bots):
1617   """Returns a list of ('bot', set(['tests']), filtered by [bots].
1618   """
1619
1620   builders_and_tests = dict((bot, set(['defaulttests'])) for bot in bots)
1621
1622   # Build up the mapping from tryserver master to bot/test.
1623   out = dict()
1624   for bot, tests in builders_and_tests.iteritems():
1625     out.setdefault(GetTryServerMasterForBot(bot), {})[bot] = tests
1626   return out
1627
1628
1629 def CheckChangeOnCommit(input_api, output_api):
1630   results = []
1631   results.extend(_CommonChecks(input_api, output_api))
1632   # TODO(thestig) temporarily disabled, doesn't work in third_party/
1633   #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
1634   #    input_api, output_api, sources))
1635   # Make sure the tree is 'open'.
1636   results.extend(input_api.canned_checks.CheckTreeIsOpen(
1637       input_api,
1638       output_api,
1639       json_url='http://chromium-status.appspot.com/current?format=json'))
1640
1641   results.extend(input_api.canned_checks.CheckChangeHasBugField(
1642       input_api, output_api))
1643   results.extend(input_api.canned_checks.CheckChangeHasDescription(
1644       input_api, output_api))
1645   return results
1646
1647
1648 def GetPreferredTryMasters(project, change):
1649   import re
1650   files = change.LocalPaths()
1651
1652   if not files or all(re.search(r'[\\\/]OWNERS$', f) for f in files):
1653     return {}
1654
1655   if all(re.search(r'\.(m|mm)$|(^|[\\\/_])mac[\\\/_.]', f) for f in files):
1656     return GetDefaultTryConfigs([
1657         'mac_chromium_compile_dbg_ng',
1658         'mac_chromium_rel_ng',
1659     ])
1660   if all(re.search('(^|[/_])win[/_.]', f) for f in files):
1661     return GetDefaultTryConfigs([
1662         'win8_chromium_rel',
1663         'win_chromium_rel_ng',
1664         'win_chromium_x64_rel_ng',
1665     ])
1666   if all(re.search(r'(^|[\\\/_])android[\\\/_.]', f) and
1667          not re.search(r'(^|[\\\/_])devtools[\\\/_.]', f) for f in files):
1668     return GetDefaultTryConfigs([
1669         'android_aosp',
1670         'android_dbg_tests_recipe',
1671     ])
1672   if all(re.search(r'[\\\/_]ios[\\\/_.]', f) for f in files):
1673     return GetDefaultTryConfigs(['ios_rel_device', 'ios_dbg_simulator'])
1674
1675   import os
1676   import json
1677   with open(os.path.join(
1678       change.RepositoryRoot(), 'testing', 'commit_queue', 'config.json')) as f:
1679     cq_config = json.load(f)
1680     cq_trybots = cq_config.get('trybots', {})
1681     builders = cq_trybots.get('launched', {})
1682     for master, master_config in cq_trybots.get('triggered', {}).iteritems():
1683       for triggered_bot in master_config:
1684         builders.get(master, {}).pop(triggered_bot, None)
1685
1686     # Explicitly iterate over copies of dicts since we mutate them.
1687     for master in builders.keys():
1688       for builder in builders[master].keys():
1689         # Do not trigger presubmit builders, since they're likely to fail
1690         # (e.g. OWNERS checks before finished code review), and we're
1691         # running local presubmit anyway.
1692         if 'presubmit' in builder:
1693           builders[master].pop(builder)
1694
1695   # Match things like path/aura/file.cc and path/file_aura.cc.
1696   # Same for chromeos.
1697   if any(re.search(r'[\\\/_](aura|chromeos)', f) for f in files):
1698     tryserver_linux = builders.setdefault('tryserver.chromium.linux', {})
1699     tryserver_linux['linux_chromium_chromeos_asan_rel_ng'] = ['defaulttests']
1700
1701   return builders