PRESUBMIT.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Top-level presubmit script for Chromium.
   6
   7 See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
   8 for more details about the presubmit API built into gcl.
   9 """
  10
  11
  12 import re
  13 import sys
  14
  15
  16 _EXCLUDED_PATHS = (
  17     r"^breakpad[\\\/].*",
  18     r"^native_client_sdk[\\\/]src[\\\/]build_tools[\\\/]make_rules.py",
  19     r"^native_client_sdk[\\\/]src[\\\/]build_tools[\\\/]make_simple.py",
  20     r"^native_client_sdk[\\\/]src[\\\/]tools[\\\/].*.mk",
  21     r"^net[\\\/]tools[\\\/]spdyshark[\\\/].*",
  22     r"^skia[\\\/].*",
  23     r"^v8[\\\/].*",
  24     r".*MakeFile$",
  25     r".+_autogen\.h$",
  26     r".+[\\\/]pnacl_shim\.c$",
  27     r"^gpu[\\\/]config[\\\/].*_list_json\.cc$",
  28     r"^chrome[\\\/]browser[\\\/]resources[\\\/]pdf[\\\/]index.js"
  29 )
  30
  31 # The NetscapePlugIn library is excluded from pan-project as it will soon
  32 # be deleted together with the rest of the NPAPI and it's not worthwhile to
  33 # update the coding style until then.
  34 _TESTRUNNER_PATHS = (
  35     r"^content[\\\/]shell[\\\/]tools[\\\/]plugin[\\\/].*",
  36 )
  37
  38 # Fragment of a regular expression that matches C++ and Objective-C++
  39 # implementation files.
  40 _IMPLEMENTATION_EXTENSIONS = r'\.(cc|cpp|cxx|mm)$'
  41
  42 # Regular expression that matches code only used for test binaries
  43 # (best effort).
  44 _TEST_CODE_EXCLUDED_PATHS = (
  45     r'.*[\\\/](fake_|test_|mock_).+%s' % _IMPLEMENTATION_EXTENSIONS,
  46     r'.+_test_(base|support|util)%s' % _IMPLEMENTATION_EXTENSIONS,
  47     r'.+_(api|browser|kif|perf|pixel|unit|ui)?test(_[a-z]+)?%s' %
  48         _IMPLEMENTATION_EXTENSIONS,
  49     r'.+profile_sync_service_harness%s' % _IMPLEMENTATION_EXTENSIONS,
  50     r'.*[\\\/](test|tool(s)?)[\\\/].*',
  51     # content_shell is used for running layout tests.
  52     r'content[\\\/]shell[\\\/].*',
  53     # At request of folks maintaining this folder.
  54     r'chrome[\\\/]browser[\\\/]automation[\\\/].*',
  55     # Non-production example code.
  56     r'mojo[\\\/]examples[\\\/].*',
  57     # Launcher for running iOS tests on the simulator.
  58     r'testing[\\\/]iossim[\\\/]iossim\.mm$',
  59 )
  60
  61 _TEST_ONLY_WARNING = (
  62     'You might be calling functions intended only for testing from\n'
  63     'production code.  It is OK to ignore this warning if you know what\n'
  64     'you are doing, as the heuristics used to detect the situation are\n'
  65     'not perfect.  The commit queue will not block on this warning.')
  66
  67
  68 _INCLUDE_ORDER_WARNING = (
  69     'Your #include order seems to be broken. Send mail to\n'
  70     'marja@chromium.org if this is not the case.')
  71
  72
  73 _BANNED_OBJC_FUNCTIONS = (
  74     (
  75       'addTrackingRect:',
  76       (
  77        'The use of -[NSView addTrackingRect:owner:userData:assumeInside:] is'
  78        'prohibited. Please use CrTrackingArea instead.',
  79        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
  80       ),
  81       False,
  82     ),
  83     (
  84       r'/NSTrackingArea\W',
  85       (
  86        'The use of NSTrackingAreas is prohibited. Please use CrTrackingArea',
  87        'instead.',
  88        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
  89       ),
  90       False,
  91     ),
  92     (
  93       'convertPointFromBase:',
  94       (
  95        'The use of -[NSView convertPointFromBase:] is almost certainly wrong.',
  96        'Please use |convertPoint:(point) fromView:nil| instead.',
  97        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
  98       ),
  99       True,
 100     ),
 101     (
 102       'convertPointToBase:',
 103       (
 104        'The use of -[NSView convertPointToBase:] is almost certainly wrong.',
 105        'Please use |convertPoint:(point) toView:nil| instead.',
 106        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 107       ),
 108       True,
 109     ),
 110     (
 111       'convertRectFromBase:',
 112       (
 113        'The use of -[NSView convertRectFromBase:] is almost certainly wrong.',
 114        'Please use |convertRect:(point) fromView:nil| instead.',
 115        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 116       ),
 117       True,
 118     ),
 119     (
 120       'convertRectToBase:',
 121       (
 122        'The use of -[NSView convertRectToBase:] is almost certainly wrong.',
 123        'Please use |convertRect:(point) toView:nil| instead.',
 124        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 125       ),
 126       True,
 127     ),
 128     (
 129       'convertSizeFromBase:',
 130       (
 131        'The use of -[NSView convertSizeFromBase:] is almost certainly wrong.',
 132        'Please use |convertSize:(point) fromView:nil| instead.',
 133        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 134       ),
 135       True,
 136     ),
 137     (
 138       'convertSizeToBase:',
 139       (
 140        'The use of -[NSView convertSizeToBase:] is almost certainly wrong.',
 141        'Please use |convertSize:(point) toView:nil| instead.',
 142        'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
 143       ),
 144       True,
 145     ),
 146 )
 147
 148
 149 _BANNED_CPP_FUNCTIONS = (
 150     # Make sure that gtest's FRIEND_TEST() macro is not used; the
 151     # FRIEND_TEST_ALL_PREFIXES() macro from base/gtest_prod_util.h should be
 152     # used instead since that allows for FLAKY_ and DISABLED_ prefixes.
 153     (
 154       'FRIEND_TEST(',
 155       (
 156        'Chromium code should not use gtest\'s FRIEND_TEST() macro. Include',
 157        'base/gtest_prod_util.h and use FRIEND_TEST_ALL_PREFIXES() instead.',
 158       ),
 159       False,
 160       (),
 161     ),
 162     (
 163       'ScopedAllowIO',
 164       (
 165        'New code should not use ScopedAllowIO. Post a task to the blocking',
 166        'pool or the FILE thread instead.',
 167       ),
 168       True,
 169       (
 170         r"^base[\\\/]process[\\\/]process_metrics_linux\.cc$",
 171         r"^chrome[\\\/]browser[\\\/]chromeos[\\\/]boot_times_loader\.cc$",
 172         r"^components[\\\/]crash[\\\/]app[\\\/]breakpad_mac\.mm$",
 173         r"^content[\\\/]shell[\\\/]browser[\\\/]shell_browser_main\.cc$",
 174         r"^content[\\\/]shell[\\\/]browser[\\\/]shell_message_filter\.cc$",
 175         r"^mojo[\\\/]edk[\\\/]embedder[\\\/]" +
 176             r"simple_platform_shared_buffer_posix\.cc$",
 177         r"^net[\\\/]disk_cache[\\\/]cache_util\.cc$",
 178         r"^net[\\\/]url_request[\\\/]test_url_fetcher_factory\.cc$",
 179       ),
 180     ),
 181     (
 182       'SkRefPtr',
 183       (
 184         'The use of SkRefPtr is prohibited. ',
 185         'Please use skia::RefPtr instead.'
 186       ),
 187       True,
 188       (),
 189     ),
 190     (
 191       'SkAutoRef',
 192       (
 193         'The indirect use of SkRefPtr via SkAutoRef is prohibited. ',
 194         'Please use skia::RefPtr instead.'
 195       ),
 196       True,
 197       (),
 198     ),
 199     (
 200       'SkAutoTUnref',
 201       (
 202         'The use of SkAutoTUnref is dangerous because it implicitly ',
 203         'converts to a raw pointer. Please use skia::RefPtr instead.'
 204       ),
 205       True,
 206       (),
 207     ),
 208     (
 209       'SkAutoUnref',
 210       (
 211         'The indirect use of SkAutoTUnref through SkAutoUnref is dangerous ',
 212         'because it implicitly converts to a raw pointer. ',
 213         'Please use skia::RefPtr instead.'
 214       ),
 215       True,
 216       (),
 217     ),
 218     (
 219       r'/HANDLE_EINTR\(.*close',
 220       (
 221        'HANDLE_EINTR(close) is invalid. If close fails with EINTR, the file',
 222        'descriptor will be closed, and it is incorrect to retry the close.',
 223        'Either call close directly and ignore its return value, or wrap close',
 224        'in IGNORE_EINTR to use its return value. See http://crbug.com/269623'
 225       ),
 226       True,
 227       (),
 228     ),
 229     (
 230       r'/IGNORE_EINTR\((?!.*close)',
 231       (
 232        'IGNORE_EINTR is only valid when wrapping close. To wrap other system',
 233        'calls, use HANDLE_EINTR. See http://crbug.com/269623',
 234       ),
 235       True,
 236       (
 237         # Files that #define IGNORE_EINTR.
 238         r'^base[\\\/]posix[\\\/]eintr_wrapper\.h$',
 239         r'^ppapi[\\\/]tests[\\\/]test_broker\.cc$',
 240       ),
 241     ),
 242     (
 243       r'/v8::Extension\(',
 244       (
 245         'Do not introduce new v8::Extensions into the code base, use',
 246         'gin::Wrappable instead. See http://crbug.com/334679',
 247       ),
 248       True,
 249       (
 250         r'extensions[\\\/]renderer[\\\/]safe_builtins\.*',
 251       ),
 252     ),
 253 )
 254
 255 _IPC_ENUM_TRAITS_DEPRECATED = (
 256     'You are using IPC_ENUM_TRAITS() in your code. It has been deprecated.\n'
 257     'See http://www.chromium.org/Home/chromium-security/education/security-tips-for-ipc')
 258
 259
 260 _VALID_OS_MACROS = (
 261     # Please keep sorted.
 262     'OS_ANDROID',
 263     'OS_ANDROID_HOST',
 264     'OS_BSD',
 265     'OS_CAT',       # For testing.
 266     'OS_CHROMEOS',
 267     'OS_FREEBSD',
 268     'OS_IOS',
 269     'OS_LINUX',
 270     'OS_MACOSX',
 271     'OS_NACL',
 272     'OS_OPENBSD',
 273     'OS_POSIX',
 274     'OS_QNX',
 275     'OS_SOLARIS',
 276     'OS_WIN',
 277 )
 278
 279
 280 def _CheckNoProductionCodeUsingTestOnlyFunctions(input_api, output_api):
 281   """Attempts to prevent use of functions intended only for testing in
 282   non-testing code. For now this is just a best-effort implementation
 283   that ignores header files and may have some false positives. A
 284   better implementation would probably need a proper C++ parser.
 285   """
 286   # We only scan .cc files and the like, as the declaration of
 287   # for-testing functions in header files are hard to distinguish from
 288   # calls to such functions without a proper C++ parser.
 289   file_inclusion_pattern = r'.+%s' % _IMPLEMENTATION_EXTENSIONS
 290
 291   base_function_pattern = r'[ :]test::[^\s]+|ForTest(ing)?|for_test(ing)?'
 292   inclusion_pattern = input_api.re.compile(r'(%s)\s*\(' % base_function_pattern)
 293   comment_pattern = input_api.re.compile(r'//.*(%s)' % base_function_pattern)
 294   exclusion_pattern = input_api.re.compile(
 295     r'::[A-Za-z0-9_]+(%s)|(%s)[^;]+\{' % (
 296       base_function_pattern, base_function_pattern))
 297
 298   def FilterFile(affected_file):
 299     black_list = (_EXCLUDED_PATHS +
 300                   _TEST_CODE_EXCLUDED_PATHS +
 301                   input_api.DEFAULT_BLACK_LIST)
 302     return input_api.FilterSourceFile(
 303       affected_file,
 304       white_list=(file_inclusion_pattern, ),
 305       black_list=black_list)
 306
 307   problems = []
 308   for f in input_api.AffectedSourceFiles(FilterFile):
 309     local_path = f.LocalPath()
 310     for line_number, line in f.ChangedContents():
 311       if (inclusion_pattern.search(line) and
 312           not comment_pattern.search(line) and
 313           not exclusion_pattern.search(line)):
 314         problems.append(
 315           '%s:%d\n    %s' % (local_path, line_number, line.strip()))
 316
 317   if problems:
 318     return [output_api.PresubmitPromptOrNotify(_TEST_ONLY_WARNING, problems)]
 319   else:
 320     return []
 321
 322
 323 def _CheckNoIOStreamInHeaders(input_api, output_api):
 324   """Checks to make sure no .h files include <iostream>."""
 325   files = []
 326   pattern = input_api.re.compile(r'^#include\s*<iostream>',
 327                                  input_api.re.MULTILINE)
 328   for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
 329     if not f.LocalPath().endswith('.h'):
 330       continue
 331     contents = input_api.ReadFile(f)
 332     if pattern.search(contents):
 333       files.append(f)
 334
 335   if len(files):
 336     return [ output_api.PresubmitError(
 337         'Do not #include <iostream> in header files, since it inserts static '
 338         'initialization into every file including the header. Instead, '
 339         '#include <ostream>. See http://crbug.com/94794',
 340         files) ]
 341   return []
 342
 343
 344 def _CheckNoUNIT_TESTInSourceFiles(input_api, output_api):
 345   """Checks to make sure no source files use UNIT_TEST"""
 346   problems = []
 347   for f in input_api.AffectedFiles():
 348     if (not f.LocalPath().endswith(('.cc', '.mm'))):
 349       continue
 350
 351     for line_num, line in f.ChangedContents():
 352       if 'UNIT_TEST ' in line or line.endswith('UNIT_TEST'):
 353         problems.append('    %s:%d' % (f.LocalPath(), line_num))
 354
 355   if not problems:
 356     return []
 357   return [output_api.PresubmitPromptWarning('UNIT_TEST is only for headers.\n' +
 358       '\n'.join(problems))]
 359
 360
 361 def _CheckNoNewWStrings(input_api, output_api):
 362   """Checks to make sure we don't introduce use of wstrings."""
 363   problems = []
 364   for f in input_api.AffectedFiles():
 365     if (not f.LocalPath().endswith(('.cc', '.h')) or
 366         f.LocalPath().endswith(('test.cc', '_win.cc', '_win.h'))):
 367       continue
 368
 369     allowWString = False
 370     for line_num, line in f.ChangedContents():
 371       if 'presubmit: allow wstring' in line:
 372         allowWString = True
 373       elif not allowWString and 'wstring' in line:
 374         problems.append('    %s:%d' % (f.LocalPath(), line_num))
 375         allowWString = False
 376       else:
 377         allowWString = False
 378
 379   if not problems:
 380     return []
 381   return [output_api.PresubmitPromptWarning('New code should not use wstrings.'
 382       '  If you are calling a cross-platform API that accepts a wstring, '
 383       'fix the API.\n' +
 384       '\n'.join(problems))]
 385
 386
 387 def _CheckNoDEPSGIT(input_api, output_api):
 388   """Make sure .DEPS.git is never modified manually."""
 389   if any(f.LocalPath().endswith('.DEPS.git') for f in
 390       input_api.AffectedFiles()):
 391     return [output_api.PresubmitError(
 392       'Never commit changes to .DEPS.git. This file is maintained by an\n'
 393       'automated system based on what\'s in DEPS and your changes will be\n'
 394       'overwritten.\n'
 395       'See https://sites.google.com/a/chromium.org/dev/developers/how-tos/get-the-code#Rolling_DEPS\n'
 396       'for more information')]
 397   return []
 398
 399
 400 def _CheckValidHostsInDEPS(input_api, output_api):
 401   """Checks that DEPS file deps are from allowed_hosts."""
 402   # Run only if DEPS file has been modified to annoy fewer bystanders.
 403   if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()):
 404     return []
 405   # Outsource work to gclient verify
 406   try:
 407     input_api.subprocess.check_output(['gclient', 'verify'])
 408     return []
 409   except input_api.subprocess.CalledProcessError, error:
 410     return [output_api.PresubmitError(
 411         'DEPS file must have only git dependencies.',
 412         long_text=error.output)]
 413
 414
 415 def _CheckNoBannedFunctions(input_api, output_api):
 416   """Make sure that banned functions are not used."""
 417   warnings = []
 418   errors = []
 419
 420   file_filter = lambda f: f.LocalPath().endswith(('.mm', '.m', '.h'))
 421   for f in input_api.AffectedFiles(file_filter=file_filter):
 422     for line_num, line in f.ChangedContents():
 423       for func_name, message, error in _BANNED_OBJC_FUNCTIONS:
 424         matched = False
 425         if func_name[0:1] == '/':
 426           regex = func_name[1:]
 427           if input_api.re.search(regex, line):
 428             matched = True
 429         elif func_name in line:
 430             matched = True
 431         if matched:
 432           problems = warnings;
 433           if error:
 434             problems = errors;
 435           problems.append('    %s:%d:' % (f.LocalPath(), line_num))
 436           for message_line in message:
 437             problems.append('      %s' % message_line)
 438
 439   file_filter = lambda f: f.LocalPath().endswith(('.cc', '.mm', '.h'))
 440   for f in input_api.AffectedFiles(file_filter=file_filter):
 441     for line_num, line in f.ChangedContents():
 442       for func_name, message, error, excluded_paths in _BANNED_CPP_FUNCTIONS:
 443         def IsBlacklisted(affected_file, blacklist):
 444           local_path = affected_file.LocalPath()
 445           for item in blacklist:
 446             if input_api.re.match(item, local_path):
 447               return True
 448           return False
 449         if IsBlacklisted(f, excluded_paths):
 450           continue
 451         matched = False
 452         if func_name[0:1] == '/':
 453           regex = func_name[1:]
 454           if input_api.re.search(regex, line):
 455             matched = True
 456         elif func_name in line:
 457             matched = True
 458         if matched:
 459           problems = warnings;
 460           if error:
 461             problems = errors;
 462           problems.append('    %s:%d:' % (f.LocalPath(), line_num))
 463           for message_line in message:
 464             problems.append('      %s' % message_line)
 465
 466   result = []
 467   if (warnings):
 468     result.append(output_api.PresubmitPromptWarning(
 469         'Banned functions were used.\n' + '\n'.join(warnings)))
 470   if (errors):
 471     result.append(output_api.PresubmitError(
 472         'Banned functions were used.\n' + '\n'.join(errors)))
 473   return result
 474
 475
 476 def _CheckNoPragmaOnce(input_api, output_api):
 477   """Make sure that banned functions are not used."""
 478   files = []
 479   pattern = input_api.re.compile(r'^#pragma\s+once',
 480                                  input_api.re.MULTILINE)
 481   for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
 482     if not f.LocalPath().endswith('.h'):
 483       continue
 484     contents = input_api.ReadFile(f)
 485     if pattern.search(contents):
 486       files.append(f)
 487
 488   if files:
 489     return [output_api.PresubmitError(
 490         'Do not use #pragma once in header files.\n'
 491         'See http://www.chromium.org/developers/coding-style#TOC-File-headers',
 492         files)]
 493   return []
 494
 495
 496 def _CheckNoTrinaryTrueFalse(input_api, output_api):
 497   """Checks to make sure we don't introduce use of foo ? true : false."""
 498   problems = []
 499   pattern = input_api.re.compile(r'\?\s*(true|false)\s*:\s*(true|false)')
 500   for f in input_api.AffectedFiles():
 501     if not f.LocalPath().endswith(('.cc', '.h', '.inl', '.m', '.mm')):
 502       continue
 503
 504     for line_num, line in f.ChangedContents():
 505       if pattern.match(line):
 506         problems.append('    %s:%d' % (f.LocalPath(), line_num))
 507
 508   if not problems:
 509     return []
 510   return [output_api.PresubmitPromptWarning(
 511       'Please consider avoiding the "? true : false" pattern if possible.\n' +
 512       '\n'.join(problems))]
 513
 514
 515 def _CheckUnwantedDependencies(input_api, output_api):
 516   """Runs checkdeps on #include statements added in this
 517   change. Breaking - rules is an error, breaking ! rules is a
 518   warning.
 519   """
 520   # We need to wait until we have an input_api object and use this
 521   # roundabout construct to import checkdeps because this file is
 522   # eval-ed and thus doesn't have __file__.
 523   original_sys_path = sys.path
 524   try:
 525     sys.path = sys.path + [input_api.os_path.join(
 526         input_api.PresubmitLocalPath(), 'buildtools', 'checkdeps')]
 527     import checkdeps
 528     from cpp_checker import CppChecker
 529     from rules import Rule
 530   finally:
 531     # Restore sys.path to what it was before.
 532     sys.path = original_sys_path
 533
 534   added_includes = []
 535   for f in input_api.AffectedFiles():
 536     if not CppChecker.IsCppFile(f.LocalPath()):
 537       continue
 538
 539     changed_lines = [line for line_num, line in f.ChangedContents()]
 540     added_includes.append([f.LocalPath(), changed_lines])
 541
 542   deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath())
 543
 544   error_descriptions = []
 545   warning_descriptions = []
 546   for path, rule_type, rule_description in deps_checker.CheckAddedCppIncludes(
 547       added_includes):
 548     description_with_path = '%s\n    %s' % (path, rule_description)
 549     if rule_type == Rule.DISALLOW:
 550       error_descriptions.append(description_with_path)
 551     else:
 552       warning_descriptions.append(description_with_path)
 553
 554   results = []
 555   if error_descriptions:
 556     results.append(output_api.PresubmitError(
 557         'You added one or more #includes that violate checkdeps rules.',
 558         error_descriptions))
 559   if warning_descriptions:
 560     results.append(output_api.PresubmitPromptOrNotify(
 561         'You added one or more #includes of files that are temporarily\n'
 562         'allowed but being removed. Can you avoid introducing the\n'
 563         '#include? See relevant DEPS file(s) for details and contacts.',
 564         warning_descriptions))
 565   return results
 566
 567
 568 def _CheckFilePermissions(input_api, output_api):
 569   """Check that all files have their permissions properly set."""
 570   if input_api.platform == 'win32':
 571     return []
 572   args = [sys.executable, 'tools/checkperms/checkperms.py', '--root',
 573           input_api.change.RepositoryRoot()]
 574   for f in input_api.AffectedFiles():
 575     args += ['--file', f.LocalPath()]
 576   checkperms = input_api.subprocess.Popen(args,
 577                                           stdout=input_api.subprocess.PIPE)
 578   errors = checkperms.communicate()[0].strip()
 579   if errors:
 580     return [output_api.PresubmitError('checkperms.py failed.',
 581                                       errors.splitlines())]
 582   return []
 583
 584
 585 def _CheckNoAuraWindowPropertyHInHeaders(input_api, output_api):
 586   """Makes sure we don't include ui/aura/window_property.h
 587   in header files.
 588   """
 589   pattern = input_api.re.compile(r'^#include\s*"ui/aura/window_property.h"')
 590   errors = []
 591   for f in input_api.AffectedFiles():
 592     if not f.LocalPath().endswith('.h'):
 593       continue
 594     for line_num, line in f.ChangedContents():
 595       if pattern.match(line):
 596         errors.append('    %s:%d' % (f.LocalPath(), line_num))
 597
 598   results = []
 599   if errors:
 600     results.append(output_api.PresubmitError(
 601       'Header files should not include ui/aura/window_property.h', errors))
 602   return results
 603
 604
 605 def _CheckIncludeOrderForScope(scope, input_api, file_path, changed_linenums):
 606   """Checks that the lines in scope occur in the right order.
 607
 608   1. C system files in alphabetical order
 609   2. C++ system files in alphabetical order
 610   3. Project's .h files
 611   """
 612
 613   c_system_include_pattern = input_api.re.compile(r'\s*#include <.*\.h>')
 614   cpp_system_include_pattern = input_api.re.compile(r'\s*#include <.*>')
 615   custom_include_pattern = input_api.re.compile(r'\s*#include ".*')
 616
 617   C_SYSTEM_INCLUDES, CPP_SYSTEM_INCLUDES, CUSTOM_INCLUDES = range(3)
 618
 619   state = C_SYSTEM_INCLUDES
 620
 621   previous_line = ''
 622   previous_line_num = 0
 623   problem_linenums = []
 624   for line_num, line in scope:
 625     if c_system_include_pattern.match(line):
 626       if state != C_SYSTEM_INCLUDES:
 627         problem_linenums.append((line_num, previous_line_num))
 628       elif previous_line and previous_line > line:
 629         problem_linenums.append((line_num, previous_line_num))
 630     elif cpp_system_include_pattern.match(line):
 631       if state == C_SYSTEM_INCLUDES:
 632         state = CPP_SYSTEM_INCLUDES
 633       elif state == CUSTOM_INCLUDES:
 634         problem_linenums.append((line_num, previous_line_num))
 635       elif previous_line and previous_line > line:
 636         problem_linenums.append((line_num, previous_line_num))
 637     elif custom_include_pattern.match(line):
 638       if state != CUSTOM_INCLUDES:
 639         state = CUSTOM_INCLUDES
 640       elif previous_line and previous_line > line:
 641         problem_linenums.append((line_num, previous_line_num))
 642     else:
 643       problem_linenums.append(line_num)
 644     previous_line = line
 645     previous_line_num = line_num
 646
 647   warnings = []
 648   for (line_num, previous_line_num) in problem_linenums:
 649     if line_num in changed_linenums or previous_line_num in changed_linenums:
 650       warnings.append('    %s:%d' % (file_path, line_num))
 651   return warnings
 652
 653
 654 def _CheckIncludeOrderInFile(input_api, f, changed_linenums):
 655   """Checks the #include order for the given file f."""
 656
 657   system_include_pattern = input_api.re.compile(r'\s*#include \<.*')
 658   # Exclude the following includes from the check:
 659   # 1) #include <.../...>, e.g., <sys/...> includes often need to appear in a
 660   # specific order.
 661   # 2) <atlbase.h>, "build/build_config.h"
 662   excluded_include_pattern = input_api.re.compile(
 663       r'\s*#include (\<.*/.*|\<atlbase\.h\>|"build/build_config.h")')
 664   custom_include_pattern = input_api.re.compile(r'\s*#include "(?P<FILE>.*)"')
 665   # Match the final or penultimate token if it is xxxtest so we can ignore it
 666   # when considering the special first include.
 667   test_file_tag_pattern = input_api.re.compile(
 668     r'_[a-z]+test(?=(_[a-zA-Z0-9]+)?\.)')
 669   if_pattern = input_api.re.compile(
 670       r'\s*#\s*(if|elif|else|endif|define|undef).*')
 671   # Some files need specialized order of includes; exclude such files from this
 672   # check.
 673   uncheckable_includes_pattern = input_api.re.compile(
 674       r'\s*#include '
 675       '("ipc/.*macros\.h"|<windows\.h>|".*gl.*autogen.h")\s*')
 676
 677   contents = f.NewContents()
 678   warnings = []
 679   line_num = 0
 680
 681   # Handle the special first include. If the first include file is
 682   # some/path/file.h, the corresponding including file can be some/path/file.cc,
 683   # some/other/path/file.cc, some/path/file_platform.cc, some/path/file-suffix.h
 684   # etc. It's also possible that no special first include exists.
 685   # If the included file is some/path/file_platform.h the including file could
 686   # also be some/path/file_xxxtest_platform.h.
 687   including_file_base_name = test_file_tag_pattern.sub(
 688     '', input_api.os_path.basename(f.LocalPath()))
 689
 690   for line in contents:
 691     line_num += 1
 692     if system_include_pattern.match(line):
 693       # No special first include -> process the line again along with normal
 694       # includes.
 695       line_num -= 1
 696       break
 697     match = custom_include_pattern.match(line)
 698     if match:
 699       match_dict = match.groupdict()
 700       header_basename = test_file_tag_pattern.sub(
 701         '', input_api.os_path.basename(match_dict['FILE'])).replace('.h', '')
 702
 703       if header_basename not in including_file_base_name:
 704         # No special first include -> process the line again along with normal
 705         # includes.
 706         line_num -= 1
 707       break
 708
 709   # Split into scopes: Each region between #if and #endif is its own scope.
 710   scopes = []
 711   current_scope = []
 712   for line in contents[line_num:]:
 713     line_num += 1
 714     if uncheckable_includes_pattern.match(line):
 715       continue
 716     if if_pattern.match(line):
 717       scopes.append(current_scope)
 718       current_scope = []
 719     elif ((system_include_pattern.match(line) or
 720            custom_include_pattern.match(line)) and
 721           not excluded_include_pattern.match(line)):
 722       current_scope.append((line_num, line))
 723   scopes.append(current_scope)
 724
 725   for scope in scopes:
 726     warnings.extend(_CheckIncludeOrderForScope(scope, input_api, f.LocalPath(),
 727                                                changed_linenums))
 728   return warnings
 729
 730
 731 def _CheckIncludeOrder(input_api, output_api):
 732   """Checks that the #include order is correct.
 733
 734   1. The corresponding header for source files.
 735   2. C system files in alphabetical order
 736   3. C++ system files in alphabetical order
 737   4. Project's .h files in alphabetical order
 738
 739   Each region separated by #if, #elif, #else, #endif, #define and #undef follows
 740   these rules separately.
 741   """
 742   def FileFilterIncludeOrder(affected_file):
 743     black_list = (_EXCLUDED_PATHS + input_api.DEFAULT_BLACK_LIST)
 744     return input_api.FilterSourceFile(affected_file, black_list=black_list)
 745
 746   warnings = []
 747   for f in input_api.AffectedFiles(file_filter=FileFilterIncludeOrder):
 748     if f.LocalPath().endswith(('.cc', '.h')):
 749       changed_linenums = set(line_num for line_num, _ in f.ChangedContents())
 750       warnings.extend(_CheckIncludeOrderInFile(input_api, f, changed_linenums))
 751
 752   results = []
 753   if warnings:
 754     results.append(output_api.PresubmitPromptOrNotify(_INCLUDE_ORDER_WARNING,
 755                                                       warnings))
 756   return results
 757
 758
 759 def _CheckForVersionControlConflictsInFile(input_api, f):
 760   pattern = input_api.re.compile('^(?:<<<<<<<|>>>>>>>) |^=======$')
 761   errors = []
 762   for line_num, line in f.ChangedContents():
 763     if pattern.match(line):
 764       errors.append('    %s:%d %s' % (f.LocalPath(), line_num, line))
 765   return errors
 766
 767
 768 def _CheckForVersionControlConflicts(input_api, output_api):
 769   """Usually this is not intentional and will cause a compile failure."""
 770   errors = []
 771   for f in input_api.AffectedFiles():
 772     errors.extend(_CheckForVersionControlConflictsInFile(input_api, f))
 773
 774   results = []
 775   if errors:
 776     results.append(output_api.PresubmitError(
 777       'Version control conflict markers found, please resolve.', errors))
 778   return results
 779
 780
 781 def _CheckHardcodedGoogleHostsInLowerLayers(input_api, output_api):
 782   def FilterFile(affected_file):
 783     """Filter function for use with input_api.AffectedSourceFiles,
 784     below.  This filters out everything except non-test files from
 785     top-level directories that generally speaking should not hard-code
 786     service URLs (e.g. src/android_webview/, src/content/ and others).
 787     """
 788     return input_api.FilterSourceFile(
 789       affected_file,
 790       white_list=(r'^(android_webview|base|content|net)[\\\/].*', ),
 791       black_list=(_EXCLUDED_PATHS +
 792                   _TEST_CODE_EXCLUDED_PATHS +
 793                   input_api.DEFAULT_BLACK_LIST))
 794
 795   base_pattern = '"[^"]*google\.com[^"]*"'
 796   comment_pattern = input_api.re.compile('//.*%s' % base_pattern)
 797   pattern = input_api.re.compile(base_pattern)
 798   problems = []  # items are (filename, line_number, line)
 799   for f in input_api.AffectedSourceFiles(FilterFile):
 800     for line_num, line in f.ChangedContents():
 801       if not comment_pattern.search(line) and pattern.search(line):
 802         problems.append((f.LocalPath(), line_num, line))
 803
 804   if problems:
 805     return [output_api.PresubmitPromptOrNotify(
 806         'Most layers below src/chrome/ should not hardcode service URLs.\n'
 807         'Are you sure this is correct?',
 808         ['  %s:%d:  %s' % (
 809             problem[0], problem[1], problem[2]) for problem in problems])]
 810   else:
 811     return []
 812
 813
 814 def _CheckNoAbbreviationInPngFileName(input_api, output_api):
 815   """Makes sure there are no abbreviations in the name of PNG files.
 816   """
 817   pattern = input_api.re.compile(r'.*_[a-z]_.*\.png$|.*_[a-z]\.png$')
 818   errors = []
 819   for f in input_api.AffectedFiles(include_deletes=False):
 820     if pattern.match(f.LocalPath()):
 821       errors.append('    %s' % f.LocalPath())
 822
 823   results = []
 824   if errors:
 825     results.append(output_api.PresubmitError(
 826         'The name of PNG files should not have abbreviations. \n'
 827         'Use _hover.png, _center.png, instead of _h.png, _c.png.\n'
 828         'Contact oshima@chromium.org if you have questions.', errors))
 829   return results
 830
 831
 832 def _FilesToCheckForIncomingDeps(re, changed_lines):
 833   """Helper method for _CheckAddedDepsHaveTargetApprovals. Returns
 834   a set of DEPS entries that we should look up.
 835
 836   For a directory (rather than a specific filename) we fake a path to
 837   a specific filename by adding /DEPS. This is chosen as a file that
 838   will seldom or never be subject to per-file include_rules.
 839   """
 840   # We ignore deps entries on auto-generated directories.
 841   AUTO_GENERATED_DIRS = ['grit', 'jni']
 842
 843   # This pattern grabs the path without basename in the first
 844   # parentheses, and the basename (if present) in the second. It
 845   # relies on the simple heuristic that if there is a basename it will
 846   # be a header file ending in ".h".
 847   pattern = re.compile(
 848       r"""['"]\+([^'"]+?)(/[a-zA-Z0-9_]+\.h)?['"].*""")
 849   results = set()
 850   for changed_line in changed_lines:
 851     m = pattern.match(changed_line)
 852     if m:
 853       path = m.group(1)
 854       if path.split('/')[0] not in AUTO_GENERATED_DIRS:
 855         if m.group(2):
 856           results.add('%s%s' % (path, m.group(2)))
 857         else:
 858           results.add('%s/DEPS' % path)
 859   return results
 860
 861
 862 def _CheckAddedDepsHaveTargetApprovals(input_api, output_api):
 863   """When a dependency prefixed with + is added to a DEPS file, we
 864   want to make sure that the change is reviewed by an OWNER of the
 865   target file or directory, to avoid layering violations from being
 866   introduced. This check verifies that this happens.
 867   """
 868   changed_lines = set()
 869   for f in input_api.AffectedFiles():
 870     filename = input_api.os_path.basename(f.LocalPath())
 871     if filename == 'DEPS':
 872       changed_lines |= set(line.strip()
 873                            for line_num, line
 874                            in f.ChangedContents())
 875   if not changed_lines:
 876     return []
 877
 878   virtual_depended_on_files = _FilesToCheckForIncomingDeps(input_api.re,
 879                                                            changed_lines)
 880   if not virtual_depended_on_files:
 881     return []
 882
 883   if input_api.is_committing:
 884     if input_api.tbr:
 885       return [output_api.PresubmitNotifyResult(
 886           '--tbr was specified, skipping OWNERS check for DEPS additions')]
 887     if not input_api.change.issue:
 888       return [output_api.PresubmitError(
 889           "DEPS approval by OWNERS check failed: this change has "
 890           "no Rietveld issue number, so we can't check it for approvals.")]
 891     output = output_api.PresubmitError
 892   else:
 893     output = output_api.PresubmitNotifyResult
 894
 895   owners_db = input_api.owners_db
 896   owner_email, reviewers = input_api.canned_checks._RietveldOwnerAndReviewers(
 897       input_api,
 898       owners_db.email_regexp,
 899       approval_needed=input_api.is_committing)
 900
 901   owner_email = owner_email or input_api.change.author_email
 902
 903   reviewers_plus_owner = set(reviewers)
 904   if owner_email:
 905     reviewers_plus_owner.add(owner_email)
 906   missing_files = owners_db.files_not_covered_by(virtual_depended_on_files,
 907                                                  reviewers_plus_owner)
 908
 909   # We strip the /DEPS part that was added by
 910   # _FilesToCheckForIncomingDeps to fake a path to a file in a
 911   # directory.
 912   def StripDeps(path):
 913     start_deps = path.rfind('/DEPS')
 914     if start_deps != -1:
 915       return path[:start_deps]
 916     else:
 917       return path
 918   unapproved_dependencies = ["'+%s'," % StripDeps(path)
 919                              for path in missing_files]
 920
 921   if unapproved_dependencies:
 922     output_list = [
 923       output('Missing LGTM from OWNERS of dependencies added to DEPS:\n    %s' %
 924              '\n    '.join(sorted(unapproved_dependencies)))]
 925     if not input_api.is_committing:
 926       suggested_owners = owners_db.reviewers_for(missing_files, owner_email)
 927       output_list.append(output(
 928           'Suggested missing target path OWNERS:\n    %s' %
 929           '\n    '.join(suggested_owners or [])))
 930     return output_list
 931
 932   return []
 933
 934
 935 def _CheckSpamLogging(input_api, output_api):
 936   file_inclusion_pattern = r'.+%s' % _IMPLEMENTATION_EXTENSIONS
 937   black_list = (_EXCLUDED_PATHS +
 938                 _TEST_CODE_EXCLUDED_PATHS +
 939                 input_api.DEFAULT_BLACK_LIST +
 940                 (r"^base[\\\/]logging\.h$",
 941                  r"^base[\\\/]logging\.cc$",
 942                  r"^chrome[\\\/]app[\\\/]chrome_main_delegate\.cc$",
 943                  r"^chrome[\\\/]browser[\\\/]chrome_browser_main\.cc$",
 944                  r"^chrome[\\\/]browser[\\\/]ui[\\\/]startup[\\\/]"
 945                      r"startup_browser_creator\.cc$",
 946                  r"^chrome[\\\/]installer[\\\/]setup[\\\/].*",
 947                  r"chrome[\\\/]browser[\\\/]diagnostics[\\\/]" +
 948                      r"diagnostics_writer\.cc$",
 949                  r"^chrome_elf[\\\/]dll_hash[\\\/]dll_hash_main\.cc$",
 950                  r"^chromecast[\\\/]",
 951                  r"^cloud_print[\\\/]",
 952                  r"^content[\\\/]common[\\\/]gpu[\\\/]client[\\\/]"
 953                      r"gl_helper_benchmark\.cc$",
 954                  r"^courgette[\\\/]courgette_tool\.cc$",
 955                  r"^extensions[\\\/]renderer[\\\/]logging_native_handler\.cc$",
 956                  r"^native_client_sdk[\\\/]",
 957                  r"^remoting[\\\/]base[\\\/]logging\.h$",
 958                  r"^remoting[\\\/]host[\\\/].*",
 959                  r"^sandbox[\\\/]linux[\\\/].*",
 960                  r"^tools[\\\/]",
 961                  r"^ui[\\\/]aura[\\\/]bench[\\\/]bench_main\.cc$",
 962                  r"^webkit[\\\/]browser[\\\/]fileapi[\\\/]" +
 963                      r"dump_file_system.cc$",))
 964   source_file_filter = lambda x: input_api.FilterSourceFile(
 965       x, white_list=(file_inclusion_pattern,), black_list=black_list)
 966
 967   log_info = []
 968   printf = []
 969
 970   for f in input_api.AffectedSourceFiles(source_file_filter):
 971     contents = input_api.ReadFile(f, 'rb')
 972     if re.search(r"\bD?LOG\s*\(\s*INFO\s*\)", contents):
 973       log_info.append(f.LocalPath())
 974     elif re.search(r"\bD?LOG_IF\s*\(\s*INFO\s*,", contents):
 975       log_info.append(f.LocalPath())
 976
 977     if re.search(r"\bprintf\(", contents):
 978       printf.append(f.LocalPath())
 979     elif re.search(r"\bfprintf\((stdout|stderr)", contents):
 980       printf.append(f.LocalPath())
 981
 982   if log_info:
 983     return [output_api.PresubmitError(
 984       'These files spam the console log with LOG(INFO):',
 985       items=log_info)]
 986   if printf:
 987     return [output_api.PresubmitError(
 988       'These files spam the console log with printf/fprintf:',
 989       items=printf)]
 990   return []
 991
 992
 993 def _CheckForAnonymousVariables(input_api, output_api):
 994   """These types are all expected to hold locks while in scope and
 995      so should never be anonymous (which causes them to be immediately
 996      destroyed)."""
 997   they_who_must_be_named = [
 998     'base::AutoLock',
 999     'base::AutoReset',
1000     'base::AutoUnlock',
1001     'SkAutoAlphaRestore',
1002     'SkAutoBitmapShaderInstall',
1003     'SkAutoBlitterChoose',
1004     'SkAutoBounderCommit',
1005     'SkAutoCallProc',
1006     'SkAutoCanvasRestore',
1007     'SkAutoCommentBlock',
1008     'SkAutoDescriptor',
1009     'SkAutoDisableDirectionCheck',
1010     'SkAutoDisableOvalCheck',
1011     'SkAutoFree',
1012     'SkAutoGlyphCache',
1013     'SkAutoHDC',
1014     'SkAutoLockColors',
1015     'SkAutoLockPixels',
1016     'SkAutoMalloc',
1017     'SkAutoMaskFreeImage',
1018     'SkAutoMutexAcquire',
1019     'SkAutoPathBoundsUpdate',
1020     'SkAutoPDFRelease',
1021     'SkAutoRasterClipValidate',
1022     'SkAutoRef',
1023     'SkAutoTime',
1024     'SkAutoTrace',
1025     'SkAutoUnref',
1026   ]
1027   anonymous = r'(%s)\s*[({]' % '|'.join(they_who_must_be_named)
1028   # bad: base::AutoLock(lock.get());
1029   # not bad: base::AutoLock lock(lock.get());
1030   bad_pattern = input_api.re.compile(anonymous)
1031   # good: new base::AutoLock(lock.get())
1032   good_pattern = input_api.re.compile(r'\bnew\s*' + anonymous)
1033   errors = []
1034
1035   for f in input_api.AffectedFiles():
1036     if not f.LocalPath().endswith(('.cc', '.h', '.inl', '.m', '.mm')):
1037       continue
1038     for linenum, line in f.ChangedContents():
1039       if bad_pattern.search(line) and not good_pattern.search(line):
1040         errors.append('%s:%d' % (f.LocalPath(), linenum))
1041
1042   if errors:
1043     return [output_api.PresubmitError(
1044       'These lines create anonymous variables that need to be named:',
1045       items=errors)]
1046   return []
1047
1048
1049 def _CheckCygwinShell(input_api, output_api):
1050   source_file_filter = lambda x: input_api.FilterSourceFile(
1051       x, white_list=(r'.+\.(gyp|gypi)$',))
1052   cygwin_shell = []
1053
1054   for f in input_api.AffectedSourceFiles(source_file_filter):
1055     for linenum, line in f.ChangedContents():
1056       if 'msvs_cygwin_shell' in line:
1057         cygwin_shell.append(f.LocalPath())
1058         break
1059
1060   if cygwin_shell:
1061     return [output_api.PresubmitError(
1062       'These files should not use msvs_cygwin_shell (the default is 0):',
1063       items=cygwin_shell)]
1064   return []
1065
1066
1067 def _CheckUserActionUpdate(input_api, output_api):
1068   """Checks if any new user action has been added."""
1069   if any('actions.xml' == input_api.os_path.basename(f) for f in
1070          input_api.LocalPaths()):
1071     # If actions.xml is already included in the changelist, the PRESUBMIT
1072     # for actions.xml will do a more complete presubmit check.
1073     return []
1074
1075   file_filter = lambda f: f.LocalPath().endswith(('.cc', '.mm'))
1076   action_re = r'[^a-zA-Z]UserMetricsAction\("([^"]*)'
1077   current_actions = None
1078   for f in input_api.AffectedFiles(file_filter=file_filter):
1079     for line_num, line in f.ChangedContents():
1080       match = input_api.re.search(action_re, line)
1081       if match:
1082         # Loads contents in tools/metrics/actions/actions.xml to memory. It's
1083         # loaded only once.
1084         if not current_actions:
1085           with open('tools/metrics/actions/actions.xml') as actions_f:
1086             current_actions = actions_f.read()
1087         # Search for the matched user action name in |current_actions|.
1088         for action_name in match.groups():
1089           action = 'name="{0}"'.format(action_name)
1090           if action not in current_actions:
1091             return [output_api.PresubmitPromptWarning(
1092               'File %s line %d: %s is missing in '
1093               'tools/metrics/actions/actions.xml. Please run '
1094               'tools/metrics/actions/extract_actions.py to update.'
1095               % (f.LocalPath(), line_num, action_name))]
1096   return []
1097
1098
1099 def _GetJSONParseError(input_api, filename, eat_comments=True):
1100   try:
1101     contents = input_api.ReadFile(filename)
1102     if eat_comments:
1103       json_comment_eater = input_api.os_path.join(
1104           input_api.PresubmitLocalPath(),
1105           'tools', 'json_comment_eater', 'json_comment_eater.py')
1106       process = input_api.subprocess.Popen(
1107           [input_api.python_executable, json_comment_eater],
1108           stdin=input_api.subprocess.PIPE,
1109           stdout=input_api.subprocess.PIPE,
1110           universal_newlines=True)
1111       (contents, _) = process.communicate(input=contents)
1112
1113     input_api.json.loads(contents)
1114   except ValueError as e:
1115     return e
1116   return None
1117
1118
1119 def _GetIDLParseError(input_api, filename):
1120   try:
1121     contents = input_api.ReadFile(filename)
1122     idl_schema = input_api.os_path.join(
1123         input_api.PresubmitLocalPath(),
1124         'tools', 'json_schema_compiler', 'idl_schema.py')
1125     process = input_api.subprocess.Popen(
1126         [input_api.python_executable, idl_schema],
1127         stdin=input_api.subprocess.PIPE,
1128         stdout=input_api.subprocess.PIPE,
1129         stderr=input_api.subprocess.PIPE,
1130         universal_newlines=True)
1131     (_, error) = process.communicate(input=contents)
1132     return error or None
1133   except ValueError as e:
1134     return e
1135
1136
1137 def _CheckParseErrors(input_api, output_api):
1138   """Check that IDL and JSON files do not contain syntax errors."""
1139   actions = {
1140     '.idl': _GetIDLParseError,
1141     '.json': _GetJSONParseError,
1142   }
1143   # These paths contain test data and other known invalid JSON files.
1144   excluded_patterns = [
1145     r'test[\\\/]data[\\\/]',
1146     r'^components[\\\/]policy[\\\/]resources[\\\/]policy_templates\.json$',
1147   ]
1148   # Most JSON files are preprocessed and support comments, but these do not.
1149   json_no_comments_patterns = [
1150     r'^testing[\\\/]',
1151   ]
1152   # Only run IDL checker on files in these directories.
1153   idl_included_patterns = [
1154     r'^chrome[\\\/]common[\\\/]extensions[\\\/]api[\\\/]',
1155     r'^extensions[\\\/]common[\\\/]api[\\\/]',
1156   ]
1157
1158   def get_action(affected_file):
1159     filename = affected_file.LocalPath()
1160     return actions.get(input_api.os_path.splitext(filename)[1])
1161
1162   def MatchesFile(patterns, path):
1163     for pattern in patterns:
1164       if input_api.re.search(pattern, path):
1165         return True
1166     return False
1167
1168   def FilterFile(affected_file):
1169     action = get_action(affected_file)
1170     if not action:
1171       return False
1172     path = affected_file.LocalPath()
1173
1174     if MatchesFile(excluded_patterns, path):
1175       return False
1176
1177     if (action == _GetIDLParseError and
1178         not MatchesFile(idl_included_patterns, path)):
1179       return False
1180     return True
1181
1182   results = []
1183   for affected_file in input_api.AffectedFiles(
1184       file_filter=FilterFile, include_deletes=False):
1185     action = get_action(affected_file)
1186     kwargs = {}
1187     if (action == _GetJSONParseError and
1188         MatchesFile(json_no_comments_patterns, affected_file.LocalPath())):
1189       kwargs['eat_comments'] = False
1190     parse_error = action(input_api,
1191                          affected_file.AbsoluteLocalPath(),
1192                          **kwargs)
1193     if parse_error:
1194       results.append(output_api.PresubmitError('%s could not be parsed: %s' %
1195           (affected_file.LocalPath(), parse_error)))
1196   return results
1197
1198
1199 def _CheckJavaStyle(input_api, output_api):
1200   """Runs checkstyle on changed java files and returns errors if any exist."""
1201   original_sys_path = sys.path
1202   try:
1203     sys.path = sys.path + [input_api.os_path.join(
1204         input_api.PresubmitLocalPath(), 'tools', 'android', 'checkstyle')]
1205     import checkstyle
1206   finally:
1207     # Restore sys.path to what it was before.
1208     sys.path = original_sys_path
1209
1210   return checkstyle.RunCheckstyle(
1211       input_api, output_api, 'tools/android/checkstyle/chromium-style-5.0.xml')
1212
1213
1214 _DEPRECATED_CSS = [
1215   # Values
1216   ( "-webkit-box", "flex" ),
1217   ( "-webkit-inline-box", "inline-flex" ),
1218   ( "-webkit-flex", "flex" ),
1219   ( "-webkit-inline-flex", "inline-flex" ),
1220   ( "-webkit-min-content", "min-content" ),
1221   ( "-webkit-max-content", "max-content" ),
1222
1223   # Properties
1224   ( "-webkit-background-clip", "background-clip" ),
1225   ( "-webkit-background-origin", "background-origin" ),
1226   ( "-webkit-background-size", "background-size" ),
1227   ( "-webkit-box-shadow", "box-shadow" ),
1228
1229   # Functions
1230   ( "-webkit-gradient", "gradient" ),
1231   ( "-webkit-repeating-gradient", "repeating-gradient" ),
1232   ( "-webkit-linear-gradient", "linear-gradient" ),
1233   ( "-webkit-repeating-linear-gradient", "repeating-linear-gradient" ),
1234   ( "-webkit-radial-gradient", "radial-gradient" ),
1235   ( "-webkit-repeating-radial-gradient", "repeating-radial-gradient" ),
1236 ]
1237
1238 def _CheckNoDeprecatedCSS(input_api, output_api):
1239   """ Make sure that we don't use deprecated CSS
1240       properties, functions or values. Our external
1241       documentation is ignored by the hooks as it
1242       needs to be consumed by WebKit. """
1243   results = []
1244   file_inclusion_pattern = (r".+\.css$")
1245   black_list = (_EXCLUDED_PATHS +
1246                 _TEST_CODE_EXCLUDED_PATHS +
1247                 input_api.DEFAULT_BLACK_LIST +
1248                 (r"^chrome/common/extensions/docs",
1249                  r"^chrome/docs",
1250                  r"^native_client_sdk"))
1251   file_filter = lambda f: input_api.FilterSourceFile(
1252       f, white_list=file_inclusion_pattern, black_list=black_list)
1253   for fpath in input_api.AffectedFiles(file_filter=file_filter):
1254     for line_num, line in fpath.ChangedContents():
1255       for (deprecated_value, value) in _DEPRECATED_CSS:
1256         if input_api.re.search(deprecated_value, line):
1257           results.append(output_api.PresubmitError(
1258               "%s:%d: Use of deprecated CSS %s, use %s instead" %
1259               (fpath.LocalPath(), line_num, deprecated_value, value)))
1260   return results
1261
1262 def _CommonChecks(input_api, output_api):
1263   """Checks common to both upload and commit."""
1264   results = []
1265   results.extend(input_api.canned_checks.PanProjectChecks(
1266       input_api, output_api,
1267       excluded_paths=_EXCLUDED_PATHS + _TESTRUNNER_PATHS))
1268   results.extend(_CheckAuthorizedAuthor(input_api, output_api))
1269   results.extend(
1270       _CheckNoProductionCodeUsingTestOnlyFunctions(input_api, output_api))
1271   results.extend(_CheckNoIOStreamInHeaders(input_api, output_api))
1272   results.extend(_CheckNoUNIT_TESTInSourceFiles(input_api, output_api))
1273   results.extend(_CheckNoNewWStrings(input_api, output_api))
1274   results.extend(_CheckNoDEPSGIT(input_api, output_api))
1275   results.extend(_CheckNoBannedFunctions(input_api, output_api))
1276   results.extend(_CheckNoPragmaOnce(input_api, output_api))
1277   results.extend(_CheckNoTrinaryTrueFalse(input_api, output_api))
1278   results.extend(_CheckUnwantedDependencies(input_api, output_api))
1279   results.extend(_CheckFilePermissions(input_api, output_api))
1280   results.extend(_CheckNoAuraWindowPropertyHInHeaders(input_api, output_api))
1281   results.extend(_CheckIncludeOrder(input_api, output_api))
1282   results.extend(_CheckForVersionControlConflicts(input_api, output_api))
1283   results.extend(_CheckPatchFiles(input_api, output_api))
1284   results.extend(_CheckHardcodedGoogleHostsInLowerLayers(input_api, output_api))
1285   results.extend(_CheckNoAbbreviationInPngFileName(input_api, output_api))
1286   results.extend(_CheckForInvalidOSMacros(input_api, output_api))
1287   results.extend(_CheckForInvalidIfDefinedMacros(input_api, output_api))
1288   # TODO(danakj): Remove this when base/move.h is removed.
1289   results.extend(_CheckForUsingSideEffectsOfPass(input_api, output_api))
1290   results.extend(_CheckAddedDepsHaveTargetApprovals(input_api, output_api))
1291   results.extend(
1292       input_api.canned_checks.CheckChangeHasNoTabs(
1293           input_api,
1294           output_api,
1295           source_file_filter=lambda x: x.LocalPath().endswith('.grd')))
1296   results.extend(_CheckSpamLogging(input_api, output_api))
1297   results.extend(_CheckForAnonymousVariables(input_api, output_api))
1298   results.extend(_CheckCygwinShell(input_api, output_api))
1299   results.extend(_CheckUserActionUpdate(input_api, output_api))
1300   results.extend(_CheckNoDeprecatedCSS(input_api, output_api))
1301   results.extend(_CheckParseErrors(input_api, output_api))
1302   results.extend(_CheckForIPCRules(input_api, output_api))
1303
1304   if any('PRESUBMIT.py' == f.LocalPath() for f in input_api.AffectedFiles()):
1305     results.extend(input_api.canned_checks.RunUnitTestsInDirectory(
1306         input_api, output_api,
1307         input_api.PresubmitLocalPath(),
1308         whitelist=[r'^PRESUBMIT_test\.py$']))
1309   return results
1310
1311
1312 def _CheckAuthorizedAuthor(input_api, output_api):
1313   """For non-googler/chromites committers, verify the author's email address is
1314   in AUTHORS.
1315   """
1316   # TODO(maruel): Add it to input_api?
1317   import fnmatch
1318
1319   author = input_api.change.author_email
1320   if not author:
1321     input_api.logging.info('No author, skipping AUTHOR check')
1322     return []
1323   authors_path = input_api.os_path.join(
1324       input_api.PresubmitLocalPath(), 'AUTHORS')
1325   valid_authors = (
1326       input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line)
1327       for line in open(authors_path))
1328   valid_authors = [item.group(1).lower() for item in valid_authors if item]
1329   if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors):
1330     input_api.logging.info('Valid authors are %s', ', '.join(valid_authors))
1331     return [output_api.PresubmitPromptWarning(
1332         ('%s is not in AUTHORS file. If you are a new contributor, please visit'
1333         '\n'
1334         'http://www.chromium.org/developers/contributing-code and read the '
1335         '"Legal" section\n'
1336         'If you are a chromite, verify the contributor signed the CLA.') %
1337         author)]
1338   return []
1339
1340
1341 def _CheckPatchFiles(input_api, output_api):
1342   problems = [f.LocalPath() for f in input_api.AffectedFiles()
1343       if f.LocalPath().endswith(('.orig', '.rej'))]
1344   if problems:
1345     return [output_api.PresubmitError(
1346         "Don't commit .rej and .orig files.", problems)]
1347   else:
1348     return []
1349
1350
1351 def _DidYouMeanOSMacro(bad_macro):
1352   try:
1353     return {'A': 'OS_ANDROID',
1354             'B': 'OS_BSD',
1355             'C': 'OS_CHROMEOS',
1356             'F': 'OS_FREEBSD',
1357             'L': 'OS_LINUX',
1358             'M': 'OS_MACOSX',
1359             'N': 'OS_NACL',
1360             'O': 'OS_OPENBSD',
1361             'P': 'OS_POSIX',
1362             'S': 'OS_SOLARIS',
1363             'W': 'OS_WIN'}[bad_macro[3].upper()]
1364   except KeyError:
1365     return ''
1366
1367
1368 def _CheckForInvalidOSMacrosInFile(input_api, f):
1369   """Check for sensible looking, totally invalid OS macros."""
1370   preprocessor_statement = input_api.re.compile(r'^\s*#')
1371   os_macro = input_api.re.compile(r'defined\((OS_[^)]+)\)')
1372   results = []
1373   for lnum, line in f.ChangedContents():
1374     if preprocessor_statement.search(line):
1375       for match in os_macro.finditer(line):
1376         if not match.group(1) in _VALID_OS_MACROS:
1377           good = _DidYouMeanOSMacro(match.group(1))
1378           did_you_mean = ' (did you mean %s?)' % good if good else ''
1379           results.append('    %s:%d %s%s' % (f.LocalPath(),
1380                                              lnum,
1381                                              match.group(1),
1382                                              did_you_mean))
1383   return results
1384
1385
1386 def _CheckForInvalidOSMacros(input_api, output_api):
1387   """Check all affected files for invalid OS macros."""
1388   bad_macros = []
1389   for f in input_api.AffectedFiles():
1390     if not f.LocalPath().endswith(('.py', '.js', '.html', '.css')):
1391       bad_macros.extend(_CheckForInvalidOSMacrosInFile(input_api, f))
1392
1393   if not bad_macros:
1394     return []
1395
1396   return [output_api.PresubmitError(
1397       'Possibly invalid OS macro[s] found. Please fix your code\n'
1398       'or add your macro to src/PRESUBMIT.py.', bad_macros)]
1399
1400
1401 def _CheckForInvalidIfDefinedMacrosInFile(input_api, f):
1402   """Check all affected files for invalid "if defined" macros."""
1403   ALWAYS_DEFINED_MACROS = (
1404       "TARGET_CPU_PPC",
1405       "TARGET_CPU_PPC64",
1406       "TARGET_CPU_68K",
1407       "TARGET_CPU_X86",
1408       "TARGET_CPU_ARM",
1409       "TARGET_CPU_MIPS",
1410       "TARGET_CPU_SPARC",
1411       "TARGET_CPU_ALPHA",
1412       "TARGET_IPHONE_SIMULATOR",
1413       "TARGET_OS_EMBEDDED",
1414       "TARGET_OS_IPHONE",
1415       "TARGET_OS_MAC",
1416       "TARGET_OS_UNIX",
1417       "TARGET_OS_WIN32",
1418   )
1419   ifdef_macro = input_api.re.compile(r'^\s*#.*(?:ifdef\s|defined\()([^\s\)]+)')
1420   results = []
1421   for lnum, line in f.ChangedContents():
1422     for match in ifdef_macro.finditer(line):
1423       if match.group(1) in ALWAYS_DEFINED_MACROS:
1424         always_defined = ' %s is always defined. ' % match.group(1)
1425         did_you_mean = 'Did you mean \'#if %s\'?' % match.group(1)
1426         results.append('    %s:%d %s\n\t%s' % (f.LocalPath(),
1427                                                lnum,
1428                                                always_defined,
1429                                                did_you_mean))
1430   return results
1431
1432
1433 def _CheckForInvalidIfDefinedMacros(input_api, output_api):
1434   """Check all affected files for invalid "if defined" macros."""
1435   bad_macros = []
1436   for f in input_api.AffectedFiles():
1437     if f.LocalPath().endswith(('.h', '.c', '.cc', '.m', '.mm')):
1438       bad_macros.extend(_CheckForInvalidIfDefinedMacrosInFile(input_api, f))
1439
1440   if not bad_macros:
1441     return []
1442
1443   return [output_api.PresubmitError(
1444       'Found ifdef check on always-defined macro[s]. Please fix your code\n'
1445       'or check the list of ALWAYS_DEFINED_MACROS in src/PRESUBMIT.py.',
1446       bad_macros)]
1447
1448
1449 def _CheckForUsingSideEffectsOfPass(input_api, output_api):
1450   """Check all affected files for using side effects of Pass."""
1451   errors = []
1452   for f in input_api.AffectedFiles():
1453     if f.LocalPath().endswith(('.h', '.c', '.cc', '.m', '.mm')):
1454       for lnum, line in f.ChangedContents():
1455         # Disallow Foo(*my_scoped_thing.Pass()); See crbug.com/418297.
1456         if re.search(r'\*[a-zA-Z0-9_]+\.Pass\(\)', line):
1457           errors.append(output_api.PresubmitError(
1458             ('%s:%d uses *foo.Pass() to delete the contents of scoped_ptr. ' +
1459              'See crbug.com/418297.') % (f.LocalPath(), lnum)))
1460   return errors
1461
1462
1463 def _CheckForIPCRules(input_api, output_api):
1464   """Check for same IPC rules described in
1465   http://www.chromium.org/Home/chromium-security/education/security-tips-for-ipc
1466   """
1467   base_pattern = r'IPC_ENUM_TRAITS\('
1468   inclusion_pattern = input_api.re.compile(r'(%s)' % base_pattern)
1469   comment_pattern = input_api.re.compile(r'//.*(%s)' % base_pattern)
1470
1471   problems = []
1472   for f in input_api.AffectedSourceFiles(None):
1473     local_path = f.LocalPath()
1474     if not local_path.endswith('.h'):
1475       continue
1476     for line_number, line in f.ChangedContents():
1477       if inclusion_pattern.search(line) and not comment_pattern.search(line):
1478         problems.append(
1479           '%s:%d\n    %s' % (local_path, line_number, line.strip()))
1480
1481   if problems:
1482     return [output_api.PresubmitPromptWarning(
1483         _IPC_ENUM_TRAITS_DEPRECATED, problems)]
1484   else:
1485     return []
1486
1487
1488 def CheckChangeOnUpload(input_api, output_api):
1489   results = []
1490   results.extend(_CommonChecks(input_api, output_api))
1491   results.extend(_CheckValidHostsInDEPS(input_api, output_api))
1492   results.extend(_CheckJavaStyle(input_api, output_api))
1493   return results
1494
1495
1496 def GetTryServerMasterForBot(bot):
1497   """Returns the Try Server master for the given bot.
1498
1499   It tries to guess the master from the bot name, but may still fail
1500   and return None.  There is no longer a default master.
1501   """
1502   # Potentially ambiguous bot names are listed explicitly.
1503   master_map = {
1504       'linux_gpu': 'tryserver.chromium.gpu',
1505       'mac_gpu': 'tryserver.chromium.gpu',
1506       'win_gpu': 'tryserver.chromium.gpu',
1507       'chromium_presubmit': 'tryserver.chromium.linux',
1508       'blink_presubmit': 'tryserver.chromium.linux',
1509       'tools_build_presubmit': 'tryserver.chromium.linux',
1510   }
1511   master = master_map.get(bot)
1512   if not master:
1513     if 'gpu' in bot:
1514       master = 'tryserver.chromium.gpu'
1515     elif 'linux' in bot or 'android' in bot or 'presubmit' in bot:
1516       master = 'tryserver.chromium.linux'
1517     elif 'win' in bot:
1518       master = 'tryserver.chromium.win'
1519     elif 'mac' in bot or 'ios' in bot:
1520       master = 'tryserver.chromium.mac'
1521   return master
1522
1523
1524 def GetDefaultTryConfigs(bots=None):
1525   """Returns a list of ('bot', set(['tests']), optionally filtered by [bots].
1526
1527   To add tests to this list, they MUST be in the the corresponding master's
1528   gatekeeper config. For example, anything on master.chromium would be closed by
1529   tools/build/masters/master.chromium/master_gatekeeper_cfg.py.
1530
1531   If 'bots' is specified, will only return configurations for bots in that list.
1532   """
1533
1534   standard_tests = [
1535       'base_unittests',
1536       'browser_tests',
1537       'cacheinvalidation_unittests',
1538       'check_deps',
1539       'check_deps2git',
1540       'content_browsertests',
1541       'content_unittests',
1542       'crypto_unittests',
1543       'gpu_unittests',
1544       'interactive_ui_tests',
1545       'ipc_tests',
1546       'jingle_unittests',
1547       'media_unittests',
1548       'net_unittests',
1549       'ppapi_unittests',
1550       'printing_unittests',
1551       'sql_unittests',
1552       'sync_unit_tests',
1553       'unit_tests',
1554       # Broken in release.
1555       #'url_unittests',
1556       #'webkit_unit_tests',
1557   ]
1558
1559   builders_and_tests = {
1560       # TODO(maruel): Figure out a way to run 'sizes' where people can
1561       # effectively update the perf expectation correctly.  This requires a
1562       # clobber=True build running 'sizes'. 'sizes' is not accurate with
1563       # incremental build. Reference:
1564       # http://chromium.org/developers/tree-sheriffs/perf-sheriffs.
1565       # TODO(maruel): An option would be to run 'sizes' but not count a failure
1566       # of this step as a try job failure.
1567       'android_aosp': ['compile'],
1568       'android_arm64_dbg_recipe': ['slave_steps'],
1569       'android_chromium_gn_compile_dbg': ['compile'],
1570       'android_chromium_gn_compile_rel': ['compile'],
1571       'android_clang_dbg': ['slave_steps'],
1572       'android_clang_dbg_recipe': ['slave_steps'],
1573       'android_dbg_tests_recipe': ['slave_steps'],
1574       'cros_x86': ['defaulttests'],
1575       'ios_dbg_simulator': [
1576           'compile',
1577           'base_unittests',
1578           'content_unittests',
1579           'crypto_unittests',
1580           'url_unittests',
1581           'net_unittests',
1582           'sql_unittests',
1583           'ui_base_unittests',
1584           'ui_unittests',
1585       ],
1586       'ios_rel_device': ['compile'],
1587       'ios_rel_device_ninja': ['compile'],
1588       'linux_asan': ['compile'],
1589       'mac_asan': ['compile'],
1590       #TODO(stip): Change the name of this builder to reflect that it's release.
1591       'linux_gtk': standard_tests,
1592       'linux_chromeos_asan': ['compile'],
1593       'linux_chromium_chromeos_clang_dbg': ['defaulttests'],
1594       'linux_chromium_chromeos_rel_swarming': ['defaulttests'],
1595       'linux_chromium_compile_dbg': ['defaulttests'],
1596       'linux_chromium_gn_dbg': ['compile'],
1597       'linux_chromium_gn_rel': ['defaulttests'],
1598       'linux_chromium_rel_swarming': ['defaulttests'],
1599       'linux_chromium_clang_dbg': ['defaulttests'],
1600       'linux_gpu': ['defaulttests'],
1601       'linux_nacl_sdk_build': ['compile'],
1602       'mac_chromium_compile_dbg': ['defaulttests'],
1603       'mac_chromium_rel_swarming': ['defaulttests'],
1604       'mac_gpu': ['defaulttests'],
1605       'mac_nacl_sdk_build': ['compile'],
1606       'win_chromium_compile_dbg': ['defaulttests'],
1607       'win_chromium_dbg': ['defaulttests'],
1608       'win_chromium_rel_swarming': ['defaulttests'],
1609       'win_chromium_x64_rel_swarming': ['defaulttests'],
1610       'win_gpu': ['defaulttests'],
1611       'win_nacl_sdk_build': ['compile'],
1612       'win8_chromium_rel': ['defaulttests'],
1613   }
1614
1615   if bots:
1616     filtered_builders_and_tests = dict((bot, set(builders_and_tests[bot]))
1617                                        for bot in bots)
1618   else:
1619     filtered_builders_and_tests = dict(
1620         (bot, set(tests))
1621         for bot, tests in builders_and_tests.iteritems())
1622
1623   # Build up the mapping from tryserver master to bot/test.
1624   out = dict()
1625   for bot, tests in filtered_builders_and_tests.iteritems():
1626     out.setdefault(GetTryServerMasterForBot(bot), {})[bot] = tests
1627   return out
1628
1629
1630 def CheckChangeOnCommit(input_api, output_api):
1631   results = []
1632   results.extend(_CommonChecks(input_api, output_api))
1633   # TODO(thestig) temporarily disabled, doesn't work in third_party/
1634   #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
1635   #    input_api, output_api, sources))
1636   # Make sure the tree is 'open'.
1637   results.extend(input_api.canned_checks.CheckTreeIsOpen(
1638       input_api,
1639       output_api,
1640       json_url='http://chromium-status.appspot.com/current?format=json'))
1641
1642   results.extend(input_api.canned_checks.CheckChangeHasBugField(
1643       input_api, output_api))
1644   results.extend(input_api.canned_checks.CheckChangeHasDescription(
1645       input_api, output_api))
1646   return results
1647
1648
1649 def GetPreferredTryMasters(project, change):
1650   files = change.LocalPaths()
1651
1652   if not files or all(re.search(r'[\\\/]OWNERS$', f) for f in files):
1653     return {}
1654
1655   if all(re.search(r'\.(m|mm)$|(^|[\\\/_])mac[\\\/_.]', f) for f in files):
1656     return GetDefaultTryConfigs([
1657         'mac_chromium_compile_dbg',
1658         'mac_chromium_rel_swarming',
1659     ])
1660   if all(re.search('(^|[/_])win[/_.]', f) for f in files):
1661     return GetDefaultTryConfigs([
1662         'win_chromium_dbg',
1663         'win_chromium_rel_swarming',
1664         'win8_chromium_rel',
1665     ])
1666   if all(re.search(r'(^|[\\\/_])android[\\\/_.]', f) for f in files):
1667     return GetDefaultTryConfigs([
1668         'android_aosp',
1669         'android_clang_dbg',
1670         'android_dbg_tests_recipe',
1671     ])
1672   if all(re.search(r'[\\\/_]ios[\\\/_.]', f) for f in files):
1673     return GetDefaultTryConfigs(['ios_rel_device', 'ios_dbg_simulator'])
1674
1675   builders = [
1676       'android_arm64_dbg_recipe',
1677       'android_chromium_gn_compile_rel',
1678       'android_chromium_gn_compile_dbg',
1679       'android_clang_dbg',
1680       'android_clang_dbg_recipe',
1681       'android_dbg_tests_recipe',
1682       'ios_dbg_simulator',
1683       'ios_rel_device',
1684       'ios_rel_device_ninja',
1685       'linux_chromium_chromeos_rel_swarming',
1686       'linux_chromium_clang_dbg',
1687       'linux_chromium_gn_dbg',
1688       'linux_chromium_gn_rel',
1689       'linux_chromium_rel_swarming',
1690       'linux_gpu',
1691       'mac_chromium_compile_dbg',
1692       'mac_chromium_rel_swarming',
1693       'mac_gpu',
1694       'win_chromium_compile_dbg',
1695       'win_chromium_rel_swarming',
1696       'win_chromium_x64_rel_swarming',
1697       'win_gpu',
1698       'win8_chromium_rel',
1699   ]
1700
1701   # Match things like path/aura/file.cc and path/file_aura.cc.
1702   # Same for chromeos.
1703   if any(re.search(r'[\\\/_](aura|chromeos)', f) for f in files):
1704     builders.extend([
1705         'linux_chromeos_asan',
1706         'linux_chromium_chromeos_clang_dbg'
1707     ])
1708
1709   # If there are gyp changes to base, build, or chromeos, run a full cros build
1710   # in addition to the shorter linux_chromeos build. Changes to high level gyp
1711   # files have a much higher chance of breaking the cros build, which is
1712   # differnt from the linux_chromeos build that most chrome developers test
1713   # with.
1714   if any(re.search('^(base|build|chromeos).*\.gypi?$', f) for f in files):
1715     builders.extend(['cros_x86'])
1716
1717   # The AOSP bot doesn't build the chrome/ layer, so ignore any changes to it
1718   # unless they're .gyp(i) files as changes to those files can break the gyp
1719   # step on that bot.
1720   if (not all(re.search('^chrome', f) for f in files) or
1721       any(re.search('\.gypi?$', f) for f in files)):
1722     builders.extend(['android_aosp'])
1723
1724   return GetDefaultTryConfigs(builders)