tools/checkdeps/checkdeps.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that files include headers from allowed directories.
   7
   8 Checks DEPS files in the source tree for rules, and applies those rules to
   9 "#include" commands in source files. Any source file including something not
  10 permitted by the DEPS files will fail.
  11
  12 The format of the deps file:
  13
  14 First you have the normal module-level deps. These are the ones used by
  15 gclient. An example would be:
  16
  17   deps = {
  18     "base":"http://foo.bar/trunk/base"
  19   }
  20
  21 DEPS files not in the top-level of a module won't need this. Then you
  22 have any additional include rules. You can add (using "+") or subtract
  23 (using "-") from the previously specified rules (including
  24 module-level deps). You can also specify a path that is allowed for
  25 now but that we intend to remove, using "!"; this is treated the same
  26 as "+" when check_deps is run by our bots, but a presubmit step will
  27 show a warning if you add a new include of a file that is only allowed
  28 by "!".
  29
  30 Note that for .java files, there is currently no difference between
  31 "+" and "!", even in the presubmit step.
  32
  33   include_rules = {
  34     # Code should be able to use base (it's specified in the module-level
  35     # deps above), but nothing in "base/evil" because it's evil.
  36     "-base/evil",
  37
  38     # But this one subdirectory of evil is OK.
  39     "+base/evil/not",
  40
  41     # And it can include files from this other directory even though there is
  42     # no deps rule for it.
  43     "+tools/crime_fighter",
  44
  45     # This dependency is allowed for now but work is ongoing to remove it,
  46     # so you shouldn't add further dependencies on it.
  47     "!base/evil/ok_for_now.h",
  48   }
  49
  50 If you have certain include rules that should only be applied for some
  51 files within this directory and subdirectories, you can write a
  52 section named specific_include_rules that is a hash map of regular
  53 expressions to the list of rules that should apply to files matching
  54 them.  Note that such rules will always be applied before the rules
  55 from 'include_rules' have been applied, but the order in which rules
  56 associated with different regular expressions is applied is arbitrary.
  57
  58   specific_include_rules = {
  59     ".*_(unit|browser|api)test\.cc": [
  60       "+libraries/testsupport",
  61     ],
  62   }
  63
  64 DEPS files may be placed anywhere in the tree. Each one applies to all
  65 subdirectories, where there may be more DEPS files that provide additions or
  66 subtractions for their own sub-trees.
  67
  68 There is an implicit rule for the current directory (where the DEPS file lives)
  69 and all of its subdirectories. This prevents you from having to explicitly
  70 allow the current directory everywhere.  This implicit rule is applied first,
  71 so you can modify or remove it using the normal include rules.
  72
  73 The rules are processed in order. This means you can explicitly allow a higher
  74 directory and then take away permissions from sub-parts, or the reverse.
  75
  76 Note that all directory separators must be slashes (Unix-style) and not
  77 backslashes. All directories should be relative to the source root and use
  78 only lowercase.
  79 """
  80
  81 import os
  82 import optparse
  83 import subprocess
  84 import sys
  85 import copy
  86
  87 import cpp_checker
  88 import java_checker
  89 from results import NormalResultsFormatter, TemporaryRulesFormatter
  90 from rules import Rule, Rules
  91
  92
  93 # Variable name used in the DEPS file to add or subtract include files from
  94 # the module-level deps.
  95 INCLUDE_RULES_VAR_NAME = 'include_rules'
  96
  97 # Variable name used in the DEPS file to add or subtract include files
  98 # from module-level deps specific to files whose basename (last
  99 # component of path) matches a given regular expression.
 100 SPECIFIC_INCLUDE_RULES_VAR_NAME = 'specific_include_rules'
 101
 102 # Optionally present in the DEPS file to list subdirectories which should not
 103 # be checked. This allows us to skip third party code, for example.
 104 SKIP_SUBDIRS_VAR_NAME = 'skip_child_includes'
 105
 106
 107 def NormalizePath(path):
 108   """Returns a path normalized to how we write DEPS rules and compare paths.
 109   """
 110   return path.lower().replace('\\', '/')
 111
 112
 113 class DepsChecker(object):
 114   """Parses include_rules from DEPS files and can verify files in the
 115   source tree against them.
 116   """
 117
 118   def __init__(self, base_directory=None, verbose=False, being_tested=False):
 119     """Creates a new DepsChecker.
 120
 121     Args:
 122       base_directory: OS-compatible path to root of checkout, e.g. C:\chr\src.
 123       verbose: Set to true for debug output.
 124       being_tested: Set to true to ignore the DEPS file at tools/checkdeps/DEPS.
 125     """
 126     self.base_directory = base_directory
 127     if not base_directory:
 128       self.base_directory = os.path.abspath(
 129         os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..'))
 130
 131     self.verbose = verbose
 132     self.results_formatter = NormalResultsFormatter(verbose)
 133
 134     self._under_test = being_tested
 135
 136     self.git_source_directories = set()
 137     self._AddGitSourceDirectories()
 138
 139     # Map of normalized directory paths to rules to use for those
 140     # directories, or None for directories that should be skipped.
 141     self.directory_rules = {}
 142     self._ApplyDirectoryRulesAndSkipSubdirs(Rules(), self.base_directory)
 143
 144   def Report(self):
 145     """Prints a report of results, and returns an exit code for the process."""
 146     if self.results_formatter.GetResults():
 147       self.results_formatter.PrintResults()
 148       return 1
 149     print '\nSUCCESS\n'
 150     return 0
 151
 152   def _ApplyRules(self, existing_rules, includes, specific_includes, cur_dir):
 153     """Applies the given include rules, returning the new rules.
 154
 155     Args:
 156       existing_rules: A set of existing rules that will be combined.
 157       include: The list of rules from the "include_rules" section of DEPS.
 158       specific_includes: E.g. {'.*_unittest\.cc': ['+foo', '-blat']} rules
 159                          from the "specific_include_rules" section of DEPS.
 160       cur_dir: The current directory, normalized path. We will create an
 161                implicit rule that allows inclusion from this directory.
 162
 163     Returns: A new set of rules combining the existing_rules with the other
 164              arguments.
 165     """
 166     rules = copy.copy(existing_rules)
 167
 168     # First apply the implicit "allow" rule for the current directory.
 169     if cur_dir.startswith(
 170           NormalizePath(os.path.normpath(self.base_directory))):
 171       relative_dir = cur_dir[len(self.base_directory) + 1:]
 172
 173       source = relative_dir
 174       if len(source) == 0:
 175         source = 'top level'  # Make the help string a little more meaningful.
 176       rules.AddRule('+' + relative_dir, 'Default rule for ' + source)
 177     else:
 178       raise Exception('Internal error: base directory is not at the beginning' +
 179                       ' for\n  %s and base dir\n  %s' %
 180                       (cur_dir, self.base_directory))
 181
 182     def AddRuleWithDescription(rule_str, dependee_regexp=None):
 183       rule_block_name = 'include_rules'
 184       if dependee_regexp:
 185         rule_block_name = 'specific_include_rules'
 186       if not relative_dir:
 187         rule_description = 'the top level %s' % rule_block_name
 188       else:
 189         rule_description = relative_dir + "'s %s" % rule_block_name
 190       rules.AddRule(rule_str, rule_description, dependee_regexp)
 191
 192     # Apply the additional explicit rules.
 193     for (_, rule_str) in enumerate(includes):
 194       AddRuleWithDescription(rule_str)
 195
 196     # Finally, apply the specific rules.
 197     for regexp, specific_rules in specific_includes.iteritems():
 198       for rule_str in specific_rules:
 199         AddRuleWithDescription(rule_str, regexp)
 200
 201     return rules
 202
 203   def _ApplyDirectoryRules(self, existing_rules, dir_name):
 204     """Combines rules from the existing rules and the new directory.
 205
 206     Any directory can contain a DEPS file. Toplevel DEPS files can contain
 207     module dependencies which are used by gclient. We use these, along with
 208     additional include rules and implicit rules for the given directory, to
 209     come up with a combined set of rules to apply for the directory.
 210
 211     Args:
 212       existing_rules: The rules for the parent directory. We'll add-on to these.
 213       dir_name: The directory name that the deps file may live in (if
 214                 it exists).  This will also be used to generate the
 215                 implicit rules.  This is a non-normalized path.
 216
 217     Returns: A tuple containing: (1) the combined set of rules to apply to the
 218              sub-tree, and (2) a list of all subdirectories that should NOT be
 219              checked, as specified in the DEPS file (if any).
 220     """
 221     norm_dir_name = NormalizePath(dir_name)
 222
 223     # Check for a .svn directory in this directory or check this directory is
 224     # contained in git source direcotries. This will tell us if it's a source
 225     # directory and should be checked.
 226     if not (os.path.exists(os.path.join(dir_name, ".svn")) or
 227             (norm_dir_name in self.git_source_directories)):
 228       return (None, [])
 229
 230     # Check the DEPS file in this directory.
 231     if self.verbose:
 232       print 'Applying rules from', dir_name
 233     def FromImpl(_unused, _unused2):
 234       pass  # NOP function so "From" doesn't fail.
 235
 236     def FileImpl(_unused):
 237       pass  # NOP function so "File" doesn't fail.
 238
 239     class _VarImpl:
 240       def __init__(self, local_scope):
 241         self._local_scope = local_scope
 242
 243       def Lookup(self, var_name):
 244         """Implements the Var syntax."""
 245         if var_name in self._local_scope.get('vars', {}):
 246           return self._local_scope['vars'][var_name]
 247         raise Exception('Var is not defined: %s' % var_name)
 248
 249     local_scope = {}
 250     global_scope = {
 251         'File': FileImpl,
 252         'From': FromImpl,
 253         'Var': _VarImpl(local_scope).Lookup,
 254         }
 255     deps_file = os.path.join(dir_name, 'DEPS')
 256
 257     # The second conditional here is to disregard the
 258     # tools/checkdeps/DEPS file while running tests.  This DEPS file
 259     # has a skip_child_includes for 'testdata' which is necessary for
 260     # running production tests, since there are intentional DEPS
 261     # violations under the testdata directory.  On the other hand when
 262     # running tests, we absolutely need to verify the contents of that
 263     # directory to trigger those intended violations and see that they
 264     # are handled correctly.
 265     if os.path.isfile(deps_file) and (
 266         not self._under_test or not os.path.split(dir_name)[1] == 'checkdeps'):
 267       execfile(deps_file, global_scope, local_scope)
 268     elif self.verbose:
 269       print '  No deps file found in', dir_name
 270
 271     # Even if a DEPS file does not exist we still invoke ApplyRules
 272     # to apply the implicit "allow" rule for the current directory
 273     include_rules = local_scope.get(INCLUDE_RULES_VAR_NAME, [])
 274     specific_include_rules = local_scope.get(SPECIFIC_INCLUDE_RULES_VAR_NAME,
 275                                              {})
 276     skip_subdirs = local_scope.get(SKIP_SUBDIRS_VAR_NAME, [])
 277
 278     return (self._ApplyRules(existing_rules, include_rules,
 279                              specific_include_rules, norm_dir_name),
 280             skip_subdirs)
 281
 282   def _ApplyDirectoryRulesAndSkipSubdirs(self, parent_rules, dir_path):
 283     """Given |parent_rules| and a subdirectory |dir_path| from the
 284     directory that owns the |parent_rules|, add |dir_path|'s rules to
 285     |self.directory_rules|, and add None entries for any of its
 286     subdirectories that should be skipped.
 287     """
 288     directory_rules, excluded_subdirs = self._ApplyDirectoryRules(parent_rules,
 289                                                                   dir_path)
 290     self.directory_rules[NormalizePath(dir_path)] = directory_rules
 291     for subdir in excluded_subdirs:
 292       self.directory_rules[NormalizePath(
 293           os.path.normpath(os.path.join(dir_path, subdir)))] = None
 294
 295   def GetDirectoryRules(self, dir_path):
 296     """Returns a Rules object to use for the given directory, or None
 297     if the given directory should be skipped.  This takes care of
 298     first building rules for parent directories (up to
 299     self.base_directory) if needed.
 300
 301     Args:
 302       dir_path: A real (non-normalized) path to the directory you want
 303       rules for.
 304     """
 305     norm_dir_path = NormalizePath(dir_path)
 306
 307     if not norm_dir_path.startswith(
 308         NormalizePath(os.path.normpath(self.base_directory))):
 309       dir_path = os.path.join(self.base_directory, dir_path)
 310       norm_dir_path = NormalizePath(dir_path)
 311
 312     parent_dir = os.path.dirname(dir_path)
 313     parent_rules = None
 314     if not norm_dir_path in self.directory_rules:
 315       parent_rules = self.GetDirectoryRules(parent_dir)
 316
 317     # We need to check for an entry for our dir_path again, in case we
 318     # are at a path e.g. A/B/C where A/B/DEPS specifies the C
 319     # subdirectory to be skipped; in this case, the invocation to
 320     # GetDirectoryRules(parent_dir) has already filled in an entry for
 321     # A/B/C.
 322     if not norm_dir_path in self.directory_rules:
 323       if not parent_rules:
 324         # If the parent directory should be skipped, then the current
 325         # directory should also be skipped.
 326         self.directory_rules[norm_dir_path] = None
 327       else:
 328         self._ApplyDirectoryRulesAndSkipSubdirs(parent_rules, dir_path)
 329     return self.directory_rules[norm_dir_path]
 330
 331   def CheckDirectory(self, start_dir):
 332     """Checks all relevant source files in the specified directory and
 333     its subdirectories for compliance with DEPS rules throughout the
 334     tree (starting at |self.base_directory|).  |start_dir| must be a
 335     subdirectory of |self.base_directory|.
 336
 337     On completion, self.results_formatter has the results of
 338     processing, and calling Report() will print a report of results.
 339     """
 340     java = java_checker.JavaChecker(self.base_directory, self.verbose)
 341     cpp = cpp_checker.CppChecker(self.verbose)
 342     checkers = dict(
 343         (extension, checker)
 344         for checker in [java, cpp] for extension in checker.EXTENSIONS)
 345     self._CheckDirectoryImpl(checkers, start_dir)
 346
 347   def _CheckDirectoryImpl(self, checkers, dir_name):
 348     rules = self.GetDirectoryRules(dir_name)
 349     if rules == None:
 350       return
 351
 352     # Collect a list of all files and directories to check.
 353     files_to_check = []
 354     dirs_to_check = []
 355     contents = os.listdir(dir_name)
 356     for cur in contents:
 357       full_name = os.path.join(dir_name, cur)
 358       if os.path.isdir(full_name):
 359         dirs_to_check.append(full_name)
 360       elif os.path.splitext(full_name)[1] in checkers:
 361         files_to_check.append(full_name)
 362
 363     # First check all files in this directory.
 364     for cur in files_to_check:
 365       checker = checkers[os.path.splitext(cur)[1]]
 366       file_status = checker.CheckFile(rules, cur)
 367       if file_status.HasViolations():
 368         self.results_formatter.AddError(file_status)
 369
 370     # Next recurse into the subdirectories.
 371     for cur in dirs_to_check:
 372       self._CheckDirectoryImpl(checkers, cur)
 373
 374   def CheckAddedCppIncludes(self, added_includes):
 375     """This is used from PRESUBMIT.py to check new #include statements added in
 376     the change being presubmit checked.
 377
 378     Args:
 379       added_includes: ((file_path, (include_line, include_line, ...), ...)
 380
 381     Return:
 382       A list of tuples, (bad_file_path, rule_type, rule_description)
 383       where rule_type is one of Rule.DISALLOW or Rule.TEMP_ALLOW and
 384       rule_description is human-readable. Empty if no problems.
 385     """
 386     cpp = cpp_checker.CppChecker(self.verbose)
 387     problems = []
 388     for file_path, include_lines in added_includes:
 389       # TODO(joi): Make this cover Java as well.
 390       if not cpp.IsCppFile(file_path):
 391         pass
 392       rules_for_file = self.GetDirectoryRules(os.path.dirname(file_path))
 393       if rules_for_file:
 394         for line in include_lines:
 395           is_include, violation = cpp.CheckLine(
 396               rules_for_file, line, file_path, True)
 397           if violation:
 398             rule_type = violation.violated_rule.allow
 399             if rule_type != Rule.ALLOW:
 400               violation_text = NormalResultsFormatter.FormatViolation(
 401                   violation, self.verbose)
 402               problems.append((file_path, rule_type, violation_text))
 403     return problems
 404
 405   def _AddGitSourceDirectories(self):
 406     """Adds any directories containing sources managed by git to
 407     self.git_source_directories.
 408     """
 409     if not os.path.exists(os.path.join(self.base_directory, '.git')):
 410       return
 411
 412     popen_out = os.popen('cd %s && git ls-files --full-name .' %
 413                          subprocess.list2cmdline([self.base_directory]))
 414     for line in popen_out.readlines():
 415       dir_name = os.path.join(self.base_directory, os.path.dirname(line))
 416       # Add the directory as well as all the parent directories. Use
 417       # forward slashes and lower case to normalize paths.
 418       while dir_name != self.base_directory:
 419         self.git_source_directories.add(NormalizePath(dir_name))
 420         dir_name = os.path.dirname(dir_name)
 421     self.git_source_directories.add(NormalizePath(self.base_directory))
 422
 423
 424 def PrintUsage():
 425   print """Usage: python checkdeps.py [--root <root>] [tocheck]
 426
 427   --root   Specifies the repository root. This defaults to "../../.." relative
 428            to the script file. This will be correct given the normal location
 429            of the script in "<root>/tools/checkdeps".
 430
 431   tocheck  Specifies the directory, relative to root, to check. This defaults
 432            to "." so it checks everything.
 433
 434 Examples:
 435   python checkdeps.py
 436   python checkdeps.py --root c:\\source chrome"""
 437
 438
 439 def main():
 440   option_parser = optparse.OptionParser()
 441   option_parser.add_option('', '--root', default='', dest='base_directory',
 442                            help='Specifies the repository root. This defaults '
 443                            'to "../../.." relative to the script file, which '
 444                            'will normally be the repository root.')
 445   option_parser.add_option('', '--temprules', action='store_true',
 446                            default=False, help='Print rules to temporarily '
 447                            'allow files that fail dependency checking.')
 448   option_parser.add_option('-v', '--verbose', action='store_true',
 449                            default=False, help='Print debug logging')
 450   options, args = option_parser.parse_args()
 451
 452   deps_checker = DepsChecker(options.base_directory, verbose=options.verbose)
 453
 454   # Figure out which directory we have to check.
 455   start_dir = deps_checker.base_directory
 456   if len(args) == 1:
 457     # Directory specified. Start here. It's supposed to be relative to the
 458     # base directory.
 459     start_dir = os.path.abspath(
 460         os.path.join(deps_checker.base_directory, args[0]))
 461   elif len(args) >= 2:
 462     # More than one argument, we don't handle this.
 463     PrintUsage()
 464     return 1
 465
 466   print 'Using base directory:', deps_checker.base_directory
 467   print 'Checking:', start_dir
 468
 469   if options.temprules:
 470     deps_checker.results_formatter = TemporaryRulesFormatter()
 471   deps_checker.CheckDirectory(start_dir)
 472   return deps_checker.Report()
 473
 474
 475 if '__main__' == __name__:
 476   sys.exit(main())