tools/checkperms/checkperms.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure files have the right permissions.
   7
   8 Some developers have broken SCM configurations that flip the executable
   9 permission on for no good reason. Unix developers who run ls --color will then
  10 see .cc files in green and get confused.
  11
  12 - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS.
  13 - For file extensions that must not be executable, add it to
  14   NOT_EXECUTABLE_EXTENSIONS.
  15 - To ignore all the files inside a directory, add it to IGNORED_PATHS.
  16 - For file base name with ambiguous state and that should not be checked for
  17   shebang, add it to IGNORED_FILENAMES.
  18
  19 Any file not matching the above will be opened and looked if it has a shebang
  20 or an ELF header. If this does not match the executable bit on the file, the
  21 file will be flagged.
  22
  23 Note that all directory separators must be slashes (Unix-style) and not
  24 backslashes. All directories should be relative to the source root and all
  25 file paths should be only lowercase.
  26 """
  27
  28 import json
  29 import logging
  30 import optparse
  31 import os
  32 import stat
  33 import string
  34 import subprocess
  35 import sys
  36
  37 #### USER EDITABLE SECTION STARTS HERE ####
  38
  39 # Files with these extensions must have executable bit set.
  40 #
  41 # Case-sensitive.
  42 EXECUTABLE_EXTENSIONS = (
  43   'bat',
  44   'dll',
  45   'dylib',
  46   'exe',
  47 )
  48
  49 # These files must have executable bit set.
  50 #
  51 # Case-insensitive, lower-case only.
  52 EXECUTABLE_PATHS = (
  53   'chrome/test/data/app_shim/app_shim_32_bit.app/contents/'
  54       'macos/app_mode_loader',
  55   'chrome/test/data/extensions/uitest/plugins/plugin.plugin/contents/'
  56       'macos/testnetscapeplugin',
  57   'chrome/test/data/extensions/uitest/plugins_private/plugin.plugin/contents/'
  58       'macos/testnetscapeplugin',
  59 )
  60
  61 # These files must not have the executable bit set. This is mainly a performance
  62 # optimization as these files are not checked for shebang. The list was
  63 # partially generated from:
  64 # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g
  65 #
  66 # Case-sensitive.
  67 NON_EXECUTABLE_EXTENSIONS = (
  68   '1',
  69   '3ds',
  70   'S',
  71   'am',
  72   'applescript',
  73   'asm',
  74   'c',
  75   'cc',
  76   'cfg',
  77   'chromium',
  78   'cpp',
  79   'crx',
  80   'cs',
  81   'css',
  82   'cur',
  83   'def',
  84   'der',
  85   'expected',
  86   'gif',
  87   'grd',
  88   'gyp',
  89   'gypi',
  90   'h',
  91   'hh',
  92   'htm',
  93   'html',
  94   'hyph',
  95   'ico',
  96   'idl',
  97   'java',
  98   'jpg',
  99   'js',
 100   'json',
 101   'm',
 102   'm4',
 103   'mm',
 104   'mms',
 105   'mock-http-headers',
 106   'nexe',
 107   'nmf',
 108   'onc',
 109   'pat',
 110   'patch',
 111   'pdf',
 112   'pem',
 113   'plist',
 114   'png',
 115   'proto',
 116   'rc',
 117   'rfx',
 118   'rgs',
 119   'rules',
 120   'spec',
 121   'sql',
 122   'srpc',
 123   'svg',
 124   'tcl',
 125   'test',
 126   'tga',
 127   'txt',
 128   'vcproj',
 129   'vsprops',
 130   'webm',
 131   'word',
 132   'xib',
 133   'xml',
 134   'xtb',
 135   'zip',
 136 )
 137
 138 # These files must not have executable bit set.
 139 #
 140 # Case-insensitive, lower-case only.
 141 NON_EXECUTABLE_PATHS = (
 142   'build/android/tests/symbolize/liba.so',
 143   'build/android/tests/symbolize/libb.so',
 144   'chrome/installer/mac/sign_app.sh.in',
 145   'chrome/installer/mac/sign_versioned_dir.sh.in',
 146   'chrome/test/data/extensions/uitest/plugins/plugin32.so',
 147   'chrome/test/data/extensions/uitest/plugins/plugin64.so',
 148   'chrome/test/data/extensions/uitest/plugins_private/plugin32.so',
 149   'chrome/test/data/extensions/uitest/plugins_private/plugin64.so',
 150   'courgette/testdata/elf-32-1',
 151   'courgette/testdata/elf-32-2',
 152   'courgette/testdata/elf-64',
 153 )
 154
 155 # File names that are always whitelisted.  (These are mostly autoconf spew.)
 156 #
 157 # Case-sensitive.
 158 IGNORED_FILENAMES = (
 159   'config.guess',
 160   'config.sub',
 161   'configure',
 162   'depcomp',
 163   'install-sh',
 164   'missing',
 165   'mkinstalldirs',
 166   'naclsdk',
 167   'scons',
 168 )
 169
 170 # File paths starting with one of these will be ignored as well.
 171 # Please consider fixing your file permissions, rather than adding to this list.
 172 #
 173 # Case-insensitive, lower-case only.
 174 IGNORED_PATHS = (
 175   'native_client_sdk/src/build_tools/sdk_tools/third_party/fancy_urllib/'
 176       '__init__.py',
 177   'out/',
 178   # TODO(maruel): Fix these.
 179   'third_party/bintrees/',
 180   'third_party/closure_linter/',
 181   'third_party/devscripts/licensecheck.pl.vanilla',
 182   'third_party/hyphen/',
 183   'third_party/jemalloc/',
 184   'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl',
 185   'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh',
 186   'third_party/lcov/contrib/galaxy/conglomerate_functions.pl',
 187   'third_party/lcov/contrib/galaxy/gen_makefile.sh',
 188   'third_party/libevent/autogen.sh',
 189   'third_party/libevent/test/test.sh',
 190   'third_party/libxml/linux/xml2-config',
 191   'third_party/libxml/src/ltmain.sh',
 192   'third_party/mesa/',
 193   'third_party/protobuf/',
 194   'third_party/python_gflags/gflags.py',
 195   'third_party/sqlite/',
 196   'third_party/talloc/script/mksyms.sh',
 197   'third_party/tcmalloc/',
 198   'third_party/tlslite/setup.py',
 199   # TODO(nednguyen): Remove this when telemetry is moved to catapult
 200   'tools/telemetry/third_party/',
 201 )
 202
 203 #### USER EDITABLE SECTION ENDS HERE ####
 204
 205 assert set(EXECUTABLE_EXTENSIONS) & set(NON_EXECUTABLE_EXTENSIONS) == set()
 206 assert set(EXECUTABLE_PATHS) & set(NON_EXECUTABLE_PATHS) == set()
 207
 208 VALID_CHARS = set(string.ascii_lowercase + string.digits + '/-_.')
 209 for paths in (EXECUTABLE_PATHS, NON_EXECUTABLE_PATHS, IGNORED_PATHS):
 210   assert all([set(path).issubset(VALID_CHARS) for path in paths])
 211
 212
 213 def capture(cmd, cwd):
 214   """Returns the output of a command.
 215
 216   Ignores the error code or stderr.
 217   """
 218   logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd))
 219   env = os.environ.copy()
 220   env['LANGUAGE'] = 'en_US.UTF-8'
 221   p = subprocess.Popen(
 222       cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
 223   return p.communicate()[0]
 224
 225
 226 def get_git_root(dir_path):
 227   """Returns the git checkout root or None."""
 228   root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip()
 229   if root:
 230     return root
 231
 232
 233 def is_ignored(rel_path):
 234   """Returns True if rel_path is in our whitelist of files to ignore."""
 235   rel_path = rel_path.lower()
 236   return (
 237       os.path.basename(rel_path) in IGNORED_FILENAMES or
 238       rel_path.lower().startswith(IGNORED_PATHS))
 239
 240
 241 def must_be_executable(rel_path):
 242   """The file name represents a file type that must have the executable bit
 243   set.
 244   """
 245   return (os.path.splitext(rel_path)[1][1:] in EXECUTABLE_EXTENSIONS or
 246           rel_path.lower() in EXECUTABLE_PATHS)
 247
 248
 249 def must_not_be_executable(rel_path):
 250   """The file name represents a file type that must not have the executable
 251   bit set.
 252   """
 253   return (os.path.splitext(rel_path)[1][1:] in NON_EXECUTABLE_EXTENSIONS or
 254           rel_path.lower() in NON_EXECUTABLE_PATHS)
 255
 256
 257 def has_executable_bit(full_path):
 258   """Returns if any executable bit is set."""
 259   permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
 260   return bool(permission & os.stat(full_path).st_mode)
 261
 262
 263 def has_shebang_or_is_elf(full_path):
 264   """Returns if the file starts with #!/ or is an ELF binary.
 265
 266   full_path is the absolute path to the file.
 267   """
 268   with open(full_path, 'rb') as f:
 269     data = f.read(4)
 270     return (data[:3] == '#!/' or data == '#! /', data == '\x7fELF')
 271
 272
 273 def check_file(root_path, rel_path):
 274   """Checks the permissions of the file whose path is root_path + rel_path and
 275   returns an error if it is inconsistent. Returns None on success.
 276
 277   It is assumed that the file is not ignored by is_ignored().
 278
 279   If the file name is matched with must_be_executable() or
 280   must_not_be_executable(), only its executable bit is checked.
 281   Otherwise, the first few bytes of the file are read to verify if it has a
 282   shebang or ELF header and compares this with the executable bit on the file.
 283   """
 284   full_path = os.path.join(root_path, rel_path)
 285   def result_dict(error):
 286     return {
 287       'error': error,
 288       'full_path': full_path,
 289       'rel_path': rel_path,
 290     }
 291   try:
 292     bit = has_executable_bit(full_path)
 293   except OSError:
 294     # It's faster to catch exception than call os.path.islink(). Chromium
 295     # tree happens to have invalid symlinks under
 296     # third_party/openssl/openssl/test/.
 297     return None
 298
 299   if must_be_executable(rel_path):
 300     if not bit:
 301       return result_dict('Must have executable bit set')
 302     return
 303   if must_not_be_executable(rel_path):
 304     if bit:
 305       return result_dict('Must not have executable bit set')
 306     return
 307
 308   # For the others, it depends on the file header.
 309   (shebang, elf) = has_shebang_or_is_elf(full_path)
 310   if bit != (shebang or elf):
 311     if bit:
 312       return result_dict('Has executable bit but not shebang or ELF header')
 313     if shebang:
 314       return result_dict('Has shebang but not executable bit')
 315     return result_dict('Has ELF header but not executable bit')
 316
 317
 318 def check_files(root, files):
 319   gen = (check_file(root, f) for f in files if not is_ignored(f))
 320   return filter(None, gen)
 321
 322
 323 class ApiBase(object):
 324   def __init__(self, root_dir, bare_output):
 325     self.root_dir = root_dir
 326     self.bare_output = bare_output
 327     self.count = 0
 328     self.count_read_header = 0
 329
 330   def check_file(self, rel_path):
 331     logging.debug('check_file(%s)' % rel_path)
 332     self.count += 1
 333
 334     if (not must_be_executable(rel_path) and
 335         not must_not_be_executable(rel_path)):
 336       self.count_read_header += 1
 337
 338     return check_file(self.root_dir, rel_path)
 339
 340   def check_dir(self, rel_path):
 341     return self.check(rel_path)
 342
 343   def check(self, start_dir):
 344     """Check the files in start_dir, recursively check its subdirectories."""
 345     errors = []
 346     items = self.list_dir(start_dir)
 347     logging.info('check(%s) -> %d' % (start_dir, len(items)))
 348     for item in items:
 349       full_path = os.path.join(self.root_dir, start_dir, item)
 350       rel_path = full_path[len(self.root_dir) + 1:]
 351       if is_ignored(rel_path):
 352         continue
 353       if os.path.isdir(full_path):
 354         # Depth first.
 355         errors.extend(self.check_dir(rel_path))
 356       else:
 357         error = self.check_file(rel_path)
 358         if error:
 359           errors.append(error)
 360     return errors
 361
 362   def list_dir(self, start_dir):
 363     """Lists all the files and directory inside start_dir."""
 364     return sorted(
 365       x for x in os.listdir(os.path.join(self.root_dir, start_dir))
 366       if not x.startswith('.')
 367     )
 368
 369
 370 class ApiAllFilesAtOnceBase(ApiBase):
 371   _files = None
 372
 373   def list_dir(self, start_dir):
 374     """Lists all the files and directory inside start_dir."""
 375     if self._files is None:
 376       self._files = sorted(self._get_all_files())
 377       if not self.bare_output:
 378         print 'Found %s files' % len(self._files)
 379     start_dir = start_dir[len(self.root_dir) + 1:]
 380     return [
 381       x[len(start_dir):] for x in self._files if x.startswith(start_dir)
 382     ]
 383
 384   def _get_all_files(self):
 385     """Lists all the files and directory inside self._root_dir."""
 386     raise NotImplementedError()
 387
 388
 389 class ApiGit(ApiAllFilesAtOnceBase):
 390   def _get_all_files(self):
 391     return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines()
 392
 393
 394 def get_scm(dir_path, bare):
 395   """Returns a properly configured ApiBase instance."""
 396   cwd = os.getcwd()
 397   root = get_git_root(dir_path or cwd)
 398   if root:
 399     if not bare:
 400       print('Found git repository at %s' % root)
 401     return ApiGit(dir_path or root, bare)
 402
 403   # Returns a non-scm aware checker.
 404   if not bare:
 405     print('Failed to determine the SCM for %s' % dir_path)
 406   return ApiBase(dir_path or cwd, bare)
 407
 408
 409 def main():
 410   usage = """Usage: python %prog [--root <root>] [tocheck]
 411   tocheck  Specifies the directory, relative to root, to check. This defaults
 412            to "." so it checks everything.
 413
 414 Examples:
 415   python %prog
 416   python %prog --root /path/to/source chrome"""
 417
 418   parser = optparse.OptionParser(usage=usage)
 419   parser.add_option(
 420       '--root',
 421       help='Specifies the repository root. This defaults '
 422            'to the checkout repository root')
 423   parser.add_option(
 424       '-v', '--verbose', action='count', default=0, help='Print debug logging')
 425   parser.add_option(
 426       '--bare',
 427       action='store_true',
 428       default=False,
 429       help='Prints the bare filename triggering the checks')
 430   parser.add_option(
 431       '--file', action='append', dest='files',
 432       help='Specifics a list of files to check the permissions of. Only these '
 433       'files will be checked')
 434   parser.add_option('--json', help='Path to JSON output file')
 435   options, args = parser.parse_args()
 436
 437   levels = [logging.ERROR, logging.INFO, logging.DEBUG]
 438   logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)])
 439
 440   if len(args) > 1:
 441     parser.error('Too many arguments used')
 442
 443   if options.root:
 444     options.root = os.path.abspath(options.root)
 445
 446   if options.files:
 447     # --file implies --bare (for PRESUBMIT.py).
 448     options.bare = True
 449
 450     errors = check_files(options.root, options.files)
 451   else:
 452     api = get_scm(options.root, options.bare)
 453     start_dir = args[0] if args else api.root_dir
 454     errors = api.check(start_dir)
 455
 456     if not options.bare:
 457       print('Processed %s files, %d files where tested for shebang/ELF '
 458             'header' % (api.count, api.count_read_header))
 459
 460   if options.json:
 461     with open(options.json, 'w') as f:
 462       json.dump(errors, f)
 463
 464   if errors:
 465     if options.bare:
 466       print '\n'.join(e['full_path'] for e in errors)
 467     else:
 468       print '\nFAILED\n'
 469       print '\n'.join('%s: %s' % (e['full_path'], e['error']) for e in errors)
 470     return 1
 471   if not options.bare:
 472     print '\nSUCCESS\n'
 473   return 0
 474
 475
 476 if '__main__' == __name__:
 477   sys.exit(main())