tools/checkperms/checkperms.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure files have the right permissions.
   7
   8 Some developers have broken SCM configurations that flip the executable
   9 permission on for no good reason. Unix developers who run ls --color will then
  10 see .cc files in green and get confused.
  11
  12 - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS.
  13 - For file extensions that must not be executable, add it to
  14   NOT_EXECUTABLE_EXTENSIONS.
  15 - To ignore all the files inside a directory, add it to IGNORED_PATHS.
  16 - For file base name with ambiguous state and that should not be checked for
  17   shebang, add it to IGNORED_FILENAMES.
  18
  19 Any file not matching the above will be opened and looked if it has a shebang
  20 or an ELF header. If this does not match the executable bit on the file, the
  21 file will be flagged.
  22
  23 Note that all directory separators must be slashes (Unix-style) and not
  24 backslashes. All directories should be relative to the source root and all
  25 file paths should be only lowercase.
  26 """
  27
  28 import json
  29 import logging
  30 import optparse
  31 import os
  32 import stat
  33 import string
  34 import subprocess
  35 import sys
  36
  37 #### USER EDITABLE SECTION STARTS HERE ####
  38
  39 # Files with these extensions must have executable bit set.
  40 #
  41 # Case-sensitive.
  42 EXECUTABLE_EXTENSIONS = (
  43   'bat',
  44   'dll',
  45   'dylib',
  46   'exe',
  47 )
  48
  49 # These files must have executable bit set.
  50 #
  51 # Case-insensitive, lower-case only.
  52 EXECUTABLE_PATHS = (
  53   'chrome/test/data/app_shim/app_shim_32_bit.app/contents/'
  54       'macos/app_mode_loader',
  55   'chrome/test/data/extensions/uitest/plugins/plugin.plugin/contents/'
  56       'macos/testnetscapeplugin',
  57   'chrome/test/data/extensions/uitest/plugins_private/plugin.plugin/contents/'
  58       'macos/testnetscapeplugin',
  59 )
  60
  61 # These files must not have the executable bit set. This is mainly a performance
  62 # optimization as these files are not checked for shebang. The list was
  63 # partially generated from:
  64 # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g
  65 #
  66 # Case-sensitive.
  67 NON_EXECUTABLE_EXTENSIONS = (
  68   '1',
  69   '3ds',
  70   'S',
  71   'am',
  72   'applescript',
  73   'asm',
  74   'c',
  75   'cc',
  76   'cfg',
  77   'chromium',
  78   'cpp',
  79   'crx',
  80   'cs',
  81   'css',
  82   'cur',
  83   'def',
  84   'der',
  85   'expected',
  86   'gif',
  87   'grd',
  88   'gyp',
  89   'gypi',
  90   'h',
  91   'hh',
  92   'htm',
  93   'html',
  94   'hyph',
  95   'ico',
  96   'idl',
  97   'java',
  98   'jpg',
  99   'js',
 100   'json',
 101   'm',
 102   'm4',
 103   'mm',
 104   'mms',
 105   'mock-http-headers',
 106   'nexe',
 107   'nmf',
 108   'onc',
 109   'pat',
 110   'patch',
 111   'pdf',
 112   'pem',
 113   'plist',
 114   'png',
 115   'proto',
 116   'rc',
 117   'rfx',
 118   'rgs',
 119   'rules',
 120   'spec',
 121   'sql',
 122   'srpc',
 123   'svg',
 124   'tcl',
 125   'test',
 126   'tga',
 127   'txt',
 128   'vcproj',
 129   'vsprops',
 130   'webm',
 131   'word',
 132   'xib',
 133   'xml',
 134   'xtb',
 135   'zip',
 136 )
 137
 138 # These files must not have executable bit set.
 139 #
 140 # Case-insensitive, lower-case only.
 141 NON_EXECUTABLE_PATHS = (
 142   'build/android/tests/symbolize/liba.so',
 143   'build/android/tests/symbolize/libb.so',
 144   'chrome/installer/mac/sign_app.sh.in',
 145   'chrome/installer/mac/sign_versioned_dir.sh.in',
 146   'chrome/test/data/extensions/uitest/plugins/plugin32.so',
 147   'chrome/test/data/extensions/uitest/plugins/plugin64.so',
 148   'chrome/test/data/extensions/uitest/plugins_private/plugin32.so',
 149   'chrome/test/data/extensions/uitest/plugins_private/plugin64.so',
 150   'courgette/testdata/elf-32-1',
 151   'courgette/testdata/elf-32-2',
 152   'courgette/testdata/elf-64',
 153 )
 154
 155 # File names that are always whitelisted.  (These are mostly autoconf spew.)
 156 #
 157 # Case-sensitive.
 158 IGNORED_FILENAMES = (
 159   'config.guess',
 160   'config.sub',
 161   'configure',
 162   'depcomp',
 163   'install-sh',
 164   'missing',
 165   'mkinstalldirs',
 166   'naclsdk',
 167   'scons',
 168 )
 169
 170 # File paths starting with one of these will be ignored as well.
 171 # Please consider fixing your file permissions, rather than adding to this list.
 172 #
 173 # Case-insensitive, lower-case only.
 174 IGNORED_PATHS = (
 175   'native_client_sdk/src/build_tools/sdk_tools/third_party/fancy_urllib/'
 176       '__init__.py',
 177   'out/',
 178   # TODO(maruel): Fix these.
 179   'third_party/bintrees/',
 180   'third_party/closure_linter/',
 181   'third_party/devscripts/licensecheck.pl.vanilla',
 182   'third_party/hyphen/',
 183   'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl',
 184   'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh',
 185   'third_party/lcov/contrib/galaxy/conglomerate_functions.pl',
 186   'third_party/lcov/contrib/galaxy/gen_makefile.sh',
 187   'third_party/libevent/autogen.sh',
 188   'third_party/libevent/test/test.sh',
 189   'third_party/libxml/linux/xml2-config',
 190   'third_party/libxml/src/ltmain.sh',
 191   'third_party/mesa/',
 192   'third_party/protobuf/',
 193   'third_party/python_gflags/gflags.py',
 194   'third_party/sqlite/',
 195   'third_party/talloc/script/mksyms.sh',
 196   'third_party/tcmalloc/',
 197   'third_party/tlslite/setup.py',
 198   # TODO(nednguyen): Remove this when telemetry is moved to catapult
 199   'tools/telemetry/third_party/',
 200 )
 201
 202 #### USER EDITABLE SECTION ENDS HERE ####
 203
 204 assert set(EXECUTABLE_EXTENSIONS) & set(NON_EXECUTABLE_EXTENSIONS) == set()
 205 assert set(EXECUTABLE_PATHS) & set(NON_EXECUTABLE_PATHS) == set()
 206
 207 VALID_CHARS = set(string.ascii_lowercase + string.digits + '/-_.')
 208 for paths in (EXECUTABLE_PATHS, NON_EXECUTABLE_PATHS, IGNORED_PATHS):
 209   assert all([set(path).issubset(VALID_CHARS) for path in paths])
 210
 211
 212 def capture(cmd, cwd):
 213   """Returns the output of a command.
 214
 215   Ignores the error code or stderr.
 216   """
 217   logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd))
 218   env = os.environ.copy()
 219   env['LANGUAGE'] = 'en_US.UTF-8'
 220   p = subprocess.Popen(
 221       cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
 222   return p.communicate()[0]
 223
 224
 225 def get_git_root(dir_path):
 226   """Returns the git checkout root or None."""
 227   root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip()
 228   if root:
 229     return root
 230
 231
 232 def is_ignored(rel_path):
 233   """Returns True if rel_path is in our whitelist of files to ignore."""
 234   rel_path = rel_path.lower()
 235   return (
 236       os.path.basename(rel_path) in IGNORED_FILENAMES or
 237       rel_path.lower().startswith(IGNORED_PATHS))
 238
 239
 240 def must_be_executable(rel_path):
 241   """The file name represents a file type that must have the executable bit
 242   set.
 243   """
 244   return (os.path.splitext(rel_path)[1][1:] in EXECUTABLE_EXTENSIONS or
 245           rel_path.lower() in EXECUTABLE_PATHS)
 246
 247
 248 def must_not_be_executable(rel_path):
 249   """The file name represents a file type that must not have the executable
 250   bit set.
 251   """
 252   return (os.path.splitext(rel_path)[1][1:] in NON_EXECUTABLE_EXTENSIONS or
 253           rel_path.lower() in NON_EXECUTABLE_PATHS)
 254
 255
 256 def has_executable_bit(full_path):
 257   """Returns if any executable bit is set."""
 258   permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
 259   return bool(permission & os.stat(full_path).st_mode)
 260
 261
 262 def has_shebang_or_is_elf(full_path):
 263   """Returns if the file starts with #!/ or is an ELF binary.
 264
 265   full_path is the absolute path to the file.
 266   """
 267   with open(full_path, 'rb') as f:
 268     data = f.read(4)
 269     return (data[:3] == '#!/' or data == '#! /', data == '\x7fELF')
 270
 271
 272 def check_file(root_path, rel_path):
 273   """Checks the permissions of the file whose path is root_path + rel_path and
 274   returns an error if it is inconsistent. Returns None on success.
 275
 276   It is assumed that the file is not ignored by is_ignored().
 277
 278   If the file name is matched with must_be_executable() or
 279   must_not_be_executable(), only its executable bit is checked.
 280   Otherwise, the first few bytes of the file are read to verify if it has a
 281   shebang or ELF header and compares this with the executable bit on the file.
 282   """
 283   full_path = os.path.join(root_path, rel_path)
 284   def result_dict(error):
 285     return {
 286       'error': error,
 287       'full_path': full_path,
 288       'rel_path': rel_path,
 289     }
 290   try:
 291     bit = has_executable_bit(full_path)
 292   except OSError:
 293     # It's faster to catch exception than call os.path.islink(). Chromium
 294     # tree happens to have invalid symlinks under
 295     # third_party/openssl/openssl/test/.
 296     return None
 297
 298   if must_be_executable(rel_path):
 299     if not bit:
 300       return result_dict('Must have executable bit set')
 301     return
 302   if must_not_be_executable(rel_path):
 303     if bit:
 304       return result_dict('Must not have executable bit set')
 305     return
 306
 307   # For the others, it depends on the file header.
 308   (shebang, elf) = has_shebang_or_is_elf(full_path)
 309   if bit != (shebang or elf):
 310     if bit:
 311       return result_dict('Has executable bit but not shebang or ELF header')
 312     if shebang:
 313       return result_dict('Has shebang but not executable bit')
 314     return result_dict('Has ELF header but not executable bit')
 315
 316
 317 def check_files(root, files):
 318   gen = (check_file(root, f) for f in files if not is_ignored(f))
 319   return filter(None, gen)
 320
 321
 322 class ApiBase(object):
 323   def __init__(self, root_dir, bare_output):
 324     self.root_dir = root_dir
 325     self.bare_output = bare_output
 326     self.count = 0
 327     self.count_read_header = 0
 328
 329   def check_file(self, rel_path):
 330     logging.debug('check_file(%s)' % rel_path)
 331     self.count += 1
 332
 333     if (not must_be_executable(rel_path) and
 334         not must_not_be_executable(rel_path)):
 335       self.count_read_header += 1
 336
 337     return check_file(self.root_dir, rel_path)
 338
 339   def check_dir(self, rel_path):
 340     return self.check(rel_path)
 341
 342   def check(self, start_dir):
 343     """Check the files in start_dir, recursively check its subdirectories."""
 344     errors = []
 345     items = self.list_dir(start_dir)
 346     logging.info('check(%s) -> %d' % (start_dir, len(items)))
 347     for item in items:
 348       full_path = os.path.join(self.root_dir, start_dir, item)
 349       rel_path = full_path[len(self.root_dir) + 1:]
 350       if is_ignored(rel_path):
 351         continue
 352       if os.path.isdir(full_path):
 353         # Depth first.
 354         errors.extend(self.check_dir(rel_path))
 355       else:
 356         error = self.check_file(rel_path)
 357         if error:
 358           errors.append(error)
 359     return errors
 360
 361   def list_dir(self, start_dir):
 362     """Lists all the files and directory inside start_dir."""
 363     return sorted(
 364       x for x in os.listdir(os.path.join(self.root_dir, start_dir))
 365       if not x.startswith('.')
 366     )
 367
 368
 369 class ApiAllFilesAtOnceBase(ApiBase):
 370   _files = None
 371
 372   def list_dir(self, start_dir):
 373     """Lists all the files and directory inside start_dir."""
 374     if self._files is None:
 375       self._files = sorted(self._get_all_files())
 376       if not self.bare_output:
 377         print 'Found %s files' % len(self._files)
 378     start_dir = start_dir[len(self.root_dir) + 1:]
 379     return [
 380       x[len(start_dir):] for x in self._files if x.startswith(start_dir)
 381     ]
 382
 383   def _get_all_files(self):
 384     """Lists all the files and directory inside self._root_dir."""
 385     raise NotImplementedError()
 386
 387
 388 class ApiGit(ApiAllFilesAtOnceBase):
 389   def _get_all_files(self):
 390     return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines()
 391
 392
 393 def get_scm(dir_path, bare):
 394   """Returns a properly configured ApiBase instance."""
 395   cwd = os.getcwd()
 396   root = get_git_root(dir_path or cwd)
 397   if root:
 398     if not bare:
 399       print('Found git repository at %s' % root)
 400     return ApiGit(dir_path or root, bare)
 401
 402   # Returns a non-scm aware checker.
 403   if not bare:
 404     print('Failed to determine the SCM for %s' % dir_path)
 405   return ApiBase(dir_path or cwd, bare)
 406
 407
 408 def main():
 409   usage = """Usage: python %prog [--root <root>] [tocheck]
 410   tocheck  Specifies the directory, relative to root, to check. This defaults
 411            to "." so it checks everything.
 412
 413 Examples:
 414   python %prog
 415   python %prog --root /path/to/source chrome"""
 416
 417   parser = optparse.OptionParser(usage=usage)
 418   parser.add_option(
 419       '--root',
 420       help='Specifies the repository root. This defaults '
 421            'to the checkout repository root')
 422   parser.add_option(
 423       '-v', '--verbose', action='count', default=0, help='Print debug logging')
 424   parser.add_option(
 425       '--bare',
 426       action='store_true',
 427       default=False,
 428       help='Prints the bare filename triggering the checks')
 429   parser.add_option(
 430       '--file', action='append', dest='files',
 431       help='Specifics a list of files to check the permissions of. Only these '
 432       'files will be checked')
 433   parser.add_option('--json', help='Path to JSON output file')
 434   options, args = parser.parse_args()
 435
 436   levels = [logging.ERROR, logging.INFO, logging.DEBUG]
 437   logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)])
 438
 439   if len(args) > 1:
 440     parser.error('Too many arguments used')
 441
 442   if options.root:
 443     options.root = os.path.abspath(options.root)
 444
 445   if options.files:
 446     # --file implies --bare (for PRESUBMIT.py).
 447     options.bare = True
 448
 449     errors = check_files(options.root, options.files)
 450   else:
 451     api = get_scm(options.root, options.bare)
 452     start_dir = args[0] if args else api.root_dir
 453     errors = api.check(start_dir)
 454
 455     if not options.bare:
 456       print('Processed %s files, %d files where tested for shebang/ELF '
 457             'header' % (api.count, api.count_read_header))
 458
 459   if options.json:
 460     with open(options.json, 'w') as f:
 461       json.dump(errors, f)
 462
 463   if errors:
 464     if options.bare:
 465       print '\n'.join(e['full_path'] for e in errors)
 466     else:
 467       print '\nFAILED\n'
 468       print '\n'.join('%s: %s' % (e['full_path'], e['error']) for e in errors)
 469     return 1
 470   if not options.bare:
 471     print '\nSUCCESS\n'
 472   return 0
 473
 474
 475 if '__main__' == __name__:
 476   sys.exit(main())