tools/checkperms/checkperms.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure files have the right permissions.
   7
   8 Some developers have broken SCM configurations that flip the svn:executable
   9 permission on for no good reason. Unix developers who run ls --color will then
  10 see .cc files in green and get confused.
  11
  12 - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS.
  13 - For file extensions that must not be executable, add it to
  14   NOT_EXECUTABLE_EXTENSIONS.
  15 - To ignore all the files inside a directory, add it to IGNORED_PATHS.
  16 - For file base name with ambiguous state and that should not be checked for
  17   shebang, add it to IGNORED_FILENAMES.
  18
  19 Any file not matching the above will be opened and looked if it has a shebang.
  20 It this doesn't match the executable bit on the file, the file will be flagged.
  21
  22 Note that all directory separators must be slashes (Unix-style) and not
  23 backslashes. All directories should be relative to the source root and all
  24 file paths should be only lowercase.
  25 """
  26
  27 import logging
  28 import optparse
  29 import os
  30 import stat
  31 import subprocess
  32 import sys
  33
  34 #### USER EDITABLE SECTION STARTS HERE ####
  35
  36 # Files with these extensions must have executable bit set.
  37 EXECUTABLE_EXTENSIONS = (
  38   'bat',
  39   'dll',
  40   'dylib',
  41   'exe',
  42 )
  43
  44 # These files must have executable bit set.
  45 EXECUTABLE_PATHS = (
  46   # TODO(maruel): Detect ELF files.
  47   'chrome/installer/mac/sign_app.sh.in',
  48   'chrome/installer/mac/sign_versioned_dir.sh.in',
  49 )
  50
  51 # These files must not have the executable bit set. This is mainly a performance
  52 # optimization as these files are not checked for shebang. The list was
  53 # partially generated from:
  54 # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g
  55 NON_EXECUTABLE_EXTENSIONS = (
  56   '1',
  57   '3ds',
  58   'S',
  59   'am',
  60   'applescript',
  61   'asm',
  62   'c',
  63   'cc',
  64   'cfg',
  65   'chromium',
  66   'cpp',
  67   'crx',
  68   'cs',
  69   'css',
  70   'cur',
  71   'def',
  72   'der',
  73   'expected',
  74   'gif',
  75   'grd',
  76   'gyp',
  77   'gypi',
  78   'h',
  79   'hh',
  80   'htm',
  81   'html',
  82   'hyph',
  83   'ico',
  84   'idl',
  85   'java',
  86   'jpg',
  87   'js',
  88   'json',
  89   'm',
  90   'm4',
  91   'mm',
  92   'mms',
  93   'mock-http-headers',
  94   'nmf',
  95   'onc',
  96   'pat',
  97   'patch',
  98   'pdf',
  99   'pem',
 100   'plist',
 101   'png',
 102   'proto',
 103   'rc',
 104   'rfx',
 105   'rgs',
 106   'rules',
 107   'spec',
 108   'sql',
 109   'srpc',
 110   'svg',
 111   'tcl',
 112   'test',
 113   'tga',
 114   'txt',
 115   'vcproj',
 116   'vsprops',
 117   'webm',
 118   'word',
 119   'xib',
 120   'xml',
 121   'xtb',
 122   'zip',
 123 )
 124
 125 # File names that are always whitelisted.  (These are all autoconf spew.)
 126 IGNORED_FILENAMES = (
 127   'config.guess',
 128   'config.sub',
 129   'configure',
 130   'depcomp',
 131   'install-sh',
 132   'missing',
 133   'mkinstalldirs',
 134   'naclsdk',
 135   'scons',
 136 )
 137
 138 # File paths starting with one of these will be ignored as well.
 139 # Please consider fixing your file permissions, rather than adding to this list.
 140 IGNORED_PATHS = (
 141   # TODO(maruel): Detect ELF files.
 142   'chrome/test/data/extensions/uitest/plugins/plugin.plugin/contents/'
 143       'macos/testnetscapeplugin',
 144   'chrome/test/data/extensions/uitest/plugins_private/plugin.plugin/contents/'
 145       'macos/testnetscapeplugin',
 146   'chrome/installer/mac/sign_app.sh.in',
 147   'chrome/installer/mac/sign_versioned_dir.sh.in',
 148   'native_client_sdk/src/build_tools/sdk_tools/third_party/',
 149   'out/',
 150   # TODO(maruel): Fix these.
 151   'third_party/android_testrunner/',
 152   'third_party/bintrees/',
 153   'third_party/closure_linter/',
 154   'third_party/devscripts/licensecheck.pl.vanilla',
 155   'third_party/hyphen/',
 156   'third_party/jemalloc/',
 157   'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl',
 158   'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh',
 159   'third_party/lcov/contrib/galaxy/conglomerate_functions.pl',
 160   'third_party/lcov/contrib/galaxy/gen_makefile.sh',
 161   'third_party/libevent/autogen.sh',
 162   'third_party/libevent/test/test.sh',
 163   'third_party/libxml/linux/xml2-config',
 164   'third_party/libxml/src/ltmain.sh',
 165   'third_party/mesa/',
 166   'third_party/protobuf/',
 167   'third_party/python_gflags/gflags.py',
 168   'third_party/sqlite/',
 169   'third_party/talloc/script/mksyms.sh',
 170   'third_party/tcmalloc/',
 171   'third_party/tlslite/setup.py',
 172 )
 173
 174 #### USER EDITABLE SECTION ENDS HERE ####
 175
 176 assert set(EXECUTABLE_EXTENSIONS) & set(NON_EXECUTABLE_EXTENSIONS) == set()
 177
 178
 179 def capture(cmd, cwd):
 180   """Returns the output of a command.
 181
 182   Ignores the error code or stderr.
 183   """
 184   logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd))
 185   env = os.environ.copy()
 186   env['LANGUAGE'] = 'en_US.UTF-8'
 187   p = subprocess.Popen(
 188       cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
 189   return p.communicate()[0]
 190
 191
 192 def get_svn_info(dir_path):
 193   """Returns svn meta-data for a svn checkout."""
 194   if not os.path.isdir(dir_path):
 195     return {}
 196   out = capture(['svn', 'info', '.', '--non-interactive'], dir_path)
 197   return dict(l.split(': ', 1) for l in out.splitlines() if l)
 198
 199
 200 def get_svn_url(dir_path):
 201   return get_svn_info(dir_path).get('URL')
 202
 203
 204 def get_svn_root(dir_path):
 205   """Returns the svn checkout root or None."""
 206   svn_url = get_svn_url(dir_path)
 207   if not svn_url:
 208     return None
 209   logging.info('svn url: %s' % svn_url)
 210   while True:
 211     parent = os.path.dirname(dir_path)
 212     if parent == dir_path:
 213       return None
 214     svn_url = svn_url.rsplit('/', 1)[0]
 215     if svn_url != get_svn_url(parent):
 216       return dir_path
 217     dir_path = parent
 218
 219
 220 def get_git_root(dir_path):
 221   """Returns the git checkout root or None."""
 222   root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip()
 223   if root:
 224     return root
 225
 226
 227 def is_ignored(rel_path):
 228   """Returns True if rel_path is in our whitelist of files to ignore."""
 229   rel_path = rel_path.lower()
 230   return (
 231       os.path.basename(rel_path) in IGNORED_FILENAMES or
 232       rel_path.startswith(IGNORED_PATHS))
 233
 234
 235 def must_be_executable(rel_path):
 236   """The file name represents a file type that must have the executable bit
 237   set.
 238   """
 239   return (
 240       os.path.splitext(rel_path)[1][1:].lower() in EXECUTABLE_EXTENSIONS or
 241       rel_path in EXECUTABLE_PATHS)
 242
 243
 244 def must_not_be_executable(rel_path):
 245   """The file name represents a file type that must not have the executable
 246   bit set.
 247   """
 248   return os.path.splitext(rel_path)[1][1:].lower() in NON_EXECUTABLE_EXTENSIONS
 249
 250
 251 def has_executable_bit(full_path):
 252   """Returns if any executable bit is set."""
 253   permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
 254   return bool(permission & os.stat(full_path).st_mode)
 255
 256
 257 def has_shebang(full_path):
 258   """Returns if the file starts with #!/.
 259
 260   file_path is the absolute path to the file.
 261   """
 262   with open(full_path, 'rb') as f:
 263     return f.read(3) == '#!/'
 264
 265 def check_file(full_path, bare_output):
 266   """Checks file_path's permissions and returns an error if it is
 267   inconsistent.
 268
 269   It is assumed that the file is not ignored by is_ignored().
 270
 271   If the file name is matched with must_be_executable() or
 272   must_not_be_executable(), only its executable bit is checked.
 273   Otherwise, the 3 first bytes of the file are read to verify if it has a
 274   shebang and compares this with the executable bit on the file.
 275   """
 276   try:
 277     bit = has_executable_bit(full_path)
 278   except OSError:
 279     # It's faster to catch exception than call os.path.islink(). Chromium
 280     # tree happens to have invalid symlinks under
 281     # third_party/openssl/openssl/test/.
 282     return None
 283
 284   if must_be_executable(full_path):
 285     if not bit:
 286       if bare_output:
 287         return full_path
 288       return '%s: Must have executable bit set' % full_path
 289     return
 290   if must_not_be_executable(full_path):
 291     if bit:
 292       if bare_output:
 293         return full_path
 294       return '%s: Must not have executable bit set' % full_path
 295     return
 296
 297   # For the others, it depends on the shebang.
 298   shebang = has_shebang(full_path)
 299   if bit != shebang:
 300     if bare_output:
 301       return full_path
 302     if bit:
 303       return '%s: Has executable bit but not shebang' % full_path
 304     else:
 305       return '%s: Has shebang but not executable bit' % full_path
 306
 307
 308 def check_files(root, files, bare_output):
 309   errors = []
 310   for file_path in files:
 311     if is_ignored(file_path):
 312       continue
 313
 314     full_file_path = os.path.join(root, file_path)
 315
 316     error = check_file(full_file_path, bare_output)
 317     if error:
 318       errors.append(error)
 319
 320   return errors
 321
 322 class ApiBase(object):
 323   def __init__(self, root_dir, bare_output):
 324     self.root_dir = root_dir
 325     self.bare_output = bare_output
 326     self.count = 0
 327     self.count_shebang = 0
 328
 329   def check_file(self, rel_path):
 330     logging.debug('check_file(%s)' % rel_path)
 331     self.count += 1
 332
 333     if (not must_be_executable(rel_path) and
 334         not must_not_be_executable(rel_path)):
 335       self.count_shebang += 1
 336
 337     full_path = os.path.join(self.root_dir, rel_path)
 338     return check_file(full_path, self.bare_output)
 339
 340   def check_dir(self, rel_path):
 341     return self.check(rel_path)
 342
 343   def check(self, start_dir):
 344     """Check the files in start_dir, recursively check its subdirectories."""
 345     errors = []
 346     items = self.list_dir(start_dir)
 347     logging.info('check(%s) -> %d' % (start_dir, len(items)))
 348     for item in items:
 349       full_path = os.path.join(self.root_dir, start_dir, item)
 350       rel_path = full_path[len(self.root_dir) + 1:]
 351       if is_ignored(rel_path):
 352         continue
 353       if os.path.isdir(full_path):
 354         # Depth first.
 355         errors.extend(self.check_dir(rel_path))
 356       else:
 357         error = self.check_file(rel_path)
 358         if error:
 359           errors.append(error)
 360     return errors
 361
 362   def list_dir(self, start_dir):
 363     """Lists all the files and directory inside start_dir."""
 364     return sorted(
 365       x for x in os.listdir(os.path.join(self.root_dir, start_dir))
 366       if not x.startswith('.')
 367     )
 368
 369
 370 class ApiSvnQuick(ApiBase):
 371   """Returns all files in svn-versioned directories, independent of the fact if
 372   they are versionned.
 373
 374   Uses svn info in each directory to determine which directories should be
 375   crawled.
 376   """
 377   def __init__(self, *args):
 378     super(ApiSvnQuick, self).__init__(*args)
 379     self.url = get_svn_url(self.root_dir)
 380
 381   def check_dir(self, rel_path):
 382     url = self.url + '/' + rel_path
 383     if get_svn_url(os.path.join(self.root_dir, rel_path)) != url:
 384       return []
 385     return super(ApiSvnQuick, self).check_dir(rel_path)
 386
 387
 388 class ApiAllFilesAtOnceBase(ApiBase):
 389   _files = None
 390
 391   def list_dir(self, start_dir):
 392     """Lists all the files and directory inside start_dir."""
 393     if self._files is None:
 394       self._files = sorted(self._get_all_files())
 395       if not self.bare_output:
 396         print 'Found %s files' % len(self._files)
 397     start_dir = start_dir[len(self.root_dir) + 1:]
 398     return [
 399       x[len(start_dir):] for x in self._files if x.startswith(start_dir)
 400     ]
 401
 402   def _get_all_files(self):
 403     """Lists all the files and directory inside self._root_dir."""
 404     raise NotImplementedError()
 405
 406
 407 class ApiSvn(ApiAllFilesAtOnceBase):
 408   """Returns all the subversion controlled files.
 409
 410   Warning: svn ls is abnormally slow.
 411   """
 412   def _get_all_files(self):
 413     cmd = ['svn', 'ls', '--non-interactive', '--recursive']
 414     return (
 415         x for x in capture(cmd, self.root_dir).splitlines()
 416         if not x.endswith(os.path.sep))
 417
 418
 419 class ApiGit(ApiAllFilesAtOnceBase):
 420   def _get_all_files(self):
 421     return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines()
 422
 423
 424 def get_scm(dir_path, bare):
 425   """Returns a properly configured ApiBase instance."""
 426   cwd = os.getcwd()
 427   root = get_svn_root(dir_path or cwd)
 428   if root:
 429     if not bare:
 430       print('Found subversion checkout at %s' % root)
 431     return ApiSvnQuick(dir_path or root, bare)
 432   root = get_git_root(dir_path or cwd)
 433   if root:
 434     if not bare:
 435       print('Found git repository at %s' % root)
 436     return ApiGit(dir_path or root, bare)
 437
 438   # Returns a non-scm aware checker.
 439   if not bare:
 440     print('Failed to determine the SCM for %s' % dir_path)
 441   return ApiBase(dir_path or cwd, bare)
 442
 443
 444 def main():
 445   usage = """Usage: python %prog [--root <root>] [tocheck]
 446   tocheck  Specifies the directory, relative to root, to check. This defaults
 447            to "." so it checks everything.
 448
 449 Examples:
 450   python %prog
 451   python %prog --root /path/to/source chrome"""
 452
 453   parser = optparse.OptionParser(usage=usage)
 454   parser.add_option(
 455       '--root',
 456       help='Specifies the repository root. This defaults '
 457            'to the checkout repository root')
 458   parser.add_option(
 459       '-v', '--verbose', action='count', default=0, help='Print debug logging')
 460   parser.add_option(
 461       '--bare',
 462       action='store_true',
 463       default=False,
 464       help='Prints the bare filename triggering the checks')
 465   parser.add_option(
 466       '--file', action='append', dest='files',
 467       help='Specifics a list of files to check the permissions of. Only these '
 468       'files will be checked')
 469   options, args = parser.parse_args()
 470
 471   levels = [logging.ERROR, logging.INFO, logging.DEBUG]
 472   logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)])
 473
 474   if len(args) > 1:
 475     parser.error('Too many arguments used')
 476
 477   if options.root:
 478     options.root = os.path.abspath(options.root)
 479
 480   if options.files:
 481     errors = check_files(options.root, options.files, options.bare)
 482     print '\n'.join(errors)
 483     return bool(errors)
 484
 485   api = get_scm(options.root, options.bare)
 486   if args:
 487     start_dir = args[0]
 488   else:
 489     start_dir = api.root_dir
 490
 491   errors = api.check(start_dir)
 492
 493   if not options.bare:
 494     print 'Processed %s files, %d files where tested for shebang' % (
 495         api.count, api.count_shebang)
 496
 497   if errors:
 498     if not options.bare:
 499       print '\nFAILED\n'
 500     print '\n'.join(errors)
 501     return 1
 502   if not options.bare:
 503     print '\nSUCCESS\n'
 504   return 0
 505
 506
 507 if '__main__' == __name__:
 508   sys.exit(main())