android_webview/tools/webview_licenses.py

   1 #!/usr/bin/python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Checks third-party licenses for the purposes of the Android WebView build.
   7
   8 The Android tree includes a snapshot of Chromium in order to power the system
   9 WebView.  This tool checks that all code uses open-source licenses compatible
  10 with Android, and that we meet the requirements of those licenses. It can also
  11 be used to generate an Android NOTICE file for the third-party code.
  12
  13 It makes use of src/tools/licenses.py and the README.chromium files on which
  14 it depends. It also makes use of a data file, third_party_files_whitelist.txt,
  15 which whitelists indicidual files which contain third-party code but which
  16 aren't in a third-party directory with a README.chromium file.
  17 """
  18
  19 import glob
  20 import imp
  21 import multiprocessing
  22 import optparse
  23 import os
  24 import re
  25 import sys
  26 import textwrap
  27
  28
  29 REPOSITORY_ROOT = os.path.abspath(os.path.join(
  30     os.path.dirname(__file__), '..', '..'))
  31
  32 # Import third_party/PRESUBMIT.py via imp to avoid importing a random
  33 # PRESUBMIT.py from $PATH, also make sure we don't generate a .pyc file.
  34 sys.dont_write_bytecode = True
  35 third_party = \
  36   imp.load_source('PRESUBMIT', \
  37                   os.path.join(REPOSITORY_ROOT, 'third_party', 'PRESUBMIT.py'))
  38
  39 sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
  40 import licenses
  41
  42 import copyright_scanner
  43 import known_issues
  44
  45 class InputApi(object):
  46   def __init__(self):
  47     self.os_path = os.path
  48     self.os_walk = os.walk
  49     self.re = re
  50     self.ReadFile = _ReadFile
  51     self.change = InputApiChange()
  52
  53 class InputApiChange(object):
  54   def __init__(self):
  55     self.RepositoryRoot = lambda: REPOSITORY_ROOT
  56
  57
  58 def GetIncompatibleDirectories():
  59   """Gets a list of third-party directories which use licenses incompatible
  60   with Android. This is used by the snapshot tool.
  61   Returns:
  62     A list of directories.
  63   """
  64
  65   result = []
  66   for directory in _FindThirdPartyDirs():
  67     if directory in known_issues.KNOWN_ISSUES:
  68       result.append(directory)
  69       continue
  70     try:
  71       metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
  72                                    require_license_file=False)
  73     except licenses.LicenseError as e:
  74       print 'Got LicenseError while scanning ' + directory
  75       raise
  76     if metadata.get('License Android Compatible', 'no').upper() == 'YES':
  77       continue
  78     license = re.split(' [Ll]icenses?$', metadata['License'])[0]
  79     if not third_party.LicenseIsCompatibleWithAndroid(InputApi(), license):
  80       result.append(directory)
  81   return result
  82
  83 def GetUnknownIncompatibleDirectories():
  84   """Gets a list of third-party directories which use licenses incompatible
  85   with Android which are not present in the known_issues.py file.
  86   This is used by the AOSP bot.
  87   Returns:
  88     A list of directories.
  89   """
  90   incompatible_directories = frozenset(GetIncompatibleDirectories())
  91   known_incompatible = []
  92   for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
  93     for exclude in exclude_list:
  94       if glob.has_magic(exclude):
  95         exclude_dirname = os.path.dirname(exclude)
  96         if glob.has_magic(exclude_dirname):
  97           print ('Exclude path %s contains an unexpected glob expression,' \
  98                  ' skipping.' % exclude)
  99         exclude = exclude_dirname
 100       known_incompatible.append(os.path.normpath(os.path.join(path, exclude)))
 101   known_incompatible = frozenset(known_incompatible)
 102   return incompatible_directories.difference(known_incompatible)
 103
 104
 105 class ScanResult(object):
 106   Ok, Warnings, Errors = range(3)
 107
 108 # Needs to be a top-level function for multiprocessing
 109 def _FindCopyrightViolations(files_to_scan_as_string):
 110   return copyright_scanner.FindCopyrightViolations(
 111     InputApi(), REPOSITORY_ROOT, files_to_scan_as_string)
 112
 113 def _ShardList(l, shard_len):
 114   return [l[i:i + shard_len] for i in range(0, len(l), shard_len)]
 115
 116 def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files):
 117   """Checks that all files which are not in a listed third-party directory,
 118   and which do not use the standard Chromium license, are whitelisted.
 119   Args:
 120     excluded_dirs_list: The list of directories to exclude from scanning.
 121     whitelisted_files: The whitelist of files.
 122   Returns:
 123     ScanResult.Ok if all files with non-standard license headers are whitelisted
 124     and the whitelist contains no stale entries;
 125     ScanResult.Warnings if there are stale entries;
 126     ScanResult.Errors if new non-whitelisted entries found.
 127   """
 128   input_api = InputApi()
 129   files_to_scan = copyright_scanner.FindFiles(
 130     input_api, REPOSITORY_ROOT, ['.'], excluded_dirs_list)
 131   sharded_files_to_scan = _ShardList(files_to_scan, 2000)
 132   pool = multiprocessing.Pool()
 133   offending_files_chunks = pool.map_async(
 134       _FindCopyrightViolations, sharded_files_to_scan).get(999999)
 135   pool.close()
 136   pool.join()
 137   # Flatten out the result
 138   offending_files = \
 139     [item for sublist in offending_files_chunks for item in sublist]
 140
 141   (unknown, missing, stale) = copyright_scanner.AnalyzeScanResults(
 142     input_api, whitelisted_files, offending_files)
 143
 144   if unknown:
 145     print 'The following files contain a third-party license but are not in ' \
 146           'a listed third-party directory and are not whitelisted. You must ' \
 147           'add the following files to the whitelist.\n%s' % \
 148           '\n'.join(sorted(unknown))
 149   if missing:
 150     print 'The following files are whitelisted, but do not exist.\n%s' % \
 151         '\n'.join(sorted(missing))
 152   if stale:
 153     print 'The following files are whitelisted unnecessarily. You must ' \
 154           'remove the following files from the whitelist.\n%s' % \
 155           '\n'.join(sorted(stale))
 156
 157   if unknown:
 158     return ScanResult.Errors
 159   elif stale or missing:
 160     return ScanResult.Warnings
 161   else:
 162     return ScanResult.Ok
 163
 164
 165 def _ReadFile(full_path, mode='rU'):
 166   """Reads a file from disk. This emulates presubmit InputApi.ReadFile func.
 167   Args:
 168     full_path: The path of the file to read.
 169   Returns:
 170     The contents of the file as a string.
 171   """
 172
 173   with open(full_path, mode) as f:
 174     return f.read()
 175
 176
 177 def _ReadLocalFile(path, mode='rb'):
 178   """Reads a file from disk.
 179   Args:
 180     path: The path of the file to read, relative to the root of the repository.
 181   Returns:
 182     The contents of the file as a string.
 183   """
 184
 185   return _ReadFile(os.path.join(REPOSITORY_ROOT, path), mode)
 186
 187
 188 def _FindThirdPartyDirs():
 189   """Gets the list of third-party directories.
 190   Returns:
 191     The list of third-party directories.
 192   """
 193
 194   # Please don't add here paths that have problems with license files,
 195   # as they will end up included in Android WebView snapshot.
 196   # Instead, add them into known_issues.py.
 197   prune_paths = [
 198     # Temporary until we figure out how not to check out quickoffice on the
 199     # Android license check bot. Tracked in crbug.com/350472.
 200     os.path.join('chrome', 'browser', 'resources', 'chromeos', 'quickoffice'),
 201     # Placeholder directory, no third-party code.
 202     os.path.join('third_party', 'adobe'),
 203     # Apache 2.0 license. See
 204     # https://code.google.com/p/chromium/issues/detail?id=140478.
 205     os.path.join('third_party', 'bidichecker'),
 206     # Isn't checked out on clients
 207     os.path.join('third_party', 'gles2_conform'),
 208     # The llvm-build doesn't exist for non-clang builder
 209     os.path.join('third_party', 'llvm-build'),
 210     # Binaries doesn't apply to android
 211     os.path.join('third_party', 'widevine'),
 212     # third_party directories in this tree aren't actually third party, but
 213     # provide a way to shadow experimental buildfiles into those directories.
 214     os.path.join('build', 'secondary'),
 215     # Not shipped, Chromium code
 216     os.path.join('tools', 'swarming_client'),
 217   ]
 218   third_party_dirs = licenses.FindThirdPartyDirs(prune_paths, REPOSITORY_ROOT)
 219   return licenses.FilterDirsWithFiles(third_party_dirs, REPOSITORY_ROOT)
 220
 221
 222 def _Scan():
 223   """Checks that license meta-data is present for all third-party code and
 224      that all non third-party code doesn't contain external copyrighted code.
 225   Returns:
 226     ScanResult.Ok if everything is in order;
 227     ScanResult.Warnings if there are non-fatal problems (e.g. stale whitelist
 228       entries)
 229     ScanResult.Errors otherwise.
 230   """
 231
 232   third_party_dirs = _FindThirdPartyDirs()
 233
 234   # First, check designated third-party directories using src/tools/licenses.py.
 235   all_licenses_valid = True
 236   for path in sorted(third_party_dirs):
 237     try:
 238       licenses.ParseDir(path, REPOSITORY_ROOT)
 239     except licenses.LicenseError, e:
 240       if not (path in known_issues.KNOWN_ISSUES):
 241         print 'Got LicenseError "%s" while scanning %s' % (e, path)
 242         all_licenses_valid = False
 243
 244   # Second, check for non-standard license text.
 245   whitelisted_files = copyright_scanner.LoadWhitelistedFilesList(InputApi())
 246   licenses_check = _CheckLicenseHeaders(third_party_dirs, whitelisted_files)
 247
 248   return licenses_check if all_licenses_valid else ScanResult.Errors
 249
 250
 251 def GenerateNoticeFile():
 252   """Generates the contents of an Android NOTICE file for the third-party code.
 253   This is used by the snapshot tool.
 254   Returns:
 255     The contents of the NOTICE file.
 256   """
 257
 258   third_party_dirs = _FindThirdPartyDirs()
 259
 260   # Don't forget Chromium's LICENSE file
 261   content = [_ReadLocalFile('LICENSE')]
 262
 263   # We provide attribution for all third-party directories.
 264   # TODO(steveblock): Limit this to only code used by the WebView binary.
 265   for directory in sorted(third_party_dirs):
 266     metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
 267                                  require_license_file=False)
 268     license_file = metadata['License File']
 269     if license_file and license_file != licenses.NOT_SHIPPED:
 270       content.append(_ReadLocalFile(license_file))
 271
 272   return '\n'.join(content)
 273
 274
 275 def _ProcessIncompatibleResult(incompatible_directories):
 276   if incompatible_directories:
 277     print ("Incompatibly licensed directories found:\n" +
 278            "\n".join(sorted(incompatible_directories)))
 279     return ScanResult.Errors
 280   return ScanResult.Ok
 281
 282 def main():
 283   class FormatterWithNewLines(optparse.IndentedHelpFormatter):
 284     def format_description(self, description):
 285       paras = description.split('\n')
 286       formatted_paras = [textwrap.fill(para, self.width) for para in paras]
 287       return '\n'.join(formatted_paras) + '\n'
 288
 289   parser = optparse.OptionParser(formatter=FormatterWithNewLines(),
 290                                  usage='%prog [options]')
 291   parser.description = (__doc__ +
 292                         '\nCommands:\n'
 293                         '  scan Check licenses.\n'
 294                         '  notice Generate Android NOTICE file on stdout.\n'
 295                         '  incompatible_directories Scan for incompatibly'
 296                         ' licensed directories.\n'
 297                         '  all_incompatible_directories Scan for incompatibly'
 298                         ' licensed directories (even those in'
 299                         ' known_issues.py).\n'
 300                         '  display_copyrights Display autorship on the files'
 301                         ' using names provided via stdin.\n')
 302   (_, args) = parser.parse_args()
 303   if len(args) != 1:
 304     parser.print_help()
 305     return ScanResult.Errors
 306
 307   if args[0] == 'scan':
 308     scan_result = _Scan()
 309     if scan_result == ScanResult.Ok:
 310       print 'OK!'
 311     return scan_result
 312   elif args[0] == 'notice':
 313     print GenerateNoticeFile()
 314     return ScanResult.Ok
 315   elif args[0] == 'incompatible_directories':
 316     return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories())
 317   elif args[0] == 'all_incompatible_directories':
 318     return _ProcessIncompatibleResult(GetIncompatibleDirectories())
 319   elif args[0] == 'display_copyrights':
 320     files = sys.stdin.read().splitlines()
 321     for f, c in \
 322         zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)):
 323       print f, '\t', ' / '.join(sorted(c))
 324     return ScanResult.Ok
 325   parser.print_help()
 326   return ScanResult.Errors
 327
 328 if __name__ == '__main__':
 329   sys.exit(main())