android_webview/tools/webview_licenses.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Checks third-party licenses for the purposes of the Android WebView build.
   7
   8 The Android tree includes a snapshot of Chromium in order to power the system
   9 WebView.  This tool checks that all code uses open-source licenses compatible
  10 with Android, and that we meet the requirements of those licenses. It can also
  11 be used to generate an Android NOTICE file for the third-party code.
  12
  13 It makes use of src/tools/licenses.py and the README.chromium files on which
  14 it depends. It also makes use of a data file, third_party_files_whitelist.txt,
  15 which whitelists indicidual files which contain third-party code but which
  16 aren't in a third-party directory with a README.chromium file.
  17 """
  18
  19 import optparse
  20 import os
  21 import re
  22 import subprocess
  23 import sys
  24 import textwrap
  25
  26
  27 REPOSITORY_ROOT = os.path.abspath(os.path.join(
  28     os.path.dirname(__file__), '..', '..'))
  29
  30 sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
  31 import licenses
  32
  33 import known_issues
  34
  35 def GetIncompatibleDirectories():
  36   """Gets a list of third-party directories which use licenses incompatible
  37   with Android. This is used by the snapshot tool.
  38   Returns:
  39     A list of directories.
  40   """
  41
  42   whitelist = [
  43     'Apache( Version)? 2(\.0)?',
  44     '(New )?BSD( [23]-Clause)?( with advertising clause)?',
  45     'L?GPL ?v?2(\.[01])?( or later)?',
  46     'MIT(/X11)?(-like)?',
  47     'MPL 1\.1 ?/ ?GPL 2(\.0)? ?/ ?LGPL 2\.1',
  48     'MPL 2(\.0)?',
  49     'Microsoft Limited Public License',
  50     'Microsoft Permissive License',
  51     'Public Domain',
  52     'SGI Free Software License B',
  53     'X11',
  54   ]
  55   regex = '^(%s)$' % '|'.join(whitelist)
  56   result = []
  57   for directory in _FindThirdPartyDirs():
  58     if directory in known_issues.KNOWN_ISSUES:
  59       result.append(directory)
  60       continue
  61     try:
  62       metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
  63                                    require_license_file=False)
  64     except licenses.LicenseError as e:
  65       print 'Got LicenseError while scanning ' + directory
  66       raise
  67     if metadata.get('License Android Compatible', 'no').upper() == 'YES':
  68       continue
  69     license = re.split(' [Ll]icenses?$', metadata['License'])[0]
  70     tokens = [x.strip() for x in re.split(' and |,', license) if len(x) > 0]
  71     for token in tokens:
  72       if not re.match(regex, token, re.IGNORECASE):
  73         result.append(directory)
  74         break
  75   return result
  76
  77 class ScanResult(object):
  78   Ok, Warnings, Errors = range(3)
  79
  80 def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files):
  81   """Checks that all files which are not in a listed third-party directory,
  82   and which do not use the standard Chromium license, are whitelisted.
  83   Args:
  84     excluded_dirs_list: The list of directories to exclude from scanning.
  85     whitelisted_files: The whitelist of files.
  86   Returns:
  87     ScanResult.Ok if all files with non-standard license headers are whitelisted
  88     and the whitelist contains no stale entries;
  89     ScanResult.Warnings if there are stale entries;
  90     ScanResult.Errors if new non-whitelisted entries found.
  91   """
  92
  93   excluded_dirs_list = [d for d in excluded_dirs_list if not 'third_party' in d]
  94   # Using a commond pattern for third-partyies makes the ignore regexp shorter
  95   excluded_dirs_list.append('third_party')
  96   # VCS dirs
  97   excluded_dirs_list.append('.git')
  98   excluded_dirs_list.append('.svn')
  99   # Build output
 100   excluded_dirs_list.append('out/Debug')
 101   excluded_dirs_list.append('out/Release')
 102   # 'Copyright' appears in license agreements
 103   excluded_dirs_list.append('chrome/app/resources')
 104   # This is a test output directory
 105   excluded_dirs_list.append('chrome/tools/test/reference_build')
 106   # This is tests directory, doesn't exist in the snapshot
 107   excluded_dirs_list.append('content/test/data')
 108   # This is a test output directory
 109   excluded_dirs_list.append('data/dom_perf')
 110   # Histogram tools, doesn't exist in the snapshot
 111   excluded_dirs_list.append('tools/histograms')
 112   # Arm sysroot tools, doesn't exist in the snapshot
 113   excluded_dirs_list.append('arm-sysroot')
 114   # Data is not part of open source chromium, but are included on some bots.
 115   excluded_dirs_list.append('data')
 116
 117   args = ['android_webview/tools/find_copyrights.pl',
 118           '.'
 119           ] + excluded_dirs_list
 120   p = subprocess.Popen(args=args, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE)
 121   lines = p.communicate()[0].splitlines()
 122
 123   offending_files = []
 124   allowed_copyrights = '^(?:\*No copyright\*' \
 125       '|20[0-9][0-9](?:-20[0-9][0-9])? The Chromium Authors\. ' \
 126       'All rights reserved.*)$'
 127   allowed_copyrights_re = re.compile(allowed_copyrights)
 128   for l in lines:
 129     entries = l.split('\t')
 130     if entries[1] == "GENERATED FILE":
 131       continue
 132     copyrights = entries[1].split(' / ')
 133     for c in copyrights:
 134       if c and not allowed_copyrights_re.match(c):
 135         offending_files.append(os.path.normpath(entries[0]))
 136         break
 137
 138   unknown = set(offending_files) - set(whitelisted_files)
 139   if unknown:
 140     print 'The following files contain a third-party license but are not in ' \
 141           'a listed third-party directory and are not whitelisted. You must ' \
 142           'add the following files to the whitelist.\n%s' % \
 143           '\n'.join(sorted(unknown))
 144
 145   stale = set(whitelisted_files) - set(offending_files)
 146   if stale:
 147     print 'The following files are whitelisted unnecessarily. You must ' \
 148           ' remove the following files from the whitelist.\n%s' % \
 149           '\n'.join(sorted(stale))
 150
 151   if unknown:
 152     return ScanResult.Errors
 153   elif stale:
 154     return ScanResult.Warnings
 155   else:
 156     return ScanResult.Ok
 157
 158
 159 def _ReadFile(path):
 160   """Reads a file from disk.
 161   Args:
 162     path: The path of the file to read, relative to the root of the repository.
 163   Returns:
 164     The contents of the file as a string.
 165   """
 166
 167   return open(os.path.join(REPOSITORY_ROOT, path), 'rb').read()
 168
 169
 170 def _FindThirdPartyDirs():
 171   """Gets the list of third-party directories.
 172   Returns:
 173     The list of third-party directories.
 174   """
 175
 176   # Please don't add here paths that have problems with license files,
 177   # as they will end up included in Android WebView snapshot.
 178   # Instead, add them into known_issues.py.
 179   prune_paths = [
 180     # Placeholder directory, no third-party code.
 181     os.path.join('third_party', 'adobe'),
 182     # Apache 2.0 license. See
 183     # https://code.google.com/p/chromium/issues/detail?id=140478.
 184     os.path.join('third_party', 'bidichecker'),
 185     # Isn't checked out on clients
 186     os.path.join('third_party', 'gles2_conform'),
 187     # The llvm-build doesn't exist for non-clang builder
 188     os.path.join('third_party', 'llvm-build'),
 189     # Binaries doesn't apply to android
 190     os.path.join('third_party', 'widevine'),
 191     # third_party directories in this tree aren't actually third party, but
 192     # provide a way to shadow experimental buildfiles into those directories.
 193     os.path.join('tools', 'gn', 'secondary'),
 194     # Not shipped, Chromium code
 195     os.path.join('tools', 'swarm_client'),
 196   ]
 197   third_party_dirs = licenses.FindThirdPartyDirs(prune_paths, REPOSITORY_ROOT)
 198   return licenses.FilterDirsWithFiles(third_party_dirs, REPOSITORY_ROOT)
 199
 200
 201 def _Scan():
 202   """Checks that license meta-data is present for all third-party code and
 203      that all non third-party code doesn't contain external copyrighted code.
 204   Returns:
 205     ScanResult.Ok if everything is in order;
 206     ScanResult.Warnings if there are non-fatal problems (e.g. stale whitelist
 207       entries)
 208     ScanResult.Errors otherwise.
 209   """
 210
 211   third_party_dirs = _FindThirdPartyDirs()
 212
 213   # First, check designated third-party directories using src/tools/licenses.py.
 214   all_licenses_valid = True
 215   for path in sorted(third_party_dirs):
 216     try:
 217       licenses.ParseDir(path, REPOSITORY_ROOT)
 218     except licenses.LicenseError, e:
 219       if not (path in known_issues.KNOWN_ISSUES):
 220         print 'Got LicenseError "%s" while scanning %s' % (e, path)
 221         all_licenses_valid = False
 222
 223   # Second, check for non-standard license text.
 224   files_data = _ReadFile(os.path.join('android_webview', 'tools',
 225                                       'third_party_files_whitelist.txt'))
 226   whitelisted_files = []
 227   for line in files_data.splitlines():
 228     match = re.match(r'([^#\s]+)', line)
 229     if match:
 230       whitelisted_files.append(match.group(1))
 231   licenses_check = _CheckLicenseHeaders(third_party_dirs, whitelisted_files)
 232
 233   return licenses_check if all_licenses_valid else ScanResult.Errors
 234
 235
 236 def GenerateNoticeFile():
 237   """Generates the contents of an Android NOTICE file for the third-party code.
 238   This is used by the snapshot tool.
 239   Returns:
 240     The contents of the NOTICE file.
 241   """
 242
 243   third_party_dirs = _FindThirdPartyDirs()
 244
 245   # Don't forget Chromium's LICENSE file
 246   content = [_ReadFile('LICENSE')]
 247
 248   # We provide attribution for all third-party directories.
 249   # TODO(steveblock): Limit this to only code used by the WebView binary.
 250   for directory in sorted(third_party_dirs):
 251     metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
 252                                  require_license_file=False)
 253     license_file = metadata['License File']
 254     if license_file and license_file != licenses.NOT_SHIPPED:
 255       content.append(_ReadFile(license_file))
 256
 257   return '\n'.join(content)
 258
 259
 260 def main():
 261   class FormatterWithNewLines(optparse.IndentedHelpFormatter):
 262     def format_description(self, description):
 263       paras = description.split('\n')
 264       formatted_paras = [textwrap.fill(para, self.width) for para in paras]
 265       return '\n'.join(formatted_paras) + '\n'
 266
 267   parser = optparse.OptionParser(formatter=FormatterWithNewLines(),
 268                                  usage='%prog [options]')
 269   parser.description = (__doc__ +
 270                        '\nCommands:\n' \
 271                        '  scan  Check licenses.\n' \
 272                        '  notice Generate Android NOTICE file on stdout')
 273   (options, args) = parser.parse_args()
 274   if len(args) != 1:
 275     parser.print_help()
 276     return ScanResult.Errors
 277
 278   if args[0] == 'scan':
 279     scan_result = _Scan()
 280     if scan_result == ScanResult.Ok:
 281       print 'OK!'
 282     return scan_result
 283   elif args[0] == 'notice':
 284     print GenerateNoticeFile()
 285     return ScanResult.Ok
 286
 287   parser.print_help()
 288   return ScanResult.Errors
 289
 290 if __name__ == '__main__':
 291   sys.exit(main())