tools/licenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Utility for checking and processing licensing information in third_party
   7 directories.
   8
   9 Usage: licenses.py <command>
  10
  11 Commands:
  12   scan     scan third_party directories, verifying that we have licensing info
  13   credits  generate about:credits on stdout
  14
  15 (You can also import this as a module.)
  16 """
  17
  18 import cgi
  19 import os
  20 import sys
  21
  22 # Paths from the root of the tree to directories to skip.
  23 PRUNE_PATHS = set([
  24     # Same module occurs in crypto/third_party/nss and net/third_party/nss, so
  25     # skip this one.
  26     os.path.join('third_party','nss'),
  27
  28     # Placeholder directory only, not third-party code.
  29     os.path.join('third_party','adobe'),
  30
  31     # Build files only, not third-party code.
  32     os.path.join('third_party','widevine'),
  33
  34     # Only binaries, used during development.
  35     os.path.join('third_party','valgrind'),
  36
  37     # Used for development and test, not in the shipping product.
  38     os.path.join('third_party','bison'),
  39     os.path.join('third_party','cygwin'),
  40     os.path.join('third_party','gnu_binutils'),
  41     os.path.join('third_party','gold'),
  42     os.path.join('third_party','gperf'),
  43     os.path.join('third_party','lighttpd'),
  44     os.path.join('third_party','llvm'),
  45     os.path.join('third_party','llvm-build'),
  46     os.path.join('third_party','mingw-w64'),
  47     os.path.join('third_party','nacl_sdk_binaries'),
  48     os.path.join('third_party','pefile'),
  49     os.path.join('third_party','perl'),
  50     os.path.join('third_party','psyco_win32'),
  51     os.path.join('third_party','pylib'),
  52     os.path.join('third_party','python_26'),
  53     os.path.join('third_party','pywebsocket'),
  54     os.path.join('third_party','syzygy'),
  55     os.path.join('tools','gn'),
  56
  57     # Chromium code in third_party.
  58     os.path.join('third_party','fuzzymatch'),
  59     os.path.join('tools', 'swarm_client'),
  60
  61     # Stuff pulled in from chrome-internal for official builds/tools.
  62     os.path.join('third_party', 'clear_cache'),
  63     os.path.join('third_party', 'gnu'),
  64     os.path.join('third_party', 'googlemac'),
  65     os.path.join('third_party', 'pcre'),
  66     os.path.join('third_party', 'psutils'),
  67     os.path.join('third_party', 'sawbuck'),
  68
  69     # Redistribution does not require attribution in documentation.
  70     os.path.join('third_party','directxsdk'),
  71     os.path.join('third_party','platformsdk_win2008_6_1'),
  72     os.path.join('third_party','platformsdk_win7'),
  73 ])
  74
  75 # Directories we don't scan through.
  76 VCS_METADATA_DIRS = ('.svn', '.git')
  77 PRUNE_DIRS = (VCS_METADATA_DIRS +
  78               ('out', 'Debug', 'Release',  # build files
  79                'layout_tests'))            # lots of subdirs
  80
  81 ADDITIONAL_PATHS = (
  82     os.path.join('breakpad'),
  83     os.path.join('chrome', 'common', 'extensions', 'docs', 'examples'),
  84     os.path.join('chrome', 'test', 'chromeos', 'autotest'),
  85     os.path.join('chrome', 'test', 'data'),
  86     os.path.join('native_client'),
  87     os.path.join('native_client_sdk'),
  88     os.path.join('net', 'tools', 'spdyshark'),
  89     os.path.join('ppapi'),
  90     os.path.join('sandbox', 'linux', 'seccomp-legacy'),
  91     os.path.join('sdch', 'open-vcdiff'),
  92     os.path.join('testing', 'gmock'),
  93     os.path.join('testing', 'gtest'),
  94     # The directory with the word list for Chinese and Japanese segmentation
  95     # with different license terms than ICU.
  96     os.path.join('third_party','icu','source','data','brkitr'),
  97     os.path.join('tools', 'grit'),
  98     os.path.join('tools', 'gyp'),
  99     os.path.join('tools', 'page_cycler', 'acid3'),
 100     os.path.join('url', 'third_party', 'mozilla'),
 101     os.path.join('v8'),
 102     # Fake directory so we can include the strongtalk license.
 103     os.path.join('v8', 'strongtalk'),
 104 )
 105
 106
 107 # Directories where we check out directly from upstream, and therefore
 108 # can't provide a README.chromium.  Please prefer a README.chromium
 109 # wherever possible.
 110 SPECIAL_CASES = {
 111     os.path.join('native_client'): {
 112         "Name": "native client",
 113         "URL": "http://code.google.com/p/nativeclient",
 114         "License": "BSD",
 115     },
 116     os.path.join('sandbox', 'linux', 'seccomp-legacy'): {
 117         "Name": "seccompsandbox",
 118         "URL": "http://code.google.com/p/seccompsandbox",
 119         "License": "BSD",
 120     },
 121     os.path.join('sdch', 'open-vcdiff'): {
 122         "Name": "open-vcdiff",
 123         "URL": "http://code.google.com/p/open-vcdiff",
 124         "License": "Apache 2.0, MIT, GPL v2 and custom licenses",
 125         "License Android Compatible": "yes",
 126     },
 127     os.path.join('testing', 'gmock'): {
 128         "Name": "gmock",
 129         "URL": "http://code.google.com/p/googlemock",
 130         "License": "BSD",
 131         "License File": "NOT_SHIPPED",
 132     },
 133     os.path.join('testing', 'gtest'): {
 134         "Name": "gtest",
 135         "URL": "http://code.google.com/p/googletest",
 136         "License": "BSD",
 137         "License File": "NOT_SHIPPED",
 138     },
 139     os.path.join('third_party', 'angle'): {
 140         "Name": "Almost Native Graphics Layer Engine",
 141         "URL": "http://code.google.com/p/angleproject/",
 142         "License": "BSD",
 143     },
 144     os.path.join('third_party', 'cros_system_api'): {
 145         "Name": "Chromium OS system API",
 146         "URL": "http://www.chromium.org/chromium-os",
 147         "License": "BSD",
 148         # Absolute path here is resolved as relative to the source root.
 149         "License File": "/LICENSE.chromium_os",
 150     },
 151     os.path.join('third_party', 'GTM'): {
 152         "Name": "Google Toolbox for Mac",
 153         "URL": "http://code.google.com/p/google-toolbox-for-mac/",
 154         "License": "Apache 2.0",
 155         "License File": "COPYING",
 156     },
 157     os.path.join('third_party', 'lss'): {
 158         "Name": "linux-syscall-support",
 159         "URL": "http://code.google.com/p/linux-syscall-support/",
 160         "License": "BSD",
 161         "License File": "/LICENSE",
 162     },
 163     os.path.join('third_party', 'ots'): {
 164         "Name": "OTS (OpenType Sanitizer)",
 165         "URL": "http://code.google.com/p/ots/",
 166         "License": "BSD",
 167     },
 168     os.path.join('third_party', 'pdfsqueeze'): {
 169         "Name": "pdfsqueeze",
 170         "URL": "http://code.google.com/p/pdfsqueeze/",
 171         "License": "Apache 2.0",
 172         "License File": "COPYING",
 173     },
 174     os.path.join('third_party', 'ppapi'): {
 175         "Name": "ppapi",
 176         "URL": "http://code.google.com/p/ppapi/",
 177     },
 178     os.path.join('third_party', 'scons-2.0.1'): {
 179         "Name": "scons-2.0.1",
 180         "URL": "http://www.scons.org",
 181         "License": "MIT",
 182         "License File": "NOT_SHIPPED",
 183     },
 184     os.path.join('third_party', 'trace-viewer'): {
 185         "Name": "trace-viewer",
 186         "URL": "http://code.google.com/p/trace-viewer",
 187         "License": "BSD",
 188         "License File": "NOT_SHIPPED",
 189     },
 190     os.path.join('third_party', 'v8-i18n'): {
 191         "Name": "Internationalization Library for v8",
 192         "URL": "http://code.google.com/p/v8-i18n/",
 193         "License": "Apache 2.0",
 194     },
 195     os.path.join('third_party', 'WebKit'): {
 196         "Name": "WebKit",
 197         "URL": "http://webkit.org/",
 198         "License": "BSD and GPL v2",
 199         # Absolute path here is resolved as relative to the source root.
 200         "License File": "/webkit/LICENSE",
 201     },
 202     os.path.join('third_party', 'webpagereplay'): {
 203         "Name": "webpagereplay",
 204         "URL": "http://code.google.com/p/web-page-replay",
 205         "License": "Apache 2.0",
 206         "License File": "NOT_SHIPPED",
 207     },
 208     os.path.join('tools', 'grit'): {
 209         "Name": "grit",
 210         "URL": "http://code.google.com/p/grit-i18n",
 211         "License": "BSD",
 212         "License File": "NOT_SHIPPED",
 213     },
 214     os.path.join('tools', 'gyp'): {
 215         "Name": "gyp",
 216         "URL": "http://code.google.com/p/gyp",
 217         "License": "BSD",
 218         "License File": "NOT_SHIPPED",
 219     },
 220     os.path.join('v8'): {
 221         "Name": "V8 JavaScript Engine",
 222         "URL": "http://code.google.com/p/v8",
 223         "License": "BSD",
 224     },
 225     os.path.join('v8', 'strongtalk'): {
 226         "Name": "Strongtalk",
 227         "URL": "http://www.strongtalk.org/",
 228         "License": "BSD",
 229         # Absolute path here is resolved as relative to the source root.
 230         "License File": "/v8/LICENSE.strongtalk",
 231     },
 232 }
 233
 234 # Special value for 'License File' field used to indicate that the license file
 235 # should not be used in about:credits.
 236 NOT_SHIPPED = "NOT_SHIPPED"
 237
 238
 239 class LicenseError(Exception):
 240     """We raise this exception when a directory's licensing info isn't
 241     fully filled out."""
 242     pass
 243
 244 def AbsolutePath(path, filename, root):
 245     """Convert a path in README.chromium to be absolute based on the source
 246     root."""
 247     if filename.startswith('/'):
 248         # Absolute-looking paths are relative to the source root
 249         # (which is the directory we're run from).
 250         absolute_path = os.path.join(root, filename[1:])
 251     else:
 252         absolute_path = os.path.join(root, path, filename)
 253     if os.path.exists(absolute_path):
 254         return absolute_path
 255     return None
 256
 257 def ParseDir(path, root, require_license_file=True):
 258     """Examine a third_party/foo component and extract its metadata."""
 259
 260     # Parse metadata fields out of README.chromium.
 261     # We examine "LICENSE" for the license file by default.
 262     metadata = {
 263         "License File": "LICENSE",  # Relative path to license text.
 264         "Name": None,               # Short name (for header on about:credits).
 265         "URL": None,                # Project home page.
 266         "License": None,            # Software license.
 267         }
 268
 269     # Relative path to a file containing some html we're required to place in
 270     # about:credits.
 271     optional_keys = ["Required Text", "License Android Compatible"]
 272
 273     if path in SPECIAL_CASES:
 274         metadata.update(SPECIAL_CASES[path])
 275     else:
 276         # Try to find README.chromium.
 277         readme_path = os.path.join(root, path, 'README.chromium')
 278         if not os.path.exists(readme_path):
 279             raise LicenseError("missing README.chromium or licenses.py "
 280                                "SPECIAL_CASES entry")
 281
 282         for line in open(readme_path):
 283             line = line.strip()
 284             if not line:
 285                 break
 286             for key in metadata.keys() + optional_keys:
 287                 field = key + ": "
 288                 if line.startswith(field):
 289                     metadata[key] = line[len(field):]
 290
 291     # Check that all expected metadata is present.
 292     for key, value in metadata.iteritems():
 293         if not value:
 294             raise LicenseError("couldn't find '" + key + "' line "
 295                                "in README.chromium or licences.py "
 296                                "SPECIAL_CASES")
 297
 298     # Special-case modules that aren't in the shipping product, so don't need
 299     # their license in about:credits.
 300     if metadata["License File"] != NOT_SHIPPED:
 301         # Check that the license file exists.
 302         for filename in (metadata["License File"], "COPYING"):
 303             license_path = AbsolutePath(path, filename, root)
 304             if license_path is not None:
 305                 break
 306
 307         if require_license_file and not license_path:
 308             raise LicenseError("License file not found. "
 309                                "Either add a file named LICENSE, "
 310                                "import upstream's COPYING if available, "
 311                                "or add a 'License File:' line to "
 312                                "README.chromium with the appropriate path.")
 313         metadata["License File"] = license_path
 314
 315     if "Required Text" in metadata:
 316         required_path = AbsolutePath(path, metadata["Required Text"], root)
 317         if required_path is not None:
 318             metadata["Required Text"] = required_path
 319         else:
 320             raise LicenseError("Required text file listed but not found.")
 321
 322     return metadata
 323
 324
 325 def ContainsFiles(path, root):
 326     """Determines whether any files exist in a directory or in any of its
 327     subdirectories."""
 328     for _, dirs, files in os.walk(os.path.join(root, path)):
 329         if files:
 330             return True
 331         for vcs_metadata in VCS_METADATA_DIRS:
 332             if vcs_metadata in dirs:
 333                 dirs.remove(vcs_metadata)
 334     return False
 335
 336
 337 def FilterDirsWithFiles(dirs_list, root):
 338     # If a directory contains no files, assume it's a DEPS directory for a
 339     # project not used by our current configuration and skip it.
 340     return [x for x in dirs_list if ContainsFiles(x, root)]
 341
 342
 343 def FindThirdPartyDirs(prune_paths, root):
 344     """Find all third_party directories underneath the source root."""
 345     third_party_dirs = []
 346     for path, dirs, files in os.walk(root):
 347         path = path[len(root)+1:]  # Pretty up the path.
 348
 349         if path in prune_paths:
 350             dirs[:] = []
 351             continue
 352
 353         # Prune out directories we want to skip.
 354         # (Note that we loop over PRUNE_DIRS so we're not iterating over a
 355         # list that we're simultaneously mutating.)
 356         for skip in PRUNE_DIRS:
 357             if skip in dirs:
 358                 dirs.remove(skip)
 359
 360         if os.path.basename(path) == 'third_party':
 361             # Add all subdirectories that are not marked for skipping.
 362             for dir in dirs:
 363                 dirpath = os.path.join(path, dir)
 364                 if dirpath not in prune_paths:
 365                     third_party_dirs.append(dirpath)
 366
 367             # Don't recurse into any subdirs from here.
 368             dirs[:] = []
 369             continue
 370
 371         # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular
 372         # third_party/foo paths.
 373         if path in ADDITIONAL_PATHS:
 374             dirs[:] = []
 375
 376     for dir in ADDITIONAL_PATHS:
 377         if dir not in prune_paths:
 378             third_party_dirs.append(dir)
 379
 380     return third_party_dirs
 381
 382
 383 def ScanThirdPartyDirs(root=None):
 384     """Scan a list of directories and report on any problems we find."""
 385     if root is None:
 386       root = os.getcwd()
 387     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
 388     third_party_dirs = FilterDirsWithFiles(third_party_dirs, root)
 389
 390     errors = []
 391     for path in sorted(third_party_dirs):
 392         try:
 393             metadata = ParseDir(path, root)
 394         except LicenseError, e:
 395             errors.append((path, e.args[0]))
 396             continue
 397
 398     for path, error in sorted(errors):
 399         print path + ": " + error
 400
 401     return len(errors) == 0
 402
 403
 404 def GenerateCredits():
 405     """Generate about:credits."""
 406
 407     if len(sys.argv) not in (2, 3):
 408       print 'usage: licenses.py credits [output_file]'
 409       return False
 410
 411     def EvaluateTemplate(template, env, escape=True):
 412         """Expand a template with variables like {{foo}} using a
 413         dictionary of expansions."""
 414         for key, val in env.items():
 415             if escape and not key.endswith("_unescaped"):
 416                 val = cgi.escape(val)
 417             template = template.replace('{{%s}}' % key, val)
 418         return template
 419
 420     root = os.path.join(os.path.dirname(__file__), '..')
 421     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
 422
 423     entry_template = open(os.path.join(root, 'chrome', 'browser', 'resources',
 424                                        'about_credits_entry.tmpl'), 'rb').read()
 425     entries = []
 426     for path in sorted(third_party_dirs):
 427         try:
 428             metadata = ParseDir(path, root)
 429         except LicenseError:
 430             # TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240).
 431             continue
 432         if metadata['License File'] == NOT_SHIPPED:
 433             continue
 434         env = {
 435             'name': metadata['Name'],
 436             'url': metadata['URL'],
 437             'license': open(metadata['License File'], 'rb').read(),
 438             'license_unescaped': '',
 439         }
 440         if 'Required Text' in metadata:
 441             required_text = open(metadata['Required Text'], 'rb').read()
 442             env["license_unescaped"] = required_text
 443         entries.append(EvaluateTemplate(entry_template, env))
 444
 445     file_template = open(os.path.join(root, 'chrome', 'browser', 'resources',
 446                                       'about_credits.tmpl'), 'rb').read()
 447     template_contents = "<!-- Generated by licenses.py; do not edit. -->"
 448     template_contents += EvaluateTemplate(file_template,
 449                                           {'entries': '\n'.join(entries)},
 450                                           escape=False)
 451
 452     if len(sys.argv) == 3:
 453       with open(sys.argv[2], 'w') as output_file:
 454         output_file.write(template_contents)
 455     elif len(sys.argv) == 2:
 456       print template_contents
 457
 458     return True
 459
 460
 461 def main():
 462     command = 'help'
 463     if len(sys.argv) > 1:
 464         command = sys.argv[1]
 465
 466     if command == 'scan':
 467         if not ScanThirdPartyDirs():
 468             return 1
 469     elif command == 'credits':
 470         if not GenerateCredits():
 471             return 1
 472     else:
 473         print __doc__
 474         return 1
 475
 476
 477 if __name__ == '__main__':
 478   sys.exit(main())