tools/licenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Utility for checking and processing licensing information in third_party
   7 directories.
   8
   9 Usage: licenses.py <command>
  10
  11 Commands:
  12   scan     scan third_party directories, verifying that we have licensing info
  13   credits  generate about:credits on stdout
  14
  15 (You can also import this as a module.)
  16 """
  17
  18 import argparse
  19 import cgi
  20 import os
  21 import sys
  22
  23 # Paths from the root of the tree to directories to skip.
  24 PRUNE_PATHS = set([
  25     # Same module occurs in crypto/third_party/nss and net/third_party/nss, so
  26     # skip this one.
  27     os.path.join('third_party','nss'),
  28
  29     # Placeholder directory only, not third-party code.
  30     os.path.join('third_party','adobe'),
  31
  32     # Apache 2.0 license. See crbug.com/140478
  33     os.path.join('third_party','bidichecker'),
  34
  35     # Build files only, not third-party code.
  36     os.path.join('third_party','widevine'),
  37
  38     # Only binaries, used during development.
  39     os.path.join('third_party','valgrind'),
  40
  41     # Used for development and test, not in the shipping product.
  42     os.path.join('build','secondary'),
  43     os.path.join('third_party','bison'),
  44     os.path.join('third_party','blanketjs'),
  45     os.path.join('third_party','cygwin'),
  46     os.path.join('third_party','gles2_conform'),
  47     os.path.join('third_party','gnu_binutils'),
  48     os.path.join('third_party','gold'),
  49     os.path.join('third_party','gperf'),
  50     os.path.join('third_party','lighttpd'),
  51     os.path.join('third_party','llvm'),
  52     os.path.join('third_party','llvm-build'),
  53     os.path.join('third_party','mingw-w64'),
  54     os.path.join('third_party','nacl_sdk_binaries'),
  55     os.path.join('third_party','pefile'),
  56     os.path.join('third_party','perl'),
  57     os.path.join('third_party','psyco_win32'),
  58     os.path.join('third_party','pylib'),
  59     os.path.join('third_party','pywebsocket'),
  60     os.path.join('third_party','qunit'),
  61     os.path.join('third_party','sinonjs'),
  62     os.path.join('third_party','syzygy'),
  63     os.path.join('tools', 'profile_chrome', 'third_party'),
  64
  65     # Chromium code in third_party.
  66     os.path.join('third_party','fuzzymatch'),
  67     os.path.join('tools', 'swarming_client'),
  68
  69     # Stuff pulled in from chrome-internal for official builds/tools.
  70     os.path.join('third_party', 'clear_cache'),
  71     os.path.join('third_party', 'gnu'),
  72     os.path.join('third_party', 'googlemac'),
  73     os.path.join('third_party', 'pcre'),
  74     os.path.join('third_party', 'psutils'),
  75     os.path.join('third_party', 'sawbuck'),
  76     # See crbug.com/350472
  77     os.path.join('chrome', 'browser', 'resources', 'chromeos', 'quickoffice'),
  78     # Chrome for Android proprietary code.
  79     os.path.join('clank'),
  80
  81     # Redistribution does not require attribution in documentation.
  82     os.path.join('third_party','directxsdk'),
  83     os.path.join('third_party','platformsdk_win2008_6_1'),
  84     os.path.join('third_party','platformsdk_win7'),
  85
  86     # For testing only, presents on some bots.
  87     os.path.join('isolate_deps_dir'),
  88 ])
  89
  90 # Directories we don't scan through.
  91 VCS_METADATA_DIRS = ('.svn', '.git')
  92 PRUNE_DIRS = (VCS_METADATA_DIRS +
  93               ('out', 'Debug', 'Release',  # build files
  94                'layout_tests'))            # lots of subdirs
  95
  96 ADDITIONAL_PATHS = (
  97     os.path.join('breakpad'),
  98     os.path.join('chrome', 'common', 'extensions', 'docs', 'examples'),
  99     os.path.join('chrome', 'test', 'chromeos', 'autotest'),
 100     os.path.join('chrome', 'test', 'data'),
 101     os.path.join('native_client'),
 102     os.path.join('net', 'tools', 'spdyshark'),
 103     os.path.join('sdch', 'open-vcdiff'),
 104     os.path.join('testing', 'gmock'),
 105     os.path.join('testing', 'gtest'),
 106     os.path.join('tools', 'grit'),
 107     os.path.join('tools', 'gyp'),
 108     os.path.join('tools', 'page_cycler', 'acid3'),
 109     os.path.join('url', 'third_party', 'mozilla'),
 110     os.path.join('v8'),
 111     # Fake directories to include the strongtalk and fdlibm licenses.
 112     os.path.join('v8', 'strongtalk'),
 113     os.path.join('v8', 'fdlibm'),
 114 )
 115
 116
 117 # Directories where we check out directly from upstream, and therefore
 118 # can't provide a README.chromium.  Please prefer a README.chromium
 119 # wherever possible.
 120 SPECIAL_CASES = {
 121     os.path.join('native_client'): {
 122         "Name": "native client",
 123         "URL": "http://code.google.com/p/nativeclient",
 124         "License": "BSD",
 125     },
 126     os.path.join('sdch', 'open-vcdiff'): {
 127         "Name": "open-vcdiff",
 128         "URL": "https://github.com.com/google/open-vcdiff",
 129         "License": "Apache 2.0, MIT, GPL v2 and custom licenses",
 130         "License Android Compatible": "yes",
 131     },
 132     os.path.join('testing', 'gmock'): {
 133         "Name": "gmock",
 134         "URL": "http://code.google.com/p/googlemock",
 135         "License": "BSD",
 136         "License File": "NOT_SHIPPED",
 137     },
 138     os.path.join('testing', 'gtest'): {
 139         "Name": "gtest",
 140         "URL": "http://code.google.com/p/googletest",
 141         "License": "BSD",
 142         "License File": "NOT_SHIPPED",
 143     },
 144     os.path.join('third_party', 'angle'): {
 145         "Name": "Almost Native Graphics Layer Engine",
 146         "URL": "http://code.google.com/p/angleproject/",
 147         "License": "BSD",
 148     },
 149     os.path.join('third_party', 'cros_system_api'): {
 150         "Name": "Chromium OS system API",
 151         "URL": "http://www.chromium.org/chromium-os",
 152         "License": "BSD",
 153         # Absolute path here is resolved as relative to the source root.
 154         "License File": "/LICENSE.chromium_os",
 155     },
 156     os.path.join('third_party', 'lss'): {
 157         "Name": "linux-syscall-support",
 158         "URL": "http://code.google.com/p/linux-syscall-support/",
 159         "License": "BSD",
 160         "License File": "/LICENSE",
 161     },
 162     os.path.join('third_party', 'ots'): {
 163         "Name": "OTS (OpenType Sanitizer)",
 164         "URL": "http://code.google.com/p/ots/",
 165         "License": "BSD",
 166     },
 167     os.path.join('third_party', 'pdfium'): {
 168         "Name": "PDFium",
 169         "URL": "http://code.google.com/p/pdfium/",
 170         "License": "BSD",
 171     },
 172     os.path.join('third_party', 'pdfsqueeze'): {
 173         "Name": "pdfsqueeze",
 174         "URL": "http://code.google.com/p/pdfsqueeze/",
 175         "License": "Apache 2.0",
 176         "License File": "COPYING",
 177     },
 178     os.path.join('third_party', 'ppapi'): {
 179         "Name": "ppapi",
 180         "URL": "http://code.google.com/p/ppapi/",
 181     },
 182     os.path.join('third_party', 'scons-2.0.1'): {
 183         "Name": "scons-2.0.1",
 184         "URL": "http://www.scons.org",
 185         "License": "MIT",
 186         "License File": "NOT_SHIPPED",
 187     },
 188     os.path.join('third_party', 'catapult'): {
 189         "Name": "catapult",
 190         "URL": "https://github.com/catapult-project/catapult",
 191         "License": "BSD",
 192         "License File": "NOT_SHIPPED",
 193     },
 194     os.path.join('third_party', 'v8-i18n'): {
 195         "Name": "Internationalization Library for v8",
 196         "URL": "http://code.google.com/p/v8-i18n/",
 197         "License": "Apache 2.0",
 198     },
 199     os.path.join('third_party', 'WebKit'): {
 200         "Name": "WebKit",
 201         "URL": "http://webkit.org/",
 202         "License": "BSD and GPL v2",
 203         # Absolute path here is resolved as relative to the source root.
 204         "License File": "/third_party/WebKit/LICENSE_FOR_ABOUT_CREDITS",
 205     },
 206     os.path.join('third_party', 'webpagereplay'): {
 207         "Name": "webpagereplay",
 208         "URL": "http://code.google.com/p/web-page-replay",
 209         "License": "Apache 2.0",
 210         "License File": "NOT_SHIPPED",
 211     },
 212     os.path.join('tools', 'grit'): {
 213         "Name": "grit",
 214         "URL": "http://code.google.com/p/grit-i18n",
 215         "License": "BSD",
 216         "License File": "NOT_SHIPPED",
 217     },
 218     os.path.join('tools', 'gyp'): {
 219         "Name": "gyp",
 220         "URL": "http://code.google.com/p/gyp",
 221         "License": "BSD",
 222         "License File": "NOT_SHIPPED",
 223     },
 224     os.path.join('v8'): {
 225         "Name": "V8 JavaScript Engine",
 226         "URL": "http://code.google.com/p/v8",
 227         "License": "BSD",
 228     },
 229     os.path.join('v8', 'strongtalk'): {
 230         "Name": "Strongtalk",
 231         "URL": "http://www.strongtalk.org/",
 232         "License": "BSD",
 233         # Absolute path here is resolved as relative to the source root.
 234         "License File": "/v8/LICENSE.strongtalk",
 235     },
 236     os.path.join('v8', 'fdlibm'): {
 237         "Name": "fdlibm",
 238         "URL": "http://www.netlib.org/fdlibm/",
 239         "License": "Freely Distributable",
 240         # Absolute path here is resolved as relative to the source root.
 241         "License File" : "/v8/src/third_party/fdlibm/LICENSE",
 242         "License Android Compatible" : "yes",
 243     },
 244     os.path.join('third_party', 'khronos_glcts'): {
 245         # These sources are not shipped, are not public, and it isn't
 246         # clear why they're tripping the license check.
 247         "Name": "khronos_glcts",
 248         "URL": "http://no-public-url",
 249         "License": "Khronos",
 250         "License File": "NOT_SHIPPED",
 251     },
 252     os.path.join('tools', 'telemetry', 'third_party', 'gsutil'): {
 253         "Name": "gsutil",
 254         "URL": "https://cloud.google.com/storage/docs/gsutil",
 255         "License": "Apache 2.0",
 256         "License File": "NOT_SHIPPED",
 257     },
 258 }
 259
 260 # Special value for 'License File' field used to indicate that the license file
 261 # should not be used in about:credits.
 262 NOT_SHIPPED = "NOT_SHIPPED"
 263
 264
 265 class LicenseError(Exception):
 266     """We raise this exception when a directory's licensing info isn't
 267     fully filled out."""
 268     pass
 269
 270 def AbsolutePath(path, filename, root):
 271     """Convert a path in README.chromium to be absolute based on the source
 272     root."""
 273     if filename.startswith('/'):
 274         # Absolute-looking paths are relative to the source root
 275         # (which is the directory we're run from).
 276         absolute_path = os.path.join(root, filename[1:])
 277     else:
 278         absolute_path = os.path.join(root, path, filename)
 279     if os.path.exists(absolute_path):
 280         return absolute_path
 281     return None
 282
 283 def ParseDir(path, root, require_license_file=True, optional_keys=None):
 284     """Examine a third_party/foo component and extract its metadata."""
 285
 286     # Parse metadata fields out of README.chromium.
 287     # We examine "LICENSE" for the license file by default.
 288     metadata = {
 289         "License File": "LICENSE",  # Relative path to license text.
 290         "Name": None,               # Short name (for header on about:credits).
 291         "URL": None,                # Project home page.
 292         "License": None,            # Software license.
 293         }
 294
 295     if optional_keys is None:
 296         optional_keys = []
 297
 298     if path in SPECIAL_CASES:
 299         metadata.update(SPECIAL_CASES[path])
 300     else:
 301         # Try to find README.chromium.
 302         readme_path = os.path.join(root, path, 'README.chromium')
 303         if not os.path.exists(readme_path):
 304             raise LicenseError("missing README.chromium or licenses.py "
 305                                "SPECIAL_CASES entry")
 306
 307         for line in open(readme_path):
 308             line = line.strip()
 309             if not line:
 310                 break
 311             for key in metadata.keys() + optional_keys:
 312                 field = key + ": "
 313                 if line.startswith(field):
 314                     metadata[key] = line[len(field):]
 315
 316     # Check that all expected metadata is present.
 317     for key, value in metadata.iteritems():
 318         if not value:
 319             raise LicenseError("couldn't find '" + key + "' line "
 320                                "in README.chromium or licences.py "
 321                                "SPECIAL_CASES")
 322
 323     # Special-case modules that aren't in the shipping product, so don't need
 324     # their license in about:credits.
 325     if metadata["License File"] != NOT_SHIPPED:
 326         # Check that the license file exists.
 327         for filename in (metadata["License File"], "COPYING"):
 328             license_path = AbsolutePath(path, filename, root)
 329             if license_path is not None:
 330                 break
 331
 332         if require_license_file and not license_path:
 333             raise LicenseError("License file not found. "
 334                                "Either add a file named LICENSE, "
 335                                "import upstream's COPYING if available, "
 336                                "or add a 'License File:' line to "
 337                                "README.chromium with the appropriate path.")
 338         metadata["License File"] = license_path
 339
 340     return metadata
 341
 342
 343 def ContainsFiles(path, root):
 344     """Determines whether any files exist in a directory or in any of its
 345     subdirectories."""
 346     for _, dirs, files in os.walk(os.path.join(root, path)):
 347         if files:
 348             return True
 349         for vcs_metadata in VCS_METADATA_DIRS:
 350             if vcs_metadata in dirs:
 351                 dirs.remove(vcs_metadata)
 352     return False
 353
 354
 355 def FilterDirsWithFiles(dirs_list, root):
 356     # If a directory contains no files, assume it's a DEPS directory for a
 357     # project not used by our current configuration and skip it.
 358     return [x for x in dirs_list if ContainsFiles(x, root)]
 359
 360
 361 def FindThirdPartyDirs(prune_paths, root):
 362     """Find all third_party directories underneath the source root."""
 363     third_party_dirs = set()
 364     for path, dirs, files in os.walk(root):
 365         path = path[len(root)+1:]  # Pretty up the path.
 366
 367         if path in prune_paths:
 368             dirs[:] = []
 369             continue
 370
 371         # Prune out directories we want to skip.
 372         # (Note that we loop over PRUNE_DIRS so we're not iterating over a
 373         # list that we're simultaneously mutating.)
 374         for skip in PRUNE_DIRS:
 375             if skip in dirs:
 376                 dirs.remove(skip)
 377
 378         if os.path.basename(path) == 'third_party':
 379             # Add all subdirectories that are not marked for skipping.
 380             for dir in dirs:
 381                 dirpath = os.path.join(path, dir)
 382                 if dirpath not in prune_paths:
 383                     third_party_dirs.add(dirpath)
 384
 385             # Don't recurse into any subdirs from here.
 386             dirs[:] = []
 387             continue
 388
 389         # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular
 390         # third_party/foo paths.
 391         if path in ADDITIONAL_PATHS:
 392             dirs[:] = []
 393
 394     for dir in ADDITIONAL_PATHS:
 395         if dir not in prune_paths:
 396             third_party_dirs.add(dir)
 397
 398     return third_party_dirs
 399
 400
 401 def FindThirdPartyDirsWithFiles(root):
 402     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
 403     return FilterDirsWithFiles(third_party_dirs, root)
 404
 405
 406 def ScanThirdPartyDirs(root=None):
 407     """Scan a list of directories and report on any problems we find."""
 408     if root is None:
 409       root = os.getcwd()
 410     third_party_dirs = FindThirdPartyDirsWithFiles(root)
 411
 412     errors = []
 413     for path in sorted(third_party_dirs):
 414         try:
 415             metadata = ParseDir(path, root)
 416         except LicenseError, e:
 417             errors.append((path, e.args[0]))
 418             continue
 419
 420     for path, error in sorted(errors):
 421         print path + ": " + error
 422
 423     return len(errors) == 0
 424
 425
 426 def GenerateCredits(file_template_file, entry_template_file, output_file):
 427     """Generate about:credits."""
 428
 429     def EvaluateTemplate(template, env, escape=True):
 430         """Expand a template with variables like {{foo}} using a
 431         dictionary of expansions."""
 432         for key, val in env.items():
 433             if escape:
 434                 val = cgi.escape(val)
 435             template = template.replace('{{%s}}' % key, val)
 436         return template
 437
 438     root = os.path.join(os.path.dirname(__file__), '..')
 439     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root)
 440
 441     if not file_template_file:
 442         file_template_file = os.path.join(root, 'chrome', 'browser',
 443                                           'resources', 'about_credits.tmpl')
 444     if not entry_template_file:
 445         entry_template_file = os.path.join(root, 'chrome', 'browser',
 446                                            'resources',
 447                                            'about_credits_entry.tmpl')
 448
 449     entry_template = open(entry_template_file).read()
 450     entries = []
 451     for path in third_party_dirs:
 452         try:
 453             metadata = ParseDir(path, root)
 454         except LicenseError:
 455             # TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240).
 456             continue
 457         if metadata['License File'] == NOT_SHIPPED:
 458             continue
 459         env = {
 460             'name': metadata['Name'],
 461             'url': metadata['URL'],
 462             'license': open(metadata['License File'], 'rb').read(),
 463         }
 464         entry = {
 465             'name': metadata['Name'],
 466             'content': EvaluateTemplate(entry_template, env),
 467         }
 468         entries.append(entry)
 469
 470     entries.sort(key=lambda entry: (entry['name'], entry['content']))
 471     entries_contents = '\n'.join([entry['content'] for entry in entries])
 472     file_template = open(file_template_file).read()
 473     template_contents = "<!-- Generated by licenses.py; do not edit. -->"
 474     template_contents += EvaluateTemplate(file_template,
 475                                           {'entries': entries_contents},
 476                                           escape=False)
 477
 478     if output_file:
 479       with open(output_file, 'w') as output:
 480         output.write(template_contents)
 481     else:
 482       print template_contents
 483
 484     return True
 485
 486
 487 def main():
 488     parser = argparse.ArgumentParser()
 489     parser.add_argument('--file-template',
 490                         help='Template HTML to use for the license page.')
 491     parser.add_argument('--entry-template',
 492                         help='Template HTML to use for each license.')
 493     parser.add_argument('command', choices=['help', 'scan', 'credits'])
 494     parser.add_argument('output_file', nargs='?')
 495     args = parser.parse_args()
 496
 497     if args.command == 'scan':
 498         if not ScanThirdPartyDirs():
 499             return 1
 500     elif args.command == 'credits':
 501         if not GenerateCredits(args.file_template, args.entry_template,
 502                                args.output_file):
 503             return 1
 504     else:
 505         print __doc__
 506         return 1
 507
 508
 509 if __name__ == '__main__':
 510   sys.exit(main())