tools/licenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Utility for checking and processing licensing information in third_party
   7 directories.
   8
   9 Usage: licenses.py <command>
  10
  11 Commands:
  12   scan     scan third_party directories, verifying that we have licensing info
  13   credits  generate about:credits on stdout
  14
  15 (You can also import this as a module.)
  16 """
  17
  18 import cgi
  19 import os
  20 import sys
  21
  22 # Paths from the root of the tree to directories to skip.
  23 PRUNE_PATHS = set([
  24     # Same module occurs in crypto/third_party/nss and net/third_party/nss, so
  25     # skip this one.
  26     os.path.join('third_party','nss'),
  27
  28     # Placeholder directory only, not third-party code.
  29     os.path.join('third_party','adobe'),
  30
  31     # Only binaries, used during development.
  32     os.path.join('third_party','valgrind'),
  33
  34     # Used for development and test, not in the shipping product.
  35     os.path.join('third_party','bidichecker'),
  36     os.path.join('third_party','cygwin'),
  37     os.path.join('third_party','gold'),
  38     os.path.join('third_party','lighttpd'),
  39     os.path.join('third_party','mingw-w64'),
  40     os.path.join('third_party','pefile'),
  41     os.path.join('third_party','python_26'),
  42     os.path.join('third_party','pywebsocket'),
  43
  44     # Stuff pulled in from chrome-internal for official builds/tools.
  45     os.path.join('third_party', 'clear_cache'),
  46     os.path.join('third_party', 'gnu'),
  47     os.path.join('third_party', 'googlemac'),
  48     os.path.join('third_party', 'pcre'),
  49     os.path.join('third_party', 'psutils'),
  50     os.path.join('third_party', 'sawbuck'),
  51
  52     # Redistribution does not require attribution in documentation.
  53     os.path.join('third_party','directxsdk'),
  54     os.path.join('third_party','platformsdk_win2008_6_1'),
  55     os.path.join('third_party','platformsdk_win7'),
  56 ])
  57
  58 # Directories we don't scan through.
  59 PRUNE_DIRS = ('.svn', '.git',             # VCS metadata
  60               'out', 'Debug', 'Release',  # build files
  61               'layout_tests')             # lots of subdirs
  62
  63 ADDITIONAL_PATHS = (
  64     os.path.join('breakpad'),
  65     os.path.join('chrome', 'common', 'extensions', 'docs', 'examples'),
  66     os.path.join('chrome', 'test', 'chromeos', 'autotest'),
  67     os.path.join('chrome', 'test', 'data'),
  68     os.path.join('googleurl'),
  69     os.path.join('native_client'),
  70     os.path.join('native_client_sdk'),
  71     os.path.join('net', 'tools', 'spdyshark'),
  72     os.path.join('ppapi'),
  73     os.path.join('sandbox', 'linux', 'seccomp-legacy'),
  74     os.path.join('sdch', 'open-vcdiff'),
  75     os.path.join('testing', 'gmock'),
  76     os.path.join('testing', 'gtest'),
  77     # The directory with the word list for Chinese and Japanese segmentation
  78     # with different license terms than ICU.
  79     os.path.join('third_party','icu','source','data','brkitr'),
  80     os.path.join('tools', 'grit'),
  81     os.path.join('tools', 'gyp'),
  82     os.path.join('tools', 'page_cycler', 'acid3'),
  83     os.path.join('v8'),
  84     # Fake directory so we can include the strongtalk license.
  85     os.path.join('v8', 'strongtalk'),
  86 )
  87
  88
  89 # Directories where we check out directly from upstream, and therefore
  90 # can't provide a README.chromium.  Please prefer a README.chromium
  91 # wherever possible.
  92 SPECIAL_CASES = {
  93     os.path.join('googleurl'): {
  94         "Name": "google-url",
  95         "URL": "http://code.google.com/p/google-url/",
  96         "License": "BSD and MPL 1.1/GPL 2.0/LGPL 2.1",
  97         "License File": "LICENSE.txt",
  98     },
  99     os.path.join('native_client'): {
 100         "Name": "native client",
 101         "URL": "http://code.google.com/p/nativeclient",
 102         "License": "BSD",
 103     },
 104     os.path.join('sandbox', 'linux', 'seccomp-legacy'): {
 105         "Name": "seccompsandbox",
 106         "URL": "http://code.google.com/p/seccompsandbox",
 107         "License": "BSD",
 108     },
 109     os.path.join('sdch', 'open-vcdiff'): {
 110         "Name": "open-vcdiff",
 111         "URL": "http://code.google.com/p/open-vcdiff",
 112         "License": "Apache 2.0, MIT, GPL v2 and custom licenses",
 113         "License Android Compatible": "yes",
 114     },
 115     os.path.join('testing', 'gmock'): {
 116         "Name": "gmock",
 117         "URL": "http://code.google.com/p/googlemock",
 118         "License": "BSD",
 119     },
 120     os.path.join('testing', 'gtest'): {
 121         "Name": "gtest",
 122         "URL": "http://code.google.com/p/googletest",
 123         "License": "BSD",
 124     },
 125     os.path.join('third_party', 'angle'): {
 126         "Name": "Almost Native Graphics Layer Engine",
 127         "URL": "http://code.google.com/p/angleproject/",
 128         "License": "BSD",
 129     },
 130     os.path.join('third_party', 'cros_system_api'): {
 131         "Name": "Chromium OS system API",
 132         "URL": "http://www.chromium.org/chromium-os",
 133         "License": "BSD",
 134         # Absolute path here is resolved as relative to the source root.
 135         "License File": "/LICENSE.chromium_os",
 136     },
 137     os.path.join('third_party', 'GTM'): {
 138         "Name": "Google Toolbox for Mac",
 139         "URL": "http://code.google.com/p/google-toolbox-for-mac/",
 140         "License": "Apache 2.0",
 141         "License File": "COPYING",
 142     },
 143     os.path.join('third_party', 'lss'): {
 144         "Name": "linux-syscall-support",
 145         "URL": "http://code.google.com/p/lss/",
 146         "License": "BSD",
 147         "License File": "/LICENSE",
 148     },
 149     os.path.join('third_party', 'ots'): {
 150         "Name": "OTS (OpenType Sanitizer)",
 151         "URL": "http://code.google.com/p/ots/",
 152         "License": "BSD",
 153     },
 154     os.path.join('third_party', 'pdfsqueeze'): {
 155         "Name": "pdfsqueeze",
 156         "URL": "http://code.google.com/p/pdfsqueeze/",
 157         "License": "Apache 2.0",
 158         "License File": "COPYING",
 159     },
 160     os.path.join('third_party', 'ppapi'): {
 161         "Name": "ppapi",
 162         "URL": "http://code.google.com/p/ppapi/",
 163     },
 164     os.path.join('third_party', 'scons-2.0.1'): {
 165         "Name": "scons-2.0.1",
 166         "URL": "http://www.scons.org",
 167         "License": "MIT",
 168     },
 169     os.path.join('third_party', 'trace-viewer'): {
 170         "Name": "trace-viewer",
 171         "URL": "http://code.google.com/p/trace-viewer",
 172         "License": "BSD",
 173     },
 174     os.path.join('third_party', 'v8-i18n'): {
 175         "Name": "Internationalization Library for v8",
 176         "URL": "http://code.google.com/p/v8-i18n/",
 177         "License": "Apache 2.0, BSD and others",
 178     },
 179     os.path.join('third_party', 'WebKit'): {
 180         "Name": "WebKit",
 181         "URL": "http://webkit.org/",
 182         "License": "BSD and GPL v2",
 183         # Absolute path here is resolved as relative to the source root.
 184         "License File": "/webkit/LICENSE",
 185     },
 186     os.path.join('third_party', 'webpagereplay'): {
 187         "Name": "webpagereplay",
 188         "URL": "http://code.google.com/p/web-page-replay",
 189         "License": "Apache 2.0",
 190     },
 191     os.path.join('tools', 'grit'): {
 192         "Name": "grit",
 193         "URL": "http://code.google.com/p/grit-i18n",
 194         "License": "BSD",
 195     },
 196     os.path.join('tools', 'gyp'): {
 197         "Name": "gyp",
 198         "URL": "http://code.google.com/p/gyp",
 199         "License": "BSD",
 200     },
 201     os.path.join('v8'): {
 202         "Name": "V8 JavaScript Engine",
 203         "URL": "http://code.google.com/p/v8",
 204         "License": "BSD",
 205     },
 206     os.path.join('v8', 'strongtalk'): {
 207         "Name": "Strongtalk",
 208         "URL": "http://www.strongtalk.org/",
 209         "License": "BSD",
 210         # Absolute path here is resolved as relative to the source root.
 211         "License File": "/v8/LICENSE.strongtalk",
 212     },
 213 }
 214
 215 # Special value for 'License File' field used to indicate that the license file
 216 # should not be used in about:credits.
 217 NOT_SHIPPED = "NOT_SHIPPED"
 218
 219
 220 class LicenseError(Exception):
 221     """We raise this exception when a directory's licensing info isn't
 222     fully filled out."""
 223     pass
 224
 225 def AbsolutePath(path, filename):
 226     """Convert a path in README.chromium to be absolute based on the source
 227     root."""
 228     if filename.startswith('/'):
 229         # Absolute-looking paths are relative to the source root
 230         # (which is the directory we're run from).
 231         absolute_path = os.path.join(os.getcwd(), filename[1:])
 232     else:
 233         absolute_path = os.path.join(path, filename)
 234     if os.path.exists(absolute_path):
 235         return absolute_path
 236     return None
 237
 238 def ParseDir(path, require_license_file=True):
 239     """Examine a third_party/foo component and extract its metadata."""
 240
 241     # Parse metadata fields out of README.chromium.
 242     # We examine "LICENSE" for the license file by default.
 243     metadata = {
 244         "License File": "LICENSE",  # Relative path to license text.
 245         "Name": None,               # Short name (for header on about:credits).
 246         "URL": None,                # Project home page.
 247         "License": None,            # Software license.
 248         }
 249
 250     # Relative path to a file containing some html we're required to place in
 251     # about:credits.
 252     optional_keys = ["Required Text", "License Android Compatible"]
 253
 254     if path in SPECIAL_CASES:
 255         metadata.update(SPECIAL_CASES[path])
 256     else:
 257         # Try to find README.chromium.
 258         readme_path = os.path.join(path, 'README.chromium')
 259         if not os.path.exists(readme_path):
 260             raise LicenseError("missing README.chromium or licenses.py "
 261                                "SPECIAL_CASES entry")
 262
 263         for line in open(readme_path):
 264             line = line.strip()
 265             if not line:
 266                 break
 267             for key in metadata.keys() + optional_keys:
 268                 field = key + ": "
 269                 if line.startswith(field):
 270                     metadata[key] = line[len(field):]
 271
 272     # Check that all expected metadata is present.
 273     for key, value in metadata.iteritems():
 274         if not value:
 275             raise LicenseError("couldn't find '" + key + "' line "
 276                                "in README.chromium or licences.py "
 277                                "SPECIAL_CASES")
 278
 279     # Special-case modules that aren't in the shipping product, so don't need
 280     # their license in about:credits.
 281     if metadata["License File"] != NOT_SHIPPED:
 282         # Check that the license file exists.
 283         for filename in (metadata["License File"], "COPYING"):
 284             license_path = AbsolutePath(path, filename)
 285             if license_path is not None:
 286                 break
 287
 288         if require_license_file and not license_path:
 289             raise LicenseError("License file not found. "
 290                                "Either add a file named LICENSE, "
 291                                "import upstream's COPYING if available, "
 292                                "or add a 'License File:' line to "
 293                                "README.chromium with the appropriate path.")
 294         metadata["License File"] = license_path
 295
 296     if "Required Text" in metadata:
 297         required_path = AbsolutePath(path, metadata["Required Text"])
 298         if required_path is not None:
 299             metadata["Required Text"] = required_path
 300         else:
 301             raise LicenseError("Required text file listed but not found.")
 302
 303     return metadata
 304
 305
 306 def ContainsFiles(path):
 307     """Determines whether any files exist in a directory or in any of its
 308     subdirectories."""
 309     for _, _, files in os.walk(path):
 310         if files:
 311             return True
 312     return False
 313
 314
 315 def FindThirdPartyDirs(prune_paths):
 316     """Find all third_party directories underneath the current directory."""
 317     third_party_dirs = []
 318     for path, dirs, files in os.walk('.'):
 319         path = path[len('./'):]  # Pretty up the path.
 320
 321         if path in prune_paths:
 322             dirs[:] = []
 323             continue
 324
 325         # Prune out directories we want to skip.
 326         # (Note that we loop over PRUNE_DIRS so we're not iterating over a
 327         # list that we're simultaneously mutating.)
 328         for skip in PRUNE_DIRS:
 329             if skip in dirs:
 330                 dirs.remove(skip)
 331
 332         if os.path.basename(path) == 'third_party':
 333             # Add all subdirectories that are not marked for skipping.
 334             for dir in dirs:
 335                 dirpath = os.path.join(path, dir)
 336                 if dirpath not in prune_paths:
 337                     third_party_dirs.append(dirpath)
 338
 339             # Don't recurse into any subdirs from here.
 340             dirs[:] = []
 341             continue
 342
 343         # Don't recurse into paths in ADDITIONAL_PATHS, like we do with regular
 344         # third_party/foo paths.
 345         if path in ADDITIONAL_PATHS:
 346             dirs[:] = []
 347
 348     for dir in ADDITIONAL_PATHS:
 349         third_party_dirs.append(dir)
 350
 351     # If a directory contains no files, assume it's a DEPS directory for a
 352     # project not used by our current configuration and skip it.
 353     return [x for x in third_party_dirs if ContainsFiles(x)]
 354
 355
 356 def ScanThirdPartyDirs():
 357     """Scan a list of directories and report on any problems we find."""
 358     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS)
 359
 360     errors = []
 361     for path in sorted(third_party_dirs):
 362         try:
 363             metadata = ParseDir(path)
 364         except LicenseError, e:
 365             errors.append((path, e.args[0]))
 366             continue
 367
 368     for path, error in sorted(errors):
 369         print path + ": " + error
 370
 371     return len(errors) == 0
 372
 373
 374 def GenerateCredits():
 375     """Generate about:credits, dumping the result to stdout."""
 376
 377     def EvaluateTemplate(template, env, escape=True):
 378         """Expand a template with variables like {{foo}} using a
 379         dictionary of expansions."""
 380         for key, val in env.items():
 381             if escape and not key.endswith("_unescaped"):
 382                 val = cgi.escape(val)
 383             template = template.replace('{{%s}}' % key, val)
 384         return template
 385
 386     third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS)
 387
 388     entry_template = open('chrome/browser/resources/about_credits_entry.tmpl',
 389                           'rb').read()
 390     entries = []
 391     for path in sorted(third_party_dirs):
 392         try:
 393             metadata = ParseDir(path)
 394         except LicenseError:
 395             print >>sys.stderr, ("WARNING: licensing info for " + path +
 396                                  " is incomplete, skipping.")
 397             continue
 398         if metadata['License File'] == NOT_SHIPPED:
 399             print >>sys.stderr, ("Path " + path + " marked as " + NOT_SHIPPED +
 400                                  ", skipping.")
 401             continue
 402         env = {
 403             'name': metadata['Name'],
 404             'url': metadata['URL'],
 405             'license': open(metadata['License File'], 'rb').read(),
 406             'license_unescaped': '',
 407         }
 408         if 'Required Text' in metadata:
 409             required_text = open(metadata['Required Text'], 'rb').read()
 410             env["license_unescaped"] = required_text
 411         entries.append(EvaluateTemplate(entry_template, env))
 412
 413     file_template = open('chrome/browser/resources/about_credits.tmpl',
 414                          'rb').read()
 415     print "<!-- Generated by licenses.py; do not edit. -->"
 416     print EvaluateTemplate(file_template, {'entries': '\n'.join(entries)},
 417                            escape=False)
 418
 419
 420 def main():
 421     command = 'help'
 422     if len(sys.argv) > 1:
 423         command = sys.argv[1]
 424
 425     if command == 'scan':
 426         if not ScanThirdPartyDirs():
 427             return 1
 428     elif command == 'credits':
 429         if not GenerateCredits():
 430             return 1
 431     else:
 432         print __doc__
 433         return 1
 434
 435
 436 if __name__ == '__main__':
 437   sys.exit(main())