tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 116         'UNKNOWN',
 117     ],
 118     # This contains files copied from elsewhere from the tree. Since the copied
 119     # directories might have suppressions below (like simplejson), whitelist the
 120     # whole directory. This is also not shipped code.
 121     'chrome/common/extensions/docs/server2/third_party': [
 122         'UNKNOWN',
 123     ],
 124     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 125         'UNKNOWN',
 126     ],
 127     'native_client': [  # http://crbug.com/98099
 128         'UNKNOWN',
 129     ],
 130     'native_client/toolchain': [
 131         'BSD GPL (v2 or later)',
 132         'BSD (2 clause) GPL (v2 or later)',
 133         'BSD (3 clause) GPL (v2 or later)',
 134         'BSL (v1.0) GPL',
 135         'BSL (v1.0) GPL (v3.1)',
 136         'GPL',
 137         'GPL (unversioned/unknown version)',
 138         'GPL (v2)',
 139         'GPL (v2 or later)',
 140         'GPL (v3.1)',
 141         'GPL (v3 or later)',
 142     ],
 143     'third_party/WebKit': [
 144         'UNKNOWN',
 145     ],
 146
 147     # http://code.google.com/p/angleproject/issues/detail?id=217
 148     'third_party/angle': [
 149         'UNKNOWN',
 150     ],
 151
 152     # http://crbug.com/222828
 153     # http://bugs.python.org/issue17514
 154     'third_party/chromite/third_party/argparse.py': [
 155         'UNKNOWN',
 156     ],
 157
 158     # http://crbug.com/326117
 159     # https://bitbucket.org/chrisatlee/poster/issue/21
 160     'third_party/chromite/third_party/poster': [
 161         'UNKNOWN',
 162     ],
 163
 164     # http://crbug.com/333508
 165     'third_party/clang_format/script': [
 166         'UNKNOWN',
 167     ],
 168
 169     # http://crbug.com/333508
 170     'buildtools/clang_format/script': [
 171         'UNKNOWN',
 172     ],
 173
 174     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 175     'third_party/cython': [
 176         'UNKNOWN',
 177     ],
 178
 179     'third_party/devscripts': [
 180         'GPL (v2 or later)',
 181     ],
 182     'third_party/expat/files/lib': [  # http://crbug.com/98121
 183         'UNKNOWN',
 184     ],
 185     'third_party/ffmpeg': [
 186         'GPL',
 187         'GPL (v2)',
 188         'GPL (v2 or later)',
 189         'GPL (v3 or later)',
 190         'UNKNOWN',  # http://crbug.com/98123
 191     ],
 192     'third_party/fontconfig': [
 193         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 194         'UNKNOWN',
 195     ],
 196     'third_party/freetype2': [ # http://crbug.com/177319
 197         'UNKNOWN',
 198     ],
 199     'third_party/hunspell': [  # http://crbug.com/98134
 200         'UNKNOWN',
 201     ],
 202     'third_party/iccjpeg': [  # http://crbug.com/98137
 203         'UNKNOWN',
 204     ],
 205     'third_party/icu': [  # http://crbug.com/98301
 206         'UNKNOWN',
 207     ],
 208     'third_party/lcov': [  # http://crbug.com/98304
 209         'UNKNOWN',
 210     ],
 211     'third_party/lcov/contrib/galaxy/genflat.pl': [
 212         'GPL (v2 or later)',
 213     ],
 214     'third_party/libc++/trunk/test': [
 215         # http://llvm.org/bugs/show_bug.cgi?id=18291
 216         'UNKNOWN',
 217     ],
 218     'third_party/libevent': [  # http://crbug.com/98309
 219         'UNKNOWN',
 220     ],
 221     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 222         'UNKNOWN',
 223     ],
 224     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 225         'UNKNOWN',
 226     ],
 227
 228     # Many liblouis files are mirrored but not used in the NaCl module.
 229     # They are not excluded from the mirror because of lack of infrastructure
 230     # support.  Getting license headers added to the files where missing is
 231     # tracked in https://github.com/liblouis/liblouis/issues/22.
 232     'third_party/liblouis/src': [
 233         'GPL (v3 or later)',
 234         'UNKNOWN',
 235     ],
 236
 237     'third_party/libpng': [  # http://crbug.com/98318
 238         'UNKNOWN',
 239     ],
 240
 241     # The following files lack license headers, but are trivial.
 242     'third_party/libusb/src/libusb/os/poll_posix.h': [
 243         'UNKNOWN',
 244     ],
 245
 246     'third_party/libvpx/source': [  # http://crbug.com/98319
 247         'UNKNOWN',
 248     ],
 249     'third_party/libxml': [
 250         'UNKNOWN',
 251     ],
 252     'third_party/libxslt': [
 253         'UNKNOWN',
 254     ],
 255     'third_party/lzma_sdk': [
 256         'UNKNOWN',
 257     ],
 258     'third_party/mesa/src': [
 259         'GPL (v2)',
 260         'GPL (v3 or later)',
 261         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 262         'UNKNOWN',  # http://crbug.com/98450
 263     ],
 264     'third_party/modp_b64': [
 265         'UNKNOWN',
 266     ],
 267     'third_party/openmax_dl/dl' : [
 268         'Khronos Group',
 269     ],
 270     'third_party/openssl': [  # http://crbug.com/98451
 271         'UNKNOWN',
 272     ],
 273     'third_party/boringssl': [
 274         # There are some files in BoringSSL which came from OpenSSL and have no
 275         # license in them. We don't wish to add the license header ourselves
 276         # thus we don't expect to pass license checks.
 277         'UNKNOWN',
 278     ],
 279     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 280         'UNKNOWN',
 281     ],
 282     'third_party/molokocacao': [  # http://crbug.com/98453
 283         'UNKNOWN',
 284     ],
 285     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 286         'UNKNOWN',
 287     ],
 288     'third_party/protobuf': [  # http://crbug.com/98455
 289         'UNKNOWN',
 290     ],
 291
 292     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 293     # BSD 2-clause license.
 294     'third_party/pycoverage': [
 295         'UNKNOWN',
 296     ],
 297
 298     'third_party/pyelftools': [ # http://crbug.com/222831
 299         'UNKNOWN',
 300     ],
 301     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 302         'UNKNOWN',
 303     ],
 304     'third_party/simplejson': [
 305         'UNKNOWN',
 306     ],
 307     'third_party/skia': [  # http://crbug.com/98463
 308         'UNKNOWN',
 309     ],
 310     'third_party/snappy/src': [  # http://crbug.com/98464
 311         'UNKNOWN',
 312     ],
 313     'third_party/smhasher/src': [  # http://crbug.com/98465
 314         'UNKNOWN',
 315     ],
 316     'third_party/speech-dispatcher/libspeechd.h': [
 317         'GPL (v2 or later)',
 318     ],
 319     'third_party/sqlite': [
 320         'UNKNOWN',
 321     ],
 322
 323     # http://crbug.com/334668
 324     # MIT license.
 325     'tools/swarming_client/third_party/httplib2': [
 326         'UNKNOWN',
 327     ],
 328
 329     # http://crbug.com/334668
 330     # Apache v2.0.
 331     'tools/swarming_client/third_party/oauth2client': [
 332         'UNKNOWN',
 333     ],
 334
 335     # https://github.com/kennethreitz/requests/issues/1610
 336     'tools/swarming_client/third_party/requests': [
 337         'UNKNOWN',
 338     ],
 339
 340     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 341         'UNKNOWN',
 342     ],
 343     'third_party/talloc': [
 344         'GPL (v3 or later)',
 345         'UNKNOWN',  # http://crbug.com/98588
 346     ],
 347     'third_party/tcmalloc': [
 348         'UNKNOWN',  # http://crbug.com/98589
 349     ],
 350     'third_party/tlslite': [
 351         'UNKNOWN',
 352     ],
 353     'third_party/webdriver': [  # http://crbug.com/98590
 354         'UNKNOWN',
 355     ],
 356
 357     # https://github.com/html5lib/html5lib-python/issues/125
 358     # https://github.com/KhronosGroup/WebGL/issues/435
 359     'third_party/webgl/src': [
 360         'UNKNOWN',
 361     ],
 362
 363     'third_party/webrtc': [  # http://crbug.com/98592
 364         'UNKNOWN',
 365     ],
 366     'third_party/xdg-utils': [  # http://crbug.com/98593
 367         'UNKNOWN',
 368     ],
 369     'third_party/yasm/source': [  # http://crbug.com/98594
 370         'UNKNOWN',
 371     ],
 372     'third_party/zlib/contrib/minizip': [
 373         'UNKNOWN',
 374     ],
 375     'third_party/zlib/trees.h': [
 376         'UNKNOWN',
 377     ],
 378     'tools/emacs': [  # http://crbug.com/98595
 379         'UNKNOWN',
 380     ],
 381     'tools/gyp/test': [
 382         'UNKNOWN',
 383     ],
 384     'tools/python/google/__init__.py': [
 385         'UNKNOWN',
 386     ],
 387     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 388         'UNKNOWN',
 389     ],
 390     'tools/symsrc/pefile.py': [
 391         'UNKNOWN',
 392     ],
 393     # Not shipped, downloaded on trybots sometimes.
 394     'tools/telemetry/third_party/gsutil': [
 395         'BSD MIT/X11 (BSD like)',
 396         'UNKNOWN',
 397     ],
 398     'tools/telemetry/third_party/pyserial': [
 399         # https://sourceforge.net/p/pyserial/feature-requests/35/
 400         'UNKNOWN',
 401     ],
 402     'v8/test/cctest': [  # http://crbug.com/98597
 403         'UNKNOWN',
 404     ],
 405     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 406         'UNKNOWN',
 407     ],
 408 }
 409
 410
 411 def check_licenses(options, args):
 412   # Figure out which directory we have to check.
 413   if len(args) == 0:
 414     # No directory to check specified, use the repository root.
 415     start_dir = options.base_directory
 416   elif len(args) == 1:
 417     # Directory specified. Start here. It's supposed to be relative to the
 418     # base directory.
 419     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 420   else:
 421     # More than one argument, we don't handle this.
 422     PrintUsage()
 423     return 1
 424
 425   print "Using base directory:", options.base_directory
 426   print "Checking:", start_dir
 427   print
 428
 429   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 430                                                    'third_party',
 431                                                    'devscripts',
 432                                                    'licensecheck.pl'))
 433
 434   licensecheck = subprocess.Popen([licensecheck_path,
 435                                    '-l', '100',
 436                                    '-r', start_dir],
 437                                   stdout=subprocess.PIPE,
 438                                   stderr=subprocess.PIPE)
 439   stdout, stderr = licensecheck.communicate()
 440   if options.verbose:
 441     print '----------- licensecheck stdout -----------'
 442     print stdout
 443     print '--------- end licensecheck stdout ---------'
 444   if licensecheck.returncode != 0 or stderr:
 445     print '----------- licensecheck stderr -----------'
 446     print stderr
 447     print '--------- end licensecheck stderr ---------'
 448     print "\nFAILED\n"
 449     return 1
 450
 451   used_suppressions = set()
 452   errors = []
 453
 454   for line in stdout.splitlines():
 455     filename, license = line.split(':', 1)
 456     filename = os.path.relpath(filename.strip(), options.base_directory)
 457
 458     # All files in the build output directory are generated one way or another.
 459     # There's no need to check them.
 460     if filename.startswith('out/'):
 461       continue
 462
 463     # For now we're just interested in the license.
 464     license = license.replace('*No copyright*', '').strip()
 465
 466     # Skip generated files.
 467     if 'GENERATED FILE' in license:
 468       continue
 469
 470     if license in WHITELISTED_LICENSES:
 471       continue
 472
 473     if not options.ignore_suppressions:
 474       matched_prefixes = [
 475           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 476           if filename.startswith(prefix) and
 477           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 478       if matched_prefixes:
 479         used_suppressions.update(set(matched_prefixes))
 480         continue
 481
 482     errors.append({'filename': filename, 'license': license})
 483
 484   if options.json:
 485     with open(options.json, 'w') as f:
 486       json.dump(errors, f)
 487
 488   if errors:
 489     for error in errors:
 490       print "'%s' has non-whitelisted license '%s'" % (
 491           error['filename'], error['license'])
 492     print "\nFAILED\n"
 493     print "Please read",
 494     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 495     print "for more info how to handle the failure."
 496     print
 497     print "Please respect OWNERS of checklicenses.py. Changes violating"
 498     print "this requirement may be reverted."
 499
 500     # Do not print unused suppressions so that above message is clearly
 501     # visible and gets proper attention. Too much unrelated output
 502     # would be distracting and make the important points easier to miss.
 503
 504     return 1
 505
 506   print "\nSUCCESS\n"
 507
 508   if not len(args):
 509     unused_suppressions = set(
 510         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 511             used_suppressions)
 512     if unused_suppressions:
 513       print "\nNOTE: unused suppressions detected:\n"
 514       print '\n'.join(unused_suppressions)
 515
 516   return 0
 517
 518
 519 def main():
 520   default_root = os.path.abspath(
 521       os.path.join(os.path.dirname(__file__), '..', '..'))
 522   option_parser = optparse.OptionParser()
 523   option_parser.add_option('--root', default=default_root,
 524                            dest='base_directory',
 525                            help='Specifies the repository root. This defaults '
 526                            'to "../.." relative to the script file, which '
 527                            'will normally be the repository root.')
 528   option_parser.add_option('-v', '--verbose', action='store_true',
 529                            default=False, help='Print debug logging')
 530   option_parser.add_option('--ignore-suppressions',
 531                            action='store_true',
 532                            default=False,
 533                            help='Ignore path-specific license whitelist.')
 534   option_parser.add_option('--json', help='Path to JSON output file')
 535   options, args = option_parser.parse_args()
 536   return check_licenses(options, args)
 537
 538
 539 if '__main__' == __name__:
 540   sys.exit(main())