tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'buildtools/third_party/libc++/trunk/test': [
 116         # http://llvm.org/bugs/show_bug.cgi?id=18291
 117         'UNKNOWN',
 118     ],
 119
 120     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 121         'UNKNOWN',
 122     ],
 123     # This contains files copied from elsewhere from the tree. Since the copied
 124     # directories might have suppressions below (like simplejson), whitelist the
 125     # whole directory. This is also not shipped code.
 126     'chrome/common/extensions/docs/server2/third_party': [
 127         'UNKNOWN',
 128     ],
 129     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 130         'UNKNOWN',
 131     ],
 132     'courgette/third_party/qsufsort.h': [  # http://crbug.com/98095
 133         'UNKNOWN',
 134     ],
 135     'native_client': [  # http://crbug.com/98099
 136         'UNKNOWN',
 137     ],
 138     'native_client/toolchain': [
 139         'BSD GPL (v2 or later)',
 140         'BSD MIT/X11 (BSD like)',
 141         'BSD (2 clause) GPL (v2 or later)',
 142         'BSD (3 clause) GPL (v2 or later)',
 143         'BSD (4 clause) ISC',
 144         'BSL (v1.0) GPL',
 145         'BSL (v1.0) GPL (v3.1)',
 146         'GPL',
 147         'GPL (unversioned/unknown version)',
 148         'GPL (v2)',
 149         'GPL (v2 or later)',
 150         'GPL (v3.1)',
 151         'GPL (v3 or later)',
 152         'MPL (v1.1) LGPL (unversioned/unknown version)',
 153     ],
 154     'third_party/WebKit': [
 155         'UNKNOWN',
 156     ],
 157
 158     # http://code.google.com/p/angleproject/issues/detail?id=217
 159     'third_party/angle': [
 160         'UNKNOWN',
 161     ],
 162
 163     # http://crbug.com/222828
 164     # http://bugs.python.org/issue17514
 165     'third_party/chromite/third_party/argparse.py': [
 166         'UNKNOWN',
 167     ],
 168
 169     # http://crbug.com/326117
 170     # https://bitbucket.org/chrisatlee/poster/issue/21
 171     'third_party/chromite/third_party/poster': [
 172         'UNKNOWN',
 173     ],
 174
 175     # http://crbug.com/333508
 176     'third_party/clang_format/script': [
 177         'UNKNOWN',
 178     ],
 179
 180     # http://crbug.com/333508
 181     'buildtools/clang_format/script': [
 182         'UNKNOWN',
 183     ],
 184
 185     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 186     'third_party/cython': [
 187         'UNKNOWN',
 188     ],
 189
 190     'third_party/devscripts': [
 191         'GPL (v2 or later)',
 192     ],
 193     'third_party/catapult/tracing/third_party/devscripts': [
 194         'GPL (v2 or later)',
 195     ],
 196
 197     # https://github.com/shazow/apiclient/issues/8
 198     # MIT license.
 199     'third_party/catapult/third_party/apiclient': [
 200         'UNKNOWN',
 201     ],
 202     'third_party/catapult/dashboard/third_party/apiclient': [
 203         'UNKNOWN',
 204     ],
 205
 206     # https://bugs.launchpad.net/beautifulsoup/+bug/1481316
 207     # MIT license.
 208     'third_party/catapult/third_party/beautifulsoup': [
 209         'UNKNOWN'
 210     ],
 211     'third_party/catapult/dashboard/third_party/beautifulsoup': [
 212         'UNKNOWN'
 213     ],
 214
 215     # https://code.google.com/p/graphy/issues/detail?id=6
 216     # Apache (v2.0)
 217     'third_party/catapult/third_party/graphy': [
 218         'UNKNOWN',
 219     ],
 220     'third_party/catapult/dashboard/third_party/graphy': [
 221         'UNKNOWN',
 222     ],
 223
 224     # https://github.com/GoogleCloudPlatform/appengine-mapreduce/issues/71
 225     # Apache (v2.0)
 226     'third_party/catapult/third_party/mapreduce': [
 227         'UNKNOWN',
 228     ],
 229     'third_party/catapult/dashboard/third_party/mapreduce': [
 230         'UNKNOWN',
 231     ],
 232
 233     # https://code.google.com/p/webapp-improved/issues/detail?id=103
 234     # Apache (v2.0).
 235     'third_party/catapult/third_party/webapp2': [
 236         'UNKNOWN',
 237     ],
 238     'third_party/catapult/dashboard/third_party/webapp2': [
 239         'UNKNOWN',
 240     ],
 241
 242     # https://github.com/Pylons/webob/issues/211
 243     # MIT license.
 244     'third_party/catapult/third_party/WebOb': [
 245         'UNKNOWN',
 246     ],
 247     'third_party/catapult/dashboard/third_party/WebOb': [
 248         'UNKNOWN',
 249     ],
 250
 251     # https://github.com/Pylons/webtest/issues/141
 252     # MIT license.
 253     'third_party/catapult/third_party/webtest': [
 254         'UNKNOWN',
 255     ],
 256     'third_party/catapult/dashboard/third_party/webtest': [
 257         'UNKNOWN',
 258     ],
 259
 260     # https://bitbucket.org/ianb/paste/issues/12/add-license-headers-to-source-files
 261     # MIT license.
 262     'third_party/catapult/third_party/Paste': [
 263         'UNKNOWN',
 264     ],
 265     'third_party/catapult/dashboard/third_party/Paste': [
 266         'UNKNOWN',
 267     ],
 268
 269     # https://github.com/google/oauth2client/issues/231
 270     # Apache v2.0.
 271     'third_party/catapult/third_party/oauth2client': [
 272         'UNKNOWN',
 273     ],
 274     'third_party/catapult/dashboard/third_party/oauth2client': [
 275         'UNKNOWN',
 276     ],
 277
 278     # https://bitbucket.org/gutworth/six/issues/129/add-license-headers-to-source-files
 279     # MIT license.
 280     'third_party/catapult/third_party/six': [
 281         'UNKNOWN',
 282     ],
 283     'third_party/catapult/dashboard/third_party/six': [
 284         'UNKNOWN',
 285     ],
 286
 287     'third_party/expat/files/lib': [  # http://crbug.com/98121
 288         'UNKNOWN',
 289     ],
 290     'third_party/ffmpeg': [
 291         'GPL',
 292         'GPL (v2)',
 293         'GPL (v2 or later)',
 294         'GPL (v3 or later)',
 295         'UNKNOWN',  # http://crbug.com/98123
 296     ],
 297     'third_party/fontconfig': [
 298         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 299         'UNKNOWN',
 300     ],
 301     'third_party/freetype2': [ # http://crbug.com/177319
 302         'UNKNOWN',
 303     ],
 304     'third_party/hunspell': [  # http://crbug.com/98134
 305         'UNKNOWN',
 306     ],
 307     'third_party/iccjpeg': [  # http://crbug.com/98137
 308         'UNKNOWN',
 309     ],
 310     'third_party/icu': [  # http://crbug.com/98301
 311         'UNKNOWN',
 312     ],
 313     'third_party/jsoncpp/source': [
 314         # https://github.com/open-source-parsers/jsoncpp/issues/234
 315         'UNKNOWN',
 316     ],
 317     'third_party/junit/src': [
 318         # https://github.com/junit-team/junit/issues/1132
 319         'UNKNOWN',
 320     ],
 321     'third_party/lcov': [  # http://crbug.com/98304
 322         'UNKNOWN',
 323     ],
 324     'third_party/lcov/contrib/galaxy/genflat.pl': [
 325         'GPL (v2 or later)',
 326     ],
 327     'third_party/libevent': [  # http://crbug.com/98309
 328         'UNKNOWN',
 329     ],
 330     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 331         'UNKNOWN',
 332     ],
 333     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 334         'UNKNOWN',
 335     ],
 336
 337     # Many liblouis files are mirrored but not used in the NaCl module.
 338     # They are not excluded from the mirror because of lack of infrastructure
 339     # support.  Getting license headers added to the files where missing is
 340     # tracked in https://github.com/liblouis/liblouis/issues/22.
 341     'third_party/liblouis/src': [
 342         'GPL (v3 or later)',
 343         'UNKNOWN',
 344     ],
 345
 346     'third_party/libpng': [  # http://crbug.com/98318
 347         'UNKNOWN',
 348     ],
 349
 350     # The following files lack license headers, but are trivial.
 351     'third_party/libusb/src/libusb/os/poll_posix.h': [
 352         'UNKNOWN',
 353     ],
 354
 355     'third_party/libvpx/source': [  # http://crbug.com/98319
 356         'UNKNOWN',
 357     ],
 358     'third_party/libxml': [
 359         'UNKNOWN',
 360     ],
 361     'third_party/libxslt': [
 362         'UNKNOWN',
 363     ],
 364     'third_party/lzma_sdk': [
 365         'UNKNOWN',
 366     ],
 367     'third_party/mesa/src': [
 368         'GPL (v2)',
 369         'GPL (v3 or later)',
 370         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 371         'UNKNOWN',  # http://crbug.com/98450
 372     ],
 373     'third_party/modp_b64': [
 374         'UNKNOWN',
 375     ],
 376     'third_party/openmax_dl/dl' : [
 377         'Khronos Group',
 378     ],
 379     'third_party/openssl': [  # http://crbug.com/98451
 380         'UNKNOWN',
 381     ],
 382     'third_party/boringssl': [
 383         # There are some files in BoringSSL which came from OpenSSL and have no
 384         # license in them. We don't wish to add the license header ourselves
 385         # thus we don't expect to pass license checks.
 386         'UNKNOWN',
 387     ],
 388     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 389         'UNKNOWN',
 390     ],
 391     'third_party/molokocacao': [  # http://crbug.com/98453
 392         'UNKNOWN',
 393     ],
 394     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 395         'UNKNOWN',
 396     ],
 397     'third_party/protobuf': [  # http://crbug.com/98455
 398         'UNKNOWN',
 399     ],
 400
 401     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 402     # BSD 2-clause license.
 403     'third_party/pycoverage': [
 404         'UNKNOWN',
 405     ],
 406
 407     'third_party/pyelftools': [ # http://crbug.com/222831
 408         'UNKNOWN',
 409     ],
 410     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 411         'UNKNOWN',
 412     ],
 413     'third_party/simplejson': [
 414         'UNKNOWN',
 415     ],
 416     'third_party/skia': [  # http://crbug.com/98463
 417         'UNKNOWN',
 418     ],
 419     'third_party/snappy/src': [  # http://crbug.com/98464
 420         'UNKNOWN',
 421     ],
 422     'third_party/smhasher/src': [  # http://crbug.com/98465
 423         'UNKNOWN',
 424     ],
 425     'third_party/speech-dispatcher/libspeechd.h': [
 426         'GPL (v2 or later)',
 427     ],
 428     'third_party/sqlite': [
 429         'UNKNOWN',
 430     ],
 431
 432     # http://crbug.com/334668
 433     # MIT license.
 434     'tools/swarming_client/third_party/httplib2': [
 435         'UNKNOWN',
 436     ],
 437
 438     # http://crbug.com/334668
 439     # Apache v2.0.
 440     'tools/swarming_client/third_party/oauth2client': [
 441         'UNKNOWN',
 442     ],
 443
 444     # http://crbug.com/471372
 445     # BSD
 446     'tools/swarming_client/third_party/pyasn1': [
 447         'UNKNOWN',
 448     ],
 449
 450     # http://crbug.com/471372
 451     # Apache v2.0.
 452     'tools/swarming_client/third_party/rsa': [
 453         'UNKNOWN',
 454     ],
 455
 456     # https://github.com/kennethreitz/requests/issues/1610
 457     'tools/swarming_client/third_party/requests': [
 458         'UNKNOWN',
 459     ],
 460
 461     'third_party/talloc': [
 462         'GPL (v3 or later)',
 463         'UNKNOWN',  # http://crbug.com/98588
 464     ],
 465     'third_party/tcmalloc': [
 466         'UNKNOWN',  # http://crbug.com/98589
 467     ],
 468     'third_party/tlslite': [
 469         'UNKNOWN',
 470     ],
 471     'third_party/webdriver': [  # http://crbug.com/98590
 472         'UNKNOWN',
 473     ],
 474
 475     # https://github.com/html5lib/html5lib-python/issues/125
 476     # https://github.com/KhronosGroup/WebGL/issues/435
 477     'third_party/webgl/src': [
 478         'UNKNOWN',
 479     ],
 480
 481     'third_party/webrtc': [  # http://crbug.com/98592
 482         'UNKNOWN',
 483     ],
 484     'third_party/xdg-utils': [  # http://crbug.com/98593
 485         'UNKNOWN',
 486     ],
 487     'third_party/yasm/source': [  # http://crbug.com/98594
 488         'UNKNOWN',
 489     ],
 490     'third_party/zlib/contrib/minizip': [
 491         'UNKNOWN',
 492     ],
 493     'third_party/zlib/trees.h': [
 494         'UNKNOWN',
 495     ],
 496     'tools/emacs': [  # http://crbug.com/98595
 497         'UNKNOWN',
 498     ],
 499     'tools/gyp/test': [
 500         'UNKNOWN',
 501     ],
 502     'tools/python/google/__init__.py': [
 503         'UNKNOWN',
 504     ],
 505     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 506         'UNKNOWN',
 507     ],
 508     'tools/symsrc/pefile.py': [
 509         'UNKNOWN',
 510     ],
 511     # Not shipped, downloaded on trybots sometimes.
 512     'tools/telemetry/third_party/gsutil': [
 513         'BSD MIT/X11 (BSD like)',
 514         'UNKNOWN',
 515     ],
 516     'tools/telemetry/third_party/pyserial': [
 517         # https://sourceforge.net/p/pyserial/feature-requests/35/
 518         'UNKNOWN',
 519     ],
 520     'v8/test/cctest': [  # http://crbug.com/98597
 521         'UNKNOWN',
 522     ],
 523     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 524         'UNKNOWN',
 525     ],
 526 }
 527
 528
 529 def check_licenses(options, args):
 530   # Figure out which directory we have to check.
 531   if len(args) == 0:
 532     # No directory to check specified, use the repository root.
 533     start_dir = options.base_directory
 534   elif len(args) == 1:
 535     # Directory specified. Start here. It's supposed to be relative to the
 536     # base directory.
 537     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 538   else:
 539     # More than one argument, we don't handle this.
 540     PrintUsage()
 541     return 1
 542
 543   print "Using base directory:", options.base_directory
 544   print "Checking:", start_dir
 545   print
 546
 547   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 548                                                    'third_party',
 549                                                    'devscripts',
 550                                                    'licensecheck.pl'))
 551
 552   licensecheck = subprocess.Popen([licensecheck_path,
 553                                    '-l', '100',
 554                                    '-r', start_dir],
 555                                   stdout=subprocess.PIPE,
 556                                   stderr=subprocess.PIPE)
 557   stdout, stderr = licensecheck.communicate()
 558   if options.verbose:
 559     print '----------- licensecheck stdout -----------'
 560     print stdout
 561     print '--------- end licensecheck stdout ---------'
 562   if licensecheck.returncode != 0 or stderr:
 563     print '----------- licensecheck stderr -----------'
 564     print stderr
 565     print '--------- end licensecheck stderr ---------'
 566     print "\nFAILED\n"
 567     return 1
 568
 569   used_suppressions = set()
 570   errors = []
 571
 572   for line in stdout.splitlines():
 573     filename, license = line.split(':', 1)
 574     filename = os.path.relpath(filename.strip(), options.base_directory)
 575
 576     # All files in the build output directory are generated one way or another.
 577     # There's no need to check them.
 578     if filename.startswith('out/'):
 579       continue
 580
 581     # For now we're just interested in the license.
 582     license = license.replace('*No copyright*', '').strip()
 583
 584     # Skip generated files.
 585     if 'GENERATED FILE' in license:
 586       continue
 587
 588     if license in WHITELISTED_LICENSES:
 589       continue
 590
 591     if not options.ignore_suppressions:
 592       matched_prefixes = [
 593           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 594           if filename.startswith(prefix) and
 595           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 596       if matched_prefixes:
 597         used_suppressions.update(set(matched_prefixes))
 598         continue
 599
 600     errors.append({'filename': filename, 'license': license})
 601
 602   if options.json:
 603     with open(options.json, 'w') as f:
 604       json.dump(errors, f)
 605
 606   if errors:
 607     for error in errors:
 608       print "'%s' has non-whitelisted license '%s'" % (
 609           error['filename'], error['license'])
 610     print "\nFAILED\n"
 611     print "Please read",
 612     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 613     print "for more info how to handle the failure."
 614     print
 615     print "Please respect OWNERS of checklicenses.py. Changes violating"
 616     print "this requirement may be reverted."
 617
 618     # Do not print unused suppressions so that above message is clearly
 619     # visible and gets proper attention. Too much unrelated output
 620     # would be distracting and make the important points easier to miss.
 621
 622     return 1
 623
 624   print "\nSUCCESS\n"
 625
 626   if not len(args):
 627     unused_suppressions = set(
 628         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 629             used_suppressions)
 630     if unused_suppressions:
 631       print "\nNOTE: unused suppressions detected:\n"
 632       print '\n'.join(unused_suppressions)
 633
 634   return 0
 635
 636
 637 def main():
 638   default_root = os.path.abspath(
 639       os.path.join(os.path.dirname(__file__), '..', '..'))
 640   option_parser = optparse.OptionParser()
 641   option_parser.add_option('--root', default=default_root,
 642                            dest='base_directory',
 643                            help='Specifies the repository root. This defaults '
 644                            'to "../.." relative to the script file, which '
 645                            'will normally be the repository root.')
 646   option_parser.add_option('-v', '--verbose', action='store_true',
 647                            default=False, help='Print debug logging')
 648   option_parser.add_option('--ignore-suppressions',
 649                            action='store_true',
 650                            default=False,
 651                            help='Ignore path-specific license whitelist.')
 652   option_parser.add_option('--json', help='Path to JSON output file')
 653   options, args = option_parser.parse_args()
 654   return check_licenses(options, args)
 655
 656
 657 if '__main__' == __name__:
 658   sys.exit(main())