tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'buildtools/third_party/libc++/trunk/test': [
 116         # http://llvm.org/bugs/show_bug.cgi?id=18291
 117         'UNKNOWN',
 118     ],
 119
 120     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 121         'UNKNOWN',
 122     ],
 123     # This contains files copied from elsewhere from the tree. Since the copied
 124     # directories might have suppressions below (like simplejson), whitelist the
 125     # whole directory. This is also not shipped code.
 126     'chrome/common/extensions/docs/server2/third_party': [
 127         'UNKNOWN',
 128     ],
 129     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 130         'UNKNOWN',
 131     ],
 132     'native_client': [  # http://crbug.com/98099
 133         'UNKNOWN',
 134     ],
 135     'native_client/toolchain': [
 136         'BSD GPL (v2 or later)',
 137         'BSD MIT/X11 (BSD like)',
 138         'BSD (2 clause) GPL (v2 or later)',
 139         'BSD (3 clause) GPL (v2 or later)',
 140         'BSD (4 clause) ISC',
 141         'BSL (v1.0) GPL',
 142         'BSL (v1.0) GPL (v3.1)',
 143         'GPL',
 144         'GPL (unversioned/unknown version)',
 145         'GPL (v2)',
 146         'GPL (v2 or later)',
 147         'GPL (v3.1)',
 148         'GPL (v3 or later)',
 149         'MPL (v1.1) LGPL (unversioned/unknown version)',
 150     ],
 151     'third_party/WebKit': [
 152         'UNKNOWN',
 153     ],
 154
 155     # http://code.google.com/p/angleproject/issues/detail?id=217
 156     'third_party/angle': [
 157         'UNKNOWN',
 158     ],
 159
 160     # http://crbug.com/222828
 161     # http://bugs.python.org/issue17514
 162     'third_party/chromite/third_party/argparse.py': [
 163         'UNKNOWN',
 164     ],
 165
 166     # http://crbug.com/326117
 167     # https://bitbucket.org/chrisatlee/poster/issue/21
 168     'third_party/chromite/third_party/poster': [
 169         'UNKNOWN',
 170     ],
 171
 172     # http://crbug.com/333508
 173     'third_party/clang_format/script': [
 174         'UNKNOWN',
 175     ],
 176
 177     # http://crbug.com/333508
 178     'buildtools/clang_format/script': [
 179         'UNKNOWN',
 180     ],
 181
 182     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 183     'third_party/cython': [
 184         'UNKNOWN',
 185     ],
 186
 187     'third_party/devscripts': [
 188         'GPL (v2 or later)',
 189     ],
 190     'third_party/catapult/tracing/third_party/devscripts': [
 191         'GPL (v2 or later)',
 192     ],
 193
 194     # https://github.com/shazow/apiclient/issues/8
 195     # MIT license.
 196     'third_party/catapult/third_party/apiclient': [
 197         'UNKNOWN',
 198     ],
 199     'third_party/catapult/dashboard/third_party/apiclient': [
 200         'UNKNOWN',
 201     ],
 202
 203     # https://bugs.launchpad.net/beautifulsoup/+bug/1481316
 204     # MIT license.
 205     'third_party/catapult/third_party/beautifulsoup': [
 206         'UNKNOWN'
 207     ],
 208     'third_party/catapult/dashboard/third_party/beautifulsoup': [
 209         'UNKNOWN'
 210     ],
 211
 212     # https://code.google.com/p/graphy/issues/detail?id=6
 213     # Apache (v2.0)
 214     'third_party/catapult/third_party/graphy': [
 215         'UNKNOWN',
 216     ],
 217     'third_party/catapult/dashboard/third_party/graphy': [
 218         'UNKNOWN',
 219     ],
 220
 221     # https://github.com/GoogleCloudPlatform/appengine-mapreduce/issues/71
 222     # Apache (v2.0)
 223     'third_party/catapult/third_party/mapreduce': [
 224         'UNKNOWN',
 225     ],
 226     'third_party/catapult/dashboard/third_party/mapreduce': [
 227         'UNKNOWN',
 228     ],
 229
 230     # https://code.google.com/p/webapp-improved/issues/detail?id=103
 231     # Apache (v2.0).
 232     'third_party/catapult/third_party/webapp2': [
 233         'UNKNOWN',
 234     ],
 235     'third_party/catapult/dashboard/third_party/webapp2': [
 236         'UNKNOWN',
 237     ],
 238
 239     # https://github.com/Pylons/webob/issues/211
 240     # MIT license.
 241     'third_party/catapult/third_party/WebOb': [
 242         'UNKNOWN',
 243     ],
 244     'third_party/catapult/dashboard/third_party/WebOb': [
 245         'UNKNOWN',
 246     ],
 247
 248     # https://github.com/Pylons/webtest/issues/141
 249     # MIT license.
 250     'third_party/catapult/third_party/webtest': [
 251         'UNKNOWN',
 252     ],
 253     'third_party/catapult/dashboard/third_party/webtest': [
 254         'UNKNOWN',
 255     ],
 256
 257     # https://bitbucket.org/ianb/paste/issues/12/add-license-headers-to-source-files
 258     # MIT license.
 259     'third_party/catapult/third_party/Paste': [
 260         'UNKNOWN',
 261     ],
 262     'third_party/catapult/dashboard/third_party/Paste': [
 263         'UNKNOWN',
 264     ],
 265
 266     # https://github.com/google/oauth2client/issues/231
 267     # Apache v2.0.
 268     'third_party/catapult/third_party/oauth2client': [
 269         'UNKNOWN',
 270     ],
 271     'third_party/catapult/dashboard/third_party/oauth2client': [
 272         'UNKNOWN',
 273     ],
 274
 275     # https://bitbucket.org/gutworth/six/issues/129/add-license-headers-to-source-files
 276     # MIT license.
 277     'third_party/catapult/third_party/six': [
 278         'UNKNOWN',
 279     ],
 280     'third_party/catapult/dashboard/third_party/six': [
 281         'UNKNOWN',
 282     ],
 283
 284     'third_party/expat/files/lib': [  # http://crbug.com/98121
 285         'UNKNOWN',
 286     ],
 287     'third_party/ffmpeg': [
 288         'GPL',
 289         'GPL (v2)',
 290         'GPL (v2 or later)',
 291         'GPL (v3 or later)',
 292         'UNKNOWN',  # http://crbug.com/98123
 293     ],
 294     'third_party/fontconfig': [
 295         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 296         'UNKNOWN',
 297     ],
 298     'third_party/freetype2': [ # http://crbug.com/177319
 299         'UNKNOWN',
 300     ],
 301     'third_party/hunspell': [  # http://crbug.com/98134
 302         'UNKNOWN',
 303     ],
 304     'third_party/iccjpeg': [  # http://crbug.com/98137
 305         'UNKNOWN',
 306     ],
 307     'third_party/icu': [  # http://crbug.com/98301
 308         'UNKNOWN',
 309     ],
 310     'third_party/jsoncpp/source': [
 311         # https://github.com/open-source-parsers/jsoncpp/issues/234
 312         'UNKNOWN',
 313     ],
 314     'third_party/junit/src': [
 315         # https://github.com/junit-team/junit/issues/1132
 316         'UNKNOWN',
 317     ],
 318     'third_party/lcov': [  # http://crbug.com/98304
 319         'UNKNOWN',
 320     ],
 321     'third_party/lcov/contrib/galaxy/genflat.pl': [
 322         'GPL (v2 or later)',
 323     ],
 324     'third_party/libevent': [  # http://crbug.com/98309
 325         'UNKNOWN',
 326     ],
 327     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 328         'UNKNOWN',
 329     ],
 330     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 331         'UNKNOWN',
 332     ],
 333
 334     # Many liblouis files are mirrored but not used in the NaCl module.
 335     # They are not excluded from the mirror because of lack of infrastructure
 336     # support.  Getting license headers added to the files where missing is
 337     # tracked in https://github.com/liblouis/liblouis/issues/22.
 338     'third_party/liblouis/src': [
 339         'GPL (v3 or later)',
 340         'UNKNOWN',
 341     ],
 342
 343     'third_party/libpng': [  # http://crbug.com/98318
 344         'UNKNOWN',
 345     ],
 346
 347     # The following files lack license headers, but are trivial.
 348     'third_party/libusb/src/libusb/os/poll_posix.h': [
 349         'UNKNOWN',
 350     ],
 351
 352     'third_party/libvpx/source': [  # http://crbug.com/98319
 353         'UNKNOWN',
 354     ],
 355     'third_party/libxml': [
 356         'UNKNOWN',
 357     ],
 358     'third_party/libxslt': [
 359         'UNKNOWN',
 360     ],
 361     'third_party/lzma_sdk': [
 362         'UNKNOWN',
 363     ],
 364     'third_party/mesa/src': [
 365         'GPL (v2)',
 366         'GPL (v3 or later)',
 367         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 368         'UNKNOWN',  # http://crbug.com/98450
 369     ],
 370     'third_party/modp_b64': [
 371         'UNKNOWN',
 372     ],
 373     'third_party/openmax_dl/dl' : [
 374         'Khronos Group',
 375     ],
 376     'third_party/openssl': [  # http://crbug.com/98451
 377         'UNKNOWN',
 378     ],
 379     'third_party/boringssl': [
 380         # There are some files in BoringSSL which came from OpenSSL and have no
 381         # license in them. We don't wish to add the license header ourselves
 382         # thus we don't expect to pass license checks.
 383         'UNKNOWN',
 384     ],
 385     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 386         'UNKNOWN',
 387     ],
 388     'third_party/molokocacao': [  # http://crbug.com/98453
 389         'UNKNOWN',
 390     ],
 391     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 392         'UNKNOWN',
 393     ],
 394     'third_party/protobuf': [  # http://crbug.com/98455
 395         'UNKNOWN',
 396     ],
 397
 398     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 399     # BSD 2-clause license.
 400     'third_party/pycoverage': [
 401         'UNKNOWN',
 402     ],
 403
 404     'third_party/pyelftools': [ # http://crbug.com/222831
 405         'UNKNOWN',
 406     ],
 407     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 408         'UNKNOWN',
 409     ],
 410     'third_party/simplejson': [
 411         'UNKNOWN',
 412     ],
 413     'third_party/skia': [  # http://crbug.com/98463
 414         'UNKNOWN',
 415     ],
 416     'third_party/snappy/src': [  # http://crbug.com/98464
 417         'UNKNOWN',
 418     ],
 419     'third_party/smhasher/src': [  # http://crbug.com/98465
 420         'UNKNOWN',
 421     ],
 422     'third_party/speech-dispatcher/libspeechd.h': [
 423         'GPL (v2 or later)',
 424     ],
 425     'third_party/sqlite': [
 426         'UNKNOWN',
 427     ],
 428
 429     # http://crbug.com/334668
 430     # MIT license.
 431     'tools/swarming_client/third_party/httplib2': [
 432         'UNKNOWN',
 433     ],
 434
 435     # http://crbug.com/334668
 436     # Apache v2.0.
 437     'tools/swarming_client/third_party/oauth2client': [
 438         'UNKNOWN',
 439     ],
 440
 441     # http://crbug.com/471372
 442     # BSD
 443     'tools/swarming_client/third_party/pyasn1': [
 444         'UNKNOWN',
 445     ],
 446
 447     # http://crbug.com/471372
 448     # Apache v2.0.
 449     'tools/swarming_client/third_party/rsa': [
 450         'UNKNOWN',
 451     ],
 452
 453     # https://github.com/kennethreitz/requests/issues/1610
 454     'tools/swarming_client/third_party/requests': [
 455         'UNKNOWN',
 456     ],
 457
 458     'third_party/talloc': [
 459         'GPL (v3 or later)',
 460         'UNKNOWN',  # http://crbug.com/98588
 461     ],
 462     'third_party/tcmalloc': [
 463         'UNKNOWN',  # http://crbug.com/98589
 464     ],
 465     'third_party/tlslite': [
 466         'UNKNOWN',
 467     ],
 468     'third_party/webdriver': [  # http://crbug.com/98590
 469         'UNKNOWN',
 470     ],
 471
 472     # https://github.com/html5lib/html5lib-python/issues/125
 473     # https://github.com/KhronosGroup/WebGL/issues/435
 474     'third_party/webgl/src': [
 475         'UNKNOWN',
 476     ],
 477
 478     'third_party/webrtc': [  # http://crbug.com/98592
 479         'UNKNOWN',
 480     ],
 481     'third_party/xdg-utils': [  # http://crbug.com/98593
 482         'UNKNOWN',
 483     ],
 484     'third_party/yasm/source': [  # http://crbug.com/98594
 485         'UNKNOWN',
 486     ],
 487     'third_party/zlib/contrib/minizip': [
 488         'UNKNOWN',
 489     ],
 490     'third_party/zlib/trees.h': [
 491         'UNKNOWN',
 492     ],
 493     'tools/emacs': [  # http://crbug.com/98595
 494         'UNKNOWN',
 495     ],
 496     'tools/gyp/test': [
 497         'UNKNOWN',
 498     ],
 499     'tools/python/google/__init__.py': [
 500         'UNKNOWN',
 501     ],
 502     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 503         'UNKNOWN',
 504     ],
 505     'tools/symsrc/pefile.py': [
 506         'UNKNOWN',
 507     ],
 508     # Not shipped, downloaded on trybots sometimes.
 509     'tools/telemetry/third_party/gsutil': [
 510         'BSD MIT/X11 (BSD like)',
 511         'UNKNOWN',
 512     ],
 513     'tools/telemetry/third_party/pyserial': [
 514         # https://sourceforge.net/p/pyserial/feature-requests/35/
 515         'UNKNOWN',
 516     ],
 517     'v8/test/cctest': [  # http://crbug.com/98597
 518         'UNKNOWN',
 519     ],
 520     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 521         'UNKNOWN',
 522     ],
 523 }
 524
 525
 526 def check_licenses(options, args):
 527   # Figure out which directory we have to check.
 528   if len(args) == 0:
 529     # No directory to check specified, use the repository root.
 530     start_dir = options.base_directory
 531   elif len(args) == 1:
 532     # Directory specified. Start here. It's supposed to be relative to the
 533     # base directory.
 534     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 535   else:
 536     # More than one argument, we don't handle this.
 537     PrintUsage()
 538     return 1
 539
 540   print "Using base directory:", options.base_directory
 541   print "Checking:", start_dir
 542   print
 543
 544   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 545                                                    'third_party',
 546                                                    'devscripts',
 547                                                    'licensecheck.pl'))
 548
 549   licensecheck = subprocess.Popen([licensecheck_path,
 550                                    '-l', '100',
 551                                    '-r', start_dir],
 552                                   stdout=subprocess.PIPE,
 553                                   stderr=subprocess.PIPE)
 554   stdout, stderr = licensecheck.communicate()
 555   if options.verbose:
 556     print '----------- licensecheck stdout -----------'
 557     print stdout
 558     print '--------- end licensecheck stdout ---------'
 559   if licensecheck.returncode != 0 or stderr:
 560     print '----------- licensecheck stderr -----------'
 561     print stderr
 562     print '--------- end licensecheck stderr ---------'
 563     print "\nFAILED\n"
 564     return 1
 565
 566   used_suppressions = set()
 567   errors = []
 568
 569   for line in stdout.splitlines():
 570     filename, license = line.split(':', 1)
 571     filename = os.path.relpath(filename.strip(), options.base_directory)
 572
 573     # All files in the build output directory are generated one way or another.
 574     # There's no need to check them.
 575     if filename.startswith('out/'):
 576       continue
 577
 578     # For now we're just interested in the license.
 579     license = license.replace('*No copyright*', '').strip()
 580
 581     # Skip generated files.
 582     if 'GENERATED FILE' in license:
 583       continue
 584
 585     if license in WHITELISTED_LICENSES:
 586       continue
 587
 588     if not options.ignore_suppressions:
 589       matched_prefixes = [
 590           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 591           if filename.startswith(prefix) and
 592           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 593       if matched_prefixes:
 594         used_suppressions.update(set(matched_prefixes))
 595         continue
 596
 597     errors.append({'filename': filename, 'license': license})
 598
 599   if options.json:
 600     with open(options.json, 'w') as f:
 601       json.dump(errors, f)
 602
 603   if errors:
 604     for error in errors:
 605       print "'%s' has non-whitelisted license '%s'" % (
 606           error['filename'], error['license'])
 607     print "\nFAILED\n"
 608     print "Please read",
 609     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 610     print "for more info how to handle the failure."
 611     print
 612     print "Please respect OWNERS of checklicenses.py. Changes violating"
 613     print "this requirement may be reverted."
 614
 615     # Do not print unused suppressions so that above message is clearly
 616     # visible and gets proper attention. Too much unrelated output
 617     # would be distracting and make the important points easier to miss.
 618
 619     return 1
 620
 621   print "\nSUCCESS\n"
 622
 623   if not len(args):
 624     unused_suppressions = set(
 625         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 626             used_suppressions)
 627     if unused_suppressions:
 628       print "\nNOTE: unused suppressions detected:\n"
 629       print '\n'.join(unused_suppressions)
 630
 631   return 0
 632
 633
 634 def main():
 635   default_root = os.path.abspath(
 636       os.path.join(os.path.dirname(__file__), '..', '..'))
 637   option_parser = optparse.OptionParser()
 638   option_parser.add_option('--root', default=default_root,
 639                            dest='base_directory',
 640                            help='Specifies the repository root. This defaults '
 641                            'to "../.." relative to the script file, which '
 642                            'will normally be the repository root.')
 643   option_parser.add_option('-v', '--verbose', action='store_true',
 644                            default=False, help='Print debug logging')
 645   option_parser.add_option('--ignore-suppressions',
 646                            action='store_true',
 647                            default=False,
 648                            help='Ignore path-specific license whitelist.')
 649   option_parser.add_option('--json', help='Path to JSON output file')
 650   options, args = option_parser.parse_args()
 651   return check_licenses(options, args)
 652
 653
 654 if '__main__' == __name__:
 655   sys.exit(main())