tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'buildtools/third_party/libc++/trunk/test': [
 116         # http://llvm.org/bugs/show_bug.cgi?id=18291
 117         'UNKNOWN',
 118     ],
 119
 120     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 121         'UNKNOWN',
 122     ],
 123     # This contains files copied from elsewhere from the tree. Since the copied
 124     # directories might have suppressions below (like simplejson), whitelist the
 125     # whole directory. This is also not shipped code.
 126     'chrome/common/extensions/docs/server2/third_party': [
 127         'UNKNOWN',
 128     ],
 129     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 130         'UNKNOWN',
 131     ],
 132     'native_client': [  # http://crbug.com/98099
 133         'UNKNOWN',
 134     ],
 135     'native_client/toolchain': [
 136         'BSD GPL (v2 or later)',
 137         'BSD MIT/X11 (BSD like)',
 138         'BSD (2 clause) GPL (v2 or later)',
 139         'BSD (3 clause) GPL (v2 or later)',
 140         'BSD (4 clause) ISC',
 141         'BSL (v1.0) GPL',
 142         'BSL (v1.0) GPL (v3.1)',
 143         'GPL',
 144         'GPL (unversioned/unknown version)',
 145         'GPL (v2)',
 146         'GPL (v2 or later)',
 147         'GPL (v3.1)',
 148         'GPL (v3 or later)',
 149         'MPL (v1.1) LGPL (unversioned/unknown version)',
 150     ],
 151     'third_party/WebKit': [
 152         'UNKNOWN',
 153     ],
 154
 155     # http://code.google.com/p/angleproject/issues/detail?id=217
 156     'third_party/angle': [
 157         'UNKNOWN',
 158     ],
 159
 160     # http://crbug.com/222828
 161     # http://bugs.python.org/issue17514
 162     'third_party/chromite/third_party/argparse.py': [
 163         'UNKNOWN',
 164     ],
 165
 166     # http://crbug.com/326117
 167     # https://bitbucket.org/chrisatlee/poster/issue/21
 168     'third_party/chromite/third_party/poster': [
 169         'UNKNOWN',
 170     ],
 171
 172     # http://crbug.com/333508
 173     'third_party/clang_format/script': [
 174         'UNKNOWN',
 175     ],
 176
 177     # http://crbug.com/333508
 178     'buildtools/clang_format/script': [
 179         'UNKNOWN',
 180     ],
 181
 182     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 183     'third_party/cython': [
 184         'UNKNOWN',
 185     ],
 186
 187     'third_party/devscripts': [
 188         'GPL (v2 or later)',
 189     ],
 190     'third_party/expat/files/lib': [  # http://crbug.com/98121
 191         'UNKNOWN',
 192     ],
 193     'third_party/ffmpeg': [
 194         'GPL',
 195         'GPL (v2)',
 196         'GPL (v2 or later)',
 197         'GPL (v3 or later)',
 198         'UNKNOWN',  # http://crbug.com/98123
 199     ],
 200     'third_party/fontconfig': [
 201         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 202         'UNKNOWN',
 203     ],
 204     'third_party/freetype2': [ # http://crbug.com/177319
 205         'UNKNOWN',
 206     ],
 207     'third_party/hunspell': [  # http://crbug.com/98134
 208         'UNKNOWN',
 209     ],
 210     'third_party/iccjpeg': [  # http://crbug.com/98137
 211         'UNKNOWN',
 212     ],
 213     'third_party/icu': [  # http://crbug.com/98301
 214         'UNKNOWN',
 215     ],
 216     'third_party/jsoncpp/source': [
 217         # https://github.com/open-source-parsers/jsoncpp/issues/234
 218         'UNKNOWN',
 219     ],
 220     'third_party/junit/src': [
 221         # https://github.com/junit-team/junit/issues/1132
 222         'UNKNOWN',
 223     ],
 224     'third_party/lcov': [  # http://crbug.com/98304
 225         'UNKNOWN',
 226     ],
 227     'third_party/lcov/contrib/galaxy/genflat.pl': [
 228         'GPL (v2 or later)',
 229     ],
 230     'third_party/libevent': [  # http://crbug.com/98309
 231         'UNKNOWN',
 232     ],
 233     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 234         'UNKNOWN',
 235     ],
 236     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 237         'UNKNOWN',
 238     ],
 239
 240     # Many liblouis files are mirrored but not used in the NaCl module.
 241     # They are not excluded from the mirror because of lack of infrastructure
 242     # support.  Getting license headers added to the files where missing is
 243     # tracked in https://github.com/liblouis/liblouis/issues/22.
 244     'third_party/liblouis/src': [
 245         'GPL (v3 or later)',
 246         'UNKNOWN',
 247     ],
 248
 249     'third_party/libpng': [  # http://crbug.com/98318
 250         'UNKNOWN',
 251     ],
 252
 253     # The following files lack license headers, but are trivial.
 254     'third_party/libusb/src/libusb/os/poll_posix.h': [
 255         'UNKNOWN',
 256     ],
 257
 258     'third_party/libvpx/source': [  # http://crbug.com/98319
 259         'UNKNOWN',
 260     ],
 261     'third_party/libxml': [
 262         'UNKNOWN',
 263     ],
 264     'third_party/libxslt': [
 265         'UNKNOWN',
 266     ],
 267     'third_party/lzma_sdk': [
 268         'UNKNOWN',
 269     ],
 270     'third_party/mesa/src': [
 271         'GPL (v2)',
 272         'GPL (v3 or later)',
 273         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 274         'UNKNOWN',  # http://crbug.com/98450
 275     ],
 276     'third_party/modp_b64': [
 277         'UNKNOWN',
 278     ],
 279     'third_party/openmax_dl/dl' : [
 280         'Khronos Group',
 281     ],
 282     'third_party/openssl': [  # http://crbug.com/98451
 283         'UNKNOWN',
 284     ],
 285     'third_party/boringssl': [
 286         # There are some files in BoringSSL which came from OpenSSL and have no
 287         # license in them. We don't wish to add the license header ourselves
 288         # thus we don't expect to pass license checks.
 289         'UNKNOWN',
 290     ],
 291     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 292         'UNKNOWN',
 293     ],
 294     'third_party/molokocacao': [  # http://crbug.com/98453
 295         'UNKNOWN',
 296     ],
 297     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 298         'UNKNOWN',
 299     ],
 300     'third_party/protobuf': [  # http://crbug.com/98455
 301         'UNKNOWN',
 302     ],
 303
 304     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 305     # BSD 2-clause license.
 306     'third_party/pycoverage': [
 307         'UNKNOWN',
 308     ],
 309
 310     'third_party/pyelftools': [ # http://crbug.com/222831
 311         'UNKNOWN',
 312     ],
 313     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 314         'UNKNOWN',
 315     ],
 316     'third_party/simplejson': [
 317         'UNKNOWN',
 318     ],
 319     'third_party/skia': [  # http://crbug.com/98463
 320         'UNKNOWN',
 321     ],
 322     'third_party/snappy/src': [  # http://crbug.com/98464
 323         'UNKNOWN',
 324     ],
 325     'third_party/smhasher/src': [  # http://crbug.com/98465
 326         'UNKNOWN',
 327     ],
 328     'third_party/speech-dispatcher/libspeechd.h': [
 329         'GPL (v2 or later)',
 330     ],
 331     'third_party/sqlite': [
 332         'UNKNOWN',
 333     ],
 334
 335     # http://crbug.com/334668
 336     # MIT license.
 337     'tools/swarming_client/third_party/httplib2': [
 338         'UNKNOWN',
 339     ],
 340
 341     # http://crbug.com/334668
 342     # Apache v2.0.
 343     'tools/swarming_client/third_party/oauth2client': [
 344         'UNKNOWN',
 345     ],
 346
 347     # http://crbug.com/471372
 348     # BSD
 349     'tools/swarming_client/third_party/pyasn1': [
 350         'UNKNOWN',
 351     ],
 352
 353     # http://crbug.com/471372
 354     # Apache v2.0.
 355     'tools/swarming_client/third_party/rsa': [
 356         'UNKNOWN',
 357     ],
 358
 359     # https://github.com/kennethreitz/requests/issues/1610
 360     'tools/swarming_client/third_party/requests': [
 361         'UNKNOWN',
 362     ],
 363
 364     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 365         'UNKNOWN',
 366     ],
 367     'third_party/talloc': [
 368         'GPL (v3 or later)',
 369         'UNKNOWN',  # http://crbug.com/98588
 370     ],
 371     'third_party/tcmalloc': [
 372         'UNKNOWN',  # http://crbug.com/98589
 373     ],
 374     'third_party/tlslite': [
 375         'UNKNOWN',
 376     ],
 377     'third_party/webdriver': [  # http://crbug.com/98590
 378         'UNKNOWN',
 379     ],
 380
 381     # https://github.com/html5lib/html5lib-python/issues/125
 382     # https://github.com/KhronosGroup/WebGL/issues/435
 383     'third_party/webgl/src': [
 384         'UNKNOWN',
 385     ],
 386
 387     'third_party/webrtc': [  # http://crbug.com/98592
 388         'UNKNOWN',
 389     ],
 390     'third_party/xdg-utils': [  # http://crbug.com/98593
 391         'UNKNOWN',
 392     ],
 393     'third_party/yasm/source': [  # http://crbug.com/98594
 394         'UNKNOWN',
 395     ],
 396     'third_party/zlib/contrib/minizip': [
 397         'UNKNOWN',
 398     ],
 399     'third_party/zlib/trees.h': [
 400         'UNKNOWN',
 401     ],
 402     'tools/emacs': [  # http://crbug.com/98595
 403         'UNKNOWN',
 404     ],
 405     'tools/gyp/test': [
 406         'UNKNOWN',
 407     ],
 408     'tools/python/google/__init__.py': [
 409         'UNKNOWN',
 410     ],
 411     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 412         'UNKNOWN',
 413     ],
 414     'tools/symsrc/pefile.py': [
 415         'UNKNOWN',
 416     ],
 417     # Not shipped, downloaded on trybots sometimes.
 418     'tools/telemetry/third_party/gsutil': [
 419         'BSD MIT/X11 (BSD like)',
 420         'UNKNOWN',
 421     ],
 422     'tools/telemetry/third_party/pyserial': [
 423         # https://sourceforge.net/p/pyserial/feature-requests/35/
 424         'UNKNOWN',
 425     ],
 426     'v8/test/cctest': [  # http://crbug.com/98597
 427         'UNKNOWN',
 428     ],
 429     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 430         'UNKNOWN',
 431     ],
 432 }
 433
 434
 435 def check_licenses(options, args):
 436   # Figure out which directory we have to check.
 437   if len(args) == 0:
 438     # No directory to check specified, use the repository root.
 439     start_dir = options.base_directory
 440   elif len(args) == 1:
 441     # Directory specified. Start here. It's supposed to be relative to the
 442     # base directory.
 443     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 444   else:
 445     # More than one argument, we don't handle this.
 446     PrintUsage()
 447     return 1
 448
 449   print "Using base directory:", options.base_directory
 450   print "Checking:", start_dir
 451   print
 452
 453   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 454                                                    'third_party',
 455                                                    'devscripts',
 456                                                    'licensecheck.pl'))
 457
 458   licensecheck = subprocess.Popen([licensecheck_path,
 459                                    '-l', '100',
 460                                    '-r', start_dir],
 461                                   stdout=subprocess.PIPE,
 462                                   stderr=subprocess.PIPE)
 463   stdout, stderr = licensecheck.communicate()
 464   if options.verbose:
 465     print '----------- licensecheck stdout -----------'
 466     print stdout
 467     print '--------- end licensecheck stdout ---------'
 468   if licensecheck.returncode != 0 or stderr:
 469     print '----------- licensecheck stderr -----------'
 470     print stderr
 471     print '--------- end licensecheck stderr ---------'
 472     print "\nFAILED\n"
 473     return 1
 474
 475   used_suppressions = set()
 476   errors = []
 477
 478   for line in stdout.splitlines():
 479     filename, license = line.split(':', 1)
 480     filename = os.path.relpath(filename.strip(), options.base_directory)
 481
 482     # All files in the build output directory are generated one way or another.
 483     # There's no need to check them.
 484     if filename.startswith('out/'):
 485       continue
 486
 487     # For now we're just interested in the license.
 488     license = license.replace('*No copyright*', '').strip()
 489
 490     # Skip generated files.
 491     if 'GENERATED FILE' in license:
 492       continue
 493
 494     if license in WHITELISTED_LICENSES:
 495       continue
 496
 497     if not options.ignore_suppressions:
 498       matched_prefixes = [
 499           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 500           if filename.startswith(prefix) and
 501           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 502       if matched_prefixes:
 503         used_suppressions.update(set(matched_prefixes))
 504         continue
 505
 506     errors.append({'filename': filename, 'license': license})
 507
 508   if options.json:
 509     with open(options.json, 'w') as f:
 510       json.dump(errors, f)
 511
 512   if errors:
 513     for error in errors:
 514       print "'%s' has non-whitelisted license '%s'" % (
 515           error['filename'], error['license'])
 516     print "\nFAILED\n"
 517     print "Please read",
 518     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 519     print "for more info how to handle the failure."
 520     print
 521     print "Please respect OWNERS of checklicenses.py. Changes violating"
 522     print "this requirement may be reverted."
 523
 524     # Do not print unused suppressions so that above message is clearly
 525     # visible and gets proper attention. Too much unrelated output
 526     # would be distracting and make the important points easier to miss.
 527
 528     return 1
 529
 530   print "\nSUCCESS\n"
 531
 532   if not len(args):
 533     unused_suppressions = set(
 534         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 535             used_suppressions)
 536     if unused_suppressions:
 537       print "\nNOTE: unused suppressions detected:\n"
 538       print '\n'.join(unused_suppressions)
 539
 540   return 0
 541
 542
 543 def main():
 544   default_root = os.path.abspath(
 545       os.path.join(os.path.dirname(__file__), '..', '..'))
 546   option_parser = optparse.OptionParser()
 547   option_parser.add_option('--root', default=default_root,
 548                            dest='base_directory',
 549                            help='Specifies the repository root. This defaults '
 550                            'to "../.." relative to the script file, which '
 551                            'will normally be the repository root.')
 552   option_parser.add_option('-v', '--verbose', action='store_true',
 553                            default=False, help='Print debug logging')
 554   option_parser.add_option('--ignore-suppressions',
 555                            action='store_true',
 556                            default=False,
 557                            help='Ignore path-specific license whitelist.')
 558   option_parser.add_option('--json', help='Path to JSON output file')
 559   options, args = option_parser.parse_args()
 560   return check_licenses(options, args)
 561
 562
 563 if '__main__' == __name__:
 564   sys.exit(main())