tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'buildtools/third_party/libc++/trunk/test': [
 116         # http://llvm.org/bugs/show_bug.cgi?id=18291
 117         'UNKNOWN',
 118     ],
 119
 120     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 121         'UNKNOWN',
 122     ],
 123     # This contains files copied from elsewhere from the tree. Since the copied
 124     # directories might have suppressions below (like simplejson), whitelist the
 125     # whole directory. This is also not shipped code.
 126     'chrome/common/extensions/docs/server2/third_party': [
 127         'UNKNOWN',
 128     ],
 129     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 130         'UNKNOWN',
 131     ],
 132     'native_client': [  # http://crbug.com/98099
 133         'UNKNOWN',
 134     ],
 135     'native_client/toolchain': [
 136         'BSD GPL (v2 or later)',
 137         'BSD MIT/X11 (BSD like)',
 138         'BSD (2 clause) GPL (v2 or later)',
 139         'BSD (3 clause) GPL (v2 or later)',
 140         'BSD (4 clause) ISC',
 141         'BSL (v1.0) GPL',
 142         'BSL (v1.0) GPL (v3.1)',
 143         'GPL',
 144         'GPL (unversioned/unknown version)',
 145         'GPL (v2)',
 146         'GPL (v2 or later)',
 147         'GPL (v3.1)',
 148         'GPL (v3 or later)',
 149         'MPL (v1.1) LGPL (unversioned/unknown version)',
 150     ],
 151     'third_party/WebKit': [
 152         'UNKNOWN',
 153     ],
 154
 155     # http://code.google.com/p/angleproject/issues/detail?id=217
 156     'third_party/angle': [
 157         'UNKNOWN',
 158     ],
 159
 160     # http://crbug.com/222828
 161     # http://bugs.python.org/issue17514
 162     'third_party/chromite/third_party/argparse.py': [
 163         'UNKNOWN',
 164     ],
 165
 166     # http://crbug.com/326117
 167     # https://bitbucket.org/chrisatlee/poster/issue/21
 168     'third_party/chromite/third_party/poster': [
 169         'UNKNOWN',
 170     ],
 171
 172     # http://crbug.com/333508
 173     'third_party/clang_format/script': [
 174         'UNKNOWN',
 175     ],
 176
 177     # http://crbug.com/333508
 178     'buildtools/clang_format/script': [
 179         'UNKNOWN',
 180     ],
 181
 182     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 183     'third_party/cython': [
 184         'UNKNOWN',
 185     ],
 186
 187     'third_party/devscripts': [
 188         'GPL (v2 or later)',
 189     ],
 190     'third_party/expat/files/lib': [  # http://crbug.com/98121
 191         'UNKNOWN',
 192     ],
 193     'third_party/ffmpeg': [
 194         'GPL',
 195         'GPL (v2)',
 196         'GPL (v2 or later)',
 197         'GPL (v3 or later)',
 198         'UNKNOWN',  # http://crbug.com/98123
 199     ],
 200     'third_party/fontconfig': [
 201         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 202         'UNKNOWN',
 203     ],
 204     'third_party/freetype2': [ # http://crbug.com/177319
 205         'UNKNOWN',
 206     ],
 207     'third_party/hunspell': [  # http://crbug.com/98134
 208         'UNKNOWN',
 209     ],
 210     'third_party/iccjpeg': [  # http://crbug.com/98137
 211         'UNKNOWN',
 212     ],
 213     'third_party/icu': [  # http://crbug.com/98301
 214         'UNKNOWN',
 215     ],
 216     'third_party/jsoncpp/source': [
 217         # https://github.com/open-source-parsers/jsoncpp/issues/234
 218         'UNKNOWN',
 219     ],
 220     'third_party/lcov': [  # http://crbug.com/98304
 221         'UNKNOWN',
 222     ],
 223     'third_party/lcov/contrib/galaxy/genflat.pl': [
 224         'GPL (v2 or later)',
 225     ],
 226     'third_party/libevent': [  # http://crbug.com/98309
 227         'UNKNOWN',
 228     ],
 229     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 230         'UNKNOWN',
 231     ],
 232     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 233         'UNKNOWN',
 234     ],
 235
 236     # Many liblouis files are mirrored but not used in the NaCl module.
 237     # They are not excluded from the mirror because of lack of infrastructure
 238     # support.  Getting license headers added to the files where missing is
 239     # tracked in https://github.com/liblouis/liblouis/issues/22.
 240     'third_party/liblouis/src': [
 241         'GPL (v3 or later)',
 242         'UNKNOWN',
 243     ],
 244
 245     'third_party/libpng': [  # http://crbug.com/98318
 246         'UNKNOWN',
 247     ],
 248
 249     # The following files lack license headers, but are trivial.
 250     'third_party/libusb/src/libusb/os/poll_posix.h': [
 251         'UNKNOWN',
 252     ],
 253
 254     'third_party/libvpx/source': [  # http://crbug.com/98319
 255         'UNKNOWN',
 256     ],
 257     'third_party/libxml': [
 258         'UNKNOWN',
 259     ],
 260     'third_party/libxslt': [
 261         'UNKNOWN',
 262     ],
 263     'third_party/lzma_sdk': [
 264         'UNKNOWN',
 265     ],
 266     'third_party/mesa/src': [
 267         'GPL (v2)',
 268         'GPL (v3 or later)',
 269         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 270         'UNKNOWN',  # http://crbug.com/98450
 271     ],
 272     'third_party/modp_b64': [
 273         'UNKNOWN',
 274     ],
 275     'third_party/openmax_dl/dl' : [
 276         'Khronos Group',
 277     ],
 278     'third_party/openssl': [  # http://crbug.com/98451
 279         'UNKNOWN',
 280     ],
 281     'third_party/boringssl': [
 282         # There are some files in BoringSSL which came from OpenSSL and have no
 283         # license in them. We don't wish to add the license header ourselves
 284         # thus we don't expect to pass license checks.
 285         'UNKNOWN',
 286     ],
 287     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 288         'UNKNOWN',
 289     ],
 290     'third_party/molokocacao': [  # http://crbug.com/98453
 291         'UNKNOWN',
 292     ],
 293     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 294         'UNKNOWN',
 295     ],
 296     'third_party/protobuf': [  # http://crbug.com/98455
 297         'UNKNOWN',
 298     ],
 299
 300     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 301     # BSD 2-clause license.
 302     'third_party/pycoverage': [
 303         'UNKNOWN',
 304     ],
 305
 306     'third_party/pyelftools': [ # http://crbug.com/222831
 307         'UNKNOWN',
 308     ],
 309     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 310         'UNKNOWN',
 311     ],
 312     'third_party/simplejson': [
 313         'UNKNOWN',
 314     ],
 315     'third_party/skia': [  # http://crbug.com/98463
 316         'UNKNOWN',
 317     ],
 318     'third_party/snappy/src': [  # http://crbug.com/98464
 319         'UNKNOWN',
 320     ],
 321     'third_party/smhasher/src': [  # http://crbug.com/98465
 322         'UNKNOWN',
 323     ],
 324     'third_party/speech-dispatcher/libspeechd.h': [
 325         'GPL (v2 or later)',
 326     ],
 327     'third_party/sqlite': [
 328         'UNKNOWN',
 329     ],
 330
 331     # http://crbug.com/334668
 332     # MIT license.
 333     'tools/swarming_client/third_party/httplib2': [
 334         'UNKNOWN',
 335     ],
 336
 337     # http://crbug.com/334668
 338     # Apache v2.0.
 339     'tools/swarming_client/third_party/oauth2client': [
 340         'UNKNOWN',
 341     ],
 342
 343     # http://crbug.com/471372
 344     # BSD
 345     'tools/swarming_client/third_party/pyasn1': [
 346         'UNKNOWN',
 347     ],
 348
 349     # http://crbug.com/471372
 350     # Apache v2.0.
 351     'tools/swarming_client/third_party/rsa': [
 352         'UNKNOWN',
 353     ],
 354
 355     # https://github.com/kennethreitz/requests/issues/1610
 356     'tools/swarming_client/third_party/requests': [
 357         'UNKNOWN',
 358     ],
 359
 360     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 361         'UNKNOWN',
 362     ],
 363     'third_party/talloc': [
 364         'GPL (v3 or later)',
 365         'UNKNOWN',  # http://crbug.com/98588
 366     ],
 367     'third_party/tcmalloc': [
 368         'UNKNOWN',  # http://crbug.com/98589
 369     ],
 370     'third_party/tlslite': [
 371         'UNKNOWN',
 372     ],
 373     'third_party/webdriver': [  # http://crbug.com/98590
 374         'UNKNOWN',
 375     ],
 376
 377     # https://github.com/html5lib/html5lib-python/issues/125
 378     # https://github.com/KhronosGroup/WebGL/issues/435
 379     'third_party/webgl/src': [
 380         'UNKNOWN',
 381     ],
 382
 383     'third_party/webrtc': [  # http://crbug.com/98592
 384         'UNKNOWN',
 385     ],
 386     'third_party/xdg-utils': [  # http://crbug.com/98593
 387         'UNKNOWN',
 388     ],
 389     'third_party/yasm/source': [  # http://crbug.com/98594
 390         'UNKNOWN',
 391     ],
 392     'third_party/zlib/contrib/minizip': [
 393         'UNKNOWN',
 394     ],
 395     'third_party/zlib/trees.h': [
 396         'UNKNOWN',
 397     ],
 398     'tools/emacs': [  # http://crbug.com/98595
 399         'UNKNOWN',
 400     ],
 401     'tools/gyp/test': [
 402         'UNKNOWN',
 403     ],
 404     'tools/python/google/__init__.py': [
 405         'UNKNOWN',
 406     ],
 407     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 408         'UNKNOWN',
 409     ],
 410     'tools/symsrc/pefile.py': [
 411         'UNKNOWN',
 412     ],
 413     # Not shipped, downloaded on trybots sometimes.
 414     'tools/telemetry/third_party/gsutil': [
 415         'BSD MIT/X11 (BSD like)',
 416         'UNKNOWN',
 417     ],
 418     'tools/telemetry/third_party/pyserial': [
 419         # https://sourceforge.net/p/pyserial/feature-requests/35/
 420         'UNKNOWN',
 421     ],
 422     'v8/test/cctest': [  # http://crbug.com/98597
 423         'UNKNOWN',
 424     ],
 425     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 426         'UNKNOWN',
 427     ],
 428 }
 429
 430
 431 def check_licenses(options, args):
 432   # Figure out which directory we have to check.
 433   if len(args) == 0:
 434     # No directory to check specified, use the repository root.
 435     start_dir = options.base_directory
 436   elif len(args) == 1:
 437     # Directory specified. Start here. It's supposed to be relative to the
 438     # base directory.
 439     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 440   else:
 441     # More than one argument, we don't handle this.
 442     PrintUsage()
 443     return 1
 444
 445   print "Using base directory:", options.base_directory
 446   print "Checking:", start_dir
 447   print
 448
 449   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 450                                                    'third_party',
 451                                                    'devscripts',
 452                                                    'licensecheck.pl'))
 453
 454   licensecheck = subprocess.Popen([licensecheck_path,
 455                                    '-l', '100',
 456                                    '-r', start_dir],
 457                                   stdout=subprocess.PIPE,
 458                                   stderr=subprocess.PIPE)
 459   stdout, stderr = licensecheck.communicate()
 460   if options.verbose:
 461     print '----------- licensecheck stdout -----------'
 462     print stdout
 463     print '--------- end licensecheck stdout ---------'
 464   if licensecheck.returncode != 0 or stderr:
 465     print '----------- licensecheck stderr -----------'
 466     print stderr
 467     print '--------- end licensecheck stderr ---------'
 468     print "\nFAILED\n"
 469     return 1
 470
 471   used_suppressions = set()
 472   errors = []
 473
 474   for line in stdout.splitlines():
 475     filename, license = line.split(':', 1)
 476     filename = os.path.relpath(filename.strip(), options.base_directory)
 477
 478     # All files in the build output directory are generated one way or another.
 479     # There's no need to check them.
 480     if filename.startswith('out/'):
 481       continue
 482
 483     # For now we're just interested in the license.
 484     license = license.replace('*No copyright*', '').strip()
 485
 486     # Skip generated files.
 487     if 'GENERATED FILE' in license:
 488       continue
 489
 490     if license in WHITELISTED_LICENSES:
 491       continue
 492
 493     if not options.ignore_suppressions:
 494       matched_prefixes = [
 495           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 496           if filename.startswith(prefix) and
 497           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 498       if matched_prefixes:
 499         used_suppressions.update(set(matched_prefixes))
 500         continue
 501
 502     errors.append({'filename': filename, 'license': license})
 503
 504   if options.json:
 505     with open(options.json, 'w') as f:
 506       json.dump(errors, f)
 507
 508   if errors:
 509     for error in errors:
 510       print "'%s' has non-whitelisted license '%s'" % (
 511           error['filename'], error['license'])
 512     print "\nFAILED\n"
 513     print "Please read",
 514     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 515     print "for more info how to handle the failure."
 516     print
 517     print "Please respect OWNERS of checklicenses.py. Changes violating"
 518     print "this requirement may be reverted."
 519
 520     # Do not print unused suppressions so that above message is clearly
 521     # visible and gets proper attention. Too much unrelated output
 522     # would be distracting and make the important points easier to miss.
 523
 524     return 1
 525
 526   print "\nSUCCESS\n"
 527
 528   if not len(args):
 529     unused_suppressions = set(
 530         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 531             used_suppressions)
 532     if unused_suppressions:
 533       print "\nNOTE: unused suppressions detected:\n"
 534       print '\n'.join(unused_suppressions)
 535
 536   return 0
 537
 538
 539 def main():
 540   default_root = os.path.abspath(
 541       os.path.join(os.path.dirname(__file__), '..', '..'))
 542   option_parser = optparse.OptionParser()
 543   option_parser.add_option('--root', default=default_root,
 544                            dest='base_directory',
 545                            help='Specifies the repository root. This defaults '
 546                            'to "../.." relative to the script file, which '
 547                            'will normally be the repository root.')
 548   option_parser.add_option('-v', '--verbose', action='store_true',
 549                            default=False, help='Print debug logging')
 550   option_parser.add_option('--ignore-suppressions',
 551                            action='store_true',
 552                            default=False,
 553                            help='Ignore path-specific license whitelist.')
 554   option_parser.add_option('--json', help='Path to JSON output file')
 555   options, args = option_parser.parse_args()
 556   return check_licenses(options, args)
 557
 558
 559 if '__main__' == __name__:
 560   sys.exit(main())