tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 116         'UNKNOWN',
 117     ],
 118     # This contains files copied from elsewhere from the tree. Since the copied
 119     # directories might have suppressions below (like simplejson), whitelist the
 120     # whole directory. This is also not shipped code.
 121     'chrome/common/extensions/docs/server2/third_party': [
 122         'UNKNOWN',
 123     ],
 124     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 125         'UNKNOWN',
 126     ],
 127     'native_client': [  # http://crbug.com/98099
 128         'UNKNOWN',
 129     ],
 130     'native_client/toolchain': [
 131         'BSD GPL (v2 or later)',
 132         'BSD (2 clause) GPL (v2 or later)',
 133         'BSD (3 clause) GPL (v2 or later)',
 134         'BSL (v1.0) GPL',
 135         'BSL (v1.0) GPL (v3.1)',
 136         'GPL',
 137         'GPL (unversioned/unknown version)',
 138         'GPL (v2)',
 139         'GPL (v2 or later)',
 140         'GPL (v3.1)',
 141         'GPL (v3 or later)',
 142     ],
 143     'third_party/WebKit': [
 144         'UNKNOWN',
 145     ],
 146
 147     # http://code.google.com/p/angleproject/issues/detail?id=217
 148     'third_party/angle': [
 149         'UNKNOWN',
 150     ],
 151
 152     # http://crbug.com/222828
 153     # http://bugs.python.org/issue17514
 154     'third_party/chromite/third_party/argparse.py': [
 155         'UNKNOWN',
 156     ],
 157
 158     # http://crbug.com/326117
 159     # https://bitbucket.org/chrisatlee/poster/issue/21
 160     'third_party/chromite/third_party/poster': [
 161         'UNKNOWN',
 162     ],
 163
 164     # http://crbug.com/333508
 165     'third_party/clang_format/script': [
 166         'UNKNOWN',
 167     ],
 168
 169     # http://crbug.com/333508
 170     'buildtools/clang_format/script': [
 171         'UNKNOWN',
 172     ],
 173
 174     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 175     'third_party/cython': [
 176         'UNKNOWN',
 177     ],
 178
 179     'third_party/devscripts': [
 180         'GPL (v2 or later)',
 181     ],
 182     'third_party/expat/files/lib': [  # http://crbug.com/98121
 183         'UNKNOWN',
 184     ],
 185     'third_party/ffmpeg': [
 186         'GPL',
 187         'GPL (v2)',
 188         'GPL (v2 or later)',
 189         'GPL (v3 or later)',
 190         'UNKNOWN',  # http://crbug.com/98123
 191     ],
 192     'third_party/fontconfig': [
 193         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 194         'UNKNOWN',
 195     ],
 196     'third_party/freetype2': [ # http://crbug.com/177319
 197         'UNKNOWN',
 198     ],
 199     'third_party/hunspell': [  # http://crbug.com/98134
 200         'UNKNOWN',
 201     ],
 202     'third_party/iccjpeg': [  # http://crbug.com/98137
 203         'UNKNOWN',
 204     ],
 205     'third_party/icu': [  # http://crbug.com/98301
 206         'UNKNOWN',
 207     ],
 208     'third_party/lcov': [  # http://crbug.com/98304
 209         'UNKNOWN',
 210     ],
 211     'third_party/lcov/contrib/galaxy/genflat.pl': [
 212         'GPL (v2 or later)',
 213     ],
 214     'third_party/libc++/trunk/include/support/solaris': [
 215         # http://llvm.org/bugs/show_bug.cgi?id=18291
 216         'UNKNOWN',
 217     ],
 218     'third_party/libc++/trunk/src/support/solaris/xlocale.c': [
 219         # http://llvm.org/bugs/show_bug.cgi?id=18291
 220         'UNKNOWN',
 221     ],
 222     'third_party/libc++/trunk/test': [
 223         # http://llvm.org/bugs/show_bug.cgi?id=18291
 224         'UNKNOWN',
 225     ],
 226     'third_party/libevent': [  # http://crbug.com/98309
 227         'UNKNOWN',
 228     ],
 229     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 230         'UNKNOWN',
 231     ],
 232     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 233         'UNKNOWN',
 234     ],
 235
 236     # Many liblouis files are mirrored but not used in the NaCl module.
 237     # They are not excluded from the mirror because of lack of infrastructure
 238     # support.  Getting license headers added to the files where missing is
 239     # tracked in https://github.com/liblouis/liblouis/issues/22.
 240     'third_party/liblouis/src': [
 241         'GPL (v3 or later)',
 242         'UNKNOWN',
 243     ],
 244
 245     'third_party/libpng': [  # http://crbug.com/98318
 246         'UNKNOWN',
 247     ],
 248
 249     # The following files lack license headers, but are trivial.
 250     'third_party/libusb/src/libusb/os/poll_posix.h': [
 251         'UNKNOWN',
 252     ],
 253
 254     'third_party/libvpx/source': [  # http://crbug.com/98319
 255         'UNKNOWN',
 256     ],
 257     'third_party/libxml': [
 258         'UNKNOWN',
 259     ],
 260     'third_party/libxslt': [
 261         'UNKNOWN',
 262     ],
 263     'third_party/lzma_sdk': [
 264         'UNKNOWN',
 265     ],
 266     'third_party/mesa/src': [
 267         'GPL (v2)',
 268         'GPL (v3 or later)',
 269         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 270         'UNKNOWN',  # http://crbug.com/98450
 271     ],
 272     'third_party/modp_b64': [
 273         'UNKNOWN',
 274     ],
 275     'third_party/openmax_dl/dl' : [
 276         'Khronos Group',
 277     ],
 278     'third_party/openssl': [  # http://crbug.com/98451
 279         'UNKNOWN',
 280     ],
 281     'third_party/boringssl': [
 282         # There are some files in BoringSSL which came from OpenSSL and have no
 283         # license in them. We don't wish to add the license header ourselves
 284         # thus we don't expect to pass license checks.
 285         'UNKNOWN',
 286     ],
 287     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 288         'UNKNOWN',
 289     ],
 290     'third_party/molokocacao': [  # http://crbug.com/98453
 291         'UNKNOWN',
 292     ],
 293     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 294         'UNKNOWN',
 295     ],
 296     'third_party/ply/__init__.py': [
 297         'UNKNOWN',
 298     ],
 299     'third_party/protobuf': [  # http://crbug.com/98455
 300         'UNKNOWN',
 301     ],
 302
 303     # http://crbug.com/222831
 304     # https://bitbucket.org/eliben/pyelftools/issue/12
 305     'third_party/pyelftools': [
 306         'UNKNOWN',
 307     ],
 308
 309     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 310         'UNKNOWN',
 311     ],
 312     'third_party/simplejson': [
 313         'UNKNOWN',
 314     ],
 315     'third_party/skia': [  # http://crbug.com/98463
 316         'UNKNOWN',
 317     ],
 318     'third_party/snappy/src': [  # http://crbug.com/98464
 319         'UNKNOWN',
 320     ],
 321     'third_party/smhasher/src': [  # http://crbug.com/98465
 322         'UNKNOWN',
 323     ],
 324     'third_party/speech-dispatcher/libspeechd.h': [
 325         'GPL (v2 or later)',
 326     ],
 327     'third_party/sqlite': [
 328         'UNKNOWN',
 329     ],
 330
 331     # http://crbug.com/334668
 332     # MIT license.
 333     'tools/swarming_client/third_party/httplib2': [
 334         'UNKNOWN',
 335     ],
 336
 337     # http://crbug.com/334668
 338     # Apache v2.0.
 339     'tools/swarming_client/third_party/oauth2client': [
 340         'UNKNOWN',
 341     ],
 342
 343     # https://github.com/kennethreitz/requests/issues/1610
 344     'tools/swarming_client/third_party/requests': [
 345         'UNKNOWN',
 346     ],
 347
 348     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 349         'UNKNOWN',
 350     ],
 351     'third_party/talloc': [
 352         'GPL (v3 or later)',
 353         'UNKNOWN',  # http://crbug.com/98588
 354     ],
 355     'third_party/tcmalloc': [
 356         'UNKNOWN',  # http://crbug.com/98589
 357     ],
 358     'third_party/tlslite': [
 359         'UNKNOWN',
 360     ],
 361     'third_party/webdriver': [  # http://crbug.com/98590
 362         'UNKNOWN',
 363     ],
 364
 365     # https://github.com/html5lib/html5lib-python/issues/125
 366     # https://github.com/KhronosGroup/WebGL/issues/435
 367     'third_party/webgl/src': [
 368         'UNKNOWN',
 369     ],
 370
 371     'third_party/webrtc': [  # http://crbug.com/98592
 372         'UNKNOWN',
 373     ],
 374     'third_party/xdg-utils': [  # http://crbug.com/98593
 375         'UNKNOWN',
 376     ],
 377     'third_party/yasm/source': [  # http://crbug.com/98594
 378         'UNKNOWN',
 379     ],
 380     'third_party/zlib/contrib/minizip': [
 381         'UNKNOWN',
 382     ],
 383     'third_party/zlib/trees.h': [
 384         'UNKNOWN',
 385     ],
 386     'tools/emacs': [  # http://crbug.com/98595
 387         'UNKNOWN',
 388     ],
 389     'tools/gyp/test': [
 390         'UNKNOWN',
 391     ],
 392     'tools/python/google/__init__.py': [
 393         'UNKNOWN',
 394     ],
 395     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 396         'UNKNOWN',
 397     ],
 398     'tools/symsrc/pefile.py': [
 399         'UNKNOWN',
 400     ],
 401     # Not shipped, downloaded on trybots sometimes.
 402     'tools/telemetry/third_party/gsutil': [
 403         'BSD MIT/X11 (BSD like)',
 404         'UNKNOWN',
 405     ],
 406     'tools/telemetry/third_party/pyserial': [
 407         # https://sourceforge.net/p/pyserial/feature-requests/35/
 408         'UNKNOWN',
 409     ],
 410     'v8/test/cctest': [  # http://crbug.com/98597
 411         'UNKNOWN',
 412     ],
 413     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 414         'UNKNOWN',
 415     ],
 416 }
 417
 418
 419 def check_licenses(options, args):
 420   # Figure out which directory we have to check.
 421   if len(args) == 0:
 422     # No directory to check specified, use the repository root.
 423     start_dir = options.base_directory
 424   elif len(args) == 1:
 425     # Directory specified. Start here. It's supposed to be relative to the
 426     # base directory.
 427     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 428   else:
 429     # More than one argument, we don't handle this.
 430     PrintUsage()
 431     return 1
 432
 433   print "Using base directory:", options.base_directory
 434   print "Checking:", start_dir
 435   print
 436
 437   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 438                                                    'third_party',
 439                                                    'devscripts',
 440                                                    'licensecheck.pl'))
 441
 442   licensecheck = subprocess.Popen([licensecheck_path,
 443                                    '-l', '100',
 444                                    '-r', start_dir],
 445                                   stdout=subprocess.PIPE,
 446                                   stderr=subprocess.PIPE)
 447   stdout, stderr = licensecheck.communicate()
 448   if options.verbose:
 449     print '----------- licensecheck stdout -----------'
 450     print stdout
 451     print '--------- end licensecheck stdout ---------'
 452   if licensecheck.returncode != 0 or stderr:
 453     print '----------- licensecheck stderr -----------'
 454     print stderr
 455     print '--------- end licensecheck stderr ---------'
 456     print "\nFAILED\n"
 457     return 1
 458
 459   used_suppressions = set()
 460   errors = []
 461
 462   for line in stdout.splitlines():
 463     filename, license = line.split(':', 1)
 464     filename = os.path.relpath(filename.strip(), options.base_directory)
 465
 466     # All files in the build output directory are generated one way or another.
 467     # There's no need to check them.
 468     if filename.startswith('out/'):
 469       continue
 470
 471     # For now we're just interested in the license.
 472     license = license.replace('*No copyright*', '').strip()
 473
 474     # Skip generated files.
 475     if 'GENERATED FILE' in license:
 476       continue
 477
 478     if license in WHITELISTED_LICENSES:
 479       continue
 480
 481     if not options.ignore_suppressions:
 482       matched_prefixes = [
 483           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 484           if filename.startswith(prefix) and
 485           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 486       if matched_prefixes:
 487         used_suppressions.update(set(matched_prefixes))
 488         continue
 489
 490     errors.append({'filename': filename, 'license': license})
 491
 492   if options.json:
 493     with open(options.json, 'w') as f:
 494       json.dump(errors, f)
 495
 496   if errors:
 497     for error in errors:
 498       print "'%s' has non-whitelisted license '%s'" % (
 499           error['filename'], error['license'])
 500     print "\nFAILED\n"
 501     print "Please read",
 502     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 503     print "for more info how to handle the failure."
 504     print
 505     print "Please respect OWNERS of checklicenses.py. Changes violating"
 506     print "this requirement may be reverted."
 507
 508     # Do not print unused suppressions so that above message is clearly
 509     # visible and gets proper attention. Too much unrelated output
 510     # would be distracting and make the important points easier to miss.
 511
 512     return 1
 513
 514   print "\nSUCCESS\n"
 515
 516   if not len(args):
 517     unused_suppressions = set(
 518         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 519             used_suppressions)
 520     if unused_suppressions:
 521       print "\nNOTE: unused suppressions detected:\n"
 522       print '\n'.join(unused_suppressions)
 523
 524   return 0
 525
 526
 527 def main():
 528   default_root = os.path.abspath(
 529       os.path.join(os.path.dirname(__file__), '..', '..'))
 530   option_parser = optparse.OptionParser()
 531   option_parser.add_option('--root', default=default_root,
 532                            dest='base_directory',
 533                            help='Specifies the repository root. This defaults '
 534                            'to "../.." relative to the script file, which '
 535                            'will normally be the repository root.')
 536   option_parser.add_option('-v', '--verbose', action='store_true',
 537                            default=False, help='Print debug logging')
 538   option_parser.add_option('--ignore-suppressions',
 539                            action='store_true',
 540                            default=False,
 541                            help='Ignore path-specific license whitelist.')
 542   option_parser.add_option('--json', help='Path to JSON output file')
 543   options, args = option_parser.parse_args()
 544   return check_licenses(options, args)
 545
 546
 547 if '__main__' == __name__:
 548   sys.exit(main())