tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 116         'UNKNOWN',
 117     ],
 118     # This contains files copied from elsewhere from the tree. Since the copied
 119     # directories might have suppressions below (like simplejson), whitelist the
 120     # whole directory. This is also not shipped code.
 121     'chrome/common/extensions/docs/server2/third_party': [
 122         'UNKNOWN',
 123     ],
 124     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 125         'UNKNOWN',
 126     ],
 127     'native_client': [  # http://crbug.com/98099
 128         'UNKNOWN',
 129     ],
 130     'native_client/toolchain': [
 131         'BSD GPL (v2 or later)',
 132         'BSD (2 clause) GPL (v2 or later)',
 133         'BSD (3 clause) GPL (v2 or later)',
 134         'BSL (v1.0) GPL',
 135         'BSL (v1.0) GPL (v3.1)',
 136         'GPL',
 137         'GPL (unversioned/unknown version)',
 138         'GPL (v2)',
 139         'GPL (v2 or later)',
 140         'GPL (v3.1)',
 141         'GPL (v3 or later)',
 142     ],
 143     'third_party/WebKit': [
 144         'UNKNOWN',
 145     ],
 146
 147     # http://code.google.com/p/angleproject/issues/detail?id=217
 148     'third_party/angle': [
 149         'UNKNOWN',
 150     ],
 151
 152     # http://crbug.com/222828
 153     # http://bugs.python.org/issue17514
 154     'third_party/chromite/third_party/argparse.py': [
 155         'UNKNOWN',
 156     ],
 157
 158     # http://crbug.com/326117
 159     # https://bitbucket.org/chrisatlee/poster/issue/21
 160     'third_party/chromite/third_party/poster': [
 161         'UNKNOWN',
 162     ],
 163
 164     # http://crbug.com/333508
 165     'third_party/clang_format/script': [
 166         'UNKNOWN',
 167     ],
 168
 169     # http://crbug.com/333508
 170     'buildtools/clang_format/script': [
 171         'UNKNOWN',
 172     ],
 173
 174     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 175     'third_party/cython': [
 176         'UNKNOWN',
 177     ],
 178
 179     'third_party/devscripts': [
 180         'GPL (v2 or later)',
 181     ],
 182     'third_party/expat/files/lib': [  # http://crbug.com/98121
 183         'UNKNOWN',
 184     ],
 185     'third_party/ffmpeg': [
 186         'GPL',
 187         'GPL (v2)',
 188         'GPL (v2 or later)',
 189         'GPL (v3 or later)',
 190         'UNKNOWN',  # http://crbug.com/98123
 191     ],
 192     'third_party/fontconfig': [
 193         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 194         'UNKNOWN',
 195     ],
 196     'third_party/freetype2': [ # http://crbug.com/177319
 197         'UNKNOWN',
 198     ],
 199     'third_party/hunspell': [  # http://crbug.com/98134
 200         'UNKNOWN',
 201     ],
 202     'third_party/iccjpeg': [  # http://crbug.com/98137
 203         'UNKNOWN',
 204     ],
 205     'third_party/icu': [  # http://crbug.com/98301
 206         'UNKNOWN',
 207     ],
 208     'third_party/lcov': [  # http://crbug.com/98304
 209         'UNKNOWN',
 210     ],
 211     'third_party/lcov/contrib/galaxy/genflat.pl': [
 212         'GPL (v2 or later)',
 213     ],
 214     'third_party/libc++/trunk/include/support/solaris': [
 215         # http://llvm.org/bugs/show_bug.cgi?id=18291
 216         'UNKNOWN',
 217     ],
 218     'third_party/libc++/trunk/src/support/solaris/xlocale.c': [
 219         # http://llvm.org/bugs/show_bug.cgi?id=18291
 220         'UNKNOWN',
 221     ],
 222     'third_party/libc++/trunk/test': [
 223         # http://llvm.org/bugs/show_bug.cgi?id=18291
 224         'UNKNOWN',
 225     ],
 226     'third_party/libevent': [  # http://crbug.com/98309
 227         'UNKNOWN',
 228     ],
 229     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 230         'UNKNOWN',
 231     ],
 232     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 233         'UNKNOWN',
 234     ],
 235
 236     # Many liblouis files are mirrored but not used in the NaCl module.
 237     # They are not excluded from the mirror because of lack of infrastructure
 238     # support.  Getting license headers added to the files where missing is
 239     # tracked in https://github.com/liblouis/liblouis/issues/22.
 240     'third_party/liblouis/src': [
 241         'GPL (v3 or later)',
 242         'UNKNOWN',
 243     ],
 244
 245     'third_party/libpng': [  # http://crbug.com/98318
 246         'UNKNOWN',
 247     ],
 248
 249     # The following files lack license headers, but are trivial.
 250     'third_party/libusb/src/libusb/os/poll_posix.h': [
 251         'UNKNOWN',
 252     ],
 253
 254     'third_party/libvpx/source': [  # http://crbug.com/98319
 255         'UNKNOWN',
 256     ],
 257     'third_party/libxml': [
 258         'UNKNOWN',
 259     ],
 260     'third_party/libxslt': [
 261         'UNKNOWN',
 262     ],
 263     'third_party/lzma_sdk': [
 264         'UNKNOWN',
 265     ],
 266     'third_party/mesa/src': [
 267         'GPL (v2)',
 268         'GPL (v3 or later)',
 269         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 270         'UNKNOWN',  # http://crbug.com/98450
 271     ],
 272     'third_party/modp_b64': [
 273         'UNKNOWN',
 274     ],
 275     'third_party/openmax_dl/dl' : [
 276         'Khronos Group',
 277     ],
 278     'third_party/openssl': [  # http://crbug.com/98451
 279         'UNKNOWN',
 280     ],
 281     'third_party/boringssl': [
 282         # There are some files in BoringSSL which came from OpenSSL and have no
 283         # license in them. We don't wish to add the license header ourselves
 284         # thus we don't expect to pass license checks.
 285         'UNKNOWN',
 286     ],
 287     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 288         'UNKNOWN',
 289     ],
 290     'third_party/molokocacao': [  # http://crbug.com/98453
 291         'UNKNOWN',
 292     ],
 293     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 294         'UNKNOWN',
 295     ],
 296     'third_party/ply/__init__.py': [
 297         'UNKNOWN',
 298     ],
 299     'third_party/protobuf': [  # http://crbug.com/98455
 300         'UNKNOWN',
 301     ],
 302
 303     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 304     # BSD 2-clause license.
 305     'third_party/pycoverage': [
 306         'UNKNOWN',
 307     ],
 308
 309     'third_party/pyelftools': [ # http://crbug.com/222831
 310         'UNKNOWN',
 311     ],
 312     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 313         'UNKNOWN',
 314     ],
 315     'third_party/simplejson': [
 316         'UNKNOWN',
 317     ],
 318     'third_party/skia': [  # http://crbug.com/98463
 319         'UNKNOWN',
 320     ],
 321     'third_party/snappy/src': [  # http://crbug.com/98464
 322         'UNKNOWN',
 323     ],
 324     'third_party/smhasher/src': [  # http://crbug.com/98465
 325         'UNKNOWN',
 326     ],
 327     'third_party/speech-dispatcher/libspeechd.h': [
 328         'GPL (v2 or later)',
 329     ],
 330     'third_party/sqlite': [
 331         'UNKNOWN',
 332     ],
 333
 334     # http://crbug.com/334668
 335     # MIT license.
 336     'tools/swarming_client/third_party/httplib2': [
 337         'UNKNOWN',
 338     ],
 339
 340     # http://crbug.com/334668
 341     # Apache v2.0.
 342     'tools/swarming_client/third_party/oauth2client': [
 343         'UNKNOWN',
 344     ],
 345
 346     # https://github.com/kennethreitz/requests/issues/1610
 347     'tools/swarming_client/third_party/requests': [
 348         'UNKNOWN',
 349     ],
 350
 351     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 352         'UNKNOWN',
 353     ],
 354     'third_party/talloc': [
 355         'GPL (v3 or later)',
 356         'UNKNOWN',  # http://crbug.com/98588
 357     ],
 358     'third_party/tcmalloc': [
 359         'UNKNOWN',  # http://crbug.com/98589
 360     ],
 361     'third_party/tlslite': [
 362         'UNKNOWN',
 363     ],
 364     'third_party/webdriver': [  # http://crbug.com/98590
 365         'UNKNOWN',
 366     ],
 367
 368     # https://github.com/html5lib/html5lib-python/issues/125
 369     # https://github.com/KhronosGroup/WebGL/issues/435
 370     'third_party/webgl/src': [
 371         'UNKNOWN',
 372     ],
 373
 374     'third_party/webrtc': [  # http://crbug.com/98592
 375         'UNKNOWN',
 376     ],
 377     'third_party/xdg-utils': [  # http://crbug.com/98593
 378         'UNKNOWN',
 379     ],
 380     'third_party/yasm/source': [  # http://crbug.com/98594
 381         'UNKNOWN',
 382     ],
 383     'third_party/zlib/contrib/minizip': [
 384         'UNKNOWN',
 385     ],
 386     'third_party/zlib/trees.h': [
 387         'UNKNOWN',
 388     ],
 389     'tools/emacs': [  # http://crbug.com/98595
 390         'UNKNOWN',
 391     ],
 392     'tools/gyp/test': [
 393         'UNKNOWN',
 394     ],
 395     'tools/python/google/__init__.py': [
 396         'UNKNOWN',
 397     ],
 398     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 399         'UNKNOWN',
 400     ],
 401     'tools/symsrc/pefile.py': [
 402         'UNKNOWN',
 403     ],
 404     # Not shipped, downloaded on trybots sometimes.
 405     'tools/telemetry/third_party/gsutil': [
 406         'BSD MIT/X11 (BSD like)',
 407         'UNKNOWN',
 408     ],
 409     'tools/telemetry/third_party/pyserial': [
 410         # https://sourceforge.net/p/pyserial/feature-requests/35/
 411         'UNKNOWN',
 412     ],
 413     'v8/test/cctest': [  # http://crbug.com/98597
 414         'UNKNOWN',
 415     ],
 416     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 417         'UNKNOWN',
 418     ],
 419 }
 420
 421
 422 def check_licenses(options, args):
 423   # Figure out which directory we have to check.
 424   if len(args) == 0:
 425     # No directory to check specified, use the repository root.
 426     start_dir = options.base_directory
 427   elif len(args) == 1:
 428     # Directory specified. Start here. It's supposed to be relative to the
 429     # base directory.
 430     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 431   else:
 432     # More than one argument, we don't handle this.
 433     PrintUsage()
 434     return 1
 435
 436   print "Using base directory:", options.base_directory
 437   print "Checking:", start_dir
 438   print
 439
 440   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 441                                                    'third_party',
 442                                                    'devscripts',
 443                                                    'licensecheck.pl'))
 444
 445   licensecheck = subprocess.Popen([licensecheck_path,
 446                                    '-l', '100',
 447                                    '-r', start_dir],
 448                                   stdout=subprocess.PIPE,
 449                                   stderr=subprocess.PIPE)
 450   stdout, stderr = licensecheck.communicate()
 451   if options.verbose:
 452     print '----------- licensecheck stdout -----------'
 453     print stdout
 454     print '--------- end licensecheck stdout ---------'
 455   if licensecheck.returncode != 0 or stderr:
 456     print '----------- licensecheck stderr -----------'
 457     print stderr
 458     print '--------- end licensecheck stderr ---------'
 459     print "\nFAILED\n"
 460     return 1
 461
 462   used_suppressions = set()
 463   errors = []
 464
 465   for line in stdout.splitlines():
 466     filename, license = line.split(':', 1)
 467     filename = os.path.relpath(filename.strip(), options.base_directory)
 468
 469     # All files in the build output directory are generated one way or another.
 470     # There's no need to check them.
 471     if filename.startswith('out/'):
 472       continue
 473
 474     # For now we're just interested in the license.
 475     license = license.replace('*No copyright*', '').strip()
 476
 477     # Skip generated files.
 478     if 'GENERATED FILE' in license:
 479       continue
 480
 481     if license in WHITELISTED_LICENSES:
 482       continue
 483
 484     if not options.ignore_suppressions:
 485       matched_prefixes = [
 486           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 487           if filename.startswith(prefix) and
 488           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 489       if matched_prefixes:
 490         used_suppressions.update(set(matched_prefixes))
 491         continue
 492
 493     errors.append({'filename': filename, 'license': license})
 494
 495   if options.json:
 496     with open(options.json, 'w') as f:
 497       json.dump(errors, f)
 498
 499   if errors:
 500     for error in errors:
 501       print "'%s' has non-whitelisted license '%s'" % (
 502           error['filename'], error['license'])
 503     print "\nFAILED\n"
 504     print "Please read",
 505     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 506     print "for more info how to handle the failure."
 507     print
 508     print "Please respect OWNERS of checklicenses.py. Changes violating"
 509     print "this requirement may be reverted."
 510
 511     # Do not print unused suppressions so that above message is clearly
 512     # visible and gets proper attention. Too much unrelated output
 513     # would be distracting and make the important points easier to miss.
 514
 515     return 1
 516
 517   print "\nSUCCESS\n"
 518
 519   if not len(args):
 520     unused_suppressions = set(
 521         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 522             used_suppressions)
 523     if unused_suppressions:
 524       print "\nNOTE: unused suppressions detected:\n"
 525       print '\n'.join(unused_suppressions)
 526
 527   return 0
 528
 529
 530 def main():
 531   default_root = os.path.abspath(
 532       os.path.join(os.path.dirname(__file__), '..', '..'))
 533   option_parser = optparse.OptionParser()
 534   option_parser.add_option('--root', default=default_root,
 535                            dest='base_directory',
 536                            help='Specifies the repository root. This defaults '
 537                            'to "../.." relative to the script file, which '
 538                            'will normally be the repository root.')
 539   option_parser.add_option('-v', '--verbose', action='store_true',
 540                            default=False, help='Print debug logging')
 541   option_parser.add_option('--ignore-suppressions',
 542                            action='store_true',
 543                            default=False,
 544                            help='Ignore path-specific license whitelist.')
 545   option_parser.add_option('--json', help='Path to JSON output file')
 546   options, args = option_parser.parse_args()
 547   return check_licenses(options, args)
 548
 549
 550 if '__main__' == __name__:
 551   sys.exit(main())