tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'buildtools/third_party/libc++/trunk/test': [
 116         # http://llvm.org/bugs/show_bug.cgi?id=18291
 117         'UNKNOWN',
 118     ],
 119
 120     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 121         'UNKNOWN',
 122     ],
 123     # This contains files copied from elsewhere from the tree. Since the copied
 124     # directories might have suppressions below (like simplejson), whitelist the
 125     # whole directory. This is also not shipped code.
 126     'chrome/common/extensions/docs/server2/third_party': [
 127         'UNKNOWN',
 128     ],
 129     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 130         'UNKNOWN',
 131     ],
 132     'native_client': [  # http://crbug.com/98099
 133         'UNKNOWN',
 134     ],
 135     'native_client/toolchain': [
 136         'BSD GPL (v2 or later)',
 137         'BSD (2 clause) GPL (v2 or later)',
 138         'BSD (3 clause) GPL (v2 or later)',
 139         'BSL (v1.0) GPL',
 140         'BSL (v1.0) GPL (v3.1)',
 141         'GPL',
 142         'GPL (unversioned/unknown version)',
 143         'GPL (v2)',
 144         'GPL (v2 or later)',
 145         'GPL (v3.1)',
 146         'GPL (v3 or later)',
 147     ],
 148     'third_party/WebKit': [
 149         'UNKNOWN',
 150     ],
 151
 152     # http://code.google.com/p/angleproject/issues/detail?id=217
 153     'third_party/angle': [
 154         'UNKNOWN',
 155     ],
 156
 157     # http://crbug.com/222828
 158     # http://bugs.python.org/issue17514
 159     'third_party/chromite/third_party/argparse.py': [
 160         'UNKNOWN',
 161     ],
 162
 163     # http://crbug.com/326117
 164     # https://bitbucket.org/chrisatlee/poster/issue/21
 165     'third_party/chromite/third_party/poster': [
 166         'UNKNOWN',
 167     ],
 168
 169     # http://crbug.com/333508
 170     'third_party/clang_format/script': [
 171         'UNKNOWN',
 172     ],
 173
 174     # http://crbug.com/333508
 175     'buildtools/clang_format/script': [
 176         'UNKNOWN',
 177     ],
 178
 179     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 180     'third_party/cython': [
 181         'UNKNOWN',
 182     ],
 183
 184     'third_party/devscripts': [
 185         'GPL (v2 or later)',
 186     ],
 187     'third_party/expat/files/lib': [  # http://crbug.com/98121
 188         'UNKNOWN',
 189     ],
 190     'third_party/ffmpeg': [
 191         'GPL',
 192         'GPL (v2)',
 193         'GPL (v2 or later)',
 194         'GPL (v3 or later)',
 195         'UNKNOWN',  # http://crbug.com/98123
 196     ],
 197     'third_party/fontconfig': [
 198         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 199         'UNKNOWN',
 200     ],
 201     'third_party/freetype2': [ # http://crbug.com/177319
 202         'UNKNOWN',
 203     ],
 204     'third_party/hunspell': [  # http://crbug.com/98134
 205         'UNKNOWN',
 206     ],
 207     'third_party/iccjpeg': [  # http://crbug.com/98137
 208         'UNKNOWN',
 209     ],
 210     'third_party/icu': [  # http://crbug.com/98301
 211         'UNKNOWN',
 212     ],
 213     'third_party/lcov': [  # http://crbug.com/98304
 214         'UNKNOWN',
 215     ],
 216     'third_party/lcov/contrib/galaxy/genflat.pl': [
 217         'GPL (v2 or later)',
 218     ],
 219     'third_party/libevent': [  # http://crbug.com/98309
 220         'UNKNOWN',
 221     ],
 222     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 223         'UNKNOWN',
 224     ],
 225     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 226         'UNKNOWN',
 227     ],
 228
 229     # Many liblouis files are mirrored but not used in the NaCl module.
 230     # They are not excluded from the mirror because of lack of infrastructure
 231     # support.  Getting license headers added to the files where missing is
 232     # tracked in https://github.com/liblouis/liblouis/issues/22.
 233     'third_party/liblouis/src': [
 234         'GPL (v3 or later)',
 235         'UNKNOWN',
 236     ],
 237
 238     'third_party/libpng': [  # http://crbug.com/98318
 239         'UNKNOWN',
 240     ],
 241
 242     # The following files lack license headers, but are trivial.
 243     'third_party/libusb/src/libusb/os/poll_posix.h': [
 244         'UNKNOWN',
 245     ],
 246
 247     'third_party/libvpx/source': [  # http://crbug.com/98319
 248         'UNKNOWN',
 249     ],
 250     'third_party/libxml': [
 251         'UNKNOWN',
 252     ],
 253     'third_party/libxslt': [
 254         'UNKNOWN',
 255     ],
 256     'third_party/lzma_sdk': [
 257         'UNKNOWN',
 258     ],
 259     'third_party/mesa/src': [
 260         'GPL (v2)',
 261         'GPL (v3 or later)',
 262         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 263         'UNKNOWN',  # http://crbug.com/98450
 264     ],
 265     'third_party/modp_b64': [
 266         'UNKNOWN',
 267     ],
 268     'third_party/openmax_dl/dl' : [
 269         'Khronos Group',
 270     ],
 271     'third_party/openssl': [  # http://crbug.com/98451
 272         'UNKNOWN',
 273     ],
 274     'third_party/boringssl': [
 275         # There are some files in BoringSSL which came from OpenSSL and have no
 276         # license in them. We don't wish to add the license header ourselves
 277         # thus we don't expect to pass license checks.
 278         'UNKNOWN',
 279     ],
 280     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 281         'UNKNOWN',
 282     ],
 283     'third_party/molokocacao': [  # http://crbug.com/98453
 284         'UNKNOWN',
 285     ],
 286     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 287         'UNKNOWN',
 288     ],
 289     'third_party/protobuf': [  # http://crbug.com/98455
 290         'UNKNOWN',
 291     ],
 292
 293     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 294     # BSD 2-clause license.
 295     'third_party/pycoverage': [
 296         'UNKNOWN',
 297     ],
 298
 299     'third_party/pyelftools': [ # http://crbug.com/222831
 300         'UNKNOWN',
 301     ],
 302     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 303         'UNKNOWN',
 304     ],
 305     'third_party/simplejson': [
 306         'UNKNOWN',
 307     ],
 308     'third_party/skia': [  # http://crbug.com/98463
 309         'UNKNOWN',
 310     ],
 311     'third_party/snappy/src': [  # http://crbug.com/98464
 312         'UNKNOWN',
 313     ],
 314     'third_party/smhasher/src': [  # http://crbug.com/98465
 315         'UNKNOWN',
 316     ],
 317     'third_party/speech-dispatcher/libspeechd.h': [
 318         'GPL (v2 or later)',
 319     ],
 320     'third_party/sqlite': [
 321         'UNKNOWN',
 322     ],
 323
 324     # http://crbug.com/334668
 325     # MIT license.
 326     'tools/swarming_client/third_party/httplib2': [
 327         'UNKNOWN',
 328     ],
 329
 330     # http://crbug.com/334668
 331     # Apache v2.0.
 332     'tools/swarming_client/third_party/oauth2client': [
 333         'UNKNOWN',
 334     ],
 335
 336     # https://github.com/kennethreitz/requests/issues/1610
 337     'tools/swarming_client/third_party/requests': [
 338         'UNKNOWN',
 339     ],
 340
 341     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 342         'UNKNOWN',
 343     ],
 344     'third_party/talloc': [
 345         'GPL (v3 or later)',
 346         'UNKNOWN',  # http://crbug.com/98588
 347     ],
 348     'third_party/tcmalloc': [
 349         'UNKNOWN',  # http://crbug.com/98589
 350     ],
 351     'third_party/tlslite': [
 352         'UNKNOWN',
 353     ],
 354     'third_party/webdriver': [  # http://crbug.com/98590
 355         'UNKNOWN',
 356     ],
 357
 358     # https://github.com/html5lib/html5lib-python/issues/125
 359     # https://github.com/KhronosGroup/WebGL/issues/435
 360     'third_party/webgl/src': [
 361         'UNKNOWN',
 362     ],
 363
 364     'third_party/webrtc': [  # http://crbug.com/98592
 365         'UNKNOWN',
 366     ],
 367     'third_party/xdg-utils': [  # http://crbug.com/98593
 368         'UNKNOWN',
 369     ],
 370     'third_party/yasm/source': [  # http://crbug.com/98594
 371         'UNKNOWN',
 372     ],
 373     'third_party/zlib/contrib/minizip': [
 374         'UNKNOWN',
 375     ],
 376     'third_party/zlib/trees.h': [
 377         'UNKNOWN',
 378     ],
 379     'tools/emacs': [  # http://crbug.com/98595
 380         'UNKNOWN',
 381     ],
 382     'tools/gyp/test': [
 383         'UNKNOWN',
 384     ],
 385     'tools/python/google/__init__.py': [
 386         'UNKNOWN',
 387     ],
 388     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 389         'UNKNOWN',
 390     ],
 391     'tools/symsrc/pefile.py': [
 392         'UNKNOWN',
 393     ],
 394     # Not shipped, downloaded on trybots sometimes.
 395     'tools/telemetry/third_party/gsutil': [
 396         'BSD MIT/X11 (BSD like)',
 397         'UNKNOWN',
 398     ],
 399     'tools/telemetry/third_party/pyserial': [
 400         # https://sourceforge.net/p/pyserial/feature-requests/35/
 401         'UNKNOWN',
 402     ],
 403     'v8/test/cctest': [  # http://crbug.com/98597
 404         'UNKNOWN',
 405     ],
 406     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 407         'UNKNOWN',
 408     ],
 409 }
 410
 411
 412 def check_licenses(options, args):
 413   # Figure out which directory we have to check.
 414   if len(args) == 0:
 415     # No directory to check specified, use the repository root.
 416     start_dir = options.base_directory
 417   elif len(args) == 1:
 418     # Directory specified. Start here. It's supposed to be relative to the
 419     # base directory.
 420     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 421   else:
 422     # More than one argument, we don't handle this.
 423     PrintUsage()
 424     return 1
 425
 426   print "Using base directory:", options.base_directory
 427   print "Checking:", start_dir
 428   print
 429
 430   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 431                                                    'third_party',
 432                                                    'devscripts',
 433                                                    'licensecheck.pl'))
 434
 435   licensecheck = subprocess.Popen([licensecheck_path,
 436                                    '-l', '100',
 437                                    '-r', start_dir],
 438                                   stdout=subprocess.PIPE,
 439                                   stderr=subprocess.PIPE)
 440   stdout, stderr = licensecheck.communicate()
 441   if options.verbose:
 442     print '----------- licensecheck stdout -----------'
 443     print stdout
 444     print '--------- end licensecheck stdout ---------'
 445   if licensecheck.returncode != 0 or stderr:
 446     print '----------- licensecheck stderr -----------'
 447     print stderr
 448     print '--------- end licensecheck stderr ---------'
 449     print "\nFAILED\n"
 450     return 1
 451
 452   used_suppressions = set()
 453   errors = []
 454
 455   for line in stdout.splitlines():
 456     filename, license = line.split(':', 1)
 457     filename = os.path.relpath(filename.strip(), options.base_directory)
 458
 459     # All files in the build output directory are generated one way or another.
 460     # There's no need to check them.
 461     if filename.startswith('out/'):
 462       continue
 463
 464     # For now we're just interested in the license.
 465     license = license.replace('*No copyright*', '').strip()
 466
 467     # Skip generated files.
 468     if 'GENERATED FILE' in license:
 469       continue
 470
 471     if license in WHITELISTED_LICENSES:
 472       continue
 473
 474     if not options.ignore_suppressions:
 475       matched_prefixes = [
 476           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 477           if filename.startswith(prefix) and
 478           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 479       if matched_prefixes:
 480         used_suppressions.update(set(matched_prefixes))
 481         continue
 482
 483     errors.append({'filename': filename, 'license': license})
 484
 485   if options.json:
 486     with open(options.json, 'w') as f:
 487       json.dump(errors, f)
 488
 489   if errors:
 490     for error in errors:
 491       print "'%s' has non-whitelisted license '%s'" % (
 492           error['filename'], error['license'])
 493     print "\nFAILED\n"
 494     print "Please read",
 495     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 496     print "for more info how to handle the failure."
 497     print
 498     print "Please respect OWNERS of checklicenses.py. Changes violating"
 499     print "this requirement may be reverted."
 500
 501     # Do not print unused suppressions so that above message is clearly
 502     # visible and gets proper attention. Too much unrelated output
 503     # would be distracting and make the important points easier to miss.
 504
 505     return 1
 506
 507   print "\nSUCCESS\n"
 508
 509   if not len(args):
 510     unused_suppressions = set(
 511         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 512             used_suppressions)
 513     if unused_suppressions:
 514       print "\nNOTE: unused suppressions detected:\n"
 515       print '\n'.join(unused_suppressions)
 516
 517   return 0
 518
 519
 520 def main():
 521   default_root = os.path.abspath(
 522       os.path.join(os.path.dirname(__file__), '..', '..'))
 523   option_parser = optparse.OptionParser()
 524   option_parser.add_option('--root', default=default_root,
 525                            dest='base_directory',
 526                            help='Specifies the repository root. This defaults '
 527                            'to "../.." relative to the script file, which '
 528                            'will normally be the repository root.')
 529   option_parser.add_option('-v', '--verbose', action='store_true',
 530                            default=False, help='Print debug logging')
 531   option_parser.add_option('--ignore-suppressions',
 532                            action='store_true',
 533                            default=False,
 534                            help='Ignore path-specific license whitelist.')
 535   option_parser.add_option('--json', help='Path to JSON output file')
 536   options, args = option_parser.parse_args()
 537   return check_licenses(options, args)
 538
 539
 540 if '__main__' == __name__:
 541   sys.exit(main())