tools/checklicenses/checklicenses.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Makes sure that all files contain proper licensing information."""
   7
   8
   9 import json
  10 import optparse
  11 import os.path
  12 import subprocess
  13 import sys
  14
  15
  16 def PrintUsage():
  17   print """Usage: python checklicenses.py [--root <root>] [tocheck]
  18   --root   Specifies the repository root. This defaults to "../.." relative
  19            to the script file. This will be correct given the normal location
  20            of the script in "<root>/tools/checklicenses".
  21
  22   --ignore-suppressions  Ignores path-specific license whitelist. Useful when
  23                          trying to remove a suppression/whitelist entry.
  24
  25   tocheck  Specifies the directory, relative to root, to check. This defaults
  26            to "." so it checks everything.
  27
  28 Examples:
  29   python checklicenses.py
  30   python checklicenses.py --root ~/chromium/src third_party"""
  31
  32
  33 WHITELISTED_LICENSES = [
  34     'Anti-Grain Geometry',
  35     'Apache (v2.0)',
  36     'Apache (v2.0) BSD (2 clause)',
  37     'Apache (v2.0) GPL (v2)',
  38     'Apple MIT',  # https://fedoraproject.org/wiki/Licensing/Apple_MIT_License
  39     'APSL (v2)',
  40     'APSL (v2) BSD (4 clause)',
  41     'BSD',
  42     'BSD (2 clause)',
  43     'BSD (2 clause) ISC',
  44     'BSD (2 clause) MIT/X11 (BSD like)',
  45     'BSD (3 clause)',
  46     'BSD (3 clause) GPL (v2)',
  47     'BSD (3 clause) ISC',
  48     'BSD (3 clause) LGPL (v2 or later)',
  49     'BSD (3 clause) LGPL (v2.1 or later)',
  50     'BSD (3 clause) MIT/X11 (BSD like)',
  51     'BSD (4 clause)',
  52     'BSD-like',
  53
  54     # TODO(phajdan.jr): Make licensecheck not print BSD-like twice.
  55     'BSD-like MIT/X11 (BSD like)',
  56
  57     'BSL (v1.0)',
  58     'FreeType (BSD like)',
  59     'FreeType (BSD like) with patent clause',
  60     'GPL (v2) LGPL (v2.1 or later)',
  61     'GPL (v2 or later) with Bison parser exception',
  62     'GPL (v2 or later) with libtool exception',
  63     'GPL (v3 or later) with Bison parser exception',
  64     'GPL with Bison parser exception',
  65     'Independent JPEG Group License',
  66     'ISC',
  67     'LGPL (unversioned/unknown version)',
  68     'LGPL (v2)',
  69     'LGPL (v2 or later)',
  70     'LGPL (v2.1)',
  71     'LGPL (v2.1 or later)',
  72     'LGPL (v3 or later)',
  73     'MIT/X11 (BSD like)',
  74     'MIT/X11 (BSD like) LGPL (v2.1 or later)',
  75     'MPL (v1.0) LGPL (v2 or later)',
  76     'MPL (v1.1)',
  77     'MPL (v1.1) BSD (3 clause) GPL (v2) LGPL (v2.1 or later)',
  78     'MPL (v1.1) BSD (3 clause) LGPL (v2.1 or later)',
  79     'MPL (v1.1) BSD-like',
  80     'MPL (v1.1) BSD-like GPL (unversioned/unknown version)',
  81     'MPL (v1.1) BSD-like GPL (v2) LGPL (v2.1 or later)',
  82     'MPL (v1.1) GPL (v2)',
  83     'MPL (v1.1) GPL (v2) LGPL (v2 or later)',
  84     'MPL (v1.1) GPL (v2) LGPL (v2.1 or later)',
  85     'MPL (v1.1) GPL (unversioned/unknown version)',
  86     'MPL (v1.1) LGPL (v2 or later)',
  87     'MPL (v1.1) LGPL (v2.1 or later)',
  88     'MPL (v2.0)',
  89     'Ms-PL',
  90     'Public domain',
  91     'Public domain BSD',
  92     'Public domain BSD (3 clause)',
  93     'Public domain BSD-like',
  94     'Public domain LGPL (v2.1 or later)',
  95     'libpng',
  96     'zlib/libpng',
  97     'SGI Free Software License B',
  98     'SunSoft (BSD like)',
  99     'University of Illinois/NCSA Open Source License (BSD like)',
 100     ('University of Illinois/NCSA Open Source License (BSD like) '
 101      'MIT/X11 (BSD like)'),
 102 ]
 103
 104
 105 PATH_SPECIFIC_WHITELISTED_LICENSES = {
 106     'base/third_party/icu': [  # http://crbug.com/98087
 107         'UNKNOWN',
 108     ],
 109
 110     # http://code.google.com/p/google-breakpad/issues/detail?id=450
 111     'breakpad/src': [
 112         'UNKNOWN',
 113     ],
 114
 115     'buildtools/third_party/libc++/trunk/test': [
 116         # http://llvm.org/bugs/show_bug.cgi?id=18291
 117         'UNKNOWN',
 118     ],
 119
 120     'chrome/common/extensions/docs/examples': [  # http://crbug.com/98092
 121         'UNKNOWN',
 122     ],
 123     # This contains files copied from elsewhere from the tree. Since the copied
 124     # directories might have suppressions below (like simplejson), whitelist the
 125     # whole directory. This is also not shipped code.
 126     'chrome/common/extensions/docs/server2/third_party': [
 127         'UNKNOWN',
 128     ],
 129     'courgette/third_party/bsdiff_create.cc': [  # http://crbug.com/98095
 130         'UNKNOWN',
 131     ],
 132     'native_client': [  # http://crbug.com/98099
 133         'UNKNOWN',
 134     ],
 135     'native_client/toolchain': [
 136         'BSD GPL (v2 or later)',
 137         'BSD (2 clause) GPL (v2 or later)',
 138         'BSD (3 clause) GPL (v2 or later)',
 139         'BSL (v1.0) GPL',
 140         'BSL (v1.0) GPL (v3.1)',
 141         'GPL',
 142         'GPL (unversioned/unknown version)',
 143         'GPL (v2)',
 144         'GPL (v2 or later)',
 145         'GPL (v3.1)',
 146         'GPL (v3 or later)',
 147     ],
 148     'third_party/WebKit': [
 149         'UNKNOWN',
 150     ],
 151
 152     # http://code.google.com/p/angleproject/issues/detail?id=217
 153     'third_party/angle': [
 154         'UNKNOWN',
 155     ],
 156
 157     # http://crbug.com/222828
 158     # http://bugs.python.org/issue17514
 159     'third_party/chromite/third_party/argparse.py': [
 160         'UNKNOWN',
 161     ],
 162
 163     # http://crbug.com/326117
 164     # https://bitbucket.org/chrisatlee/poster/issue/21
 165     'third_party/chromite/third_party/poster': [
 166         'UNKNOWN',
 167     ],
 168
 169     # http://crbug.com/333508
 170     'third_party/clang_format/script': [
 171         'UNKNOWN',
 172     ],
 173
 174     # http://crbug.com/333508
 175     'buildtools/clang_format/script': [
 176         'UNKNOWN',
 177     ],
 178
 179     # https://mail.python.org/pipermail/cython-devel/2014-July/004062.html
 180     'third_party/cython': [
 181         'UNKNOWN',
 182     ],
 183
 184     'third_party/devscripts': [
 185         'GPL (v2 or later)',
 186     ],
 187     'third_party/expat/files/lib': [  # http://crbug.com/98121
 188         'UNKNOWN',
 189     ],
 190     'third_party/ffmpeg': [
 191         'GPL',
 192         'GPL (v2)',
 193         'GPL (v2 or later)',
 194         'GPL (v3 or later)',
 195         'UNKNOWN',  # http://crbug.com/98123
 196     ],
 197     'third_party/fontconfig': [
 198         # https://bugs.freedesktop.org/show_bug.cgi?id=73401
 199         'UNKNOWN',
 200     ],
 201     'third_party/freetype2': [ # http://crbug.com/177319
 202         'UNKNOWN',
 203     ],
 204     'third_party/hunspell': [  # http://crbug.com/98134
 205         'UNKNOWN',
 206     ],
 207     'third_party/iccjpeg': [  # http://crbug.com/98137
 208         'UNKNOWN',
 209     ],
 210     'third_party/icu': [  # http://crbug.com/98301
 211         'UNKNOWN',
 212     ],
 213     'third_party/jsoncpp/source': [
 214         # https://github.com/open-source-parsers/jsoncpp/issues/234
 215         'UNKNOWN',
 216     ],
 217     'third_party/lcov': [  # http://crbug.com/98304
 218         'UNKNOWN',
 219     ],
 220     'third_party/lcov/contrib/galaxy/genflat.pl': [
 221         'GPL (v2 or later)',
 222     ],
 223     'third_party/libevent': [  # http://crbug.com/98309
 224         'UNKNOWN',
 225     ],
 226     'third_party/libjingle/source/talk': [  # http://crbug.com/98310
 227         'UNKNOWN',
 228     ],
 229     'third_party/libjpeg_turbo': [  # http://crbug.com/98314
 230         'UNKNOWN',
 231     ],
 232
 233     # Many liblouis files are mirrored but not used in the NaCl module.
 234     # They are not excluded from the mirror because of lack of infrastructure
 235     # support.  Getting license headers added to the files where missing is
 236     # tracked in https://github.com/liblouis/liblouis/issues/22.
 237     'third_party/liblouis/src': [
 238         'GPL (v3 or later)',
 239         'UNKNOWN',
 240     ],
 241
 242     'third_party/libpng': [  # http://crbug.com/98318
 243         'UNKNOWN',
 244     ],
 245
 246     # The following files lack license headers, but are trivial.
 247     'third_party/libusb/src/libusb/os/poll_posix.h': [
 248         'UNKNOWN',
 249     ],
 250
 251     'third_party/libvpx/source': [  # http://crbug.com/98319
 252         'UNKNOWN',
 253     ],
 254     'third_party/libxml': [
 255         'UNKNOWN',
 256     ],
 257     'third_party/libxslt': [
 258         'UNKNOWN',
 259     ],
 260     'third_party/lzma_sdk': [
 261         'UNKNOWN',
 262     ],
 263     'third_party/mesa/src': [
 264         'GPL (v2)',
 265         'GPL (v3 or later)',
 266         'MIT/X11 (BSD like) GPL (v3 or later) with Bison parser exception',
 267         'UNKNOWN',  # http://crbug.com/98450
 268     ],
 269     'third_party/modp_b64': [
 270         'UNKNOWN',
 271     ],
 272     'third_party/openmax_dl/dl' : [
 273         'Khronos Group',
 274     ],
 275     'third_party/openssl': [  # http://crbug.com/98451
 276         'UNKNOWN',
 277     ],
 278     'third_party/boringssl': [
 279         # There are some files in BoringSSL which came from OpenSSL and have no
 280         # license in them. We don't wish to add the license header ourselves
 281         # thus we don't expect to pass license checks.
 282         'UNKNOWN',
 283     ],
 284     'third_party/ots/tools/ttf-checksum.py': [  # http://code.google.com/p/ots/issues/detail?id=2
 285         'UNKNOWN',
 286     ],
 287     'third_party/molokocacao': [  # http://crbug.com/98453
 288         'UNKNOWN',
 289     ],
 290     'third_party/ocmock/OCMock': [  # http://crbug.com/98454
 291         'UNKNOWN',
 292     ],
 293     'third_party/protobuf': [  # http://crbug.com/98455
 294         'UNKNOWN',
 295     ],
 296
 297     # https://bitbucket.org/ned/coveragepy/issue/313/add-license-file-containing-2-3-or-4
 298     # BSD 2-clause license.
 299     'third_party/pycoverage': [
 300         'UNKNOWN',
 301     ],
 302
 303     'third_party/pyelftools': [ # http://crbug.com/222831
 304         'UNKNOWN',
 305     ],
 306     'third_party/scons-2.0.1/engine/SCons': [  # http://crbug.com/98462
 307         'UNKNOWN',
 308     ],
 309     'third_party/simplejson': [
 310         'UNKNOWN',
 311     ],
 312     'third_party/skia': [  # http://crbug.com/98463
 313         'UNKNOWN',
 314     ],
 315     'third_party/snappy/src': [  # http://crbug.com/98464
 316         'UNKNOWN',
 317     ],
 318     'third_party/smhasher/src': [  # http://crbug.com/98465
 319         'UNKNOWN',
 320     ],
 321     'third_party/speech-dispatcher/libspeechd.h': [
 322         'GPL (v2 or later)',
 323     ],
 324     'third_party/sqlite': [
 325         'UNKNOWN',
 326     ],
 327
 328     # http://crbug.com/334668
 329     # MIT license.
 330     'tools/swarming_client/third_party/httplib2': [
 331         'UNKNOWN',
 332     ],
 333
 334     # http://crbug.com/334668
 335     # Apache v2.0.
 336     'tools/swarming_client/third_party/oauth2client': [
 337         'UNKNOWN',
 338     ],
 339
 340     # http://crbug.com/471372
 341     # BSD
 342     'tools/swarming_client/third_party/pyasn1': [
 343         'UNKNOWN',
 344     ],
 345
 346     # http://crbug.com/471372
 347     # Apache v2.0.
 348     'tools/swarming_client/third_party/rsa': [
 349         'UNKNOWN',
 350     ],
 351
 352     # https://github.com/kennethreitz/requests/issues/1610
 353     'tools/swarming_client/third_party/requests': [
 354         'UNKNOWN',
 355     ],
 356
 357     'third_party/swig/Lib/linkruntime.c': [  # http://crbug.com/98585
 358         'UNKNOWN',
 359     ],
 360     'third_party/talloc': [
 361         'GPL (v3 or later)',
 362         'UNKNOWN',  # http://crbug.com/98588
 363     ],
 364     'third_party/tcmalloc': [
 365         'UNKNOWN',  # http://crbug.com/98589
 366     ],
 367     'third_party/tlslite': [
 368         'UNKNOWN',
 369     ],
 370     'third_party/webdriver': [  # http://crbug.com/98590
 371         'UNKNOWN',
 372     ],
 373
 374     # https://github.com/html5lib/html5lib-python/issues/125
 375     # https://github.com/KhronosGroup/WebGL/issues/435
 376     'third_party/webgl/src': [
 377         'UNKNOWN',
 378     ],
 379
 380     'third_party/webrtc': [  # http://crbug.com/98592
 381         'UNKNOWN',
 382     ],
 383     'third_party/xdg-utils': [  # http://crbug.com/98593
 384         'UNKNOWN',
 385     ],
 386     'third_party/yasm/source': [  # http://crbug.com/98594
 387         'UNKNOWN',
 388     ],
 389     'third_party/zlib/contrib/minizip': [
 390         'UNKNOWN',
 391     ],
 392     'third_party/zlib/trees.h': [
 393         'UNKNOWN',
 394     ],
 395     'tools/emacs': [  # http://crbug.com/98595
 396         'UNKNOWN',
 397     ],
 398     'tools/gyp/test': [
 399         'UNKNOWN',
 400     ],
 401     'tools/python/google/__init__.py': [
 402         'UNKNOWN',
 403     ],
 404     'tools/stats_viewer/Properties/AssemblyInfo.cs': [
 405         'UNKNOWN',
 406     ],
 407     'tools/symsrc/pefile.py': [
 408         'UNKNOWN',
 409     ],
 410     # Not shipped, downloaded on trybots sometimes.
 411     'tools/telemetry/third_party/gsutil': [
 412         'BSD MIT/X11 (BSD like)',
 413         'UNKNOWN',
 414     ],
 415     'tools/telemetry/third_party/pyserial': [
 416         # https://sourceforge.net/p/pyserial/feature-requests/35/
 417         'UNKNOWN',
 418     ],
 419     'v8/test/cctest': [  # http://crbug.com/98597
 420         'UNKNOWN',
 421     ],
 422     'v8/src/third_party/kernel/tools/perf/util/jitdump.h': [  # http://crbug.com/391716
 423         'UNKNOWN',
 424     ],
 425 }
 426
 427
 428 def check_licenses(options, args):
 429   # Figure out which directory we have to check.
 430   if len(args) == 0:
 431     # No directory to check specified, use the repository root.
 432     start_dir = options.base_directory
 433   elif len(args) == 1:
 434     # Directory specified. Start here. It's supposed to be relative to the
 435     # base directory.
 436     start_dir = os.path.abspath(os.path.join(options.base_directory, args[0]))
 437   else:
 438     # More than one argument, we don't handle this.
 439     PrintUsage()
 440     return 1
 441
 442   print "Using base directory:", options.base_directory
 443   print "Checking:", start_dir
 444   print
 445
 446   licensecheck_path = os.path.abspath(os.path.join(options.base_directory,
 447                                                    'third_party',
 448                                                    'devscripts',
 449                                                    'licensecheck.pl'))
 450
 451   licensecheck = subprocess.Popen([licensecheck_path,
 452                                    '-l', '100',
 453                                    '-r', start_dir],
 454                                   stdout=subprocess.PIPE,
 455                                   stderr=subprocess.PIPE)
 456   stdout, stderr = licensecheck.communicate()
 457   if options.verbose:
 458     print '----------- licensecheck stdout -----------'
 459     print stdout
 460     print '--------- end licensecheck stdout ---------'
 461   if licensecheck.returncode != 0 or stderr:
 462     print '----------- licensecheck stderr -----------'
 463     print stderr
 464     print '--------- end licensecheck stderr ---------'
 465     print "\nFAILED\n"
 466     return 1
 467
 468   used_suppressions = set()
 469   errors = []
 470
 471   for line in stdout.splitlines():
 472     filename, license = line.split(':', 1)
 473     filename = os.path.relpath(filename.strip(), options.base_directory)
 474
 475     # All files in the build output directory are generated one way or another.
 476     # There's no need to check them.
 477     if filename.startswith('out/'):
 478       continue
 479
 480     # For now we're just interested in the license.
 481     license = license.replace('*No copyright*', '').strip()
 482
 483     # Skip generated files.
 484     if 'GENERATED FILE' in license:
 485       continue
 486
 487     if license in WHITELISTED_LICENSES:
 488       continue
 489
 490     if not options.ignore_suppressions:
 491       matched_prefixes = [
 492           prefix for prefix in PATH_SPECIFIC_WHITELISTED_LICENSES
 493           if filename.startswith(prefix) and
 494           license in PATH_SPECIFIC_WHITELISTED_LICENSES[prefix]]
 495       if matched_prefixes:
 496         used_suppressions.update(set(matched_prefixes))
 497         continue
 498
 499     errors.append({'filename': filename, 'license': license})
 500
 501   if options.json:
 502     with open(options.json, 'w') as f:
 503       json.dump(errors, f)
 504
 505   if errors:
 506     for error in errors:
 507       print "'%s' has non-whitelisted license '%s'" % (
 508           error['filename'], error['license'])
 509     print "\nFAILED\n"
 510     print "Please read",
 511     print "http://www.chromium.org/developers/adding-3rd-party-libraries"
 512     print "for more info how to handle the failure."
 513     print
 514     print "Please respect OWNERS of checklicenses.py. Changes violating"
 515     print "this requirement may be reverted."
 516
 517     # Do not print unused suppressions so that above message is clearly
 518     # visible and gets proper attention. Too much unrelated output
 519     # would be distracting and make the important points easier to miss.
 520
 521     return 1
 522
 523   print "\nSUCCESS\n"
 524
 525   if not len(args):
 526     unused_suppressions = set(
 527         PATH_SPECIFIC_WHITELISTED_LICENSES.iterkeys()).difference(
 528             used_suppressions)
 529     if unused_suppressions:
 530       print "\nNOTE: unused suppressions detected:\n"
 531       print '\n'.join(unused_suppressions)
 532
 533   return 0
 534
 535
 536 def main():
 537   default_root = os.path.abspath(
 538       os.path.join(os.path.dirname(__file__), '..', '..'))
 539   option_parser = optparse.OptionParser()
 540   option_parser.add_option('--root', default=default_root,
 541                            dest='base_directory',
 542                            help='Specifies the repository root. This defaults '
 543                            'to "../.." relative to the script file, which '
 544                            'will normally be the repository root.')
 545   option_parser.add_option('-v', '--verbose', action='store_true',
 546                            default=False, help='Print debug logging')
 547   option_parser.add_option('--ignore-suppressions',
 548                            action='store_true',
 549                            default=False,
 550                            help='Ignore path-specific license whitelist.')
 551   option_parser.add_option('--json', help='Path to JSON output file')
 552   options, args = option_parser.parse_args()
 553   return check_licenses(options, args)
 554
 555
 556 if '__main__' == __name__:
 557   sys.exit(main())