tools/lint/android/lints.py

   1 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
   2 # vim: set filetype=python:
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 import glob
   8 import itertools
   9 import json
  10 import os
  11 import re
  12 import shlex
  13 import subprocess
  14 import sys
  15 import xml.etree.ElementTree as ET
  16
  17 import mozpack.path as mozpath
  18 from mozlint import result
  19 from mozpack.files import FileFinder
  20
  21 # The Gradle target invocations are serialized with a simple locking file scheme.  It's fine for
  22 # them to take a while, since the first will compile all the Java, etc, and then perform
  23 # potentially expensive static analyses.
  24 GRADLE_LOCK_MAX_WAIT_SECONDS = 20 * 60
  25
  26 EXCLUSION_FILES = [
  27     os.path.join("tools", "rewriting", "Generated.txt"),
  28     os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"),
  29 ]
  30
  31
  32 def setup(root, **setupargs):
  33     if setupargs.get("substs", {}).get("MOZ_BUILD_APP") != "mobile/android":
  34         return -1
  35
  36     if "topobjdir" not in setupargs:
  37         setupargs["log"].debug(
  38             f"Skipping {setupargs['name']}: a configured Android build is required!"
  39         )
  40         return -1
  41
  42     return 0
  43
  44
  45 def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose=True):
  46     sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android"))
  47     from gradle import gradle_lock
  48
  49     with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS):
  50         # The android-lint parameter can be used by gradle tasks to run special
  51         # logic when they are run for a lint using
  52         #   project.hasProperty('android-lint')
  53         cmd_args = (
  54             [
  55                 sys.executable,
  56                 os.path.join(topsrcdir, "mach"),
  57                 "gradle",
  58                 "--verbose",
  59                 "-Pandroid-lint",
  60                 "--",
  61             ]
  62             + tasks
  63             + extra_args
  64         )
  65
  66         cmd = " ".join(shlex.quote(arg) for arg in cmd_args)
  67         log.debug(cmd)
  68
  69         # Gradle and mozprocess do not get along well, so we use subprocess
  70         # directly.
  71         proc = subprocess.Popen(cmd_args, cwd=topsrcdir)
  72         status = None
  73         # Leave it to the subprocess to handle Ctrl+C. If it terminates as a result
  74         # of Ctrl+C, proc.wait() will return a status code, and, we get out of the
  75         # loop. If it doesn't, like e.g. gdb, we continue waiting.
  76         while status is None:
  77             try:
  78                 status = proc.wait()
  79             except KeyboardInterrupt:
  80                 pass
  81
  82         try:
  83             proc.wait()
  84         except KeyboardInterrupt:
  85             proc.kill()
  86             raise
  87
  88         return proc.returncode
  89
  90
  91 def gradlew(log, topsrcdir=None, topobjdir=None, tasks=[], cwd=None):
  92     sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android"))
  93     from gradle import gradle_lock
  94
  95     with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS):
  96         # The android-lint parameter can be used by gradle tasks to run special
  97         # logic when they are run for a lint using
  98         #   project.hasProperty('android-lint')
  99         cmd_args = ["./gradlew"] + tasks
 100
 101         cmd = " ".join(shlex.quote(arg) for arg in cmd_args)
 102         log.debug(cmd)
 103
 104         # Gradle and mozprocess do not get along well, so we use subprocess
 105         # directly.
 106         proc = subprocess.Popen(cmd_args, cwd=cwd)
 107         status = None
 108         # Leave it to the subprocess to handle Ctrl+C. If it terminates as a result
 109         # of Ctrl+C, proc.wait() will return a status code, and, we get out of the
 110         # loop. If it doesn't, like e.g. gdb, we continue waiting.
 111         while status is None:
 112             try:
 113                 status = proc.wait()
 114             except KeyboardInterrupt:
 115                 pass
 116
 117         try:
 118             proc.wait()
 119         except KeyboardInterrupt:
 120             proc.kill()
 121             raise
 122
 123         return proc.returncode
 124
 125
 126 def format(config, fix=None, **lintargs):
 127     topsrcdir = lintargs["root"]
 128     topobjdir = lintargs["topobjdir"]
 129
 130     if fix:
 131         tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FIX_TASKS"]
 132     else:
 133         tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_CHECK_TASKS"]
 134
 135     ret = gradle(
 136         lintargs["log"],
 137         topsrcdir=topsrcdir,
 138         topobjdir=topobjdir,
 139         tasks=tasks,
 140         extra_args=lintargs.get("extra_args") or [],
 141     )
 142
 143     results = []
 144     for path in lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FOLDERS"]:
 145         folder = os.path.join(
 146             topobjdir, "gradle", "build", path, "spotless", "spotlessJava"
 147         )
 148         for filename in glob.iglob(folder + "/**/*.java", recursive=True):
 149             err = {
 150                 "rule": "spotless-java",
 151                 "path": os.path.join(
 152                     topsrcdir, path, mozpath.relpath(filename, folder)
 153                 ),
 154                 "lineno": 0,
 155                 "column": 0,
 156                 "message": "Formatting error, please run ./mach lint -l android-format --fix",
 157                 "level": "error",
 158             }
 159             results.append(result.from_config(config, **err))
 160         folder = os.path.join(
 161             topobjdir, "gradle", "build", path, "spotless", "spotlessKotlin"
 162         )
 163         for filename in glob.iglob(folder + "/**/*.kt", recursive=True):
 164             err = {
 165                 "rule": "spotless-kt",
 166                 "path": os.path.join(
 167                     topsrcdir, path, mozpath.relpath(filename, folder)
 168                 ),
 169                 "lineno": 0,
 170                 "column": 0,
 171                 "message": "Formatting error, please run ./mach lint -l android-format --fix",
 172                 "level": "error",
 173             }
 174             results.append(result.from_config(config, **err))
 175
 176     if len(results) == 0 and ret != 0:
 177         # spotless seems to hit unfixed error.
 178         err = {
 179             "rule": "spotless",
 180             "path": "",
 181             "lineno": 0,
 182             "column": 0,
 183             "message": "Unexpected error",
 184             "level": "error",
 185         }
 186         results.append(result.from_config(config, **err))
 187
 188     # If --fix was passed, we just report the number of files that were changed
 189     if fix:
 190         return {"results": [], "fixed": len(results)}
 191     return results
 192
 193
 194 def fenix_format(config, fix=None, **lintargs):
 195     return report_gradlew(
 196         config,
 197         fix,
 198         os.path.join("mobile", "android", "fenix"),
 199         **lintargs,
 200     )
 201
 202
 203 def ac_format(config, fix=None, **lintargs):
 204     return report_gradlew(
 205         config,
 206         fix,
 207         os.path.join("mobile", "android", "android-components"),
 208         **lintargs,
 209     )
 210
 211
 212 def focus_format(config, fix=None, **lintargs):
 213     return report_gradlew(
 214         config,
 215         fix,
 216         os.path.join("mobile", "android", "focus-android"),
 217         **lintargs,
 218     )
 219
 220
 221 def report_gradlew(config, fix, subdir, **lintargs):
 222     topsrcdir = lintargs["root"]
 223     topobjdir = lintargs["topobjdir"]
 224
 225     if fix:
 226         tasks = ["ktlintFormat", "detekt"]
 227     else:
 228         tasks = ["ktlint", "detekt"]
 229
 230     for task in tasks:
 231         gradlew(
 232             lintargs["log"],
 233             topsrcdir=topsrcdir,
 234             topobjdir=topobjdir,
 235             tasks=[task],
 236             cwd=os.path.join(topsrcdir, subdir),
 237         )
 238
 239     reports = os.path.join(topsrcdir, subdir, "build", "reports")
 240     results = []
 241
 242     excludes = []
 243     for path in EXCLUSION_FILES:
 244         with open(os.path.join(topsrcdir, path), "r") as fh:
 245             for f in fh.readlines():
 246                 if "*" in f:
 247                     excludes.extend(glob.glob(f.strip()))
 248                 elif f.startswith(subdir):
 249                     excludes.append(f.strip())
 250
 251     try:
 252         tree = ET.parse(
 253             open(
 254                 os.path.join(
 255                     reports,
 256                     "detekt",
 257                     "detekt.xml",
 258                 ),
 259                 "rt",
 260             )
 261         )
 262         root = tree.getroot()
 263
 264         for file in root.findall("file"):
 265             name = file.get("name")
 266             if is_excluded_file(topsrcdir, excludes, name):
 267                 continue
 268             for error in file:
 269                 err = {
 270                     "rule": error.get("source"),
 271                     "path": name,
 272                     "lineno": int(error.get("line") or 0),
 273                     "column": int(error.get("column") or 0),
 274                     "message": error.get("message"),
 275                     "level": "error",
 276                 }
 277                 results.append(result.from_config(config, **err))
 278     except FileNotFoundError:
 279         pass
 280
 281     ktlint_file = "ktlint.json"
 282     if fix:
 283         ktlint_file = "ktlintFormat.json"
 284     try:
 285         issues = json.load(
 286             open(
 287                 os.path.join(
 288                     reports,
 289                     "ktlint",
 290                     ktlint_file,
 291                 ),
 292                 "rt",
 293             )
 294         )
 295
 296         for issue in issues:
 297             name = issue["file"]
 298             if is_excluded_file(topsrcdir, excludes, name):
 299                 continue
 300             for error in issue["errors"]:
 301                 err = {
 302                     "rule": error["rule"],
 303                     "path": name,
 304                     "lineno": error["line"],
 305                     "column": error["column"],
 306                     "message": error["message"],
 307                     "level": "error",
 308                 }
 309                 results.append(result.from_config(config, **err))
 310     except FileNotFoundError:
 311         pass
 312
 313     return results
 314
 315
 316 def is_excluded_file(topsrcdir, excludes, file):
 317     for path in excludes:
 318         if file.startswith(os.path.join(topsrcdir, path)):
 319             return True
 320     return False
 321
 322
 323 def api_lint(config, **lintargs):
 324     topsrcdir = lintargs["root"]
 325     topobjdir = lintargs["topobjdir"]
 326
 327     gradle(
 328         lintargs["log"],
 329         topsrcdir=topsrcdir,
 330         topobjdir=topobjdir,
 331         tasks=lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"],
 332         extra_args=lintargs.get("extra_args") or [],
 333     )
 334
 335     folder = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER"]
 336
 337     results = []
 338
 339     with open(os.path.join(topobjdir, folder, "apilint-result.json")) as f:
 340         issues = json.load(f)
 341
 342         for rule in ("compat_failures", "failures"):
 343             for r in issues[rule]:
 344                 err = {
 345                     "rule": r["rule"] if rule == "failures" else "compat_failures",
 346                     "path": r["file"],
 347                     "lineno": int(r["line"]),
 348                     "column": int(r.get("column") or 0),
 349                     "message": r["msg"],
 350                     "level": "error" if r["error"] else "warning",
 351                 }
 352                 results.append(result.from_config(config, **err))
 353
 354         for r in issues["api_changes"]:
 355             err = {
 356                 "rule": "api_changes",
 357                 "path": r["file"],
 358                 "lineno": int(r["line"]),
 359                 "column": int(r.get("column") or 0),
 360                 "message": "Unexpected api change. Please run ./mach gradle {} for more "
 361                 "information".format(
 362                     " ".join(lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"])
 363                 ),
 364             }
 365             results.append(result.from_config(config, **err))
 366
 367     return results
 368
 369
 370 def javadoc(config, **lintargs):
 371     topsrcdir = lintargs["root"]
 372     topobjdir = lintargs["topobjdir"]
 373
 374     gradle(
 375         lintargs["log"],
 376         topsrcdir=topsrcdir,
 377         topobjdir=topobjdir,
 378         tasks=lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"],
 379         extra_args=lintargs.get("extra_args") or [],
 380     )
 381
 382     output_files = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES"]
 383
 384     results = []
 385
 386     for output_file in output_files:
 387         with open(os.path.join(topobjdir, output_file)) as f:
 388             # Like: '[{"path":"/absolute/path/to/topsrcdir/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java","lineno":"462","level":"warning","message":"no @return"}]'.  # NOQA: E501
 389             issues = json.load(f)
 390
 391             for issue in issues:
 392                 # We want warnings to be errors for linting purposes.
 393                 # TODO: Bug 1316188 - resolve missing javadoc comments
 394                 issue["level"] = (
 395                     "error" if issue["message"] != ": no comment" else "warning"
 396                 )
 397                 results.append(result.from_config(config, **issue))
 398
 399     return results
 400
 401
 402 def lint(config, **lintargs):
 403     topsrcdir = lintargs["root"]
 404     topobjdir = lintargs["topobjdir"]
 405
 406     gradle(
 407         lintargs["log"],
 408         topsrcdir=topsrcdir,
 409         topobjdir=topobjdir,
 410         tasks=lintargs["substs"]["GRADLE_ANDROID_LINT_TASKS"],
 411         extra_args=lintargs.get("extra_args") or [],
 412     )
 413
 414     path = os.path.join(
 415         lintargs["topobjdir"],
 416         "gradle/build/mobile/android/geckoview/reports",
 417         "lint-results-{}.xml".format(
 418             lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"]
 419         ),
 420     )
 421     tree = ET.parse(open(path, "rt"))
 422     root = tree.getroot()
 423
 424     results = []
 425
 426     for issue in root.findall("issue"):
 427         location = issue[0]
 428         if "third_party" in location.get("file") or "thirdparty" in location.get(
 429             "file"
 430         ):
 431             continue
 432         err = {
 433             "level": issue.get("severity").lower(),
 434             "rule": issue.get("id"),
 435             "message": issue.get("message"),
 436             "path": location.get("file"),
 437             "lineno": int(location.get("line") or 0),
 438         }
 439         results.append(result.from_config(config, **err))
 440
 441     return results
 442
 443
 444 def _parse_checkstyle_output(config, topsrcdir=None, report_path=None):
 445     tree = ET.parse(open(report_path, "rt"))
 446     root = tree.getroot()
 447
 448     for file in root.findall("file"):
 449         for error in file.findall("error"):
 450             # Like <error column="42" line="22" message="Name 'mPorts' must match pattern 'xm[A-Z][A-Za-z]*$'." severity="error" source="com.puppycrawl.tools.checkstyle.checks.naming.MemberNameCheck" />.  # NOQA: E501
 451             err = {
 452                 "level": "error",
 453                 "rule": error.get("source"),
 454                 "message": error.get("message"),
 455                 "path": file.get("name"),
 456                 "lineno": int(error.get("line") or 0),
 457                 "column": int(error.get("column") or 0),
 458             }
 459             yield result.from_config(config, **err)
 460
 461
 462 def checkstyle(config, **lintargs):
 463     topsrcdir = lintargs["root"]
 464     topobjdir = lintargs["topobjdir"]
 465
 466     gradle(
 467         lintargs["log"],
 468         topsrcdir=topsrcdir,
 469         topobjdir=topobjdir,
 470         tasks=lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_TASKS"],
 471         extra_args=lintargs.get("extra_args") or [],
 472     )
 473
 474     results = []
 475
 476     for relative_path in lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES"]:
 477         report_path = os.path.join(lintargs["topobjdir"], relative_path)
 478         results.extend(
 479             _parse_checkstyle_output(
 480                 config, topsrcdir=lintargs["root"], report_path=report_path
 481             )
 482         )
 483
 484     return results
 485
 486
 487 def _parse_android_test_results(config, topsrcdir=None, report_dir=None):
 488     # A brute force way to turn a Java FQN into a path on disk.  Assumes Java
 489     # and Kotlin sources are in mobile/android for performance and simplicity.
 490     sourcepath_finder = FileFinder(os.path.join(topsrcdir, "mobile", "android"))
 491
 492     finder = FileFinder(report_dir)
 493     reports = list(finder.find("TEST-*.xml"))
 494     if not reports:
 495         raise RuntimeError("No reports found under {}".format(report_dir))
 496
 497     for report, _ in reports:
 498         tree = ET.parse(open(os.path.join(finder.base, report), "rt"))
 499         root = tree.getroot()
 500
 501         class_name = root.get(
 502             "name"
 503         )  # Like 'org.mozilla.gecko.permissions.TestPermissions'.
 504         path = (
 505             "**/" + class_name.replace(".", "/") + ".*"
 506         )  # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'.  # NOQA: E501
 507
 508         for testcase in root.findall("testcase"):
 509             function_name = testcase.get("name")
 510
 511             # Schema cribbed from http://llg.cubic.org/docs/junit/.
 512             for unexpected in itertools.chain(
 513                 testcase.findall("error"), testcase.findall("failure")
 514             ):
 515                 sourcepaths = list(sourcepath_finder.find(path))
 516                 if not sourcepaths:
 517                     raise RuntimeError(
 518                         "No sourcepath found for class {class_name}".format(
 519                             class_name=class_name
 520                         )
 521                     )
 522
 523                 for sourcepath, _ in sourcepaths:
 524                     lineno = 0
 525                     message = unexpected.get("message")
 526                     # Turn '... at org.mozilla.gecko.permissions.TestPermissions.testMultipleRequestsAreQueuedAndDispatchedSequentially(TestPermissions.java:118)' into 118.  # NOQA: E501
 527                     pattern = r"at {class_name}\.{function_name}\(.*:(\d+)\)"
 528                     pattern = pattern.format(
 529                         class_name=class_name, function_name=function_name
 530                     )
 531                     match = re.search(pattern, message)
 532                     if match:
 533                         lineno = int(match.group(1))
 534                     else:
 535                         msg = "No source line found for {class_name}.{function_name}".format(
 536                             class_name=class_name, function_name=function_name
 537                         )
 538                         raise RuntimeError(msg)
 539
 540                     err = {
 541                         "level": "error",
 542                         "rule": unexpected.get("type"),
 543                         "message": message,
 544                         "path": os.path.join(
 545                             topsrcdir, "mobile", "android", sourcepath
 546                         ),
 547                         "lineno": lineno,
 548                     }
 549                     yield result.from_config(config, **err)
 550
 551
 552 def test(config, **lintargs):
 553     topsrcdir = lintargs["root"]
 554     topobjdir = lintargs["topobjdir"]
 555
 556     gradle(
 557         lintargs["log"],
 558         topsrcdir=topsrcdir,
 559         topobjdir=topobjdir,
 560         tasks=lintargs["substs"]["GRADLE_ANDROID_TEST_TASKS"],
 561         extra_args=lintargs.get("extra_args") or [],
 562     )
 563
 564     results = []
 565
 566     def capitalize(s):
 567         # Can't use str.capitalize because it lower cases trailing letters.
 568         return (s[0].upper() + s[1:]) if s else ""
 569
 570     pairs = [("geckoview", lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"])]
 571     for project, variant in pairs:
 572         report_dir = os.path.join(
 573             lintargs["topobjdir"],
 574             "gradle/build/mobile/android/{}/test-results/test{}UnitTest".format(
 575                 project, capitalize(variant)
 576             ),
 577         )
 578     results.extend(
 579         _parse_android_test_results(
 580             config, topsrcdir=lintargs["root"], report_dir=report_dir
 581         )
 582     )
 583
 584     return results