tools/lint/test/conftest.py

   1 import logging
   2 import os
   3 import pathlib
   4 import sys
   5 from collections import defaultdict
   6
   7 import pytest
   8 from mozbuild.base import MozbuildObject
   9 from mozlint.parser import Parser
  10 from mozlint.pathutils import findobject
  11 from mozlint.result import ResultSummary
  12 from mozlog.structuredlog import StructuredLogger
  13 from mozpack import path
  14
  15 here = path.abspath(path.dirname(__file__))
  16 build = MozbuildObject.from_environment(cwd=here, virtualenv_name="python-test")
  17
  18 lintdir = path.dirname(here)
  19 sys.path.insert(0, lintdir)
  20 logger = logging.getLogger("mozlint")
  21
  22
  23 def pytest_generate_tests(metafunc):
  24     """Finds, loads and returns the config for the linter name specified by the
  25     LINTER global variable in the calling module.
  26
  27     This implies that each test file (that uses this fixture) should only be
  28     used to test a single linter. If no LINTER variable is defined, the test
  29     will fail.
  30     """
  31     if "config" in metafunc.fixturenames:
  32         if not hasattr(metafunc.module, "LINTER"):
  33             pytest.fail(
  34                 "'config' fixture used from a module that didn't set the LINTER variable"
  35             )
  36
  37         name = metafunc.module.LINTER
  38         config_path = path.join(lintdir, "{}.yml".format(name))
  39         parser = Parser(build.topsrcdir)
  40         configs = parser.parse(config_path)
  41         config_names = {config["name"] for config in configs}
  42
  43         marker = metafunc.definition.get_closest_marker("lint_config")
  44         if marker:
  45             config_name = marker.kwargs["name"]
  46             if config_name not in config_names:
  47                 pytest.fail(f"lint config {config_name} not present in {name}.yml")
  48             configs = [
  49                 config for config in configs if config["name"] == marker.kwargs["name"]
  50             ]
  51
  52         ids = [config["name"] for config in configs]
  53         metafunc.parametrize("config", configs, ids=ids)
  54
  55
  56 @pytest.fixture(scope="module")
  57 def root(request):
  58     """Return the root directory for the files of the linter under test.
  59
  60     For example, with LINTER=flake8 this would be tools/lint/test/files/flake8.
  61     """
  62     if not hasattr(request.module, "LINTER"):
  63         pytest.fail(
  64             "'root' fixture used from a module that didn't set the LINTER variable"
  65         )
  66     return path.join(here, "files", request.module.LINTER)
  67
  68
  69 @pytest.fixture(scope="module")
  70 def paths(root):
  71     """Return a function that can resolve file paths relative to the linter
  72     under test.
  73
  74     Can be used like `paths('foo.py', 'bar/baz')`. This will return a list of
  75     absolute paths under the `root` files directory.
  76     """
  77
  78     def _inner(*paths):
  79         if not paths:
  80             return [root]
  81         return [path.normpath(path.join(root, p)) for p in paths]
  82
  83     return _inner
  84
  85
  86 @pytest.fixture(autouse=True)
  87 def run_setup(config):
  88     """Make sure that if the linter named in the LINTER global variable has a
  89     setup function, it gets called before running the tests.
  90     """
  91     if "setup" not in config:
  92         return
  93
  94     if config["name"] == "clang-format":
  95         # Skip the setup for the clang-format linter, as it requires a Mach context
  96         # (which we may not have if pytest is invoked directly).
  97         return
  98
  99     log = logging.LoggerAdapter(
 100         logger, {"lintname": config.get("name"), "pid": os.getpid()}
 101     )
 102
 103     func = findobject(config["setup"])
 104     func(
 105         build.topsrcdir,
 106         virtualenv_manager=build.virtualenv_manager,
 107         virtualenv_bin_path=build.virtualenv_manager.bin_path,
 108         log=log,
 109     )
 110
 111
 112 @pytest.fixture
 113 def lint(config, root, request):
 114     """Find and return the 'lint' function for the external linter named in the
 115     LINTER global variable.
 116
 117     This will automatically pass in the 'config' and 'root' arguments if not
 118     specified.
 119     """
 120
 121     if hasattr(request.module, "fixed"):
 122         request.module.fixed = 0
 123
 124     try:
 125         func = findobject(config["payload"])
 126     except (ImportError, ValueError):
 127         pytest.fail(
 128             "could not resolve a lint function from '{}'".format(config["payload"])
 129         )
 130
 131     ResultSummary.root = root
 132
 133     def wrapper(paths, config=config, root=root, collapse_results=False, **lintargs):
 134         logger.setLevel(logging.DEBUG)
 135         lintargs["log"] = logging.LoggerAdapter(
 136             logger, {"lintname": config.get("name"), "pid": os.getpid()}
 137         )
 138
 139         results = func(paths, config, root=root, **lintargs)
 140         if hasattr(request.module, "fixed") and isinstance(results, dict):
 141             request.module.fixed += results["fixed"]
 142
 143         if isinstance(results, dict):
 144             results = results["results"]
 145
 146         if isinstance(results, (list, tuple)):
 147             results = sorted(results)
 148
 149         if not collapse_results:
 150             return results
 151
 152         ret = defaultdict(list)
 153         for r in results:
 154             ret[r.relpath].append(r)
 155         return ret
 156
 157     return wrapper
 158
 159
 160 @pytest.fixture
 161 def structuredlog_lint(config, root, logger=None):
 162     """Find and return the 'lint' function for the external linter named in the
 163     LINTER global variable. This variant of the lint function is for linters that
 164     use the 'structuredlog' type.
 165
 166     This will automatically pass in the 'config' and 'root' arguments if not
 167     specified.
 168     """
 169     try:
 170         func = findobject(config["payload"])
 171     except (ImportError, ValueError):
 172         pytest.fail(
 173             "could not resolve a lint function from '{}'".format(config["payload"])
 174         )
 175
 176     ResultSummary.root = root
 177
 178     if not logger:
 179         logger = structured_logger()
 180
 181     def wrapper(
 182         paths,
 183         config=config,
 184         root=root,
 185         logger=logger,
 186         collapse_results=False,
 187         **lintargs,
 188     ):
 189         lintargs["log"] = logging.LoggerAdapter(
 190             logger, {"lintname": config.get("name"), "pid": os.getpid()}
 191         )
 192         results = func(paths, config, root=root, logger=logger, **lintargs)
 193         if not collapse_results:
 194             return results
 195
 196         ret = defaultdict(list)
 197         for r in results:
 198             ret[r.path].append(r)
 199         return ret
 200
 201     return wrapper
 202
 203
 204 @pytest.fixture
 205 def global_lint(config, root, request):
 206     try:
 207         func = findobject(config["payload"])
 208     except (ImportError, ValueError):
 209         pytest.fail(
 210             "could not resolve a lint function from '{}'".format(config["payload"])
 211         )
 212
 213     ResultSummary.root = root
 214
 215     def wrapper(config=config, root=root, collapse_results=False, **lintargs):
 216         logger.setLevel(logging.DEBUG)
 217         lintargs["log"] = logging.LoggerAdapter(
 218             logger, {"lintname": config.get("name"), "pid": os.getpid()}
 219         )
 220         results = func(config, root=root, **lintargs)
 221         if hasattr(request.module, "fixed") and isinstance(results, dict):
 222             request.module.fixed += results["fixed"]
 223
 224         if isinstance(results, dict):
 225             results = results["results"]
 226
 227         if isinstance(results, (list, tuple)):
 228             results = sorted(results)
 229
 230         if not collapse_results:
 231             return results
 232
 233         ret = defaultdict(list)
 234         for r in results:
 235             ret[r.relpath].append(r)
 236         return ret
 237
 238     return wrapper
 239
 240
 241 @pytest.fixture
 242 def create_temp_file(tmpdir):
 243     def inner(contents, name=None):
 244         name = name or "temp.py"
 245         path = tmpdir.join(name)
 246         path.write(contents)
 247         return path.strpath
 248
 249     return inner
 250
 251
 252 @pytest.fixture
 253 def structured_logger():
 254     return StructuredLogger("logger")
 255
 256
 257 @pytest.fixture
 258 def perfdocs_sample():
 259     from test_perfdocs import (
 260         DYNAMIC_SAMPLE_CONFIG,
 261         SAMPLE_CONFIG,
 262         SAMPLE_INI,
 263         SAMPLE_METRICS_CONFIG,
 264         SAMPLE_TEST,
 265         temp_dir,
 266         temp_file,
 267     )
 268
 269     with temp_dir() as tmpdir:
 270         suite_dir = pathlib.Path(tmpdir, "suite")
 271         raptor_dir = pathlib.Path(tmpdir, "raptor")
 272         raptor_suitedir = pathlib.Path(tmpdir, "raptor", "suite")
 273         raptor_another_suitedir = pathlib.Path(tmpdir, "raptor", "another_suite")
 274         perfdocs_dir = pathlib.Path(tmpdir, "perfdocs")
 275
 276         perfdocs_dir.mkdir(parents=True, exist_ok=True)
 277         suite_dir.mkdir(parents=True, exist_ok=True)
 278         raptor_dir.mkdir(parents=True, exist_ok=True)
 279         raptor_suitedir.mkdir(parents=True, exist_ok=True)
 280         raptor_another_suitedir.mkdir(parents=True, exist_ok=True)
 281
 282         with temp_file(
 283             "perftest.toml", tempdir=suite_dir, content='["perftest_sample.js"]'
 284         ) as tmpmanifest, temp_file(
 285             "raptor_example1.ini", tempdir=raptor_suitedir, content=SAMPLE_INI
 286         ) as tmpexample1manifest, temp_file(
 287             "raptor_example2.ini", tempdir=raptor_another_suitedir, content=SAMPLE_INI
 288         ) as tmpexample2manifest, temp_file(
 289             "perftest_sample.js", tempdir=suite_dir, content=SAMPLE_TEST
 290         ) as tmptest, temp_file(
 291             "config.yml", tempdir=perfdocs_dir, content=SAMPLE_CONFIG
 292         ) as tmpconfig, temp_file(
 293             "config_2.yml", tempdir=perfdocs_dir, content=DYNAMIC_SAMPLE_CONFIG
 294         ) as tmpconfig_2, temp_file(
 295             "config_metrics.yml", tempdir=perfdocs_dir, content=SAMPLE_METRICS_CONFIG
 296         ) as tmpconfig_metrics, temp_file(
 297             "index.rst",
 298             tempdir=perfdocs_dir,
 299             content="{metrics_rst_name}{documentation}",
 300         ) as tmpindex:
 301             yield {
 302                 "top_dir": tmpdir,
 303                 "manifest": {"path": tmpmanifest},
 304                 "example1_manifest": tmpexample1manifest,
 305                 "example2_manifest": tmpexample2manifest,
 306                 "test": tmptest,
 307                 "config": tmpconfig,
 308                 "config_2": tmpconfig_2,
 309                 "config_metrics": tmpconfig_metrics,
 310                 "index": tmpindex,
 311             }