tools/lint/test/test_perfdocs_generation.py

   1 import os
   2 import pathlib
   3 from unittest import mock
   4
   5 import mozunit
   6
   7 LINTER = "perfdocs"
   8
   9
  10 def setup_sample_logger(logger, structured_logger, top_dir):
  11     from perfdocs.logger import PerfDocLogger
  12
  13     PerfDocLogger.LOGGER = structured_logger
  14     PerfDocLogger.PATHS = ["perfdocs"]
  15     PerfDocLogger.TOP_DIR = top_dir
  16
  17     import perfdocs.gatherer as gt
  18     import perfdocs.generator as gn
  19     import perfdocs.utils as utls
  20     import perfdocs.verifier as vf
  21
  22     gt.logger = logger
  23     vf.logger = logger
  24     gn.logger = logger
  25     utls.logger = logger
  26
  27
  28 @mock.patch("perfdocs.logger.PerfDocLogger")
  29 def test_perfdocs_generator_generate_perfdocs_pass(
  30     logger, structured_logger, perfdocs_sample
  31 ):
  32     from test_perfdocs import temp_file
  33
  34     top_dir = perfdocs_sample["top_dir"]
  35     setup_sample_logger(logger, structured_logger, top_dir)
  36
  37     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
  38     templates_dir.mkdir(parents=True, exist_ok=True)
  39
  40     from perfdocs.generator import Generator
  41     from perfdocs.verifier import Verifier
  42
  43     verifier = Verifier(top_dir)
  44     verifier.validate_tree()
  45
  46     generator = Generator(verifier, generate=True, workspace=top_dir)
  47     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
  48         generator.generate_perfdocs()
  49
  50     assert logger.warning.call_count == 0
  51
  52
  53 @mock.patch("perfdocs.logger.PerfDocLogger")
  54 def test_perfdocs_generator_generate_perfdocs_metrics_pass(
  55     logger, structured_logger, perfdocs_sample
  56 ):
  57     from test_perfdocs import temp_file
  58
  59     top_dir = perfdocs_sample["top_dir"]
  60     setup_sample_logger(logger, structured_logger, top_dir)
  61
  62     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
  63     templates_dir.mkdir(parents=True, exist_ok=True)
  64
  65     from perfdocs.generator import Generator
  66     from perfdocs.verifier import Verifier
  67
  68     sample_gatherer_result = {
  69         "suite": {"Example": {"metrics": ["fcp", "SpeedIndex"]}},
  70         "another_suite": {"Example": {}},
  71     }
  72     sample_test_list_result = {
  73         "suite": [{"metrics": ["fcp", "SpeedIndex"], "name": "Example"}],
  74         "another_suite": [{"name": "Example"}],
  75     }
  76
  77     with temp_file(
  78         "metrics.rst",
  79         tempdir=pathlib.Path(top_dir, "perfdocs"),
  80         content="{metrics_documentation}",
  81     ):
  82         with mock.patch(
  83             "perfdocs.framework_gatherers.RaptorGatherer.get_test_list"
  84         ) as m:
  85             m.return_value = sample_gatherer_result
  86             with perfdocs_sample["config"].open("w") as f1, perfdocs_sample[
  87                 "config_metrics"
  88             ].open("r") as f2:
  89                 # Overwrite content of config.yml with metrics config
  90                 f1.write(f2.read())
  91
  92             verifier = Verifier(top_dir)
  93             verifier.validate_tree()
  94
  95             verifier._gatherer.framework_gatherers["raptor"]._descriptions = (
  96                 sample_test_list_result
  97             )
  98
  99         generator = Generator(verifier, generate=True, workspace=top_dir)
 100         with temp_file(
 101             "index.rst", tempdir=templates_dir, content="{test_documentation}"
 102         ):
 103             generator.generate_perfdocs()
 104
 105         with pathlib.Path(generator.perfdocs_path, "raptor-metrics.rst").open() as f:
 106             metrics_content = f.read()
 107             assert "{metrics_documentation}" not in metrics_content
 108             assert "a description" in metrics_content
 109             assert "**Tests using it**" in metrics_content
 110
 111         with pathlib.Path(generator.perfdocs_path, "raptor.rst").open() as f:
 112             raptor_index_content = f.read()
 113             assert "{metrics_rst_name}" not in raptor_index_content
 114             assert "raptor-metrics" in raptor_index_content
 115
 116     assert logger.warning.call_count == 0
 117
 118
 119 @mock.patch("perfdocs.logger.PerfDocLogger")
 120 def test_perfdocs_generator_needed_regeneration(
 121     logger, structured_logger, perfdocs_sample
 122 ):
 123     top_dir = perfdocs_sample["top_dir"]
 124     setup_sample_logger(logger, structured_logger, top_dir)
 125
 126     from perfdocs.generator import Generator
 127     from perfdocs.verifier import Verifier
 128
 129     verifier = Verifier(top_dir)
 130     verifier.validate_tree()
 131
 132     generator = Generator(verifier, generate=False, workspace=top_dir)
 133     generator.generate_perfdocs()
 134
 135     expected = "PerfDocs need to be regenerated."
 136     args, _ = logger.warning.call_args
 137
 138     assert logger.warning.call_count == 1
 139     assert args[0] == expected
 140
 141
 142 @mock.patch("perfdocs.generator.get_changed_files", new=lambda x: [])
 143 @mock.patch("perfdocs.generator.ON_TRY", new=True)
 144 @mock.patch("perfdocs.logger.PerfDocLogger")
 145 def test_perfdocs_generator_needed_update(logger, structured_logger, perfdocs_sample):
 146     from test_perfdocs import temp_file
 147
 148     top_dir = perfdocs_sample["top_dir"]
 149     setup_sample_logger(logger, structured_logger, top_dir)
 150
 151     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 152     templates_dir.mkdir(parents=True, exist_ok=True)
 153
 154     from perfdocs.generator import Generator
 155     from perfdocs.verifier import Verifier
 156
 157     # Initializing perfdocs
 158     verifier = Verifier(top_dir)
 159     verifier.validate_tree()
 160
 161     generator = Generator(verifier, generate=True, workspace=top_dir)
 162     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 163         generator.generate_perfdocs()
 164
 165         # Removed file for testing and run again
 166         generator._generate = False
 167         files = [f for f in os.listdir(generator.perfdocs_path)]
 168         for f in files:
 169             os.remove(str(pathlib.Path(generator.perfdocs_path, f)))
 170
 171         generator.generate_perfdocs()
 172
 173     expected = (
 174         "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. "
 175         "You can also apply the perfdocs.diff patch file produced from this "
 176         "reviewbot test to fix the issue."
 177     )
 178     args, _ = logger.warning.call_args
 179
 180     assert logger.warning.call_count == 1
 181     assert args[0] == expected
 182
 183     # Check to ensure a diff was produced
 184     assert logger.log.call_count == 6
 185
 186     logs = [v[0][0] for v in logger.log.call_args_list]
 187     for failure_log in (
 188         "Some files are missing or are funny.",
 189         "Missing in existing docs: index.rst",
 190         "Missing in existing docs: mozperftest.rst",
 191     ):
 192         assert failure_log in logs
 193
 194
 195 @mock.patch("perfdocs.generator.get_changed_files", new=lambda x: [])
 196 @mock.patch("perfdocs.generator.ON_TRY", new=True)
 197 def test_perfdocs_generator_update_with_no_changes(structured_logger, perfdocs_sample):
 198     """This test ensures that when no changed files exist, we'll still trigger a failure."""
 199     from perfdocs.logger import PerfDocLogger
 200     from test_perfdocs import temp_file
 201
 202     top_dir = perfdocs_sample["top_dir"]
 203
 204     logger_mock = mock.MagicMock()
 205     PerfDocLogger.LOGGER = logger_mock
 206     PerfDocLogger.PATHS = ["perfdocs"]
 207     PerfDocLogger.TOP_DIR = top_dir
 208     logger = PerfDocLogger()
 209
 210     setup_sample_logger(logger, logger_mock, top_dir)
 211
 212     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 213     templates_dir.mkdir(parents=True, exist_ok=True)
 214
 215     from perfdocs.generator import Generator
 216     from perfdocs.verifier import Verifier
 217
 218     # Initializing perfdocs
 219     verifier = Verifier(top_dir)
 220     verifier.validate_tree()
 221
 222     generator = Generator(verifier, generate=True, workspace=top_dir)
 223     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 224         generator.generate_perfdocs()
 225
 226         # Removed file for testing and run again
 227         generator._generate = False
 228         files = [f for f in os.listdir(generator.perfdocs_path)]
 229         for f in files:
 230             os.remove(str(pathlib.Path(generator.perfdocs_path, f)))
 231
 232         generator.generate_perfdocs()
 233
 234     expected = (
 235         "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. "
 236         "You can also apply the perfdocs.diff patch file produced from this "
 237         "reviewbot test to fix the issue."
 238     )
 239     assert logger.LOGGER.lint_error.call_args is not None
 240     _, msg = logger.LOGGER.lint_error.call_args
 241
 242     assert logger.FAILED
 243     assert logger.LOGGER.lint_error.call_count == 1
 244     assert msg["message"] == expected
 245     assert msg["rule"] == "Flawless performance docs (unknown file)"
 246
 247
 248 @mock.patch("perfdocs.logger.PerfDocLogger")
 249 def test_perfdocs_generator_created_perfdocs(
 250     logger, structured_logger, perfdocs_sample
 251 ):
 252     from test_perfdocs import temp_file
 253
 254     top_dir = perfdocs_sample["top_dir"]
 255     setup_sample_logger(logger, structured_logger, top_dir)
 256
 257     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 258     templates_dir.mkdir(parents=True, exist_ok=True)
 259
 260     from perfdocs.generator import Generator
 261     from perfdocs.verifier import Verifier
 262
 263     verifier = Verifier(top_dir)
 264     verifier.validate_tree()
 265
 266     generator = Generator(verifier, generate=True, workspace=top_dir)
 267     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 268         perfdocs_tmpdir = generator._create_perfdocs()
 269
 270     files = [f for f in os.listdir(perfdocs_tmpdir)]
 271     files.sort()
 272     expected_files = ["index.rst", "mozperftest.rst"]
 273
 274     for i, file in enumerate(files):
 275         assert file == expected_files[i]
 276
 277     with pathlib.Path(perfdocs_tmpdir, expected_files[0]).open() as f:
 278         filedata = f.readlines()
 279     assert "".join(filedata) == "  * :doc:`mozperftest`"
 280
 281
 282 @mock.patch("perfdocs.logger.PerfDocLogger")
 283 def test_perfdocs_generator_build_perfdocs(logger, structured_logger, perfdocs_sample):
 284     top_dir = perfdocs_sample["top_dir"]
 285     setup_sample_logger(logger, structured_logger, top_dir)
 286
 287     from perfdocs.generator import Generator
 288     from perfdocs.verifier import Verifier
 289
 290     verifier = Verifier(top_dir)
 291     verifier.validate_tree()
 292
 293     generator = Generator(verifier, generate=True, workspace=top_dir)
 294     frameworks_info = generator.build_perfdocs_from_tree()
 295
 296     expected = ["dynamic", "metrics", "static"]
 297
 298     for framework in sorted(frameworks_info.keys()):
 299         for i, framework_info in enumerate(frameworks_info[framework]):
 300             assert framework_info == expected[i]
 301
 302
 303 @mock.patch("perfdocs.logger.PerfDocLogger")
 304 def test_perfdocs_generator_create_temp_dir(logger, structured_logger, perfdocs_sample):
 305     top_dir = perfdocs_sample["top_dir"]
 306     setup_sample_logger(logger, structured_logger, top_dir)
 307
 308     from perfdocs.generator import Generator
 309     from perfdocs.verifier import Verifier
 310
 311     verifier = Verifier(top_dir)
 312     verifier.validate_tree()
 313
 314     generator = Generator(verifier, generate=True, workspace=top_dir)
 315     tmpdir = generator._create_temp_dir()
 316
 317     assert pathlib.Path(tmpdir).is_dir()
 318
 319
 320 @mock.patch("perfdocs.logger.PerfDocLogger")
 321 def test_perfdocs_generator_create_temp_dir_fail(
 322     logger, structured_logger, perfdocs_sample
 323 ):
 324     top_dir = perfdocs_sample["top_dir"]
 325     setup_sample_logger(logger, structured_logger, top_dir)
 326
 327     from perfdocs.generator import Generator
 328     from perfdocs.verifier import Verifier
 329
 330     verifier = Verifier(top_dir)
 331     verifier.validate_tree()
 332
 333     generator = Generator(verifier, generate=True, workspace=top_dir)
 334     with mock.patch("perfdocs.generator.pathlib") as path_mock:
 335         path_mock.Path().mkdir.side_effect = OSError()
 336         path_mock.Path().is_dir.return_value = False
 337         tmpdir = generator._create_temp_dir()
 338
 339     expected = "Error creating temp file: "
 340     args, _ = logger.critical.call_args
 341
 342     assert not tmpdir
 343     assert logger.critical.call_count == 1
 344     assert args[0] == expected
 345
 346
 347 @mock.patch("perfdocs.logger.PerfDocLogger")
 348 def test_perfdocs_generator_save_perfdocs_pass(
 349     logger, structured_logger, perfdocs_sample
 350 ):
 351     from test_perfdocs import temp_file
 352
 353     top_dir = perfdocs_sample["top_dir"]
 354     setup_sample_logger(logger, structured_logger, top_dir)
 355
 356     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 357     templates_dir.mkdir(parents=True, exist_ok=True)
 358
 359     from perfdocs.generator import Generator
 360     from perfdocs.verifier import Verifier
 361
 362     verifier = Verifier(top_dir)
 363     verifier.validate_tree()
 364
 365     generator = Generator(verifier, generate=True, workspace=top_dir)
 366
 367     assert not generator.perfdocs_path.is_dir()
 368
 369     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 370         perfdocs_tmpdir = generator._create_perfdocs()
 371
 372     generator._save_perfdocs(perfdocs_tmpdir)
 373
 374     expected = ["index.rst", "mozperftest.rst"]
 375     files = [f for f in os.listdir(generator.perfdocs_path)]
 376     files.sort()
 377
 378     for i, file in enumerate(files):
 379         assert file == expected[i]
 380
 381
 382 @mock.patch("perfdocs.generator.shutil")
 383 @mock.patch("perfdocs.logger.PerfDocLogger")
 384 def test_perfdocs_generator_save_perfdocs_fail(
 385     logger, shutil, structured_logger, perfdocs_sample
 386 ):
 387     from test_perfdocs import temp_file
 388
 389     top_dir = perfdocs_sample["top_dir"]
 390     setup_sample_logger(logger, structured_logger, top_dir)
 391
 392     templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates")
 393     templates_dir.mkdir(parents=True, exist_ok=True)
 394
 395     from perfdocs.generator import Generator
 396     from perfdocs.verifier import Verifier
 397
 398     verifier = Verifier(top_dir)
 399     verifier.validate_tree()
 400
 401     generator = Generator(verifier, generate=True, workspace=top_dir)
 402     with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"):
 403         perfdocs_tmpdir = generator._create_perfdocs()
 404
 405     shutil.copytree = mock.Mock(side_effect=Exception())
 406     generator._save_perfdocs(perfdocs_tmpdir)
 407
 408     expected = "There was an error while saving the documentation: "
 409     args, _ = logger.critical.call_args
 410
 411     assert logger.critical.call_count == 1
 412     assert args[0] == expected
 413
 414
 415 if __name__ == "__main__":
 416     mozunit.main()